![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
1.1.1.3 root 37: #include <kern/printf.h>
1.1 root 38: #include <mach/kern_return.h>
39: #include <mach/port.h>
40: #include <mach/vm_attributes.h>
41: #include <mach/vm_param.h>
42: #include <kern/assert.h>
1.1.1.3 root 43: #include <kern/debug.h>
44: #include <kern/kalloc.h>
1.1.1.7 ! root 45: #include <kern/list.h>
1.1.1.3 root 46: #include <kern/rbtree.h>
47: #include <kern/slab.h>
48: #include <vm/pmap.h>
1.1 root 49: #include <vm/vm_fault.h>
50: #include <vm/vm_map.h>
51: #include <vm/vm_object.h>
52: #include <vm/vm_page.h>
1.1.1.3 root 53: #include <vm/vm_resident.h>
1.1 root 54: #include <vm/vm_kern.h>
55: #include <ipc/ipc_port.h>
56:
1.1.1.3 root 57: #if MACH_KDB
58: #include <ddb/db_output.h>
59: #include <vm/vm_print.h>
60: #endif /* MACH_KDB */
61:
1.1 root 62: /*
63: * Macros to copy a vm_map_entry. We must be careful to correctly
64: * manage the wired page count. vm_map_entry_copy() creates a new
65: * map entry to the same memory - the wired count in the new entry
66: * must be set to zero. vm_map_entry_copy_full() creates a new
67: * entry that is identical to the old entry. This preserves the
1.1.1.3 root 68: * wire count; it's used for map splitting and cache changing in
1.1 root 69: * vm_map_copyout.
70: */
71: #define vm_map_entry_copy(NEW,OLD) \
72: MACRO_BEGIN \
73: *(NEW) = *(OLD); \
74: (NEW)->is_shared = FALSE; \
75: (NEW)->needs_wakeup = FALSE; \
76: (NEW)->in_transition = FALSE; \
77: (NEW)->wired_count = 0; \
78: (NEW)->user_wired_count = 0; \
79: MACRO_END
80:
81: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
82:
83: /*
84: * Virtual memory maps provide for the mapping, protection,
85: * and sharing of virtual memory objects. In addition,
86: * this module provides for an efficient virtual copy of
87: * memory from one map to another.
88: *
89: * Synchronization is required prior to most operations.
90: *
91: * Maps consist of an ordered doubly-linked list of simple
1.1.1.3 root 92: * entries; a hint and a red-black tree are used to speed up lookups.
1.1 root 93: *
94: * Sharing maps have been deleted from this version of Mach.
95: * All shared objects are now mapped directly into the respective
96: * maps. This requires a change in the copy on write strategy;
97: * the asymmetric (delayed) strategy is used for shared temporary
98: * objects instead of the symmetric (shadow) strategy. This is
99: * selected by the (new) use_shared_copy bit in the object. See
100: * vm_object_copy_temporary in vm_object.c for details. All maps
101: * are now "top level" maps (either task map, kernel map or submap
1.1.1.2 root 102: * of the kernel map).
1.1 root 103: *
104: * Since portions of maps are specified by start/end addreses,
105: * which may not align with existing map entries, all
106: * routines merely "clip" entries to these start/end values.
107: * [That is, an entry is split into two, bordering at a
108: * start or end value.] Note that these clippings may not
109: * always be necessary (as the two resulting entries are then
110: * not changed); however, the clipping is done for convenience.
111: * No attempt is currently made to "glue back together" two
112: * abutting entries.
113: *
114: * The symmetric (shadow) copy strategy implements virtual copy
115: * by copying VM object references from one map to
116: * another, and then marking both regions as copy-on-write.
117: * It is important to note that only one writeable reference
118: * to a VM object region exists in any map when this strategy
119: * is used -- this means that shadow object creation can be
120: * delayed until a write operation occurs. The asymmetric (delayed)
121: * strategy allows multiple maps to have writeable references to
122: * the same region of a vm object, and hence cannot delay creating
123: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
124: * Copying of permanent objects is completely different; see
125: * vm_object_copy_strategically() in vm_object.c.
126: */
127:
1.1.1.3 root 128: struct kmem_cache vm_map_cache; /* cache for vm_map structures */
129: struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
130: struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
1.1 root 131:
132: /*
133: * Placeholder object for submap operations. This object is dropped
134: * into the range by a call to vm_map_find, and removed when
135: * vm_map_submap creates the submap.
136: */
137:
1.1.1.3 root 138: static struct vm_object vm_submap_object_store;
139: vm_object_t vm_submap_object = &vm_submap_object_store;
1.1 root 140:
141: /*
142: * vm_map_init:
143: *
144: * Initialize the vm_map module. Must be called before
145: * any other vm_map routines.
146: *
1.1.1.3 root 147: * Map and entry structures are allocated from caches -- we must
148: * initialize those caches.
1.1 root 149: *
1.1.1.6 root 150: * There are two caches of interest:
1.1 root 151: *
1.1.1.3 root 152: * vm_map_cache: used to allocate maps.
153: * vm_map_entry_cache: used to allocate map entries.
154: *
1.1.1.6 root 155: * We make sure the map entry cache allocates memory directly from the
156: * physical allocator to avoid recursion with this module.
1.1 root 157: */
158:
1.1.1.3 root 159: void vm_map_init(void)
160: {
161: kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
1.1.1.6 root 162: NULL, 0);
1.1.1.3 root 163: kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
1.1.1.6 root 164: sizeof(struct vm_map_entry), 0, NULL,
165: KMEM_CACHE_NOOFFSLAB | KMEM_CACHE_PHYSMEM);
1.1.1.3 root 166: kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
1.1.1.6 root 167: sizeof(struct vm_map_copy), 0, NULL, 0);
1.1 root 168:
169: /*
170: * Submap object is initialized by vm_object_init.
171: */
172: }
173:
1.1.1.4 root 174: void vm_map_setup(
175: vm_map_t map,
176: pmap_t pmap,
177: vm_offset_t min,
1.1.1.7 ! root 178: vm_offset_t max)
1.1.1.3 root 179: {
180: vm_map_first_entry(map) = vm_map_to_entry(map);
181: vm_map_last_entry(map) = vm_map_to_entry(map);
182: map->hdr.nentries = 0;
183: rbtree_init(&map->hdr.tree);
1.1.1.7 ! root 184: rbtree_init(&map->hdr.gap_tree);
1.1.1.3 root 185:
186: map->size = 0;
1.1.1.5 root 187: map->user_wired = 0;
1.1.1.3 root 188: map->ref_count = 1;
189: map->pmap = pmap;
190: map->min_offset = min;
191: map->max_offset = max;
192: map->wiring_required = FALSE;
193: map->wait_for_space = FALSE;
194: map->first_free = vm_map_to_entry(map);
195: map->hint = vm_map_to_entry(map);
1.1.1.7 ! root 196: map->name = NULL;
1.1.1.3 root 197: vm_map_lock_init(map);
198: simple_lock_init(&map->ref_lock);
199: simple_lock_init(&map->hint_lock);
200: }
201:
1.1 root 202: /*
203: * vm_map_create:
204: *
205: * Creates and returns a new empty VM map with
206: * the given physical map structure, and having
207: * the given lower and upper address bounds.
208: */
1.1.1.4 root 209: vm_map_t vm_map_create(
210: pmap_t pmap,
211: vm_offset_t min,
1.1.1.7 ! root 212: vm_offset_t max)
1.1 root 213: {
1.1.1.4 root 214: vm_map_t result;
1.1 root 215:
1.1.1.3 root 216: result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
1.1 root 217: if (result == VM_MAP_NULL)
218: panic("vm_map_create");
219:
1.1.1.7 ! root 220: vm_map_setup(result, pmap, min, max);
1.1 root 221:
222: return(result);
223: }
224:
1.1.1.7 ! root 225: void vm_map_lock(struct vm_map *map)
! 226: {
! 227: lock_write(&map->lock);
! 228:
! 229: /*
! 230: * XXX Memory allocation may occur while a map is locked,
! 231: * for example when clipping entries. If the system is running
! 232: * low on memory, allocating may block until pages are
! 233: * available. But if a map used by the default pager is
! 234: * kept locked, a deadlock occurs.
! 235: *
! 236: * This workaround temporarily elevates the current thread
! 237: * VM privileges to avoid that particular deadlock, and does
! 238: * so regardless of the map for convenience, and because it's
! 239: * currently impossible to predict which map the default pager
! 240: * may depend on.
! 241: *
! 242: * This workaround isn't reliable, and only makes exhaustion
! 243: * less likely. In particular pageout may cause lots of data
! 244: * to be passed between the kernel and the pagers, often
! 245: * in the form of large copy maps. Making the minimum
! 246: * number of pages depend on the total number of pages
! 247: * should make exhaustion even less likely.
! 248: */
! 249:
! 250: if (current_thread()) {
! 251: current_thread()->vm_privilege++;
! 252: assert(current_thread()->vm_privilege != 0);
! 253: }
! 254:
! 255: map->timestamp++;
! 256: }
! 257:
! 258: void vm_map_unlock(struct vm_map *map)
! 259: {
! 260: if (current_thread()) {
! 261: current_thread()->vm_privilege--;
! 262: }
! 263:
! 264: lock_write_done(&map->lock);
! 265: }
! 266:
1.1 root 267: /*
268: * vm_map_entry_create: [ internal use only ]
269: *
270: * Allocates a VM map entry for insertion in the
271: * given map (or map copy). No fields are filled.
272: */
273: #define vm_map_entry_create(map) \
274: _vm_map_entry_create(&(map)->hdr)
275:
276: #define vm_map_copy_entry_create(copy) \
277: _vm_map_entry_create(&(copy)->cpy_hdr)
278:
279: vm_map_entry_t _vm_map_entry_create(map_header)
1.1.1.4 root 280: const struct vm_map_header *map_header;
1.1 root 281: {
1.1.1.4 root 282: vm_map_entry_t entry;
1.1 root 283:
1.1.1.6 root 284: entry = (vm_map_entry_t) kmem_cache_alloc(&vm_map_entry_cache);
1.1 root 285: if (entry == VM_MAP_ENTRY_NULL)
286: panic("vm_map_entry_create");
287:
288: return(entry);
289: }
290:
291: /*
292: * vm_map_entry_dispose: [ internal use only ]
293: *
294: * Inverse of vm_map_entry_create.
295: */
296: #define vm_map_entry_dispose(map, entry) \
297: _vm_map_entry_dispose(&(map)->hdr, (entry))
298:
299: #define vm_map_copy_entry_dispose(map, entry) \
300: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
301:
302: void _vm_map_entry_dispose(map_header, entry)
1.1.1.4 root 303: const struct vm_map_header *map_header;
304: vm_map_entry_t entry;
1.1 root 305: {
1.1.1.6 root 306: (void)map_header;
1.1 root 307:
1.1.1.6 root 308: kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry);
1.1.1.3 root 309: }
310:
311: /*
312: * Red-black tree lookup/insert comparison functions
313: */
314: static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
315: const struct rbtree_node *node)
316: {
317: struct vm_map_entry *entry;
318:
319: entry = rbtree_entry(node, struct vm_map_entry, tree_node);
320:
321: if (addr < entry->vme_start)
322: return -1;
323: else if (addr < entry->vme_end)
324: return 0;
1.1 root 325: else
1.1.1.3 root 326: return 1;
327: }
328:
329: static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
330: const struct rbtree_node *b)
331: {
332: struct vm_map_entry *entry;
1.1 root 333:
1.1.1.3 root 334: entry = rbtree_entry(a, struct vm_map_entry, tree_node);
335: return vm_map_entry_cmp_lookup(entry->vme_start, b);
1.1 root 336: }
337:
338: /*
1.1.1.7 ! root 339: * Gap management functions
! 340: */
! 341: static inline int vm_map_entry_gap_cmp_lookup(vm_size_t gap_size,
! 342: const struct rbtree_node *node)
! 343: {
! 344: struct vm_map_entry *entry;
! 345:
! 346: entry = rbtree_entry(node, struct vm_map_entry, gap_node);
! 347:
! 348: if (gap_size < entry->gap_size)
! 349: return -1;
! 350: else if (gap_size == entry->gap_size)
! 351: return 0;
! 352: else
! 353: return 1;
! 354: }
! 355:
! 356: static inline int vm_map_entry_gap_cmp_insert(const struct rbtree_node *a,
! 357: const struct rbtree_node *b)
! 358: {
! 359: struct vm_map_entry *entry;
! 360:
! 361: entry = rbtree_entry(a, struct vm_map_entry, gap_node);
! 362: return vm_map_entry_gap_cmp_lookup(entry->gap_size, b);
! 363: }
! 364:
! 365: static int
! 366: vm_map_gap_valid(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 367: {
! 368: return entry != (struct vm_map_entry *)&hdr->links;
! 369: }
! 370:
! 371: static void
! 372: vm_map_gap_compute(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 373: {
! 374: struct vm_map_entry *next;
! 375:
! 376: next = entry->vme_next;
! 377:
! 378: if (vm_map_gap_valid(hdr, next)) {
! 379: entry->gap_size = next->vme_start - entry->vme_end;
! 380: } else {
! 381: entry->gap_size = hdr->vme_end - entry->vme_end;
! 382: }
! 383: }
! 384:
! 385: static void
! 386: vm_map_gap_insert_single(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 387: {
! 388: struct vm_map_entry *tmp;
! 389: struct rbtree_node *node;
! 390: unsigned long slot;
! 391:
! 392: if (!vm_map_gap_valid(hdr, entry)) {
! 393: return;
! 394: }
! 395:
! 396: vm_map_gap_compute(hdr, entry);
! 397:
! 398: if (entry->gap_size == 0) {
! 399: return;
! 400: }
! 401:
! 402: node = rbtree_lookup_slot(&hdr->gap_tree, entry->gap_size,
! 403: vm_map_entry_gap_cmp_lookup, slot);
! 404:
! 405: if (node == NULL) {
! 406: rbtree_insert_slot(&hdr->gap_tree, slot, &entry->gap_node);
! 407: list_init(&entry->gap_list);
! 408: entry->in_gap_tree = 1;
! 409: } else {
! 410: tmp = rbtree_entry(node, struct vm_map_entry, gap_node);
! 411: list_insert_tail(&tmp->gap_list, &entry->gap_list);
! 412: entry->in_gap_tree = 0;
! 413: }
! 414: }
! 415:
! 416: static void
! 417: vm_map_gap_remove_single(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 418: {
! 419: struct vm_map_entry *tmp;
! 420:
! 421: if (!vm_map_gap_valid(hdr, entry)) {
! 422: return;
! 423: }
! 424:
! 425: if (entry->gap_size == 0) {
! 426: return;
! 427: }
! 428:
! 429: if (!entry->in_gap_tree) {
! 430: list_remove(&entry->gap_list);
! 431: return;
! 432: }
! 433:
! 434: rbtree_remove(&hdr->gap_tree, &entry->gap_node);
! 435:
! 436: if (list_empty(&entry->gap_list)) {
! 437: return;
! 438: }
! 439:
! 440: tmp = list_first_entry(&entry->gap_list, struct vm_map_entry, gap_list);
! 441: assert(tmp->gap_size == entry->gap_size);
! 442: list_remove(&tmp->gap_list);
! 443: list_set_head(&tmp->gap_list, &entry->gap_list);
! 444: assert(!tmp->in_gap_tree);
! 445: rbtree_insert(&hdr->gap_tree, &tmp->gap_node,
! 446: vm_map_entry_gap_cmp_insert);
! 447: tmp->in_gap_tree = 1;
! 448: }
! 449:
! 450: static void
! 451: vm_map_gap_update(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 452: {
! 453: vm_map_gap_remove_single(hdr, entry);
! 454: vm_map_gap_insert_single(hdr, entry);
! 455: }
! 456:
! 457: static void
! 458: vm_map_gap_insert(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 459: {
! 460: vm_map_gap_remove_single(hdr, entry->vme_prev);
! 461: vm_map_gap_insert_single(hdr, entry->vme_prev);
! 462: vm_map_gap_insert_single(hdr, entry);
! 463: }
! 464:
! 465: static void
! 466: vm_map_gap_remove(struct vm_map_header *hdr, struct vm_map_entry *entry)
! 467: {
! 468: vm_map_gap_remove_single(hdr, entry);
! 469: vm_map_gap_remove_single(hdr, entry->vme_prev);
! 470: vm_map_gap_insert_single(hdr, entry->vme_prev);
! 471: }
! 472:
! 473: /*
1.1 root 474: * vm_map_entry_{un,}link:
475: *
476: * Insert/remove entries from maps (or map copies).
1.1.1.3 root 477: *
478: * The start and end addresses of the entries must be properly set
479: * before using these macros.
1.1 root 480: */
481: #define vm_map_entry_link(map, after_where, entry) \
482: _vm_map_entry_link(&(map)->hdr, after_where, entry)
483:
484: #define vm_map_copy_entry_link(copy, after_where, entry) \
485: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
486:
487: #define _vm_map_entry_link(hdr, after_where, entry) \
488: MACRO_BEGIN \
489: (hdr)->nentries++; \
490: (entry)->vme_prev = (after_where); \
491: (entry)->vme_next = (after_where)->vme_next; \
492: (entry)->vme_prev->vme_next = \
493: (entry)->vme_next->vme_prev = (entry); \
1.1.1.3 root 494: rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \
495: vm_map_entry_cmp_insert); \
1.1.1.7 ! root 496: vm_map_gap_insert((hdr), (entry)); \
1.1 root 497: MACRO_END
498:
499: #define vm_map_entry_unlink(map, entry) \
500: _vm_map_entry_unlink(&(map)->hdr, entry)
501:
502: #define vm_map_copy_entry_unlink(copy, entry) \
503: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
504:
505: #define _vm_map_entry_unlink(hdr, entry) \
506: MACRO_BEGIN \
507: (hdr)->nentries--; \
508: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
509: (entry)->vme_prev->vme_next = (entry)->vme_next; \
1.1.1.3 root 510: rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
1.1.1.7 ! root 511: vm_map_gap_remove((hdr), (entry)); \
1.1 root 512: MACRO_END
513:
514: /*
515: * vm_map_reference:
516: *
517: * Creates another valid reference to the given map.
518: *
519: */
1.1.1.4 root 520: void vm_map_reference(vm_map_t map)
1.1 root 521: {
522: if (map == VM_MAP_NULL)
523: return;
524:
525: simple_lock(&map->ref_lock);
526: map->ref_count++;
527: simple_unlock(&map->ref_lock);
528: }
529:
530: /*
531: * vm_map_deallocate:
532: *
533: * Removes a reference from the specified map,
534: * destroying it if no references remain.
535: * The map should not be locked.
536: */
1.1.1.4 root 537: void vm_map_deallocate(vm_map_t map)
1.1 root 538: {
1.1.1.4 root 539: int c;
1.1 root 540:
541: if (map == VM_MAP_NULL)
542: return;
543:
544: simple_lock(&map->ref_lock);
545: c = --map->ref_count;
546: simple_unlock(&map->ref_lock);
547:
548: if (c > 0) {
549: return;
550: }
551:
552: projected_buffer_collect(map);
553: (void) vm_map_delete(map, map->min_offset, map->max_offset);
554:
555: pmap_destroy(map->pmap);
556:
1.1.1.3 root 557: kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
1.1 root 558: }
559:
560: /*
561: * SAVE_HINT:
562: *
563: * Saves the specified entry as the hint for
564: * future lookups. Performs necessary interlocks.
565: */
566: #define SAVE_HINT(map,value) \
567: simple_lock(&(map)->hint_lock); \
568: (map)->hint = (value); \
569: simple_unlock(&(map)->hint_lock);
570:
571: /*
572: * vm_map_lookup_entry: [ internal use only ]
573: *
574: * Finds the map entry containing (or
575: * immediately preceding) the specified address
576: * in the given map; the entry is returned
577: * in the "entry" parameter. The boolean
578: * result indicates whether the address is
579: * actually contained in the map.
580: */
1.1.1.4 root 581: boolean_t vm_map_lookup_entry(
582: vm_map_t map,
583: vm_offset_t address,
584: vm_map_entry_t *entry) /* OUT */
1.1 root 585: {
1.1.1.4 root 586: struct rbtree_node *node;
587: vm_map_entry_t hint;
1.1 root 588:
589: /*
1.1.1.3 root 590: * First, make a quick check to see if we are already
591: * looking at the entry we want (which is often the case).
1.1 root 592: */
593:
594: simple_lock(&map->hint_lock);
1.1.1.3 root 595: hint = map->hint;
1.1 root 596: simple_unlock(&map->hint_lock);
597:
1.1.1.3 root 598: if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) {
599: if (address < hint->vme_end) {
600: *entry = hint;
1.1 root 601: return(TRUE);
1.1.1.3 root 602: } else {
603: vm_map_entry_t next = hint->vme_next;
604:
605: if ((next == vm_map_to_entry(map))
606: || (address < next->vme_start)) {
607: *entry = hint;
608: return(FALSE);
609: }
1.1 root 610: }
611: }
612:
613: /*
1.1.1.3 root 614: * If the hint didn't help, use the red-black tree.
1.1 root 615: */
616:
1.1.1.3 root 617: node = rbtree_lookup_nearest(&map->hdr.tree, address,
618: vm_map_entry_cmp_lookup, RBTREE_LEFT);
1.1 root 619:
1.1.1.3 root 620: if (node == NULL) {
621: *entry = vm_map_to_entry(map);
622: SAVE_HINT(map, *entry);
623: return(FALSE);
624: } else {
625: *entry = rbtree_entry(node, struct vm_map_entry, tree_node);
626: SAVE_HINT(map, *entry);
627: return((address < (*entry)->vme_end) ? TRUE : FALSE);
1.1 root 628: }
629: }
630:
631: /*
632: * Routine: invalid_user_access
633: *
634: * Verifies whether user access is valid.
635: */
636:
637: boolean_t
1.1.1.4 root 638: invalid_user_access(
639: vm_map_t map,
640: vm_offset_t start,
641: vm_offset_t end,
642: vm_prot_t prot)
1.1 root 643: {
644: vm_map_entry_t entry;
645:
646: return (map == VM_MAP_NULL || map == kernel_map ||
647: !vm_map_lookup_entry(map, start, &entry) ||
648: entry->vme_end < end ||
649: (prot & ~(entry->protection)));
650: }
651:
1.1.1.7 ! root 652: /*
! 653: * Find a range of available space from the specified map.
! 654: *
! 655: * If successful, this function returns the map entry immediately preceding
! 656: * the range, and writes the range address in startp. If the map contains
! 657: * no entry, the entry returned points to the map header.
! 658: * Otherwise, NULL is returned.
! 659: *
! 660: * If map_locked is true, this function will not wait for more space in case
! 661: * of failure. Otherwise, the map is locked.
! 662: */
! 663: static struct vm_map_entry *
! 664: vm_map_find_entry_anywhere(struct vm_map *map,
! 665: vm_size_t size,
! 666: vm_offset_t mask,
! 667: boolean_t map_locked,
! 668: vm_offset_t *startp)
! 669: {
! 670: struct vm_map_entry *entry;
! 671: struct rbtree_node *node;
! 672: vm_size_t max_size;
! 673: vm_offset_t start, end;
! 674:
! 675: assert(size != 0);
! 676:
! 677: if (!map_locked) {
! 678: vm_map_lock(map);
! 679: }
! 680:
! 681: restart:
! 682: if (map->hdr.nentries == 0) {
! 683: entry = vm_map_to_entry(map);
! 684: start = (map->min_offset + mask) & ~mask;
! 685: end = start + size;
! 686:
! 687: if ((end <= start) || (end > map->max_offset)) {
! 688: goto error;
! 689: }
! 690:
! 691: *startp = start;
! 692: return entry;
! 693: }
! 694:
! 695: entry = map->first_free;
! 696:
! 697: if (entry != vm_map_to_entry(map)) {
! 698: start = (entry->vme_end + mask) & ~mask;
! 699: end = start + size;
! 700:
! 701: if ((end > start)
! 702: && (end <= map->max_offset)
! 703: && (end <= (entry->vme_end + entry->gap_size))) {
! 704: *startp = start;
! 705: return entry;
! 706: }
! 707: }
! 708:
! 709: max_size = size + mask;
! 710:
! 711: if (max_size < size) {
! 712: goto error;
! 713: }
! 714:
! 715: node = rbtree_lookup_nearest(&map->hdr.gap_tree, max_size,
! 716: vm_map_entry_gap_cmp_lookup, RBTREE_RIGHT);
! 717:
! 718: if (node == NULL) {
! 719: if (map_locked || !map->wait_for_space) {
! 720: goto error;
! 721: }
! 722:
! 723: assert_wait((event_t)map, TRUE);
! 724: vm_map_unlock(map);
! 725: thread_block(NULL);
! 726: vm_map_lock(map);
! 727: goto restart;
! 728: }
! 729:
! 730: entry = rbtree_entry(node, struct vm_map_entry, gap_node);
! 731: assert(entry->in_gap_tree);
! 732:
! 733: if (!list_empty(&entry->gap_list)) {
! 734: entry = list_last_entry(&entry->gap_list,
! 735: struct vm_map_entry, gap_list);
! 736: }
! 737:
! 738: assert(entry->gap_size >= max_size);
! 739: start = (entry->vme_end + mask) & ~mask;
! 740: end = start + size;
! 741: assert(end > start);
! 742: assert(end <= (entry->vme_end + entry->gap_size));
! 743: *startp = start;
! 744: return entry;
! 745:
! 746: error:
! 747: printf("no more room in %p (%s)\n", map, map->name);
! 748: return NULL;
! 749: }
1.1 root 750:
751: /*
752: * Routine: vm_map_find_entry
753: * Purpose:
754: * Allocate a range in the specified virtual address map,
755: * returning the entry allocated for that range.
756: * Used by kmem_alloc, etc. Returns wired entries.
757: *
758: * The map must be locked.
759: *
760: * If an entry is allocated, the object/offset fields
761: * are initialized to zero. If an object is supplied,
762: * then an existing entry may be extended.
763: */
1.1.1.4 root 764: kern_return_t vm_map_find_entry(
765: vm_map_t map,
766: vm_offset_t *address, /* OUT */
767: vm_size_t size,
768: vm_offset_t mask,
769: vm_object_t object,
770: vm_map_entry_t *o_entry) /* OUT */
1.1 root 771: {
1.1.1.4 root 772: vm_map_entry_t entry, new_entry;
773: vm_offset_t start;
774: vm_offset_t end;
1.1 root 775:
1.1.1.7 ! root 776: entry = vm_map_find_entry_anywhere(map, size, mask, TRUE, &start);
1.1 root 777:
1.1.1.7 ! root 778: if (entry == NULL) {
! 779: return KERN_NO_SPACE;
1.1 root 780: }
781:
1.1.1.7 ! root 782: end = start + size;
! 783:
1.1 root 784: /*
785: * At this point,
786: * "start" and "end" should define the endpoints of the
787: * available new range, and
788: * "entry" should refer to the region before the new
789: * range, and
790: *
791: * the map should be locked.
792: */
793:
794: *address = start;
795:
796: /*
797: * See whether we can avoid creating a new entry by
798: * extending one of our neighbors. [So far, we only attempt to
799: * extend from below.]
800: */
801:
802: if ((object != VM_OBJECT_NULL) &&
803: (entry != vm_map_to_entry(map)) &&
804: (entry->vme_end == start) &&
805: (!entry->is_shared) &&
806: (!entry->is_sub_map) &&
807: (entry->object.vm_object == object) &&
808: (entry->needs_copy == FALSE) &&
809: (entry->inheritance == VM_INHERIT_DEFAULT) &&
810: (entry->protection == VM_PROT_DEFAULT) &&
811: (entry->max_protection == VM_PROT_ALL) &&
812: (entry->wired_count == 1) &&
813: (entry->user_wired_count == 0) &&
814: (entry->projected_on == 0)) {
815: /*
816: * Because this is a special case,
817: * we don't need to use vm_object_coalesce.
818: */
819:
820: entry->vme_end = end;
1.1.1.7 ! root 821: vm_map_gap_update(&map->hdr, entry);
1.1 root 822: new_entry = entry;
823: } else {
824: new_entry = vm_map_entry_create(map);
825:
826: new_entry->vme_start = start;
827: new_entry->vme_end = end;
828:
829: new_entry->is_shared = FALSE;
830: new_entry->is_sub_map = FALSE;
831: new_entry->object.vm_object = VM_OBJECT_NULL;
832: new_entry->offset = (vm_offset_t) 0;
833:
834: new_entry->needs_copy = FALSE;
835:
836: new_entry->inheritance = VM_INHERIT_DEFAULT;
837: new_entry->protection = VM_PROT_DEFAULT;
838: new_entry->max_protection = VM_PROT_ALL;
839: new_entry->wired_count = 1;
840: new_entry->user_wired_count = 0;
841:
842: new_entry->in_transition = FALSE;
843: new_entry->needs_wakeup = FALSE;
844: new_entry->projected_on = 0;
845:
846: /*
847: * Insert the new entry into the list
848: */
849:
850: vm_map_entry_link(map, entry, new_entry);
851: }
852:
853: map->size += size;
854:
855: /*
856: * Update the free space hint and the lookup hint
857: */
858:
859: map->first_free = new_entry;
860: SAVE_HINT(map, new_entry);
861:
862: *o_entry = new_entry;
863: return(KERN_SUCCESS);
864: }
865:
1.1.1.4 root 866: boolean_t vm_map_pmap_enter_print = FALSE;
867: boolean_t vm_map_pmap_enter_enable = FALSE;
1.1 root 868:
869: /*
870: * Routine: vm_map_pmap_enter
871: *
872: * Description:
873: * Force pages from the specified object to be entered into
874: * the pmap at the specified address if they are present.
875: * As soon as a page not found in the object the scan ends.
876: *
877: * Returns:
1.1.1.2 root 878: * Nothing.
1.1 root 879: *
880: * In/out conditions:
881: * The source map should not be locked on entry.
882: */
883: void
1.1.1.4 root 884: vm_map_pmap_enter(
885: vm_map_t map,
886: vm_offset_t addr,
887: vm_offset_t end_addr,
888: vm_object_t object,
889: vm_offset_t offset,
890: vm_prot_t protection)
1.1 root 891: {
892: while (addr < end_addr) {
1.1.1.4 root 893: vm_page_t m;
1.1 root 894:
895: vm_object_lock(object);
896: vm_object_paging_begin(object);
897:
898: m = vm_page_lookup(object, offset);
899: if (m == VM_PAGE_NULL || m->absent) {
900: vm_object_paging_end(object);
901: vm_object_unlock(object);
902: return;
903: }
904:
905: if (vm_map_pmap_enter_print) {
906: printf("vm_map_pmap_enter:");
1.1.1.3 root 907: printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
1.1 root 908: map, addr, object, offset);
909: }
910:
911: m->busy = TRUE;
912: vm_object_unlock(object);
913:
914: PMAP_ENTER(map->pmap, addr, m,
915: protection, FALSE);
916:
917: vm_object_lock(object);
918: PAGE_WAKEUP_DONE(m);
919: vm_page_lock_queues();
920: if (!m->active && !m->inactive)
921: vm_page_activate(m);
922: vm_page_unlock_queues();
923: vm_object_paging_end(object);
924: vm_object_unlock(object);
925:
926: offset += PAGE_SIZE;
927: addr += PAGE_SIZE;
928: }
929: }
930:
931: /*
932: * Routine: vm_map_enter
933: *
934: * Description:
935: * Allocate a range in the specified virtual address map.
936: * The resulting range will refer to memory defined by
937: * the given memory object and offset into that object.
938: *
939: * Arguments are as defined in the vm_map call.
940: */
941: kern_return_t vm_map_enter(
1.1.1.4 root 942: vm_map_t map,
943: vm_offset_t *address, /* IN/OUT */
944: vm_size_t size,
945: vm_offset_t mask,
946: boolean_t anywhere,
947: vm_object_t object,
948: vm_offset_t offset,
949: boolean_t needs_copy,
950: vm_prot_t cur_protection,
951: vm_prot_t max_protection,
952: vm_inherit_t inheritance)
953: {
954: vm_map_entry_t entry;
955: vm_offset_t start;
956: vm_offset_t end;
957: kern_return_t result = KERN_SUCCESS;
1.1 root 958:
959: #define RETURN(value) { result = value; goto BailOut; }
960:
1.1.1.3 root 961: if (size == 0)
962: return KERN_INVALID_ARGUMENT;
963:
1.1 root 964: start = *address;
965:
966: if (anywhere) {
1.1.1.7 ! root 967: entry = vm_map_find_entry_anywhere(map, size, mask, FALSE, &start);
1.1 root 968:
1.1.1.7 ! root 969: if (entry == NULL) {
1.1 root 970: RETURN(KERN_NO_SPACE);
971: }
972:
1.1.1.7 ! root 973: end = start + size;
1.1 root 974: *address = start;
975: } else {
976: vm_map_entry_t temp_entry;
977:
978: /*
979: * Verify that:
980: * the address doesn't itself violate
981: * the mask requirement.
982: */
983:
984: if ((start & mask) != 0)
985: return(KERN_NO_SPACE);
986:
987: vm_map_lock(map);
988:
989: /*
990: * ... the address is within bounds
991: */
992:
993: end = start + size;
994:
995: if ((start < map->min_offset) ||
996: (end > map->max_offset) ||
997: (start >= end)) {
998: RETURN(KERN_INVALID_ADDRESS);
999: }
1000:
1001: /*
1002: * ... the starting address isn't allocated
1003: */
1004:
1005: if (vm_map_lookup_entry(map, start, &temp_entry))
1006: RETURN(KERN_NO_SPACE);
1007:
1008: entry = temp_entry;
1009:
1010: /*
1011: * ... the next region doesn't overlap the
1012: * end point.
1013: */
1014:
1015: if ((entry->vme_next != vm_map_to_entry(map)) &&
1016: (entry->vme_next->vme_start < end))
1017: RETURN(KERN_NO_SPACE);
1018: }
1019:
1020: /*
1021: * At this point,
1022: * "start" and "end" should define the endpoints of the
1023: * available new range, and
1024: * "entry" should refer to the region before the new
1025: * range, and
1026: *
1027: * the map should be locked.
1028: */
1029:
1030: /*
1031: * See whether we can avoid creating a new entry (and object) by
1032: * extending one of our neighbors. [So far, we only attempt to
1033: * extend from below.]
1034: */
1035:
1036: if ((object == VM_OBJECT_NULL) &&
1037: (entry != vm_map_to_entry(map)) &&
1038: (entry->vme_end == start) &&
1039: (!entry->is_shared) &&
1040: (!entry->is_sub_map) &&
1041: (entry->inheritance == inheritance) &&
1042: (entry->protection == cur_protection) &&
1043: (entry->max_protection == max_protection) &&
1044: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
1.1.1.2 root 1045: (entry->projected_on == 0)) {
1.1 root 1046: if (vm_object_coalesce(entry->object.vm_object,
1047: VM_OBJECT_NULL,
1048: entry->offset,
1049: (vm_offset_t) 0,
1050: (vm_size_t)(entry->vme_end - entry->vme_start),
1051: (vm_size_t)(end - entry->vme_end))) {
1052:
1053: /*
1054: * Coalesced the two objects - can extend
1055: * the previous map entry to include the
1056: * new range.
1057: */
1058: map->size += (end - entry->vme_end);
1059: entry->vme_end = end;
1.1.1.7 ! root 1060: vm_map_gap_update(&map->hdr, entry);
1.1 root 1061: RETURN(KERN_SUCCESS);
1062: }
1063: }
1064:
1065: /*
1066: * Create a new entry
1067: */
1068:
1069: /**/ {
1.1.1.4 root 1070: vm_map_entry_t new_entry;
1.1 root 1071:
1072: new_entry = vm_map_entry_create(map);
1073:
1074: new_entry->vme_start = start;
1075: new_entry->vme_end = end;
1076:
1077: new_entry->is_shared = FALSE;
1078: new_entry->is_sub_map = FALSE;
1079: new_entry->object.vm_object = object;
1080: new_entry->offset = offset;
1081:
1082: new_entry->needs_copy = needs_copy;
1083:
1084: new_entry->inheritance = inheritance;
1085: new_entry->protection = cur_protection;
1086: new_entry->max_protection = max_protection;
1087: new_entry->wired_count = 0;
1088: new_entry->user_wired_count = 0;
1089:
1090: new_entry->in_transition = FALSE;
1091: new_entry->needs_wakeup = FALSE;
1092: new_entry->projected_on = 0;
1093:
1094: /*
1095: * Insert the new entry into the list
1096: */
1097:
1098: vm_map_entry_link(map, entry, new_entry);
1099: map->size += size;
1100:
1101: /*
1102: * Update the free space hint and the lookup hint
1103: */
1104:
1105: if ((map->first_free == entry) &&
1106: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
1107: >= new_entry->vme_start))
1108: map->first_free = new_entry;
1109:
1110: SAVE_HINT(map, new_entry);
1111:
1112: vm_map_unlock(map);
1113:
1114: if ((object != VM_OBJECT_NULL) &&
1115: (vm_map_pmap_enter_enable) &&
1116: (!anywhere) &&
1.1.1.2 root 1117: (!needs_copy) &&
1.1 root 1118: (size < (128*1024))) {
1.1.1.2 root 1119: vm_map_pmap_enter(map, start, end,
1.1 root 1120: object, offset, cur_protection);
1121: }
1122:
1123: return(result);
1124: /**/ }
1125:
1126: BailOut: ;
1127:
1128: vm_map_unlock(map);
1129: return(result);
1130:
1131: #undef RETURN
1132: }
1133:
1134: /*
1135: * vm_map_clip_start: [ internal use only ]
1136: *
1137: * Asserts that the given entry begins at or after
1138: * the specified address; if necessary,
1139: * it splits the entry into two.
1140: */
1141: #define vm_map_clip_start(map, entry, startaddr) \
1142: MACRO_BEGIN \
1143: if ((startaddr) > (entry)->vme_start) \
1144: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1145: MACRO_END
1146:
1147: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1148: MACRO_BEGIN \
1149: if ((startaddr) > (entry)->vme_start) \
1150: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1151: MACRO_END
1152:
1153: /*
1154: * This routine is called only when it is known that
1155: * the entry must be split.
1156: */
1.1.1.4 root 1157: void _vm_map_clip_start(
1158: struct vm_map_header *map_header,
1159: vm_map_entry_t entry,
1160: vm_offset_t start)
1.1 root 1161: {
1.1.1.4 root 1162: vm_map_entry_t new_entry;
1.1 root 1163:
1164: /*
1165: * Split off the front portion --
1166: * note that we must insert the new
1167: * entry BEFORE this one, so that
1168: * this entry has the specified starting
1169: * address.
1170: */
1171:
1172: new_entry = _vm_map_entry_create(map_header);
1173: vm_map_entry_copy_full(new_entry, entry);
1174:
1175: new_entry->vme_end = start;
1176: entry->offset += (start - entry->vme_start);
1177: entry->vme_start = start;
1178:
1179: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1180:
1181: if (entry->is_sub_map)
1182: vm_map_reference(new_entry->object.sub_map);
1183: else
1184: vm_object_reference(new_entry->object.vm_object);
1185: }
1186:
1187: /*
1188: * vm_map_clip_end: [ internal use only ]
1189: *
1190: * Asserts that the given entry ends at or before
1191: * the specified address; if necessary,
1192: * it splits the entry into two.
1193: */
1194: #define vm_map_clip_end(map, entry, endaddr) \
1195: MACRO_BEGIN \
1196: if ((endaddr) < (entry)->vme_end) \
1197: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1198: MACRO_END
1199:
1200: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1201: MACRO_BEGIN \
1202: if ((endaddr) < (entry)->vme_end) \
1203: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1204: MACRO_END
1205:
1206: /*
1207: * This routine is called only when it is known that
1208: * the entry must be split.
1209: */
1.1.1.4 root 1210: void _vm_map_clip_end(
1211: struct vm_map_header *map_header,
1212: vm_map_entry_t entry,
1213: vm_offset_t end)
1.1 root 1214: {
1.1.1.4 root 1215: vm_map_entry_t new_entry;
1.1 root 1216:
1217: /*
1218: * Create a new entry and insert it
1219: * AFTER the specified entry
1220: */
1221:
1222: new_entry = _vm_map_entry_create(map_header);
1223: vm_map_entry_copy_full(new_entry, entry);
1224:
1225: new_entry->vme_start = entry->vme_end = end;
1226: new_entry->offset += (end - entry->vme_start);
1227:
1228: _vm_map_entry_link(map_header, entry, new_entry);
1229:
1230: if (entry->is_sub_map)
1231: vm_map_reference(new_entry->object.sub_map);
1232: else
1233: vm_object_reference(new_entry->object.vm_object);
1234: }
1235:
1236: /*
1237: * VM_MAP_RANGE_CHECK: [ internal use only ]
1238: *
1239: * Asserts that the starting and ending region
1240: * addresses fall within the valid range of the map.
1241: */
1242: #define VM_MAP_RANGE_CHECK(map, start, end) \
1243: { \
1244: if (start < vm_map_min(map)) \
1245: start = vm_map_min(map); \
1246: if (end > vm_map_max(map)) \
1247: end = vm_map_max(map); \
1248: if (start > end) \
1249: start = end; \
1250: }
1251:
1252: /*
1253: * vm_map_submap: [ kernel use only ]
1254: *
1255: * Mark the given range as handled by a subordinate map.
1256: *
1257: * This range must have been created with vm_map_find using
1258: * the vm_submap_object, and no other operations may have been
1259: * performed on this range prior to calling vm_map_submap.
1260: *
1261: * Only a limited number of operations can be performed
1262: * within this rage after calling vm_map_submap:
1263: * vm_fault
1264: * [Don't try vm_map_copyin!]
1265: *
1266: * To remove a submapping, one must first remove the
1267: * range from the superior map, and then destroy the
1268: * submap (if desired). [Better yet, don't try it.]
1269: */
1.1.1.4 root 1270: kern_return_t vm_map_submap(
1271: vm_map_t map,
1272: vm_offset_t start,
1273: vm_offset_t end,
1274: vm_map_t submap)
1.1 root 1275: {
1276: vm_map_entry_t entry;
1.1.1.4 root 1277: kern_return_t result = KERN_INVALID_ARGUMENT;
1278: vm_object_t object;
1.1 root 1279:
1280: vm_map_lock(map);
1281:
1282: VM_MAP_RANGE_CHECK(map, start, end);
1283:
1284: if (vm_map_lookup_entry(map, start, &entry)) {
1285: vm_map_clip_start(map, entry, start);
1286: }
1287: else
1288: entry = entry->vme_next;
1289:
1290: vm_map_clip_end(map, entry, end);
1291:
1292: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1293: (!entry->is_sub_map) &&
1294: ((object = entry->object.vm_object) == vm_submap_object) &&
1295: (object->resident_page_count == 0) &&
1296: (object->copy == VM_OBJECT_NULL) &&
1297: (object->shadow == VM_OBJECT_NULL) &&
1298: (!object->pager_created)) {
1299: entry->object.vm_object = VM_OBJECT_NULL;
1300: vm_object_deallocate(object);
1301: entry->is_sub_map = TRUE;
1302: vm_map_reference(entry->object.sub_map = submap);
1303: result = KERN_SUCCESS;
1304: }
1305: vm_map_unlock(map);
1306:
1307: return(result);
1308: }
1309:
1310: /*
1311: * vm_map_protect:
1312: *
1313: * Sets the protection of the specified address
1314: * region in the target map. If "set_max" is
1315: * specified, the maximum protection is to be set;
1316: * otherwise, only the current protection is affected.
1317: */
1.1.1.4 root 1318: kern_return_t vm_map_protect(
1319: vm_map_t map,
1320: vm_offset_t start,
1321: vm_offset_t end,
1322: vm_prot_t new_prot,
1323: boolean_t set_max)
1.1 root 1324: {
1.1.1.4 root 1325: vm_map_entry_t current;
1326: vm_map_entry_t entry;
1.1 root 1327:
1328: vm_map_lock(map);
1329:
1330: VM_MAP_RANGE_CHECK(map, start, end);
1331:
1332: if (vm_map_lookup_entry(map, start, &entry)) {
1333: vm_map_clip_start(map, entry, start);
1334: }
1335: else
1336: entry = entry->vme_next;
1337:
1338: /*
1339: * Make a first pass to check for protection
1340: * violations.
1341: */
1342:
1343: current = entry;
1344: while ((current != vm_map_to_entry(map)) &&
1345: (current->vme_start < end)) {
1346:
1347: if (current->is_sub_map) {
1348: vm_map_unlock(map);
1349: return(KERN_INVALID_ARGUMENT);
1350: }
1351: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1352: != new_prot) {
1353: vm_map_unlock(map);
1354: return(KERN_PROTECTION_FAILURE);
1355: }
1356:
1357: current = current->vme_next;
1358: }
1359:
1360: /*
1361: * Go back and fix up protections.
1362: * [Note that clipping is not necessary the second time.]
1363: */
1364:
1365: current = entry;
1366:
1367: while ((current != vm_map_to_entry(map)) &&
1368: (current->vme_start < end)) {
1369:
1370: vm_prot_t old_prot;
1371:
1372: vm_map_clip_end(map, current, end);
1373:
1374: old_prot = current->protection;
1375: if (set_max)
1376: current->protection =
1377: (current->max_protection = new_prot) &
1378: old_prot;
1379: else
1380: current->protection = new_prot;
1381:
1382: /*
1383: * Update physical map if necessary.
1384: */
1385:
1386: if (current->protection != old_prot) {
1387: pmap_protect(map->pmap, current->vme_start,
1388: current->vme_end,
1389: current->protection);
1390: }
1391: current = current->vme_next;
1392: }
1393:
1394: vm_map_unlock(map);
1395: return(KERN_SUCCESS);
1396: }
1397:
1398: /*
1399: * vm_map_inherit:
1400: *
1401: * Sets the inheritance of the specified address
1402: * range in the target map. Inheritance
1403: * affects how the map will be shared with
1404: * child maps at the time of vm_map_fork.
1405: */
1.1.1.4 root 1406: kern_return_t vm_map_inherit(
1407: vm_map_t map,
1408: vm_offset_t start,
1409: vm_offset_t end,
1410: vm_inherit_t new_inheritance)
1.1 root 1411: {
1.1.1.4 root 1412: vm_map_entry_t entry;
1.1 root 1413: vm_map_entry_t temp_entry;
1414:
1415: vm_map_lock(map);
1416:
1417: VM_MAP_RANGE_CHECK(map, start, end);
1418:
1419: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1420: entry = temp_entry;
1421: vm_map_clip_start(map, entry, start);
1422: }
1423: else
1424: entry = temp_entry->vme_next;
1425:
1426: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1427: vm_map_clip_end(map, entry, end);
1428:
1429: entry->inheritance = new_inheritance;
1430:
1431: entry = entry->vme_next;
1432: }
1433:
1434: vm_map_unlock(map);
1435: return(KERN_SUCCESS);
1436: }
1437:
1438: /*
1439: * vm_map_pageable_common:
1440: *
1441: * Sets the pageability of the specified address
1442: * range in the target map. Regions specified
1443: * as not pageable require locked-down physical
1444: * memory and physical page maps. access_type indicates
1445: * types of accesses that must not generate page faults.
1446: * This is checked against protection of memory being locked-down.
1447: * access_type of VM_PROT_NONE makes memory pageable.
1448: *
1449: * The map must not be locked, but a reference
1450: * must remain to the map throughout the call.
1451: *
1452: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1453: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1454: */
1.1.1.4 root 1455: kern_return_t vm_map_pageable_common(
1456: vm_map_t map,
1457: vm_offset_t start,
1458: vm_offset_t end,
1459: vm_prot_t access_type,
1460: boolean_t user_wire)
1.1 root 1461: {
1.1.1.4 root 1462: vm_map_entry_t entry;
1.1 root 1463: vm_map_entry_t start_entry;
1464:
1465: vm_map_lock(map);
1466:
1467: VM_MAP_RANGE_CHECK(map, start, end);
1468:
1469: if (vm_map_lookup_entry(map, start, &start_entry)) {
1470: entry = start_entry;
1471: /*
1472: * vm_map_clip_start will be done later.
1473: */
1474: }
1475: else {
1476: /*
1477: * Start address is not in map; this is fatal.
1478: */
1479: vm_map_unlock(map);
1.1.1.7 ! root 1480: return(KERN_NO_SPACE);
1.1 root 1481: }
1482:
1483: /*
1484: * Actions are rather different for wiring and unwiring,
1485: * so we have two separate cases.
1486: */
1487:
1488: if (access_type == VM_PROT_NONE) {
1489:
1490: vm_map_clip_start(map, entry, start);
1491:
1492: /*
1.1.1.7 ! root 1493: * Unwiring. First ensure that there are no holes
! 1494: * in the specified range.
1.1 root 1495: */
1496: while ((entry != vm_map_to_entry(map)) &&
1497: (entry->vme_start < end)) {
1498:
1.1.1.7 ! root 1499: if ((entry->vme_end < end) &&
! 1500: ((entry->vme_next == vm_map_to_entry(map)) ||
! 1501: (entry->vme_next->vme_start > entry->vme_end))) {
1.1 root 1502: vm_map_unlock(map);
1.1.1.7 ! root 1503: return(KERN_NO_SPACE);
1.1 root 1504: }
1505: entry = entry->vme_next;
1506: }
1507:
1508: /*
1509: * Now decrement the wiring count for each region.
1510: * If a region becomes completely unwired,
1511: * unwire its physical pages and mappings.
1512: */
1513: entry = start_entry;
1514: while ((entry != vm_map_to_entry(map)) &&
1515: (entry->vme_start < end)) {
1516: vm_map_clip_end(map, entry, end);
1517:
1.1.1.7 ! root 1518: /* Skip unwired entries */
! 1519: if (entry->wired_count == 0) {
! 1520: assert(entry->user_wired_count == 0);
! 1521: entry = entry->vme_next;
! 1522: continue;
! 1523: }
! 1524:
1.1 root 1525: if (user_wire) {
1526: if (--(entry->user_wired_count) == 0)
1.1.1.5 root 1527: {
1528: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1529: entry->wired_count--;
1.1.1.5 root 1530: }
1.1 root 1531: }
1532: else {
1533: entry->wired_count--;
1534: }
1.1.1.2 root 1535:
1.1 root 1536: if (entry->wired_count == 0)
1537: vm_fault_unwire(map, entry);
1538:
1539: entry = entry->vme_next;
1540: }
1541: }
1542:
1543: else {
1544: /*
1545: * Wiring. We must do this in two passes:
1546: *
1547: * 1. Holding the write lock, we create any shadow
1548: * or zero-fill objects that need to be created.
1549: * Then we clip each map entry to the region to be
1550: * wired and increment its wiring count. We
1551: * create objects before clipping the map entries
1552: * to avoid object proliferation.
1553: *
1554: * 2. We downgrade to a read lock, and call
1555: * vm_fault_wire to fault in the pages for any
1556: * newly wired area (wired_count is 1).
1557: *
1558: * Downgrading to a read lock for vm_fault_wire avoids
1559: * a possible deadlock with another thread that may have
1560: * faulted on one of the pages to be wired (it would mark
1561: * the page busy, blocking us, then in turn block on the
1562: * map lock that we hold). Because of problems in the
1563: * recursive lock package, we cannot upgrade to a write
1564: * lock in vm_map_lookup. Thus, any actions that require
1565: * the write lock must be done beforehand. Because we
1566: * keep the read lock on the map, the copy-on-write
1567: * status of the entries we modify here cannot change.
1568: */
1569:
1570: /*
1571: * Pass 1.
1572: */
1573: while ((entry != vm_map_to_entry(map)) &&
1574: (entry->vme_start < end)) {
1575: vm_map_clip_end(map, entry, end);
1576:
1577: if (entry->wired_count == 0) {
1578:
1579: /*
1580: * Perform actions of vm_map_lookup that need
1581: * the write lock on the map: create a shadow
1582: * object for a copy-on-write region, or an
1583: * object for a zero-fill region.
1584: */
1585: if (entry->needs_copy &&
1586: ((entry->protection & VM_PROT_WRITE) != 0)) {
1587:
1588: vm_object_shadow(&entry->object.vm_object,
1589: &entry->offset,
1590: (vm_size_t)(entry->vme_end
1591: - entry->vme_start));
1592: entry->needs_copy = FALSE;
1593: }
1594: if (entry->object.vm_object == VM_OBJECT_NULL) {
1595: entry->object.vm_object =
1596: vm_object_allocate(
1.1.1.2 root 1597: (vm_size_t)(entry->vme_end
1.1 root 1598: - entry->vme_start));
1599: entry->offset = (vm_offset_t)0;
1600: }
1601: }
1602: vm_map_clip_start(map, entry, start);
1603: vm_map_clip_end(map, entry, end);
1604:
1605: if (user_wire) {
1606: if ((entry->user_wired_count)++ == 0)
1.1.1.5 root 1607: {
1608: map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 1609: entry->wired_count++;
1.1.1.5 root 1610: }
1.1 root 1611: }
1612: else {
1613: entry->wired_count++;
1614: }
1615:
1616: /*
1617: * Check for holes and protection mismatch.
1618: * Holes: Next entry should be contiguous unless
1619: * this is the end of the region.
1620: * Protection: Access requested must be allowed.
1621: */
1.1.1.2 root 1622: if (((entry->vme_end < end) &&
1.1 root 1623: ((entry->vme_next == vm_map_to_entry(map)) ||
1624: (entry->vme_next->vme_start > entry->vme_end))) ||
1625: ((entry->protection & access_type) != access_type)) {
1626: /*
1627: * Found a hole or protection problem.
1628: * Object creation actions
1629: * do not need to be undone, but the
1630: * wired counts need to be restored.
1631: */
1632: while ((entry != vm_map_to_entry(map)) &&
1633: (entry->vme_end > start)) {
1634: if (user_wire) {
1635: if (--(entry->user_wired_count) == 0)
1.1.1.6 root 1636: {
1.1.1.5 root 1637: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1638: entry->wired_count--;
1.1.1.6 root 1639: }
1.1 root 1640: }
1641: else {
1642: entry->wired_count--;
1643: }
1644:
1645: entry = entry->vme_prev;
1646: }
1647:
1648: vm_map_unlock(map);
1.1.1.7 ! root 1649: return(KERN_NO_SPACE);
1.1 root 1650: }
1651: entry = entry->vme_next;
1652: }
1653:
1654: /*
1655: * Pass 2.
1656: */
1657:
1658: /*
1659: * HACK HACK HACK HACK
1660: *
1661: * If we are wiring in the kernel map or a submap of it,
1662: * unlock the map to avoid deadlocks. We trust that the
1663: * kernel threads are well-behaved, and therefore will
1664: * not do anything destructive to this region of the map
1665: * while we have it unlocked. We cannot trust user threads
1666: * to do the same.
1667: *
1668: * HACK HACK HACK HACK
1669: */
1670: if (vm_map_pmap(map) == kernel_pmap) {
1671: vm_map_unlock(map); /* trust me ... */
1672: }
1673: else {
1674: vm_map_lock_set_recursive(map);
1675: vm_map_lock_write_to_read(map);
1676: }
1677:
1678: entry = start_entry;
1679: while (entry != vm_map_to_entry(map) &&
1680: entry->vme_start < end) {
1681: /*
1682: * Wiring cases:
1683: * Kernel: wired == 1 && user_wired == 0
1684: * User: wired == 1 && user_wired == 1
1685: *
1686: * Don't need to wire if either is > 1. wired = 0 &&
1687: * user_wired == 1 can't happen.
1688: */
1689:
1690: /*
1691: * XXX This assumes that the faults always succeed.
1692: */
1693: if ((entry->wired_count == 1) &&
1694: (entry->user_wired_count <= 1)) {
1695: vm_fault_wire(map, entry);
1696: }
1697: entry = entry->vme_next;
1698: }
1699:
1700: if (vm_map_pmap(map) == kernel_pmap) {
1701: vm_map_lock(map);
1702: }
1703: else {
1704: vm_map_lock_clear_recursive(map);
1705: }
1706: }
1707:
1708: vm_map_unlock(map);
1709:
1710: return(KERN_SUCCESS);
1711: }
1712:
1713: /*
1714: * vm_map_entry_delete: [ internal use only ]
1715: *
1716: * Deallocate the given entry from the target map.
1.1.1.2 root 1717: */
1.1.1.4 root 1718: void vm_map_entry_delete(
1719: vm_map_t map,
1720: vm_map_entry_t entry)
1.1 root 1721: {
1.1.1.4 root 1722: vm_offset_t s, e;
1723: vm_object_t object;
1.1 root 1724: extern vm_object_t kernel_object;
1725:
1726: s = entry->vme_start;
1727: e = entry->vme_end;
1728:
1729: /*Check if projected buffer*/
1730: if (map != kernel_map && entry->projected_on != 0) {
1731: /*Check if projected kernel entry is persistent;
1732: may only manipulate directly if it is*/
1733: if (entry->projected_on->projected_on == 0)
1734: entry->wired_count = 0; /*Avoid unwire fault*/
1735: else
1736: return;
1737: }
1738:
1739: /*
1740: * Get the object. Null objects cannot have pmap entries.
1741: */
1742:
1743: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1744:
1745: /*
1746: * Unwire before removing addresses from the pmap;
1747: * otherwise, unwiring will put the entries back in
1748: * the pmap.
1749: */
1750:
1751: if (entry->wired_count != 0) {
1752: vm_fault_unwire(map, entry);
1753: entry->wired_count = 0;
1.1.1.5 root 1754: if (entry->user_wired_count)
1755: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1756: entry->user_wired_count = 0;
1757: }
1758:
1759: /*
1760: * If the object is shared, we must remove
1761: * *all* references to this data, since we can't
1762: * find all of the physical maps which are sharing
1763: * it.
1764: */
1765:
1766: if (object == kernel_object) {
1767: vm_object_lock(object);
1768: vm_object_page_remove(object, entry->offset,
1769: entry->offset + (e - s));
1770: vm_object_unlock(object);
1771: } else if (entry->is_shared) {
1772: vm_object_pmap_remove(object,
1773: entry->offset,
1774: entry->offset + (e - s));
1775: }
1776: else {
1777: pmap_remove(map->pmap, s, e);
1778: }
1779: }
1780:
1781: /*
1782: * Deallocate the object only after removing all
1783: * pmap entries pointing to its pages.
1784: */
1785:
1786: if (entry->is_sub_map)
1787: vm_map_deallocate(entry->object.sub_map);
1788: else
1789: vm_object_deallocate(entry->object.vm_object);
1790:
1791: vm_map_entry_unlink(map, entry);
1792: map->size -= e - s;
1793:
1794: vm_map_entry_dispose(map, entry);
1795: }
1796:
1797: /*
1798: * vm_map_delete: [ internal use only ]
1799: *
1800: * Deallocates the given address range from the target
1801: * map.
1802: */
1803:
1.1.1.4 root 1804: kern_return_t vm_map_delete(
1805: vm_map_t map,
1806: vm_offset_t start,
1807: vm_offset_t end)
1.1 root 1808: {
1809: vm_map_entry_t entry;
1810: vm_map_entry_t first_entry;
1811:
1812: /*
1813: * Find the start of the region, and clip it
1814: */
1815:
1816: if (!vm_map_lookup_entry(map, start, &first_entry))
1817: entry = first_entry->vme_next;
1818: else {
1819: entry = first_entry;
1820: vm_map_clip_start(map, entry, start);
1821:
1822: /*
1823: * Fix the lookup hint now, rather than each
1824: * time though the loop.
1825: */
1826:
1827: SAVE_HINT(map, entry->vme_prev);
1828: }
1829:
1830: /*
1831: * Save the free space hint
1832: */
1833:
1834: if (map->first_free->vme_start >= start)
1835: map->first_free = entry->vme_prev;
1836:
1837: /*
1838: * Step through all entries in this region
1839: */
1840:
1841: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1842: vm_map_entry_t next;
1843:
1844: vm_map_clip_end(map, entry, end);
1845:
1846: /*
1847: * If the entry is in transition, we must wait
1848: * for it to exit that state. It could be clipped
1849: * while we leave the map unlocked.
1850: */
1851: if(entry->in_transition) {
1852: /*
1853: * Say that we are waiting, and wait for entry.
1854: */
1855: entry->needs_wakeup = TRUE;
1856: vm_map_entry_wait(map, FALSE);
1857: vm_map_lock(map);
1858:
1859: /*
1860: * The entry could have been clipped or it
1861: * may not exist anymore. look it up again.
1862: */
1863: if(!vm_map_lookup_entry(map, start, &entry)) {
1864: entry = entry->vme_next;
1865: }
1866: continue;
1867: }
1868:
1869: next = entry->vme_next;
1870:
1871: vm_map_entry_delete(map, entry);
1872: entry = next;
1873: }
1874:
1875: if (map->wait_for_space)
1876: thread_wakeup((event_t) map);
1877:
1878: return(KERN_SUCCESS);
1879: }
1880:
1881: /*
1882: * vm_map_remove:
1883: *
1884: * Remove the given address range from the target map.
1885: * This is the exported form of vm_map_delete.
1886: */
1.1.1.4 root 1887: kern_return_t vm_map_remove(
1888: vm_map_t map,
1889: vm_offset_t start,
1890: vm_offset_t end)
1.1 root 1891: {
1.1.1.4 root 1892: kern_return_t result;
1.1 root 1893:
1894: vm_map_lock(map);
1895: VM_MAP_RANGE_CHECK(map, start, end);
1896: result = vm_map_delete(map, start, end);
1897: vm_map_unlock(map);
1898:
1899: return(result);
1900: }
1901:
1902:
1903: /*
1904: * vm_map_copy_steal_pages:
1905: *
1906: * Steal all the pages from a vm_map_copy page_list by copying ones
1907: * that have not already been stolen.
1908: */
1909: void
1.1.1.4 root 1910: vm_map_copy_steal_pages(vm_map_copy_t copy)
1.1 root 1911: {
1.1.1.4 root 1912: vm_page_t m, new_m;
1913: int i;
1914: vm_object_t object;
1.1 root 1915:
1916: for (i = 0; i < copy->cpy_npages; i++) {
1917:
1918: /*
1919: * If the page is not tabled, then it's already stolen.
1920: */
1921: m = copy->cpy_page_list[i];
1922: if (!m->tabled)
1923: continue;
1924:
1925: /*
1926: * Page was not stolen, get a new
1927: * one and do the copy now.
1928: */
1.1.1.7 ! root 1929: while ((new_m = vm_page_grab()) == VM_PAGE_NULL) {
1.1 root 1930: VM_PAGE_WAIT((void(*)()) 0);
1931: }
1932:
1933: vm_page_copy(m, new_m);
1934:
1935: object = m->object;
1936: vm_object_lock(object);
1937: vm_page_lock_queues();
1938: if (!m->active && !m->inactive)
1939: vm_page_activate(m);
1940: vm_page_unlock_queues();
1941: PAGE_WAKEUP_DONE(m);
1942: vm_object_paging_end(object);
1943: vm_object_unlock(object);
1944:
1945: copy->cpy_page_list[i] = new_m;
1946: }
1947: }
1948:
1949: /*
1950: * vm_map_copy_page_discard:
1951: *
1952: * Get rid of the pages in a page_list copy. If the pages are
1953: * stolen, they are freed. If the pages are not stolen, they
1954: * are unbusied, and associated state is cleaned up.
1955: */
1.1.1.4 root 1956: void vm_map_copy_page_discard(vm_map_copy_t copy)
1.1 root 1957: {
1958: while (copy->cpy_npages > 0) {
1959: vm_page_t m;
1960:
1961: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1962: VM_PAGE_NULL) {
1963:
1964: /*
1965: * If it's not in the table, then it's
1966: * a stolen page that goes back
1967: * to the free list. Else it belongs
1968: * to some object, and we hold a
1969: * paging reference on that object.
1970: */
1971: if (!m->tabled) {
1972: VM_PAGE_FREE(m);
1973: }
1974: else {
1975: vm_object_t object;
1976:
1977: object = m->object;
1978:
1979: vm_object_lock(object);
1980: vm_page_lock_queues();
1981: if (!m->active && !m->inactive)
1982: vm_page_activate(m);
1983: vm_page_unlock_queues();
1984:
1985: PAGE_WAKEUP_DONE(m);
1986: vm_object_paging_end(object);
1987: vm_object_unlock(object);
1988: }
1989: }
1990: }
1991: }
1992:
1993: /*
1994: * Routine: vm_map_copy_discard
1995: *
1996: * Description:
1997: * Dispose of a map copy object (returned by
1998: * vm_map_copyin).
1999: */
2000: void
1.1.1.4 root 2001: vm_map_copy_discard(vm_map_copy_t copy)
1.1 root 2002: {
2003: free_next_copy:
2004: if (copy == VM_MAP_COPY_NULL)
2005: return;
2006:
2007: switch (copy->type) {
2008: case VM_MAP_COPY_ENTRY_LIST:
2009: while (vm_map_copy_first_entry(copy) !=
2010: vm_map_copy_to_entry(copy)) {
2011: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
2012:
2013: vm_map_copy_entry_unlink(copy, entry);
2014: vm_object_deallocate(entry->object.vm_object);
2015: vm_map_copy_entry_dispose(copy, entry);
2016: }
2017: break;
2018: case VM_MAP_COPY_OBJECT:
2019: vm_object_deallocate(copy->cpy_object);
2020: break;
2021: case VM_MAP_COPY_PAGE_LIST:
2022:
2023: /*
2024: * To clean this up, we have to unbusy all the pages
2025: * and release the paging references in their objects.
2026: */
2027: if (copy->cpy_npages > 0)
2028: vm_map_copy_page_discard(copy);
2029:
2030: /*
2031: * If there's a continuation, abort it. The
2032: * abort routine releases any storage.
2033: */
2034: if (vm_map_copy_has_cont(copy)) {
2035:
2036: /*
2037: * Special case: recognize
2038: * vm_map_copy_discard_cont and optimize
2039: * here to avoid tail recursion.
2040: */
2041: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1.1.1.4 root 2042: vm_map_copy_t new_copy;
1.1 root 2043:
2044: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1.1.1.3 root 2045: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2046: copy = new_copy;
2047: goto free_next_copy;
2048: }
2049: else {
2050: vm_map_copy_abort_cont(copy);
2051: }
2052: }
2053:
2054: break;
2055: }
1.1.1.3 root 2056: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2057: }
2058:
2059: /*
2060: * Routine: vm_map_copy_copy
2061: *
2062: * Description:
2063: * Move the information in a map copy object to
2064: * a new map copy object, leaving the old one
2065: * empty.
2066: *
2067: * This is used by kernel routines that need
2068: * to look at out-of-line data (in copyin form)
2069: * before deciding whether to return SUCCESS.
2070: * If the routine returns FAILURE, the original
2071: * copy object will be deallocated; therefore,
2072: * these routines must make a copy of the copy
2073: * object and leave the original empty so that
2074: * deallocation will not fail.
2075: */
2076: vm_map_copy_t
1.1.1.4 root 2077: vm_map_copy_copy(vm_map_copy_t copy)
1.1 root 2078: {
2079: vm_map_copy_t new_copy;
2080:
2081: if (copy == VM_MAP_COPY_NULL)
2082: return VM_MAP_COPY_NULL;
2083:
2084: /*
2085: * Allocate a new copy object, and copy the information
2086: * from the old one into it.
2087: */
2088:
1.1.1.3 root 2089: new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 2090: *new_copy = *copy;
2091:
2092: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
2093: /*
2094: * The links in the entry chain must be
2095: * changed to point to the new copy object.
2096: */
2097: vm_map_copy_first_entry(copy)->vme_prev
2098: = vm_map_copy_to_entry(new_copy);
2099: vm_map_copy_last_entry(copy)->vme_next
2100: = vm_map_copy_to_entry(new_copy);
2101: }
2102:
2103: /*
2104: * Change the old copy object into one that contains
2105: * nothing to be deallocated.
2106: */
2107: copy->type = VM_MAP_COPY_OBJECT;
2108: copy->cpy_object = VM_OBJECT_NULL;
2109:
2110: /*
2111: * Return the new object.
2112: */
2113: return new_copy;
2114: }
2115:
2116: /*
2117: * Routine: vm_map_copy_discard_cont
2118: *
2119: * Description:
2120: * A version of vm_map_copy_discard that can be called
2121: * as a continuation from a vm_map_copy page list.
2122: */
1.1.1.4 root 2123: kern_return_t vm_map_copy_discard_cont(
2124: vm_map_copyin_args_t cont_args,
2125: vm_map_copy_t *copy_result) /* OUT */
1.1 root 2126: {
2127: vm_map_copy_discard((vm_map_copy_t) cont_args);
2128: if (copy_result != (vm_map_copy_t *)0)
2129: *copy_result = VM_MAP_COPY_NULL;
2130: return(KERN_SUCCESS);
2131: }
2132:
2133: /*
2134: * Routine: vm_map_copy_overwrite
2135: *
2136: * Description:
2137: * Copy the memory described by the map copy
2138: * object (copy; returned by vm_map_copyin) onto
2139: * the specified destination region (dst_map, dst_addr).
2140: * The destination must be writeable.
2141: *
2142: * Unlike vm_map_copyout, this routine actually
2143: * writes over previously-mapped memory. If the
2144: * previous mapping was to a permanent (user-supplied)
2145: * memory object, it is preserved.
2146: *
2147: * The attributes (protection and inheritance) of the
2148: * destination region are preserved.
2149: *
2150: * If successful, consumes the copy object.
2151: * Otherwise, the caller is responsible for it.
2152: *
2153: * Implementation notes:
2154: * To overwrite temporary virtual memory, it is
2155: * sufficient to remove the previous mapping and insert
2156: * the new copy. This replacement is done either on
2157: * the whole region (if no permanent virtual memory
2158: * objects are embedded in the destination region) or
2159: * in individual map entries.
2160: *
2161: * To overwrite permanent virtual memory, it is
2162: * necessary to copy each page, as the external
2163: * memory management interface currently does not
2164: * provide any optimizations.
2165: *
2166: * Once a page of permanent memory has been overwritten,
2167: * it is impossible to interrupt this function; otherwise,
2168: * the call would be neither atomic nor location-independent.
2169: * The kernel-state portion of a user thread must be
2170: * interruptible.
2171: *
2172: * It may be expensive to forward all requests that might
2173: * overwrite permanent memory (vm_write, vm_copy) to
2174: * uninterruptible kernel threads. This routine may be
2175: * called by interruptible threads; however, success is
2176: * not guaranteed -- if the request cannot be performed
2177: * atomically and interruptibly, an error indication is
2178: * returned.
2179: */
1.1.1.4 root 2180: kern_return_t vm_map_copy_overwrite(
2181: vm_map_t dst_map,
2182: vm_offset_t dst_addr,
2183: vm_map_copy_t copy,
2184: boolean_t interruptible)
1.1 root 2185: {
2186: vm_size_t size;
2187: vm_offset_t start;
2188: vm_map_entry_t tmp_entry;
2189: vm_map_entry_t entry;
2190:
2191: boolean_t contains_permanent_objects = FALSE;
2192:
2193: interruptible = FALSE; /* XXX */
2194:
2195: /*
2196: * Check for null copy object.
2197: */
2198:
2199: if (copy == VM_MAP_COPY_NULL)
2200: return(KERN_SUCCESS);
2201:
2202: /*
2203: * Only works for entry lists at the moment. Will
2204: * support page lists LATER.
2205: */
2206:
2207: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2208:
2209: /*
2210: * Currently this routine only handles page-aligned
2211: * regions. Eventually, it should handle misalignments
2212: * by actually copying pages.
2213: */
2214:
2215: if (!page_aligned(copy->offset) ||
2216: !page_aligned(copy->size) ||
2217: !page_aligned(dst_addr))
2218: return(KERN_INVALID_ARGUMENT);
2219:
2220: size = copy->size;
2221:
2222: if (size == 0) {
2223: vm_map_copy_discard(copy);
2224: return(KERN_SUCCESS);
2225: }
2226:
2227: /*
2228: * Verify that the destination is all writeable
2229: * initially.
2230: */
2231: start_pass_1:
2232: vm_map_lock(dst_map);
2233: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2234: vm_map_unlock(dst_map);
2235: return(KERN_INVALID_ADDRESS);
2236: }
2237: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2238: for (entry = tmp_entry;;) {
2239: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2240: vm_map_entry_t next = entry->vme_next;
2241:
2242: if ( ! (entry->protection & VM_PROT_WRITE)) {
2243: vm_map_unlock(dst_map);
2244: return(KERN_PROTECTION_FAILURE);
2245: }
2246:
2247: /*
2248: * If the entry is in transition, we must wait
2249: * for it to exit that state. Anything could happen
2250: * when we unlock the map, so start over.
2251: */
2252: if (entry->in_transition) {
2253:
2254: /*
2255: * Say that we are waiting, and wait for entry.
2256: */
2257: entry->needs_wakeup = TRUE;
2258: vm_map_entry_wait(dst_map, FALSE);
2259:
2260: goto start_pass_1;
2261: }
2262:
2263: if (size <= sub_size)
2264: break;
2265:
2266: if ((next == vm_map_to_entry(dst_map)) ||
2267: (next->vme_start != entry->vme_end)) {
2268: vm_map_unlock(dst_map);
2269: return(KERN_INVALID_ADDRESS);
2270: }
2271:
2272:
2273: /*
2274: * Check for permanent objects in the destination.
2275: */
2276:
2277: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2278: !entry->object.vm_object->temporary)
2279: contains_permanent_objects = TRUE;
2280:
2281: size -= sub_size;
2282: entry = next;
2283: }
2284:
2285: /*
2286: * If there are permanent objects in the destination, then
2287: * the copy cannot be interrupted.
2288: */
2289:
1.1.1.3 root 2290: if (interruptible && contains_permanent_objects) {
2291: vm_map_unlock(dst_map);
1.1 root 2292: return(KERN_FAILURE); /* XXX */
1.1.1.3 root 2293: }
1.1 root 2294:
2295: /*
2296: * XXXO If there are no permanent objects in the destination,
1.1.1.2 root 2297: * XXXO and the destination map entry is not shared,
1.1 root 2298: * XXXO then the map entries can be deleted and replaced
2299: * XXXO with those from the copy. The following code is the
2300: * XXXO basic idea of what to do, but there are lots of annoying
2301: * XXXO little details about getting protection and inheritance
2302: * XXXO right. Should add protection, inheritance, and sharing checks
2303: * XXXO to the above pass and make sure that no wiring is involved.
2304: */
2305: /*
1.1.1.7 ! root 2306: * if (!contains_permanent_objects) {
1.1 root 2307: *
2308: * *
2309: * * Run over copy and adjust entries. Steal code
2310: * * from vm_map_copyout() to do this.
2311: * *
2312: *
2313: * tmp_entry = tmp_entry->vme_prev;
2314: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2315: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2316: *
2317: * vm_map_unlock(dst_map);
2318: * vm_map_copy_discard(copy);
2319: * }
2320: */
2321: /*
2322: *
2323: * Make a second pass, overwriting the data
2324: * At the beginning of each loop iteration,
2325: * the next entry to be overwritten is "tmp_entry"
2326: * (initially, the value returned from the lookup above),
2327: * and the starting address expected in that entry
2328: * is "start".
2329: */
2330:
2331: start = dst_addr;
2332:
2333: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2334: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2335: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2336: vm_object_t object;
1.1.1.2 root 2337:
1.1 root 2338: entry = tmp_entry;
2339: size = (entry->vme_end - entry->vme_start);
2340: /*
2341: * Make sure that no holes popped up in the
2342: * address map, and that the protection is
2343: * still valid, in case the map was unlocked
2344: * earlier.
2345: */
2346:
2347: if (entry->vme_start != start) {
2348: vm_map_unlock(dst_map);
2349: return(KERN_INVALID_ADDRESS);
2350: }
2351: assert(entry != vm_map_to_entry(dst_map));
2352:
2353: /*
2354: * Check protection again
2355: */
2356:
2357: if ( ! (entry->protection & VM_PROT_WRITE)) {
2358: vm_map_unlock(dst_map);
2359: return(KERN_PROTECTION_FAILURE);
2360: }
2361:
2362: /*
2363: * Adjust to source size first
2364: */
2365:
2366: if (copy_size < size) {
2367: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2368: size = copy_size;
2369: }
2370:
2371: /*
2372: * Adjust to destination size
2373: */
2374:
2375: if (size < copy_size) {
2376: vm_map_copy_clip_end(copy, copy_entry,
2377: copy_entry->vme_start + size);
2378: copy_size = size;
2379: }
2380:
2381: assert((entry->vme_end - entry->vme_start) == size);
2382: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2383: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2384:
2385: /*
2386: * If the destination contains temporary unshared memory,
2387: * we can perform the copy by throwing it away and
2388: * installing the source data.
2389: */
2390:
2391: object = entry->object.vm_object;
2392: if (!entry->is_shared &&
2393: ((object == VM_OBJECT_NULL) || object->temporary)) {
2394: vm_object_t old_object = entry->object.vm_object;
2395: vm_offset_t old_offset = entry->offset;
2396:
2397: entry->object = copy_entry->object;
2398: entry->offset = copy_entry->offset;
2399: entry->needs_copy = copy_entry->needs_copy;
2400: entry->wired_count = 0;
1.1.1.5 root 2401: if (entry->user_wired_count)
2402: dst_map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 2403: entry->user_wired_count = 0;
2404:
2405: vm_map_copy_entry_unlink(copy, copy_entry);
2406: vm_map_copy_entry_dispose(copy, copy_entry);
2407:
2408: vm_object_pmap_protect(
2409: old_object,
2410: old_offset,
2411: size,
2412: dst_map->pmap,
2413: tmp_entry->vme_start,
2414: VM_PROT_NONE);
2415:
2416: vm_object_deallocate(old_object);
2417:
2418: /*
2419: * Set up for the next iteration. The map
2420: * has not been unlocked, so the next
2421: * address should be at the end of this
2422: * entry, and the next map entry should be
2423: * the one following it.
2424: */
2425:
2426: start = tmp_entry->vme_end;
2427: tmp_entry = tmp_entry->vme_next;
2428: } else {
2429: vm_map_version_t version;
2430: vm_object_t dst_object = entry->object.vm_object;
2431: vm_offset_t dst_offset = entry->offset;
2432: kern_return_t r;
2433:
2434: /*
2435: * Take an object reference, and record
2436: * the map version information so that the
2437: * map can be safely unlocked.
2438: */
2439:
2440: vm_object_reference(dst_object);
2441:
2442: version.main_timestamp = dst_map->timestamp;
2443:
2444: vm_map_unlock(dst_map);
2445:
2446: /*
2447: * Copy as much as possible in one pass
2448: */
2449:
2450: copy_size = size;
2451: r = vm_fault_copy(
2452: copy_entry->object.vm_object,
2453: copy_entry->offset,
2454: ©_size,
2455: dst_object,
2456: dst_offset,
2457: dst_map,
2458: &version,
2459: FALSE /* XXX interruptible */ );
2460:
2461: /*
2462: * Release the object reference
2463: */
2464:
2465: vm_object_deallocate(dst_object);
2466:
2467: /*
2468: * If a hard error occurred, return it now
2469: */
2470:
2471: if (r != KERN_SUCCESS)
2472: return(r);
2473:
2474: if (copy_size != 0) {
2475: /*
2476: * Dispose of the copied region
2477: */
2478:
2479: vm_map_copy_clip_end(copy, copy_entry,
2480: copy_entry->vme_start + copy_size);
2481: vm_map_copy_entry_unlink(copy, copy_entry);
2482: vm_object_deallocate(copy_entry->object.vm_object);
2483: vm_map_copy_entry_dispose(copy, copy_entry);
2484: }
2485:
2486: /*
2487: * Pick up in the destination map where we left off.
2488: *
2489: * Use the version information to avoid a lookup
2490: * in the normal case.
2491: */
2492:
2493: start += copy_size;
2494: vm_map_lock(dst_map);
2495: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2496: /* We can safely use saved tmp_entry value */
2497:
2498: vm_map_clip_end(dst_map, tmp_entry, start);
2499: tmp_entry = tmp_entry->vme_next;
2500: } else {
2501: /* Must do lookup of tmp_entry */
2502:
2503: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2504: vm_map_unlock(dst_map);
2505: return(KERN_INVALID_ADDRESS);
2506: }
2507: vm_map_clip_start(dst_map, tmp_entry, start);
2508: }
2509: }
2510:
2511: }
2512: vm_map_unlock(dst_map);
2513:
2514: /*
2515: * Throw away the vm_map_copy object
2516: */
2517: vm_map_copy_discard(copy);
2518:
2519: return(KERN_SUCCESS);
2520: }
2521:
2522: /*
1.1.1.7 ! root 2523: * Routine: vm_map_copy_insert
1.1.1.2 root 2524: *
1.1 root 2525: * Description:
2526: * Link a copy chain ("copy") into a map at the
2527: * specified location (after "where").
2528: * Side effects:
2529: * The copy chain is destroyed.
2530: */
1.1.1.7 ! root 2531: static void
! 2532: vm_map_copy_insert(struct vm_map *map, struct vm_map_entry *where,
! 2533: struct vm_map_copy *copy)
! 2534: {
! 2535: struct vm_map_entry *entry;
! 2536:
! 2537: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
! 2538:
! 2539: for (;;) {
! 2540: entry = vm_map_copy_first_entry(copy);
! 2541:
! 2542: if (entry == vm_map_copy_to_entry(copy)) {
! 2543: break;
! 2544: }
! 2545:
! 2546: /*
! 2547: * TODO Turn copy maps into their own type so they don't
! 2548: * use any of the tree operations.
! 2549: */
! 2550: vm_map_copy_entry_unlink(copy, entry);
! 2551: vm_map_entry_link(map, where, entry);
! 2552: where = entry;
! 2553: }
! 2554:
! 2555: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t)copy);
! 2556: }
1.1 root 2557:
2558: /*
2559: * Routine: vm_map_copyout
2560: *
2561: * Description:
2562: * Copy out a copy chain ("copy") into newly-allocated
2563: * space in the destination map.
2564: *
2565: * If successful, consumes the copy object.
2566: * Otherwise, the caller is responsible for it.
2567: */
1.1.1.4 root 2568: kern_return_t vm_map_copyout(
2569: vm_map_t dst_map,
2570: vm_offset_t *dst_addr, /* OUT */
2571: vm_map_copy_t copy)
1.1 root 2572: {
2573: vm_size_t size;
2574: vm_size_t adjustment;
2575: vm_offset_t start;
2576: vm_offset_t vm_copy_start;
2577: vm_map_entry_t last;
2578: vm_map_entry_t entry;
2579:
2580: /*
2581: * Check for null copy object.
2582: */
2583:
2584: if (copy == VM_MAP_COPY_NULL) {
2585: *dst_addr = 0;
2586: return(KERN_SUCCESS);
2587: }
2588:
2589: /*
2590: * Check for special copy object, created
2591: * by vm_map_copyin_object.
2592: */
2593:
2594: if (copy->type == VM_MAP_COPY_OBJECT) {
2595: vm_object_t object = copy->cpy_object;
2596: vm_size_t offset = copy->offset;
2597: vm_size_t tmp_size = copy->size;
2598: kern_return_t kr;
2599:
2600: *dst_addr = 0;
2601: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2602: (vm_offset_t) 0, TRUE,
2603: object, offset, FALSE,
2604: VM_PROT_DEFAULT, VM_PROT_ALL,
2605: VM_INHERIT_DEFAULT);
2606: if (kr != KERN_SUCCESS)
2607: return(kr);
1.1.1.3 root 2608: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2609: return(KERN_SUCCESS);
2610: }
2611:
2612: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2613: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2614:
2615: /*
2616: * Find space for the data
2617: */
2618:
2619: vm_copy_start = trunc_page(copy->offset);
2620: size = round_page(copy->offset + copy->size) - vm_copy_start;
1.1.1.7 ! root 2621: last = vm_map_find_entry_anywhere(dst_map, size, 0, FALSE, &start);
1.1 root 2622:
1.1.1.7 ! root 2623: if (last == NULL) {
! 2624: vm_map_unlock(dst_map);
! 2625: return KERN_NO_SPACE;
1.1 root 2626: }
2627:
2628: /*
2629: * Adjust the addresses in the copy chain, and
2630: * reset the region attributes.
2631: */
2632:
2633: adjustment = start - vm_copy_start;
2634: for (entry = vm_map_copy_first_entry(copy);
2635: entry != vm_map_copy_to_entry(copy);
2636: entry = entry->vme_next) {
2637: entry->vme_start += adjustment;
2638: entry->vme_end += adjustment;
2639:
1.1.1.7 ! root 2640: /*
! 2641: * XXX There is no need to update the gap tree here.
! 2642: * See vm_map_copy_insert.
! 2643: */
! 2644:
1.1 root 2645: entry->inheritance = VM_INHERIT_DEFAULT;
2646: entry->protection = VM_PROT_DEFAULT;
2647: entry->max_protection = VM_PROT_ALL;
2648: entry->projected_on = 0;
2649:
2650: /*
2651: * If the entry is now wired,
2652: * map the pages into the destination map.
2653: */
2654: if (entry->wired_count != 0) {
1.1.1.4 root 2655: vm_offset_t va;
2656: vm_offset_t offset;
2657: vm_object_t object;
1.1 root 2658:
2659: object = entry->object.vm_object;
2660: offset = entry->offset;
2661: va = entry->vme_start;
2662:
2663: pmap_pageable(dst_map->pmap,
2664: entry->vme_start,
2665: entry->vme_end,
2666: TRUE);
2667:
2668: while (va < entry->vme_end) {
1.1.1.4 root 2669: vm_page_t m;
1.1 root 2670:
2671: /*
2672: * Look up the page in the object.
2673: * Assert that the page will be found in the
2674: * top object:
2675: * either
2676: * the object was newly created by
2677: * vm_object_copy_slowly, and has
2678: * copies of all of the pages from
2679: * the source object
2680: * or
2681: * the object was moved from the old
2682: * map entry; because the old map
2683: * entry was wired, all of the pages
2684: * were in the top-level object.
2685: * (XXX not true if we wire pages for
2686: * reading)
2687: */
2688: vm_object_lock(object);
2689: vm_object_paging_begin(object);
2690:
2691: m = vm_page_lookup(object, offset);
2692: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2693: m->absent)
2694: panic("vm_map_copyout: wiring 0x%x", m);
2695:
2696: m->busy = TRUE;
2697: vm_object_unlock(object);
2698:
2699: PMAP_ENTER(dst_map->pmap, va, m,
2700: entry->protection, TRUE);
2701:
2702: vm_object_lock(object);
2703: PAGE_WAKEUP_DONE(m);
2704: /* the page is wired, so we don't have to activate */
2705: vm_object_paging_end(object);
2706: vm_object_unlock(object);
2707:
2708: offset += PAGE_SIZE;
2709: va += PAGE_SIZE;
2710: }
2711: }
2712:
2713:
2714: }
2715:
2716: /*
2717: * Correct the page alignment for the result
2718: */
2719:
2720: *dst_addr = start + (copy->offset - vm_copy_start);
2721:
2722: /*
2723: * Update the hints and the map size
2724: */
2725:
2726: if (dst_map->first_free == last)
2727: dst_map->first_free = vm_map_copy_last_entry(copy);
2728: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2729:
2730: dst_map->size += size;
2731:
2732: /*
2733: * Link in the copy
2734: */
2735:
2736: vm_map_copy_insert(dst_map, last, copy);
2737:
2738: vm_map_unlock(dst_map);
2739:
2740: /*
2741: * XXX If wiring_required, call vm_map_pageable
2742: */
2743:
2744: return(KERN_SUCCESS);
2745: }
2746:
2747: /*
2748: *
2749: * vm_map_copyout_page_list:
2750: *
2751: * Version of vm_map_copyout() for page list vm map copies.
2752: *
2753: */
1.1.1.4 root 2754: kern_return_t vm_map_copyout_page_list(
2755: vm_map_t dst_map,
2756: vm_offset_t *dst_addr, /* OUT */
2757: vm_map_copy_t copy)
1.1 root 2758: {
2759: vm_size_t size;
2760: vm_offset_t start;
2761: vm_offset_t end;
2762: vm_offset_t offset;
2763: vm_map_entry_t last;
2764: vm_object_t object;
2765: vm_page_t *page_list, m;
2766: vm_map_entry_t entry;
2767: vm_offset_t old_last_offset;
2768: boolean_t cont_invoked, needs_wakeup = FALSE;
2769: kern_return_t result = KERN_SUCCESS;
2770: vm_map_copy_t orig_copy;
2771: vm_offset_t dst_offset;
2772: boolean_t must_wire;
2773:
2774: /*
2775: * Make sure the pages are stolen, because we are
2776: * going to put them in a new object. Assume that
2777: * all pages are identical to first in this regard.
2778: */
2779:
2780: page_list = ©->cpy_page_list[0];
2781: if ((*page_list)->tabled)
2782: vm_map_copy_steal_pages(copy);
2783:
2784: /*
2785: * Find space for the data
2786: */
2787:
2788: size = round_page(copy->offset + copy->size) -
2789: trunc_page(copy->offset);
2790:
1.1.1.7 ! root 2791: vm_map_lock(dst_map);
1.1 root 2792:
1.1.1.7 ! root 2793: last = vm_map_find_entry_anywhere(dst_map, size, 0, TRUE, &start);
1.1 root 2794:
1.1.1.7 ! root 2795: if (last == NULL) {
! 2796: vm_map_unlock(dst_map);
! 2797: return KERN_NO_SPACE;
! 2798: }
1.1 root 2799:
1.1.1.7 ! root 2800: end = start + size;
1.1 root 2801:
1.1.1.7 ! root 2802: must_wire = dst_map->wiring_required;
1.1 root 2803:
2804: /*
2805: * See whether we can avoid creating a new entry (and object) by
2806: * extending one of our neighbors. [So far, we only attempt to
2807: * extend from below.]
2808: *
2809: * The code path below here is a bit twisted. If any of the
2810: * extension checks fails, we branch to create_object. If
2811: * it all works, we fall out the bottom and goto insert_pages.
2812: */
2813: if (last == vm_map_to_entry(dst_map) ||
2814: last->vme_end != start ||
2815: last->is_shared != FALSE ||
2816: last->is_sub_map != FALSE ||
2817: last->inheritance != VM_INHERIT_DEFAULT ||
2818: last->protection != VM_PROT_DEFAULT ||
2819: last->max_protection != VM_PROT_ALL ||
2820: (must_wire ? (last->wired_count != 1 ||
2821: last->user_wired_count != 1) :
2822: (last->wired_count != 0))) {
2823: goto create_object;
2824: }
1.1.1.2 root 2825:
1.1 root 2826: /*
2827: * If this entry needs an object, make one.
2828: */
2829: if (last->object.vm_object == VM_OBJECT_NULL) {
2830: object = vm_object_allocate(
2831: (vm_size_t)(last->vme_end - last->vme_start + size));
2832: last->object.vm_object = object;
2833: last->offset = 0;
2834: vm_object_lock(object);
2835: }
2836: else {
2837: vm_offset_t prev_offset = last->offset;
2838: vm_size_t prev_size = start - last->vme_start;
2839: vm_size_t new_size;
2840:
2841: /*
2842: * This is basically vm_object_coalesce.
2843: */
2844:
2845: object = last->object.vm_object;
2846: vm_object_lock(object);
2847:
2848: /*
2849: * Try to collapse the object first
2850: */
2851: vm_object_collapse(object);
2852:
2853: /*
2854: * Can't coalesce if pages not mapped to
2855: * last may be in use anyway:
2856: * . more than one reference
2857: * . paged out
2858: * . shadows another object
2859: * . has a copy elsewhere
2860: * . paging references (pages might be in page-list)
2861: */
2862:
2863: if ((object->ref_count > 1) ||
2864: object->pager_created ||
2865: (object->shadow != VM_OBJECT_NULL) ||
2866: (object->copy != VM_OBJECT_NULL) ||
2867: (object->paging_in_progress != 0)) {
2868: vm_object_unlock(object);
2869: goto create_object;
2870: }
2871:
2872: /*
2873: * Extend the object if necessary. Don't have to call
2874: * vm_object_page_remove because the pages aren't mapped,
2875: * and vm_page_replace will free up any old ones it encounters.
2876: */
2877: new_size = prev_offset + prev_size + size;
2878: if (new_size > object->size)
2879: object->size = new_size;
2880: }
2881:
2882: /*
2883: * Coalesced the two objects - can extend
2884: * the previous map entry to include the
2885: * new range.
2886: */
2887: dst_map->size += size;
2888: last->vme_end = end;
1.1.1.7 ! root 2889: vm_map_gap_update(&dst_map->hdr, last);
1.1 root 2890:
2891: SAVE_HINT(dst_map, last);
2892:
2893: goto insert_pages;
2894:
2895: create_object:
2896:
2897: /*
2898: * Create object
2899: */
2900: object = vm_object_allocate(size);
2901:
2902: /*
2903: * Create entry
2904: */
2905:
2906: entry = vm_map_entry_create(dst_map);
2907:
2908: entry->object.vm_object = object;
2909: entry->offset = 0;
2910:
2911: entry->is_shared = FALSE;
2912: entry->is_sub_map = FALSE;
2913: entry->needs_copy = FALSE;
2914:
2915: if (must_wire) {
2916: entry->wired_count = 1;
1.1.1.5 root 2917: dst_map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 2918: entry->user_wired_count = 1;
2919: } else {
2920: entry->wired_count = 0;
2921: entry->user_wired_count = 0;
2922: }
2923:
2924: entry->in_transition = TRUE;
2925: entry->needs_wakeup = FALSE;
2926:
2927: entry->vme_start = start;
2928: entry->vme_end = start + size;
1.1.1.2 root 2929:
1.1 root 2930: entry->inheritance = VM_INHERIT_DEFAULT;
2931: entry->protection = VM_PROT_DEFAULT;
2932: entry->max_protection = VM_PROT_ALL;
2933: entry->projected_on = 0;
2934:
2935: vm_object_lock(object);
2936:
2937: /*
2938: * Update the hints and the map size
2939: */
2940: if (dst_map->first_free == last) {
2941: dst_map->first_free = entry;
2942: }
2943: SAVE_HINT(dst_map, entry);
2944: dst_map->size += size;
2945:
2946: /*
2947: * Link in the entry
2948: */
2949: vm_map_entry_link(dst_map, last, entry);
2950: last = entry;
2951:
2952: /*
1.1.1.2 root 2953: * Transfer pages into new object.
1.1 root 2954: * Scan page list in vm_map_copy.
2955: */
2956: insert_pages:
2957: dst_offset = copy->offset & PAGE_MASK;
2958: cont_invoked = FALSE;
2959: orig_copy = copy;
2960: last->in_transition = TRUE;
2961: old_last_offset = last->offset
2962: + (start - last->vme_start);
2963:
2964: vm_page_lock_queues();
2965:
2966: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2967: m = *page_list;
2968: assert(m && !m->tabled);
2969:
2970: /*
2971: * Must clear busy bit in page before inserting it.
2972: * Ok to skip wakeup logic because nobody else
2973: * can possibly know about this page.
2974: * The page is dirty in its new object.
2975: */
2976:
2977: assert(!m->wanted);
2978:
2979: m->busy = FALSE;
2980: m->dirty = TRUE;
2981: vm_page_replace(m, object, old_last_offset + offset);
2982: if (must_wire) {
2983: vm_page_wire(m);
2984: PMAP_ENTER(dst_map->pmap,
2985: last->vme_start + m->offset - last->offset,
2986: m, last->protection, TRUE);
2987: } else {
2988: vm_page_activate(m);
2989: }
2990:
2991: *page_list++ = VM_PAGE_NULL;
2992: if (--(copy->cpy_npages) == 0 &&
2993: vm_map_copy_has_cont(copy)) {
2994: vm_map_copy_t new_copy;
2995:
2996: /*
2997: * Ok to unlock map because entry is
2998: * marked in_transition.
2999: */
3000: cont_invoked = TRUE;
3001: vm_page_unlock_queues();
3002: vm_object_unlock(object);
3003: vm_map_unlock(dst_map);
3004: vm_map_copy_invoke_cont(copy, &new_copy, &result);
3005:
3006: if (result == KERN_SUCCESS) {
3007:
3008: /*
3009: * If we got back a copy with real pages,
3010: * steal them now. Either all of the
3011: * pages in the list are tabled or none
3012: * of them are; mixtures are not possible.
3013: *
3014: * Save original copy for consume on
3015: * success logic at end of routine.
3016: */
3017: if (copy != orig_copy)
3018: vm_map_copy_discard(copy);
3019:
3020: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
3021: page_list = ©->cpy_page_list[0];
3022: if ((*page_list)->tabled)
3023: vm_map_copy_steal_pages(copy);
3024: }
3025: }
3026: else {
3027: /*
3028: * Continuation failed.
3029: */
3030: vm_map_lock(dst_map);
3031: goto error;
3032: }
3033:
3034: vm_map_lock(dst_map);
3035: vm_object_lock(object);
3036: vm_page_lock_queues();
3037: }
3038: }
3039:
3040: vm_page_unlock_queues();
3041: vm_object_unlock(object);
3042:
3043: *dst_addr = start + dst_offset;
1.1.1.2 root 3044:
1.1 root 3045: /*
3046: * Clear the in transition bits. This is easy if we
3047: * didn't have a continuation.
3048: */
3049: error:
3050: if (!cont_invoked) {
3051: /*
3052: * We didn't unlock the map, so nobody could
3053: * be waiting.
3054: */
3055: last->in_transition = FALSE;
3056: assert(!last->needs_wakeup);
3057: needs_wakeup = FALSE;
3058: }
3059: else {
3060: if (!vm_map_lookup_entry(dst_map, start, &entry))
3061: panic("vm_map_copyout_page_list: missing entry");
3062:
3063: /*
3064: * Clear transition bit for all constituent entries that
3065: * were in the original entry. Also check for waiters.
3066: */
3067: while((entry != vm_map_to_entry(dst_map)) &&
3068: (entry->vme_start < end)) {
3069: assert(entry->in_transition);
3070: entry->in_transition = FALSE;
3071: if(entry->needs_wakeup) {
3072: entry->needs_wakeup = FALSE;
3073: needs_wakeup = TRUE;
3074: }
3075: entry = entry->vme_next;
3076: }
3077: }
1.1.1.2 root 3078:
1.1 root 3079: if (result != KERN_SUCCESS)
3080: vm_map_delete(dst_map, start, end);
3081:
3082: vm_map_unlock(dst_map);
3083:
3084: if (needs_wakeup)
3085: vm_map_entry_wakeup(dst_map);
3086:
3087: /*
3088: * Consume on success logic.
3089: */
3090: if (copy != orig_copy) {
1.1.1.3 root 3091: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 3092: }
3093: if (result == KERN_SUCCESS) {
1.1.1.3 root 3094: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
1.1 root 3095: }
1.1.1.2 root 3096:
1.1 root 3097: return(result);
3098: }
3099:
3100: /*
3101: * Routine: vm_map_copyin
3102: *
3103: * Description:
3104: * Copy the specified region (src_addr, len) from the
3105: * source address space (src_map), possibly removing
3106: * the region from the source address space (src_destroy).
3107: *
3108: * Returns:
3109: * A vm_map_copy_t object (copy_result), suitable for
3110: * insertion into another address space (using vm_map_copyout),
3111: * copying over another address space region (using
3112: * vm_map_copy_overwrite). If the copy is unused, it
3113: * should be destroyed (using vm_map_copy_discard).
3114: *
3115: * In/out conditions:
3116: * The source map should not be locked on entry.
3117: */
1.1.1.4 root 3118: kern_return_t vm_map_copyin(
3119: vm_map_t src_map,
3120: vm_offset_t src_addr,
3121: vm_size_t len,
3122: boolean_t src_destroy,
3123: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3124: {
3125: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3126: * in multi-level lookup, this
3127: * entry contains the actual
3128: * vm_object/offset.
3129: */
3130:
3131: vm_offset_t src_start; /* Start of current entry --
3132: * where copy is taking place now
3133: */
3134: vm_offset_t src_end; /* End of entire region to be
3135: * copied */
3136:
3137: vm_map_copy_t copy; /* Resulting copy */
3138:
3139: /*
3140: * Check for copies of zero bytes.
3141: */
3142:
3143: if (len == 0) {
3144: *copy_result = VM_MAP_COPY_NULL;
3145: return(KERN_SUCCESS);
3146: }
3147:
3148: /*
1.1.1.7 ! root 3149: * Check that the end address doesn't overflow
! 3150: */
! 3151:
! 3152: if ((src_addr + len) <= src_addr) {
! 3153: return KERN_INVALID_ADDRESS;
! 3154: }
! 3155:
! 3156: /*
1.1 root 3157: * Compute start and end of region
3158: */
3159:
3160: src_start = trunc_page(src_addr);
3161: src_end = round_page(src_addr + len);
3162:
3163: /*
1.1.1.7 ! root 3164: * XXX VM maps shouldn't end at maximum address
1.1 root 3165: */
3166:
1.1.1.7 ! root 3167: if (src_end == 0) {
! 3168: return KERN_INVALID_ADDRESS;
! 3169: }
1.1 root 3170:
3171: /*
3172: * Allocate a header element for the list.
3173: *
1.1.1.2 root 3174: * Use the start and end in the header to
1.1 root 3175: * remember the endpoints prior to rounding.
3176: */
3177:
1.1.1.3 root 3178: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3179: vm_map_copy_first_entry(copy) =
3180: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3181: copy->type = VM_MAP_COPY_ENTRY_LIST;
3182: copy->cpy_hdr.nentries = 0;
1.1.1.3 root 3183: rbtree_init(©->cpy_hdr.tree);
1.1.1.7 ! root 3184: rbtree_init(©->cpy_hdr.gap_tree);
1.1 root 3185:
3186: copy->offset = src_addr;
3187: copy->size = len;
1.1.1.2 root 3188:
1.1 root 3189: #define RETURN(x) \
3190: MACRO_BEGIN \
3191: vm_map_unlock(src_map); \
3192: vm_map_copy_discard(copy); \
3193: MACRO_RETURN(x); \
3194: MACRO_END
3195:
3196: /*
3197: * Find the beginning of the region.
3198: */
3199:
3200: vm_map_lock(src_map);
3201:
3202: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3203: RETURN(KERN_INVALID_ADDRESS);
3204: vm_map_clip_start(src_map, tmp_entry, src_start);
3205:
3206: /*
3207: * Go through entries until we get to the end.
3208: */
3209:
3210: while (TRUE) {
3211: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3212: vm_size_t src_size; /* Size of source
3213: * map entry (in both
3214: * maps)
3215: */
3216:
3217: vm_object_t src_object; /* Object to copy */
3218: vm_offset_t src_offset;
3219:
3220: boolean_t src_needs_copy; /* Should source map
3221: * be made read-only
3222: * for copy-on-write?
3223: */
3224:
3225: vm_map_entry_t new_entry; /* Map entry for copy */
3226: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3227:
3228: boolean_t was_wired; /* Was source wired? */
3229: vm_map_version_t version; /* Version before locks
3230: * dropped to make copy
3231: */
3232:
3233: /*
3234: * Verify that the region can be read.
3235: */
3236:
3237: if (! (src_entry->protection & VM_PROT_READ))
3238: RETURN(KERN_PROTECTION_FAILURE);
3239:
3240: /*
3241: * Clip against the endpoints of the entire region.
3242: */
3243:
3244: vm_map_clip_end(src_map, src_entry, src_end);
3245:
3246: src_size = src_entry->vme_end - src_start;
3247: src_object = src_entry->object.vm_object;
3248: src_offset = src_entry->offset;
3249: was_wired = (src_entry->wired_count != 0);
3250:
3251: /*
3252: * Create a new address map entry to
3253: * hold the result. Fill in the fields from
3254: * the appropriate source entries.
3255: */
3256:
3257: new_entry = vm_map_copy_entry_create(copy);
3258: vm_map_entry_copy(new_entry, src_entry);
3259:
3260: /*
3261: * Attempt non-blocking copy-on-write optimizations.
3262: */
3263:
3264: if (src_destroy &&
3265: (src_object == VM_OBJECT_NULL ||
3266: (src_object->temporary && !src_object->use_shared_copy)))
3267: {
3268: /*
3269: * If we are destroying the source, and the object
3270: * is temporary, and not shared writable,
3271: * we can move the object reference
3272: * from the source to the copy. The copy is
3273: * copy-on-write only if the source is.
3274: * We make another reference to the object, because
3275: * destroying the source entry will deallocate it.
3276: */
3277: vm_object_reference(src_object);
3278:
3279: /*
3280: * Copy is always unwired. vm_map_copy_entry
3281: * set its wired count to zero.
3282: */
3283:
3284: goto CopySuccessful;
3285: }
3286:
3287: if (!was_wired &&
3288: vm_object_copy_temporary(
3289: &new_entry->object.vm_object,
3290: &new_entry->offset,
3291: &src_needs_copy,
3292: &new_entry_needs_copy)) {
3293:
3294: new_entry->needs_copy = new_entry_needs_copy;
3295:
3296: /*
3297: * Handle copy-on-write obligations
3298: */
3299:
3300: if (src_needs_copy && !tmp_entry->needs_copy) {
3301: vm_object_pmap_protect(
3302: src_object,
3303: src_offset,
3304: src_size,
3305: (src_entry->is_shared ? PMAP_NULL
3306: : src_map->pmap),
3307: src_entry->vme_start,
3308: src_entry->protection &
3309: ~VM_PROT_WRITE);
3310:
3311: tmp_entry->needs_copy = TRUE;
3312: }
3313:
3314: /*
3315: * The map has never been unlocked, so it's safe to
3316: * move to the next entry rather than doing another
3317: * lookup.
3318: */
3319:
3320: goto CopySuccessful;
3321: }
3322:
3323: new_entry->needs_copy = FALSE;
3324:
3325: /*
3326: * Take an object reference, so that we may
3327: * release the map lock(s).
3328: */
3329:
3330: assert(src_object != VM_OBJECT_NULL);
3331: vm_object_reference(src_object);
3332:
3333: /*
3334: * Record the timestamp for later verification.
3335: * Unlock the map.
3336: */
3337:
3338: version.main_timestamp = src_map->timestamp;
3339: vm_map_unlock(src_map);
3340:
3341: /*
3342: * Perform the copy
3343: */
3344:
3345: if (was_wired) {
3346: vm_object_lock(src_object);
3347: (void) vm_object_copy_slowly(
3348: src_object,
3349: src_offset,
3350: src_size,
3351: FALSE,
3352: &new_entry->object.vm_object);
3353: new_entry->offset = 0;
3354: new_entry->needs_copy = FALSE;
3355: } else {
3356: kern_return_t result;
3357:
3358: result = vm_object_copy_strategically(src_object,
3359: src_offset,
3360: src_size,
3361: &new_entry->object.vm_object,
3362: &new_entry->offset,
3363: &new_entry_needs_copy);
3364:
3365: new_entry->needs_copy = new_entry_needs_copy;
1.1.1.2 root 3366:
1.1 root 3367:
3368: if (result != KERN_SUCCESS) {
3369: vm_map_copy_entry_dispose(copy, new_entry);
3370:
3371: vm_map_lock(src_map);
3372: RETURN(result);
3373: }
3374:
3375: }
3376:
3377: /*
3378: * Throw away the extra reference
3379: */
3380:
3381: vm_object_deallocate(src_object);
3382:
3383: /*
3384: * Verify that the map has not substantially
3385: * changed while the copy was being made.
3386: */
3387:
3388: vm_map_lock(src_map); /* Increments timestamp once! */
3389:
3390: if ((version.main_timestamp + 1) == src_map->timestamp)
3391: goto CopySuccessful;
3392:
3393: /*
3394: * Simple version comparison failed.
3395: *
3396: * Retry the lookup and verify that the
3397: * same object/offset are still present.
3398: *
3399: * [Note: a memory manager that colludes with
3400: * the calling task can detect that we have
3401: * cheated. While the map was unlocked, the
3402: * mapping could have been changed and restored.]
3403: */
3404:
3405: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3406: vm_map_copy_entry_dispose(copy, new_entry);
3407: RETURN(KERN_INVALID_ADDRESS);
3408: }
3409:
3410: src_entry = tmp_entry;
3411: vm_map_clip_start(src_map, src_entry, src_start);
3412:
3413: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3414: goto VerificationFailed;
3415:
3416: if (src_entry->vme_end < new_entry->vme_end)
3417: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3418:
3419: if ((src_entry->object.vm_object != src_object) ||
3420: (src_entry->offset != src_offset) ) {
3421:
3422: /*
3423: * Verification failed.
3424: *
3425: * Start over with this top-level entry.
3426: */
3427:
3428: VerificationFailed: ;
3429:
3430: vm_object_deallocate(new_entry->object.vm_object);
3431: vm_map_copy_entry_dispose(copy, new_entry);
3432: tmp_entry = src_entry;
3433: continue;
3434: }
3435:
3436: /*
3437: * Verification succeeded.
3438: */
3439:
3440: CopySuccessful: ;
3441:
3442: /*
3443: * Link in the new copy entry.
3444: */
3445:
3446: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3447: new_entry);
1.1.1.2 root 3448:
1.1 root 3449: /*
3450: * Determine whether the entire region
3451: * has been copied.
3452: */
3453: src_start = new_entry->vme_end;
3454: if ((src_start >= src_end) && (src_end != 0))
3455: break;
3456:
3457: /*
3458: * Verify that there are no gaps in the region
3459: */
3460:
3461: tmp_entry = src_entry->vme_next;
3462: if (tmp_entry->vme_start != src_start)
3463: RETURN(KERN_INVALID_ADDRESS);
3464: }
3465:
3466: /*
3467: * If the source should be destroyed, do it now, since the
1.1.1.2 root 3468: * copy was successful.
1.1 root 3469: */
3470: if (src_destroy)
3471: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3472:
3473: vm_map_unlock(src_map);
3474:
3475: *copy_result = copy;
3476: return(KERN_SUCCESS);
3477:
3478: #undef RETURN
3479: }
3480:
3481: /*
3482: * vm_map_copyin_object:
3483: *
3484: * Create a copy object from an object.
3485: * Our caller donates an object reference.
3486: */
3487:
1.1.1.4 root 3488: kern_return_t vm_map_copyin_object(
3489: vm_object_t object,
3490: vm_offset_t offset, /* offset of region in object */
3491: vm_size_t size, /* size of region in object */
3492: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3493: {
3494: vm_map_copy_t copy; /* Resulting copy */
3495:
3496: /*
3497: * We drop the object into a special copy object
3498: * that contains the object directly. These copy objects
1.1.1.7 ! root 3499: * are distinguished by links.
1.1 root 3500: */
3501:
1.1.1.3 root 3502: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3503: vm_map_copy_first_entry(copy) =
3504: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3505: copy->type = VM_MAP_COPY_OBJECT;
3506: copy->cpy_object = object;
3507: copy->offset = offset;
3508: copy->size = size;
3509:
3510: *copy_result = copy;
3511: return(KERN_SUCCESS);
3512: }
3513:
3514: /*
3515: * vm_map_copyin_page_list_cont:
3516: *
3517: * Continuation routine for vm_map_copyin_page_list.
1.1.1.2 root 3518: *
1.1 root 3519: * If vm_map_copyin_page_list can't fit the entire vm range
3520: * into a single page list object, it creates a continuation.
3521: * When the target of the operation has used the pages in the
3522: * initial page list, it invokes the continuation, which calls
3523: * this routine. If an error happens, the continuation is aborted
3524: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3525: * pages are discarded from the initial page list before invoking
3526: * the continuation.
3527: *
3528: * NOTE: This is not the same sort of continuation used by
3529: * the scheduler.
3530: */
3531:
1.1.1.4 root 3532: kern_return_t vm_map_copyin_page_list_cont(
3533: vm_map_copyin_args_t cont_args,
3534: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3535: {
3536: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
1.1.1.4 root 3537: boolean_t do_abort, src_destroy, src_destroy_only;
1.1 root 3538:
3539: /*
3540: * Check for cases that only require memory destruction.
3541: */
3542: do_abort = (copy_result == (vm_map_copy_t *) 0);
3543: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3544: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3545:
3546: if (do_abort || src_destroy_only) {
1.1.1.2 root 3547: if (src_destroy)
1.1 root 3548: result = vm_map_remove(cont_args->map,
3549: cont_args->destroy_addr,
3550: cont_args->destroy_addr + cont_args->destroy_len);
3551: if (!do_abort)
3552: *copy_result = VM_MAP_COPY_NULL;
3553: }
3554: else {
3555: result = vm_map_copyin_page_list(cont_args->map,
3556: cont_args->src_addr, cont_args->src_len, src_destroy,
3557: cont_args->steal_pages, copy_result, TRUE);
3558:
3559: if (src_destroy && !cont_args->steal_pages &&
3560: vm_map_copy_has_cont(*copy_result)) {
3561: vm_map_copyin_args_t new_args;
3562: /*
3563: * Transfer old destroy info.
3564: */
3565: new_args = (vm_map_copyin_args_t)
3566: (*copy_result)->cpy_cont_args;
3567: new_args->destroy_addr = cont_args->destroy_addr;
3568: new_args->destroy_len = cont_args->destroy_len;
3569: }
3570: }
1.1.1.2 root 3571:
1.1 root 3572: vm_map_deallocate(cont_args->map);
3573: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3574:
3575: return(result);
3576: }
3577:
3578: /*
3579: * vm_map_copyin_page_list:
3580: *
3581: * This is a variant of vm_map_copyin that copies in a list of pages.
3582: * If steal_pages is TRUE, the pages are only in the returned list.
3583: * If steal_pages is FALSE, the pages are busy and still in their
3584: * objects. A continuation may be returned if not all the pages fit:
3585: * the recipient of this copy_result must be prepared to deal with it.
3586: */
3587:
1.1.1.4 root 3588: kern_return_t vm_map_copyin_page_list(
3589: vm_map_t src_map,
3590: vm_offset_t src_addr,
3591: vm_size_t len,
3592: boolean_t src_destroy,
3593: boolean_t steal_pages,
3594: vm_map_copy_t *copy_result, /* OUT */
3595: boolean_t is_cont)
1.1 root 3596: {
3597: vm_map_entry_t src_entry;
3598: vm_page_t m;
3599: vm_offset_t src_start;
3600: vm_offset_t src_end;
3601: vm_size_t src_size;
3602: vm_object_t src_object;
3603: vm_offset_t src_offset;
3604: vm_offset_t src_last_offset;
3605: vm_map_copy_t copy; /* Resulting copy */
3606: kern_return_t result = KERN_SUCCESS;
3607: boolean_t need_map_lookup;
3608: vm_map_copyin_args_t cont_args;
3609:
3610: /*
3611: * If steal_pages is FALSE, this leaves busy pages in
3612: * the object. A continuation must be used if src_destroy
3613: * is true in this case (!steal_pages && src_destroy).
3614: *
3615: * XXX Still have a more general problem of what happens
3616: * XXX if the same page occurs twice in a list. Deadlock
3617: * XXX can happen if vm_fault_page was called. A
3618: * XXX possible solution is to use a continuation if vm_fault_page
3619: * XXX is called and we cross a map entry boundary.
3620: */
3621:
3622: /*
3623: * Check for copies of zero bytes.
3624: */
3625:
3626: if (len == 0) {
3627: *copy_result = VM_MAP_COPY_NULL;
3628: return(KERN_SUCCESS);
3629: }
3630:
3631: /*
1.1.1.7 ! root 3632: * Check that the end address doesn't overflow
! 3633: */
! 3634:
! 3635: if ((src_addr + len) <= src_addr) {
! 3636: return KERN_INVALID_ADDRESS;
! 3637: }
! 3638:
! 3639: /*
1.1 root 3640: * Compute start and end of region
3641: */
3642:
3643: src_start = trunc_page(src_addr);
3644: src_end = round_page(src_addr + len);
3645:
3646: /*
1.1.1.7 ! root 3647: * XXX VM maps shouldn't end at maximum address
1.1 root 3648: */
3649:
1.1.1.7 ! root 3650: if (src_end == 0) {
1.1 root 3651: return KERN_INVALID_ADDRESS;
3652: }
3653:
3654: /*
3655: * Allocate a header element for the page list.
3656: *
3657: * Record original offset and size, as caller may not
3658: * be page-aligned.
3659: */
3660:
1.1.1.3 root 3661: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3662: copy->type = VM_MAP_COPY_PAGE_LIST;
3663: copy->cpy_npages = 0;
3664: copy->offset = src_addr;
3665: copy->size = len;
3666: copy->cpy_cont = ((kern_return_t (*)()) 0);
3667: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
1.1.1.2 root 3668:
1.1 root 3669: /*
3670: * Find the beginning of the region.
3671: */
3672:
3673: do_map_lookup:
3674:
3675: vm_map_lock(src_map);
3676:
3677: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3678: result = KERN_INVALID_ADDRESS;
3679: goto error;
3680: }
3681: need_map_lookup = FALSE;
3682:
3683: /*
3684: * Go through entries until we get to the end.
3685: */
3686:
3687: while (TRUE) {
3688:
3689: if (! (src_entry->protection & VM_PROT_READ)) {
3690: result = KERN_PROTECTION_FAILURE;
3691: goto error;
3692: }
3693:
3694: if (src_end > src_entry->vme_end)
3695: src_size = src_entry->vme_end - src_start;
3696: else
3697: src_size = src_end - src_start;
3698:
3699: src_object = src_entry->object.vm_object;
3700: src_offset = src_entry->offset +
3701: (src_start - src_entry->vme_start);
3702:
3703: /*
3704: * If src_object is NULL, allocate it now;
3705: * we're going to fault on it shortly.
3706: */
3707: if (src_object == VM_OBJECT_NULL) {
3708: src_object = vm_object_allocate((vm_size_t)
3709: src_entry->vme_end -
3710: src_entry->vme_start);
3711: src_entry->object.vm_object = src_object;
3712: }
3713:
3714: /*
3715: * Iterate over pages. Fault in ones that aren't present.
3716: */
3717: src_last_offset = src_offset + src_size;
3718: for (; (src_offset < src_last_offset && !need_map_lookup);
3719: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3720:
3721: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3722: make_continuation:
3723: /*
3724: * At this point we have the max number of
3725: * pages busy for this thread that we're
3726: * willing to allow. Stop here and record
3727: * arguments for the remainder. Note:
3728: * this means that this routine isn't atomic,
3729: * but that's the breaks. Note that only
3730: * the first vm_map_copy_t that comes back
3731: * from this routine has the right offset
3732: * and size; those from continuations are
3733: * page rounded, and short by the amount
3734: * already done.
3735: *
3736: * Reset src_end so the src_destroy
3737: * code at the bottom doesn't do
3738: * something stupid.
3739: */
3740:
1.1.1.2 root 3741: cont_args = (vm_map_copyin_args_t)
1.1 root 3742: kalloc(sizeof(vm_map_copyin_args_data_t));
3743: cont_args->map = src_map;
3744: vm_map_reference(src_map);
3745: cont_args->src_addr = src_start;
3746: cont_args->src_len = len - (src_start - src_addr);
3747: if (src_destroy) {
3748: cont_args->destroy_addr = cont_args->src_addr;
3749: cont_args->destroy_len = cont_args->src_len;
3750: }
3751: else {
3752: cont_args->destroy_addr = (vm_offset_t) 0;
3753: cont_args->destroy_len = (vm_offset_t) 0;
3754: }
3755: cont_args->steal_pages = steal_pages;
3756:
3757: copy->cpy_cont_args = (char *) cont_args;
3758: copy->cpy_cont = vm_map_copyin_page_list_cont;
3759:
3760: src_end = src_start;
3761: vm_map_clip_end(src_map, src_entry, src_end);
3762: break;
3763: }
3764:
3765: /*
3766: * Try to find the page of data.
3767: */
3768: vm_object_lock(src_object);
3769: vm_object_paging_begin(src_object);
3770: if (((m = vm_page_lookup(src_object, src_offset)) !=
3771: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3772: !m->absent && !m->error) {
3773:
3774: /*
3775: * This is the page. Mark it busy
3776: * and keep the paging reference on
3777: * the object whilst we do our thing.
3778: */
3779: m->busy = TRUE;
3780:
3781: /*
3782: * Also write-protect the page, so
3783: * that the map`s owner cannot change
3784: * the data. The busy bit will prevent
3785: * faults on the page from succeeding
3786: * until the copy is released; after
3787: * that, the page can be re-entered
3788: * as writable, since we didn`t alter
3789: * the map entry. This scheme is a
3790: * cheap copy-on-write.
3791: *
3792: * Don`t forget the protection and
3793: * the page_lock value!
3794: *
3795: * If the source is being destroyed
3796: * AND not shared writable, we don`t
3797: * have to protect the page, since
3798: * we will destroy the (only)
3799: * writable mapping later.
3800: */
3801: if (!src_destroy ||
3802: src_object->use_shared_copy)
3803: {
3804: pmap_page_protect(m->phys_addr,
3805: src_entry->protection
3806: & ~m->page_lock
3807: & ~VM_PROT_WRITE);
3808: }
3809:
3810: }
3811: else {
3812: vm_prot_t result_prot;
3813: vm_page_t top_page;
3814: kern_return_t kr;
1.1.1.2 root 3815:
1.1 root 3816: /*
3817: * Have to fault the page in; must
3818: * unlock the map to do so. While
3819: * the map is unlocked, anything
3820: * can happen, we must lookup the
3821: * map entry before continuing.
3822: */
3823: vm_map_unlock(src_map);
3824: need_map_lookup = TRUE;
3825: retry:
3826: result_prot = VM_PROT_READ;
1.1.1.2 root 3827:
1.1 root 3828: kr = vm_fault_page(src_object, src_offset,
3829: VM_PROT_READ, FALSE, FALSE,
3830: &result_prot, &m, &top_page,
3831: FALSE, (void (*)()) 0);
3832: /*
3833: * Cope with what happened.
3834: */
3835: switch (kr) {
3836: case VM_FAULT_SUCCESS:
3837: break;
3838: case VM_FAULT_INTERRUPTED: /* ??? */
3839: case VM_FAULT_RETRY:
3840: vm_object_lock(src_object);
3841: vm_object_paging_begin(src_object);
3842: goto retry;
3843: case VM_FAULT_MEMORY_SHORTAGE:
3844: VM_PAGE_WAIT((void (*)()) 0);
3845: vm_object_lock(src_object);
3846: vm_object_paging_begin(src_object);
3847: goto retry;
3848: case VM_FAULT_FICTITIOUS_SHORTAGE:
3849: vm_page_more_fictitious();
3850: vm_object_lock(src_object);
3851: vm_object_paging_begin(src_object);
3852: goto retry;
3853: case VM_FAULT_MEMORY_ERROR:
3854: /*
3855: * Something broke. If this
3856: * is a continuation, return
3857: * a partial result if possible,
3858: * else fail the whole thing.
3859: * In the continuation case, the
3860: * next continuation call will
3861: * get this error if it persists.
3862: */
3863: vm_map_lock(src_map);
3864: if (is_cont &&
3865: copy->cpy_npages != 0)
3866: goto make_continuation;
3867:
3868: result = KERN_MEMORY_ERROR;
3869: goto error;
3870: }
1.1.1.2 root 3871:
1.1 root 3872: if (top_page != VM_PAGE_NULL) {
3873: vm_object_lock(src_object);
3874: VM_PAGE_FREE(top_page);
3875: vm_object_paging_end(src_object);
3876: vm_object_unlock(src_object);
3877: }
3878:
3879: /*
3880: * We do not need to write-protect
3881: * the page, since it cannot have
3882: * been in the pmap (and we did not
3883: * enter it above). The busy bit
3884: * will protect the page from being
3885: * entered as writable until it is
3886: * unlocked.
3887: */
3888:
3889: }
3890:
3891: /*
3892: * The page is busy, its object is locked, and
3893: * we have a paging reference on it. Either
3894: * the map is locked, or need_map_lookup is
3895: * TRUE.
1.1.1.2 root 3896: *
1.1 root 3897: * Put the page in the page list.
3898: */
3899: copy->cpy_page_list[copy->cpy_npages++] = m;
3900: vm_object_unlock(m->object);
3901: }
1.1.1.2 root 3902:
1.1 root 3903: /*
3904: * DETERMINE whether the entire region
3905: * has been copied.
3906: */
3907: if (src_start >= src_end && src_end != 0) {
3908: if (need_map_lookup)
3909: vm_map_lock(src_map);
3910: break;
3911: }
3912:
3913: /*
3914: * If need_map_lookup is TRUE, have to start over with
3915: * another map lookup. Note that we dropped the map
3916: * lock (to call vm_fault_page) above only in this case.
3917: */
3918: if (need_map_lookup)
3919: goto do_map_lookup;
3920:
3921: /*
3922: * Verify that there are no gaps in the region
3923: */
3924:
3925: src_start = src_entry->vme_end;
3926: src_entry = src_entry->vme_next;
3927: if (src_entry->vme_start != src_start) {
3928: result = KERN_INVALID_ADDRESS;
3929: goto error;
3930: }
3931: }
3932:
3933: /*
3934: * If steal_pages is true, make sure all
3935: * pages in the copy are not in any object
3936: * We try to remove them from the original
3937: * object, but we may have to copy them.
3938: *
3939: * At this point every page in the list is busy
3940: * and holds a paging reference to its object.
3941: * When we're done stealing, every page is busy,
3942: * and in no object (m->tabled == FALSE).
3943: */
3944: src_start = trunc_page(src_addr);
3945: if (steal_pages) {
1.1.1.4 root 3946: int i;
1.1 root 3947: vm_offset_t unwire_end;
3948:
3949: unwire_end = src_start;
3950: for (i = 0; i < copy->cpy_npages; i++) {
3951:
3952: /*
3953: * Remove the page from its object if it
3954: * can be stolen. It can be stolen if:
3955: *
1.1.1.2 root 3956: * (1) The source is being destroyed,
1.1 root 3957: * the object is temporary, and
3958: * not shared.
3959: * (2) The page is not precious.
3960: *
3961: * The not shared check consists of two
3962: * parts: (a) there are no objects that
3963: * shadow this object. (b) it is not the
3964: * object in any shared map entries (i.e.,
3965: * use_shared_copy is not set).
3966: *
3967: * The first check (a) means that we can't
3968: * steal pages from objects that are not
3969: * at the top of their shadow chains. This
3970: * should not be a frequent occurrence.
3971: *
3972: * Stealing wired pages requires telling the
3973: * pmap module to let go of them.
1.1.1.2 root 3974: *
1.1 root 3975: * NOTE: stealing clean pages from objects
3976: * whose mappings survive requires a call to
3977: * the pmap module. Maybe later.
3978: */
3979: m = copy->cpy_page_list[i];
3980: src_object = m->object;
3981: vm_object_lock(src_object);
3982:
3983: if (src_destroy &&
3984: src_object->temporary &&
3985: (!src_object->shadowed) &&
3986: (!src_object->use_shared_copy) &&
3987: !m->precious) {
3988: vm_offset_t page_vaddr;
1.1.1.2 root 3989:
1.1 root 3990: page_vaddr = src_start + (i * PAGE_SIZE);
3991: if (m->wire_count > 0) {
3992:
3993: assert(m->wire_count == 1);
3994: /*
3995: * In order to steal a wired
3996: * page, we have to unwire it
3997: * first. We do this inline
3998: * here because we have the page.
3999: *
4000: * Step 1: Unwire the map entry.
4001: * Also tell the pmap module
4002: * that this piece of the
4003: * pmap is pageable.
4004: */
4005: vm_object_unlock(src_object);
4006: if (page_vaddr >= unwire_end) {
4007: if (!vm_map_lookup_entry(src_map,
4008: page_vaddr, &src_entry))
4009: panic("vm_map_copyin_page_list: missing wired map entry");
4010:
4011: vm_map_clip_start(src_map, src_entry,
4012: page_vaddr);
4013: vm_map_clip_end(src_map, src_entry,
4014: src_start + src_size);
4015:
4016: assert(src_entry->wired_count > 0);
4017: src_entry->wired_count = 0;
1.1.1.5 root 4018: if (src_entry->user_wired_count)
4019: src_map->user_wired -= src_entry->vme_end - src_entry->vme_start;
1.1 root 4020: src_entry->user_wired_count = 0;
4021: unwire_end = src_entry->vme_end;
4022: pmap_pageable(vm_map_pmap(src_map),
4023: page_vaddr, unwire_end, TRUE);
4024: }
4025:
4026: /*
4027: * Step 2: Unwire the page.
4028: * pmap_remove handles this for us.
4029: */
4030: vm_object_lock(src_object);
4031: }
4032:
4033: /*
4034: * Don't need to remove the mapping;
4035: * vm_map_delete will handle it.
4036: *
4037: * Steal the page. Setting the wire count
4038: * to zero is vm_page_unwire without
4039: * activating the page.
4040: */
4041: vm_page_lock_queues();
4042: vm_page_remove(m);
4043: if (m->wire_count > 0) {
4044: m->wire_count = 0;
4045: vm_page_wire_count--;
4046: } else {
4047: VM_PAGE_QUEUES_REMOVE(m);
4048: }
4049: vm_page_unlock_queues();
4050: }
4051: else {
4052: /*
4053: * Have to copy this page. Have to
4054: * unlock the map while copying,
4055: * hence no further page stealing.
4056: * Hence just copy all the pages.
4057: * Unlock the map while copying;
4058: * This means no further page stealing.
4059: */
4060: vm_object_unlock(src_object);
4061: vm_map_unlock(src_map);
4062:
4063: vm_map_copy_steal_pages(copy);
4064:
4065: vm_map_lock(src_map);
4066: break;
4067: }
4068:
4069: vm_object_paging_end(src_object);
4070: vm_object_unlock(src_object);
4071: }
4072:
4073: /*
4074: * If the source should be destroyed, do it now, since the
4075: * copy was successful.
4076: */
4077:
4078: if (src_destroy) {
4079: (void) vm_map_delete(src_map, src_start, src_end);
4080: }
4081: }
4082: else {
4083: /*
4084: * !steal_pages leaves busy pages in the map.
4085: * This will cause src_destroy to hang. Use
4086: * a continuation to prevent this.
4087: */
4088: if (src_destroy && !vm_map_copy_has_cont(copy)) {
1.1.1.2 root 4089: cont_args = (vm_map_copyin_args_t)
1.1 root 4090: kalloc(sizeof(vm_map_copyin_args_data_t));
4091: vm_map_reference(src_map);
4092: cont_args->map = src_map;
4093: cont_args->src_addr = (vm_offset_t) 0;
4094: cont_args->src_len = (vm_size_t) 0;
4095: cont_args->destroy_addr = src_start;
4096: cont_args->destroy_len = src_end - src_start;
4097: cont_args->steal_pages = FALSE;
4098:
4099: copy->cpy_cont_args = (char *) cont_args;
4100: copy->cpy_cont = vm_map_copyin_page_list_cont;
4101: }
1.1.1.2 root 4102:
1.1 root 4103: }
4104:
4105: vm_map_unlock(src_map);
4106:
4107: *copy_result = copy;
4108: return(result);
4109:
4110: error:
4111: vm_map_unlock(src_map);
4112: vm_map_copy_discard(copy);
4113: return(result);
4114: }
4115:
4116: /*
4117: * vm_map_fork:
4118: *
4119: * Create and return a new map based on the old
4120: * map, according to the inheritance values on the
4121: * regions in that map.
4122: *
4123: * The source map must not be locked.
4124: */
1.1.1.4 root 4125: vm_map_t vm_map_fork(vm_map_t old_map)
1.1 root 4126: {
4127: vm_map_t new_map;
4128: vm_map_entry_t old_entry;
4129: vm_map_entry_t new_entry;
4130: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4131: vm_size_t new_size = 0;
4132: vm_size_t entry_size;
4133: vm_object_t object;
4134:
1.1.1.7 ! root 4135: if (new_pmap == PMAP_NULL)
! 4136: return VM_MAP_NULL;
! 4137:
1.1 root 4138: vm_map_lock(old_map);
4139:
4140: new_map = vm_map_create(new_pmap,
4141: old_map->min_offset,
1.1.1.7 ! root 4142: old_map->max_offset);
1.1 root 4143:
4144: for (
4145: old_entry = vm_map_first_entry(old_map);
4146: old_entry != vm_map_to_entry(old_map);
4147: ) {
4148: if (old_entry->is_sub_map)
4149: panic("vm_map_fork: encountered a submap");
4150:
4151: entry_size = (old_entry->vme_end - old_entry->vme_start);
4152:
4153: switch (old_entry->inheritance) {
4154: case VM_INHERIT_NONE:
4155: break;
4156:
4157: case VM_INHERIT_SHARE:
4158: /*
4159: * New sharing code. New map entry
4160: * references original object. Temporary
4161: * objects use asynchronous copy algorithm for
4162: * future copies. First make sure we have
4163: * the right object. If we need a shadow,
4164: * or someone else already has one, then
4165: * make a new shadow and share it.
4166: */
4167:
4168: object = old_entry->object.vm_object;
4169: if (object == VM_OBJECT_NULL) {
4170: object = vm_object_allocate(
4171: (vm_size_t)(old_entry->vme_end -
4172: old_entry->vme_start));
4173: old_entry->offset = 0;
4174: old_entry->object.vm_object = object;
4175: assert(!old_entry->needs_copy);
4176: }
4177: else if (old_entry->needs_copy || object->shadowed ||
4178: (object->temporary && !old_entry->is_shared &&
4179: object->size > (vm_size_t)(old_entry->vme_end -
4180: old_entry->vme_start))) {
4181:
4182: assert(object->temporary);
4183: assert(!(object->shadowed && old_entry->is_shared));
4184: vm_object_shadow(
4185: &old_entry->object.vm_object,
4186: &old_entry->offset,
4187: (vm_size_t) (old_entry->vme_end -
4188: old_entry->vme_start));
1.1.1.2 root 4189:
1.1 root 4190: /*
4191: * If we're making a shadow for other than
4192: * copy on write reasons, then we have
4193: * to remove write permission.
4194: */
4195:
4196: if (!old_entry->needs_copy &&
4197: (old_entry->protection & VM_PROT_WRITE)) {
4198: pmap_protect(vm_map_pmap(old_map),
4199: old_entry->vme_start,
4200: old_entry->vme_end,
4201: old_entry->protection &
4202: ~VM_PROT_WRITE);
4203: }
4204: old_entry->needs_copy = FALSE;
4205: object = old_entry->object.vm_object;
4206: }
4207:
4208: /*
4209: * Set use_shared_copy to indicate that
4210: * object must use shared (delayed) copy-on
4211: * write. This is ignored for permanent objects.
4212: * Bump the reference count for the new entry
4213: */
4214:
4215: vm_object_lock(object);
4216: object->use_shared_copy = TRUE;
4217: object->ref_count++;
4218: vm_object_unlock(object);
4219:
1.1.1.3 root 4220: new_entry = vm_map_entry_create(new_map);
4221:
1.1 root 4222: if (old_entry->projected_on != 0) {
4223: /*
4224: * If entry is projected buffer, clone the
4225: * entry exactly.
4226: */
4227:
4228: vm_map_entry_copy_full(new_entry, old_entry);
4229:
4230: } else {
4231: /*
4232: * Clone the entry, using object ref from above.
4233: * Mark both entries as shared.
4234: */
4235:
4236: vm_map_entry_copy(new_entry, old_entry);
4237: old_entry->is_shared = TRUE;
4238: new_entry->is_shared = TRUE;
4239: }
4240:
4241: /*
4242: * Insert the entry into the new map -- we
4243: * know we're inserting at the end of the new
4244: * map.
4245: */
4246:
4247: vm_map_entry_link(
4248: new_map,
4249: vm_map_last_entry(new_map),
4250: new_entry);
4251:
4252: /*
4253: * Update the physical map
4254: */
4255:
4256: pmap_copy(new_map->pmap, old_map->pmap,
4257: new_entry->vme_start,
4258: entry_size,
4259: old_entry->vme_start);
4260:
4261: new_size += entry_size;
4262: break;
4263:
4264: case VM_INHERIT_COPY:
4265: if (old_entry->wired_count == 0) {
4266: boolean_t src_needs_copy;
4267: boolean_t new_entry_needs_copy;
4268:
4269: new_entry = vm_map_entry_create(new_map);
4270: vm_map_entry_copy(new_entry, old_entry);
4271:
4272: if (vm_object_copy_temporary(
4273: &new_entry->object.vm_object,
4274: &new_entry->offset,
4275: &src_needs_copy,
4276: &new_entry_needs_copy)) {
4277:
4278: /*
4279: * Handle copy-on-write obligations
4280: */
4281:
4282: if (src_needs_copy && !old_entry->needs_copy) {
4283: vm_object_pmap_protect(
4284: old_entry->object.vm_object,
4285: old_entry->offset,
4286: entry_size,
4287: (old_entry->is_shared ?
4288: PMAP_NULL :
4289: old_map->pmap),
4290: old_entry->vme_start,
4291: old_entry->protection &
4292: ~VM_PROT_WRITE);
4293:
4294: old_entry->needs_copy = TRUE;
4295: }
4296:
4297: new_entry->needs_copy = new_entry_needs_copy;
4298:
4299: /*
4300: * Insert the entry at the end
4301: * of the map.
4302: */
4303:
4304: vm_map_entry_link(new_map,
4305: vm_map_last_entry(new_map),
4306: new_entry);
4307:
4308:
4309: new_size += entry_size;
4310: break;
4311: }
4312:
4313: vm_map_entry_dispose(new_map, new_entry);
4314: }
4315:
4316: /* INNER BLOCK (copy cannot be optimized) */ {
4317:
4318: vm_offset_t start = old_entry->vme_start;
4319: vm_map_copy_t copy;
4320: vm_map_entry_t last = vm_map_last_entry(new_map);
4321:
4322: vm_map_unlock(old_map);
4323: if (vm_map_copyin(old_map,
4324: start,
4325: entry_size,
4326: FALSE,
1.1.1.2 root 4327: ©)
1.1 root 4328: != KERN_SUCCESS) {
4329: vm_map_lock(old_map);
4330: if (!vm_map_lookup_entry(old_map, start, &last))
4331: last = last->vme_next;
4332: old_entry = last;
4333: /*
4334: * For some error returns, want to
4335: * skip to the next element.
4336: */
4337:
4338: continue;
4339: }
4340:
4341: /*
4342: * Insert the copy into the new map
4343: */
4344:
4345: vm_map_copy_insert(new_map, last, copy);
4346: new_size += entry_size;
4347:
4348: /*
4349: * Pick up the traversal at the end of
4350: * the copied region.
4351: */
4352:
4353: vm_map_lock(old_map);
4354: start += entry_size;
4355: if (!vm_map_lookup_entry(old_map, start, &last))
4356: last = last->vme_next;
4357: else
4358: vm_map_clip_start(old_map, last, start);
4359: old_entry = last;
4360:
4361: continue;
4362: /* INNER BLOCK (copy cannot be optimized) */ }
4363: }
4364: old_entry = old_entry->vme_next;
4365: }
4366:
4367: new_map->size = new_size;
4368: vm_map_unlock(old_map);
4369:
4370: return(new_map);
4371: }
4372:
4373: /*
4374: * vm_map_lookup:
4375: *
4376: * Finds the VM object, offset, and
4377: * protection for a given virtual address in the
4378: * specified map, assuming a page fault of the
4379: * type specified.
4380: *
4381: * Returns the (object, offset, protection) for
4382: * this address, whether it is wired down, and whether
4383: * this map has the only reference to the data in question.
4384: * In order to later verify this lookup, a "version"
4385: * is returned.
4386: *
4387: * The map should not be locked; it will not be
4388: * locked on exit. In order to guarantee the
4389: * existence of the returned object, it is returned
4390: * locked.
4391: *
4392: * If a lookup is requested with "write protection"
4393: * specified, the map may be changed to perform virtual
4394: * copying operations, although the data referenced will
4395: * remain the same.
4396: */
1.1.1.4 root 4397: kern_return_t vm_map_lookup(
4398: vm_map_t *var_map, /* IN/OUT */
4399: vm_offset_t vaddr,
4400: vm_prot_t fault_type,
4401:
4402: vm_map_version_t *out_version, /* OUT */
4403: vm_object_t *object, /* OUT */
4404: vm_offset_t *offset, /* OUT */
4405: vm_prot_t *out_prot, /* OUT */
4406: boolean_t *wired) /* OUT */
4407: {
4408: vm_map_entry_t entry;
4409: vm_map_t map = *var_map;
4410: vm_prot_t prot;
1.1 root 4411:
4412: RetryLookup: ;
4413:
4414: /*
4415: * Lookup the faulting address.
4416: */
4417:
4418: vm_map_lock_read(map);
4419:
4420: #define RETURN(why) \
4421: { \
4422: vm_map_unlock_read(map); \
4423: return(why); \
4424: }
4425:
4426: /*
4427: * If the map has an interesting hint, try it before calling
4428: * full blown lookup routine.
4429: */
4430:
4431: simple_lock(&map->hint_lock);
4432: entry = map->hint;
4433: simple_unlock(&map->hint_lock);
4434:
4435: if ((entry == vm_map_to_entry(map)) ||
4436: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4437: vm_map_entry_t tmp_entry;
4438:
4439: /*
4440: * Entry was either not a valid hint, or the vaddr
4441: * was not contained in the entry, so do a full lookup.
4442: */
4443: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4444: RETURN(KERN_INVALID_ADDRESS);
4445:
4446: entry = tmp_entry;
4447: }
4448:
4449: /*
4450: * Handle submaps.
4451: */
4452:
4453: if (entry->is_sub_map) {
4454: vm_map_t old_map = map;
4455:
4456: *var_map = map = entry->object.sub_map;
4457: vm_map_unlock_read(old_map);
4458: goto RetryLookup;
4459: }
1.1.1.2 root 4460:
1.1 root 4461: /*
4462: * Check whether this task is allowed to have
4463: * this page.
4464: */
4465:
4466: prot = entry->protection;
4467:
1.1.1.3 root 4468: if ((fault_type & (prot)) != fault_type) {
1.1 root 4469: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4470: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4471: } else {
4472: RETURN(KERN_PROTECTION_FAILURE);
4473: }
1.1.1.3 root 4474: }
1.1 root 4475:
4476: /*
4477: * If this page is not pageable, we have to get
4478: * it for all possible accesses.
4479: */
4480:
1.1.1.3 root 4481: if ((*wired = (entry->wired_count != 0)))
1.1 root 4482: prot = fault_type = entry->protection;
4483:
4484: /*
4485: * If the entry was copy-on-write, we either ...
4486: */
4487:
4488: if (entry->needs_copy) {
4489: /*
4490: * If we want to write the page, we may as well
4491: * handle that now since we've got the map locked.
4492: *
4493: * If we don't need to write the page, we just
4494: * demote the permissions allowed.
4495: */
4496:
4497: if (fault_type & VM_PROT_WRITE) {
4498: /*
4499: * Make a new object, and place it in the
4500: * object chain. Note that no new references
4501: * have appeared -- one just moved from the
4502: * map to the new object.
4503: */
4504:
4505: if (vm_map_lock_read_to_write(map)) {
4506: goto RetryLookup;
4507: }
4508: map->timestamp++;
4509:
4510: vm_object_shadow(
4511: &entry->object.vm_object,
4512: &entry->offset,
4513: (vm_size_t) (entry->vme_end - entry->vme_start));
1.1.1.2 root 4514:
1.1 root 4515: entry->needs_copy = FALSE;
1.1.1.2 root 4516:
1.1 root 4517: vm_map_lock_write_to_read(map);
4518: }
4519: else {
4520: /*
4521: * We're attempting to read a copy-on-write
4522: * page -- don't allow writes.
4523: */
4524:
4525: prot &= (~VM_PROT_WRITE);
4526: }
4527: }
4528:
4529: /*
4530: * Create an object if necessary.
4531: */
4532: if (entry->object.vm_object == VM_OBJECT_NULL) {
4533:
4534: if (vm_map_lock_read_to_write(map)) {
4535: goto RetryLookup;
4536: }
4537:
4538: entry->object.vm_object = vm_object_allocate(
4539: (vm_size_t)(entry->vme_end - entry->vme_start));
4540: entry->offset = 0;
4541: vm_map_lock_write_to_read(map);
4542: }
4543:
4544: /*
4545: * Return the object/offset from this entry. If the entry
4546: * was copy-on-write or empty, it has been fixed up. Also
4547: * return the protection.
4548: */
4549:
4550: *offset = (vaddr - entry->vme_start) + entry->offset;
4551: *object = entry->object.vm_object;
4552: *out_prot = prot;
4553:
4554: /*
4555: * Lock the object to prevent it from disappearing
4556: */
4557:
4558: vm_object_lock(*object);
4559:
4560: /*
4561: * Save the version number and unlock the map.
4562: */
4563:
4564: out_version->main_timestamp = map->timestamp;
4565:
4566: RETURN(KERN_SUCCESS);
1.1.1.2 root 4567:
1.1 root 4568: #undef RETURN
4569: }
4570:
4571: /*
4572: * vm_map_verify:
4573: *
4574: * Verifies that the map in question has not changed
4575: * since the given version. If successful, the map
4576: * will not change until vm_map_verify_done() is called.
4577: */
1.1.1.4 root 4578: boolean_t vm_map_verify(
4579: vm_map_t map,
4580: vm_map_version_t *version) /* REF */
1.1 root 4581: {
4582: boolean_t result;
4583:
4584: vm_map_lock_read(map);
4585: result = (map->timestamp == version->main_timestamp);
4586:
4587: if (!result)
4588: vm_map_unlock_read(map);
4589:
4590: return(result);
4591: }
4592:
4593: /*
4594: * vm_map_verify_done:
4595: *
4596: * Releases locks acquired by a vm_map_verify.
4597: *
4598: * This is now a macro in vm/vm_map.h. It does a
4599: * vm_map_unlock_read on the map.
4600: */
4601:
4602: /*
4603: * vm_region:
4604: *
4605: * User call to obtain information about a region in
4606: * a task's address map.
4607: */
4608:
1.1.1.4 root 4609: kern_return_t vm_region(
4610: vm_map_t map,
4611: vm_offset_t *address, /* IN/OUT */
4612: vm_size_t *size, /* OUT */
4613: vm_prot_t *protection, /* OUT */
4614: vm_prot_t *max_protection, /* OUT */
4615: vm_inherit_t *inheritance, /* OUT */
4616: boolean_t *is_shared, /* OUT */
4617: ipc_port_t *object_name, /* OUT */
4618: vm_offset_t *offset_in_object) /* OUT */
1.1 root 4619: {
4620: vm_map_entry_t tmp_entry;
4621: vm_map_entry_t entry;
4622: vm_offset_t tmp_offset;
4623: vm_offset_t start;
4624:
4625: if (map == VM_MAP_NULL)
4626: return(KERN_INVALID_ARGUMENT);
4627:
4628: start = *address;
4629:
4630: vm_map_lock_read(map);
4631: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4632: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4633: vm_map_unlock_read(map);
4634: return(KERN_NO_SPACE);
4635: }
4636: } else {
4637: entry = tmp_entry;
4638: }
4639:
4640: start = entry->vme_start;
4641: *protection = entry->protection;
4642: *max_protection = entry->max_protection;
4643: *inheritance = entry->inheritance;
4644: *address = start;
4645: *size = (entry->vme_end - start);
4646:
4647: tmp_offset = entry->offset;
4648:
4649:
4650: if (entry->is_sub_map) {
4651: *is_shared = FALSE;
4652: *object_name = IP_NULL;
4653: *offset_in_object = tmp_offset;
4654: } else {
4655: *is_shared = entry->is_shared;
4656: *object_name = vm_object_name(entry->object.vm_object);
4657: *offset_in_object = tmp_offset;
4658: }
4659:
4660: vm_map_unlock_read(map);
4661:
4662: return(KERN_SUCCESS);
4663: }
4664:
4665: /*
4666: * Routine: vm_map_simplify
4667: *
4668: * Description:
4669: * Attempt to simplify the map representation in
4670: * the vicinity of the given starting address.
4671: * Note:
4672: * This routine is intended primarily to keep the
4673: * kernel maps more compact -- they generally don't
4674: * benefit from the "expand a map entry" technology
4675: * at allocation time because the adjacent entry
4676: * is often wired down.
4677: */
1.1.1.4 root 4678: void vm_map_simplify(
4679: vm_map_t map,
4680: vm_offset_t start)
1.1 root 4681: {
4682: vm_map_entry_t this_entry;
4683: vm_map_entry_t prev_entry;
4684:
4685: vm_map_lock(map);
4686: if (
4687: (vm_map_lookup_entry(map, start, &this_entry)) &&
4688: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4689:
4690: (prev_entry->vme_end == start) &&
4691:
4692: (prev_entry->is_shared == FALSE) &&
4693: (prev_entry->is_sub_map == FALSE) &&
4694:
4695: (this_entry->is_shared == FALSE) &&
4696: (this_entry->is_sub_map == FALSE) &&
4697:
4698: (prev_entry->inheritance == this_entry->inheritance) &&
4699: (prev_entry->protection == this_entry->protection) &&
4700: (prev_entry->max_protection == this_entry->max_protection) &&
4701: (prev_entry->wired_count == this_entry->wired_count) &&
4702: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4703:
4704: (prev_entry->needs_copy == this_entry->needs_copy) &&
4705:
4706: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4707: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4708: == this_entry->offset) &&
4709: (prev_entry->projected_on == 0) &&
1.1.1.2 root 4710: (this_entry->projected_on == 0)
1.1 root 4711: ) {
4712: if (map->first_free == this_entry)
4713: map->first_free = prev_entry;
4714:
4715: SAVE_HINT(map, prev_entry);
4716: prev_entry->vme_end = this_entry->vme_end;
1.1.1.7 ! root 4717: vm_map_entry_unlink(map, this_entry);
1.1 root 4718: vm_object_deallocate(this_entry->object.vm_object);
4719: vm_map_entry_dispose(map, this_entry);
4720: }
4721: vm_map_unlock(map);
4722: }
4723:
4724:
4725: /*
4726: * Routine: vm_map_machine_attribute
4727: * Purpose:
4728: * Provide machine-specific attributes to mappings,
4729: * such as cachability etc. for machines that provide
4730: * them. NUMA architectures and machines with big/strange
4731: * caches will use this.
4732: * Note:
4733: * Responsibilities for locking and checking are handled here,
4734: * everything else in the pmap module. If any non-volatile
4735: * information must be kept, the pmap module should handle
4736: * it itself. [This assumes that attributes do not
4737: * need to be inherited, which seems ok to me]
4738: */
1.1.1.4 root 4739: kern_return_t vm_map_machine_attribute(
4740: vm_map_t map,
4741: vm_offset_t address,
4742: vm_size_t size,
4743: vm_machine_attribute_t attribute,
4744: vm_machine_attribute_val_t* value) /* IN/OUT */
1.1 root 4745: {
4746: kern_return_t ret;
4747:
4748: if (address < vm_map_min(map) ||
4749: (address + size) > vm_map_max(map))
4750: return KERN_INVALID_ARGUMENT;
4751:
4752: vm_map_lock(map);
4753:
4754: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4755:
4756: vm_map_unlock(map);
4757:
4758: return ret;
4759: }
4760:
4761:
4762: #if MACH_KDB
4763:
4764: #define printf kdbprintf
4765:
4766: /*
4767: * vm_map_print: [ debug ]
4768: */
1.1.1.6 root 4769: void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const char *modif)
1.1 root 4770: {
1.1.1.6 root 4771: vm_map_t map;
1.1.1.4 root 4772: vm_map_entry_t entry;
1.1 root 4773:
1.1.1.6 root 4774: if (!have_addr)
4775: map = current_thread()->task->map;
4776: else
4777: map = (vm_map_t)addr;
4778:
1.1.1.7 ! root 4779: iprintf("Map 0x%X: name=\"%s\", pmap=0x%X,",
! 4780: (vm_offset_t) map, map->name, (vm_offset_t) (map->pmap));
1.1 root 4781: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4782: printf("version=%d\n", map->timestamp);
1.1.1.6 root 4783: indent += 1;
1.1 root 4784: for (entry = vm_map_first_entry(map);
4785: entry != vm_map_to_entry(map);
4786: entry = entry->vme_next) {
4787: static char *inheritance_name[3] = { "share", "copy", "none"};
4788:
4789: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
1.1.1.6 root 4790: printf("start=0x%X, end=0x%X\n",
1.1 root 4791: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
1.1.1.6 root 4792: iprintf("prot=%X/%X/%s, ",
1.1 root 4793: entry->protection,
4794: entry->max_protection,
4795: inheritance_name[entry->inheritance]);
4796: if (entry->wired_count != 0) {
4797: printf("wired(");
4798: if (entry->user_wired_count != 0)
4799: printf("u");
4800: if (entry->wired_count >
4801: ((entry->user_wired_count == 0) ? 0 : 1))
4802: printf("k");
4803: printf(") ");
4804: }
4805: if (entry->in_transition) {
4806: printf("in transition");
4807: if (entry->needs_wakeup)
4808: printf("(wake request)");
4809: printf(", ");
4810: }
4811: if (entry->is_sub_map) {
4812: printf("submap=0x%X, offset=0x%X\n",
4813: (vm_offset_t) entry->object.sub_map,
4814: (vm_offset_t) entry->offset);
4815: } else {
4816: printf("object=0x%X, offset=0x%X",
4817: (vm_offset_t) entry->object.vm_object,
4818: (vm_offset_t) entry->offset);
4819: if (entry->is_shared)
4820: printf(", shared");
4821: if (entry->needs_copy)
4822: printf(", copy needed");
4823: printf("\n");
4824:
4825: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4826: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
1.1.1.6 root 4827: indent += 1;
1.1 root 4828: vm_object_print(entry->object.vm_object);
1.1.1.6 root 4829: indent -= 1;
1.1 root 4830: }
4831: }
4832: }
1.1.1.6 root 4833: indent -= 1;
1.1 root 4834: }
4835:
4836: /*
4837: * Routine: vm_map_copy_print
4838: * Purpose:
4839: * Pretty-print a copy object for ddb.
4840: */
4841:
4842: void vm_map_copy_print(copy)
1.1.1.4 root 4843: const vm_map_copy_t copy;
1.1 root 4844: {
4845: int i, npages;
4846:
4847: printf("copy object 0x%x\n", copy);
4848:
1.1.1.6 root 4849: indent += 1;
1.1 root 4850:
4851: iprintf("type=%d", copy->type);
4852: switch (copy->type) {
4853: case VM_MAP_COPY_ENTRY_LIST:
4854: printf("[entry_list]");
4855: break;
1.1.1.2 root 4856:
1.1 root 4857: case VM_MAP_COPY_OBJECT:
4858: printf("[object]");
4859: break;
1.1.1.2 root 4860:
1.1 root 4861: case VM_MAP_COPY_PAGE_LIST:
4862: printf("[page_list]");
4863: break;
4864:
4865: default:
4866: printf("[bad type]");
4867: break;
4868: }
4869: printf(", offset=0x%x", copy->offset);
4870: printf(", size=0x%x\n", copy->size);
4871:
4872: switch (copy->type) {
4873: case VM_MAP_COPY_ENTRY_LIST:
4874: /* XXX add stuff here */
4875: break;
4876:
4877: case VM_MAP_COPY_OBJECT:
4878: iprintf("object=0x%x\n", copy->cpy_object);
4879: break;
4880:
4881: case VM_MAP_COPY_PAGE_LIST:
4882: iprintf("npages=%d", copy->cpy_npages);
4883: printf(", cont=%x", copy->cpy_cont);
4884: printf(", cont_args=%x\n", copy->cpy_cont_args);
4885: if (copy->cpy_npages < 0) {
4886: npages = 0;
4887: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4888: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4889: } else {
4890: npages = copy->cpy_npages;
4891: }
4892: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4893: for (i = 0; i < npages - 1; i++) {
4894: printf("0x%x, ", copy->cpy_page_list[i]);
4895: }
4896: if (npages > 0) {
4897: printf("0x%x", copy->cpy_page_list[npages - 1]);
4898: }
4899: printf("}\n");
4900: break;
4901: }
4902:
1.1.1.6 root 4903: indent -= 1;
1.1 root 4904: }
1.1.1.2 root 4905: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.