![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
1.1.1.3 root 37: #include <kern/printf.h>
1.1 root 38: #include <mach/kern_return.h>
39: #include <mach/port.h>
40: #include <mach/vm_attributes.h>
41: #include <mach/vm_param.h>
42: #include <kern/assert.h>
1.1.1.3 root 43: #include <kern/debug.h>
44: #include <kern/kalloc.h>
45: #include <kern/rbtree.h>
46: #include <kern/slab.h>
47: #include <vm/pmap.h>
1.1 root 48: #include <vm/vm_fault.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
1.1.1.3 root 52: #include <vm/vm_resident.h>
1.1 root 53: #include <vm/vm_kern.h>
54: #include <ipc/ipc_port.h>
55:
1.1.1.3 root 56: #if MACH_KDB
57: #include <ddb/db_output.h>
58: #include <vm/vm_print.h>
59: #endif /* MACH_KDB */
60:
1.1 root 61: /*
62: * Macros to copy a vm_map_entry. We must be careful to correctly
63: * manage the wired page count. vm_map_entry_copy() creates a new
64: * map entry to the same memory - the wired count in the new entry
65: * must be set to zero. vm_map_entry_copy_full() creates a new
66: * entry that is identical to the old entry. This preserves the
1.1.1.3 root 67: * wire count; it's used for map splitting and cache changing in
1.1 root 68: * vm_map_copyout.
69: */
70: #define vm_map_entry_copy(NEW,OLD) \
71: MACRO_BEGIN \
72: *(NEW) = *(OLD); \
73: (NEW)->is_shared = FALSE; \
74: (NEW)->needs_wakeup = FALSE; \
75: (NEW)->in_transition = FALSE; \
76: (NEW)->wired_count = 0; \
77: (NEW)->user_wired_count = 0; \
78: MACRO_END
79:
80: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
81:
82: /*
83: * Virtual memory maps provide for the mapping, protection,
84: * and sharing of virtual memory objects. In addition,
85: * this module provides for an efficient virtual copy of
86: * memory from one map to another.
87: *
88: * Synchronization is required prior to most operations.
89: *
90: * Maps consist of an ordered doubly-linked list of simple
1.1.1.3 root 91: * entries; a hint and a red-black tree are used to speed up lookups.
1.1 root 92: *
93: * Sharing maps have been deleted from this version of Mach.
94: * All shared objects are now mapped directly into the respective
95: * maps. This requires a change in the copy on write strategy;
96: * the asymmetric (delayed) strategy is used for shared temporary
97: * objects instead of the symmetric (shadow) strategy. This is
98: * selected by the (new) use_shared_copy bit in the object. See
99: * vm_object_copy_temporary in vm_object.c for details. All maps
100: * are now "top level" maps (either task map, kernel map or submap
1.1.1.2 root 101: * of the kernel map).
1.1 root 102: *
103: * Since portions of maps are specified by start/end addreses,
104: * which may not align with existing map entries, all
105: * routines merely "clip" entries to these start/end values.
106: * [That is, an entry is split into two, bordering at a
107: * start or end value.] Note that these clippings may not
108: * always be necessary (as the two resulting entries are then
109: * not changed); however, the clipping is done for convenience.
110: * No attempt is currently made to "glue back together" two
111: * abutting entries.
112: *
113: * The symmetric (shadow) copy strategy implements virtual copy
114: * by copying VM object references from one map to
115: * another, and then marking both regions as copy-on-write.
116: * It is important to note that only one writeable reference
117: * to a VM object region exists in any map when this strategy
118: * is used -- this means that shadow object creation can be
119: * delayed until a write operation occurs. The asymmetric (delayed)
120: * strategy allows multiple maps to have writeable references to
121: * the same region of a vm object, and hence cannot delay creating
122: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
123: * Copying of permanent objects is completely different; see
124: * vm_object_copy_strategically() in vm_object.c.
125: */
126:
1.1.1.3 root 127: struct kmem_cache vm_map_cache; /* cache for vm_map structures */
128: struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
129: struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
130: struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
1.1 root 131:
132: /*
133: * Placeholder object for submap operations. This object is dropped
134: * into the range by a call to vm_map_find, and removed when
135: * vm_map_submap creates the submap.
136: */
137:
1.1.1.3 root 138: static struct vm_object vm_submap_object_store;
139: vm_object_t vm_submap_object = &vm_submap_object_store;
1.1 root 140:
141: /*
142: * vm_map_init:
143: *
144: * Initialize the vm_map module. Must be called before
145: * any other vm_map routines.
146: *
1.1.1.3 root 147: * Map and entry structures are allocated from caches -- we must
148: * initialize those caches.
1.1 root 149: *
1.1.1.3 root 150: * There are three caches of interest:
1.1 root 151: *
1.1.1.3 root 152: * vm_map_cache: used to allocate maps.
153: * vm_map_entry_cache: used to allocate map entries.
154: * vm_map_kentry_cache: used to allocate map entries for the kernel.
155: *
156: * Kernel map entries are allocated from a special cache, using a custom
157: * page allocation function to avoid recursion. It would be difficult
158: * (perhaps impossible) for the kernel to allocate more memory to an entry
159: * cache when it became empty since the very act of allocating memory
160: * implies the creation of a new entry.
1.1 root 161: */
162:
163: vm_offset_t kentry_data;
1.1.1.3 root 164: vm_size_t kentry_data_size = KENTRY_DATA_SIZE;
1.1 root 165:
1.1.1.3 root 166: static vm_offset_t kentry_pagealloc(vm_size_t size)
1.1 root 167: {
1.1.1.3 root 168: vm_offset_t result;
1.1 root 169:
1.1.1.3 root 170: if (size > kentry_data_size)
171: panic("vm_map: kentry memory exhausted");
1.1 root 172:
1.1.1.3 root 173: result = kentry_data;
174: kentry_data += size;
175: kentry_data_size -= size;
176: return result;
177: }
178:
179: void vm_map_init(void)
180: {
181: kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
182: NULL, NULL, NULL, 0);
183: kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
184: sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0);
185: kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
186: sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc,
187: NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB
188: | KMEM_CACHE_NORECLAIM);
189: kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
190: sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0);
1.1 root 191:
192: /*
193: * Submap object is initialized by vm_object_init.
194: */
195: }
196:
1.1.1.4 ! root 197: void vm_map_setup(
! 198: vm_map_t map,
! 199: pmap_t pmap,
! 200: vm_offset_t min,
! 201: vm_offset_t max,
! 202: boolean_t pageable)
1.1.1.3 root 203: {
204: vm_map_first_entry(map) = vm_map_to_entry(map);
205: vm_map_last_entry(map) = vm_map_to_entry(map);
206: map->hdr.nentries = 0;
207: map->hdr.entries_pageable = pageable;
208: rbtree_init(&map->hdr.tree);
209:
210: map->size = 0;
211: map->ref_count = 1;
212: map->pmap = pmap;
213: map->min_offset = min;
214: map->max_offset = max;
215: map->wiring_required = FALSE;
216: map->wait_for_space = FALSE;
217: map->first_free = vm_map_to_entry(map);
218: map->hint = vm_map_to_entry(map);
219: vm_map_lock_init(map);
220: simple_lock_init(&map->ref_lock);
221: simple_lock_init(&map->hint_lock);
222: }
223:
1.1 root 224: /*
225: * vm_map_create:
226: *
227: * Creates and returns a new empty VM map with
228: * the given physical map structure, and having
229: * the given lower and upper address bounds.
230: */
1.1.1.4 ! root 231: vm_map_t vm_map_create(
! 232: pmap_t pmap,
! 233: vm_offset_t min,
! 234: vm_offset_t max,
! 235: boolean_t pageable)
1.1 root 236: {
1.1.1.4 ! root 237: vm_map_t result;
1.1 root 238:
1.1.1.3 root 239: result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
1.1 root 240: if (result == VM_MAP_NULL)
241: panic("vm_map_create");
242:
1.1.1.3 root 243: vm_map_setup(result, pmap, min, max, pageable);
1.1 root 244:
245: return(result);
246: }
247:
248: /*
249: * vm_map_entry_create: [ internal use only ]
250: *
251: * Allocates a VM map entry for insertion in the
252: * given map (or map copy). No fields are filled.
253: */
254: #define vm_map_entry_create(map) \
255: _vm_map_entry_create(&(map)->hdr)
256:
257: #define vm_map_copy_entry_create(copy) \
258: _vm_map_entry_create(&(copy)->cpy_hdr)
259:
260: vm_map_entry_t _vm_map_entry_create(map_header)
1.1.1.4 ! root 261: const struct vm_map_header *map_header;
1.1 root 262: {
1.1.1.4 ! root 263: kmem_cache_t cache;
! 264: vm_map_entry_t entry;
1.1 root 265:
266: if (map_header->entries_pageable)
1.1.1.3 root 267: cache = &vm_map_entry_cache;
1.1 root 268: else
1.1.1.3 root 269: cache = &vm_map_kentry_cache;
1.1 root 270:
1.1.1.3 root 271: entry = (vm_map_entry_t) kmem_cache_alloc(cache);
1.1 root 272: if (entry == VM_MAP_ENTRY_NULL)
273: panic("vm_map_entry_create");
274:
275: return(entry);
276: }
277:
278: /*
279: * vm_map_entry_dispose: [ internal use only ]
280: *
281: * Inverse of vm_map_entry_create.
282: */
283: #define vm_map_entry_dispose(map, entry) \
284: _vm_map_entry_dispose(&(map)->hdr, (entry))
285:
286: #define vm_map_copy_entry_dispose(map, entry) \
287: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
288:
289: void _vm_map_entry_dispose(map_header, entry)
1.1.1.4 ! root 290: const struct vm_map_header *map_header;
! 291: vm_map_entry_t entry;
1.1 root 292: {
1.1.1.4 ! root 293: kmem_cache_t cache;
1.1 root 294:
295: if (map_header->entries_pageable)
1.1.1.3 root 296: cache = &vm_map_entry_cache;
297: else
298: cache = &vm_map_kentry_cache;
299:
300: kmem_cache_free(cache, (vm_offset_t) entry);
301: }
302:
303: /*
304: * Red-black tree lookup/insert comparison functions
305: */
306: static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
307: const struct rbtree_node *node)
308: {
309: struct vm_map_entry *entry;
310:
311: entry = rbtree_entry(node, struct vm_map_entry, tree_node);
312:
313: if (addr < entry->vme_start)
314: return -1;
315: else if (addr < entry->vme_end)
316: return 0;
1.1 root 317: else
1.1.1.3 root 318: return 1;
319: }
320:
321: static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
322: const struct rbtree_node *b)
323: {
324: struct vm_map_entry *entry;
1.1 root 325:
1.1.1.3 root 326: entry = rbtree_entry(a, struct vm_map_entry, tree_node);
327: return vm_map_entry_cmp_lookup(entry->vme_start, b);
1.1 root 328: }
329:
330: /*
331: * vm_map_entry_{un,}link:
332: *
333: * Insert/remove entries from maps (or map copies).
1.1.1.3 root 334: *
335: * The start and end addresses of the entries must be properly set
336: * before using these macros.
1.1 root 337: */
338: #define vm_map_entry_link(map, after_where, entry) \
339: _vm_map_entry_link(&(map)->hdr, after_where, entry)
340:
341: #define vm_map_copy_entry_link(copy, after_where, entry) \
342: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
343:
344: #define _vm_map_entry_link(hdr, after_where, entry) \
345: MACRO_BEGIN \
346: (hdr)->nentries++; \
347: (entry)->vme_prev = (after_where); \
348: (entry)->vme_next = (after_where)->vme_next; \
349: (entry)->vme_prev->vme_next = \
350: (entry)->vme_next->vme_prev = (entry); \
1.1.1.3 root 351: rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \
352: vm_map_entry_cmp_insert); \
1.1 root 353: MACRO_END
354:
355: #define vm_map_entry_unlink(map, entry) \
356: _vm_map_entry_unlink(&(map)->hdr, entry)
357:
358: #define vm_map_copy_entry_unlink(copy, entry) \
359: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
360:
361: #define _vm_map_entry_unlink(hdr, entry) \
362: MACRO_BEGIN \
363: (hdr)->nentries--; \
364: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
365: (entry)->vme_prev->vme_next = (entry)->vme_next; \
1.1.1.3 root 366: rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
1.1 root 367: MACRO_END
368:
369: /*
370: * vm_map_reference:
371: *
372: * Creates another valid reference to the given map.
373: *
374: */
1.1.1.4 ! root 375: void vm_map_reference(vm_map_t map)
1.1 root 376: {
377: if (map == VM_MAP_NULL)
378: return;
379:
380: simple_lock(&map->ref_lock);
381: map->ref_count++;
382: simple_unlock(&map->ref_lock);
383: }
384:
385: /*
386: * vm_map_deallocate:
387: *
388: * Removes a reference from the specified map,
389: * destroying it if no references remain.
390: * The map should not be locked.
391: */
1.1.1.4 ! root 392: void vm_map_deallocate(vm_map_t map)
1.1 root 393: {
1.1.1.4 ! root 394: int c;
1.1 root 395:
396: if (map == VM_MAP_NULL)
397: return;
398:
399: simple_lock(&map->ref_lock);
400: c = --map->ref_count;
401: simple_unlock(&map->ref_lock);
402:
403: if (c > 0) {
404: return;
405: }
406:
407: projected_buffer_collect(map);
408: (void) vm_map_delete(map, map->min_offset, map->max_offset);
409:
410: pmap_destroy(map->pmap);
411:
1.1.1.3 root 412: kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
1.1 root 413: }
414:
415: /*
416: * SAVE_HINT:
417: *
418: * Saves the specified entry as the hint for
419: * future lookups. Performs necessary interlocks.
420: */
421: #define SAVE_HINT(map,value) \
422: simple_lock(&(map)->hint_lock); \
423: (map)->hint = (value); \
424: simple_unlock(&(map)->hint_lock);
425:
426: /*
427: * vm_map_lookup_entry: [ internal use only ]
428: *
429: * Finds the map entry containing (or
430: * immediately preceding) the specified address
431: * in the given map; the entry is returned
432: * in the "entry" parameter. The boolean
433: * result indicates whether the address is
434: * actually contained in the map.
435: */
1.1.1.4 ! root 436: boolean_t vm_map_lookup_entry(
! 437: vm_map_t map,
! 438: vm_offset_t address,
! 439: vm_map_entry_t *entry) /* OUT */
1.1 root 440: {
1.1.1.4 ! root 441: struct rbtree_node *node;
! 442: vm_map_entry_t hint;
1.1 root 443:
444: /*
1.1.1.3 root 445: * First, make a quick check to see if we are already
446: * looking at the entry we want (which is often the case).
1.1 root 447: */
448:
449: simple_lock(&map->hint_lock);
1.1.1.3 root 450: hint = map->hint;
1.1 root 451: simple_unlock(&map->hint_lock);
452:
1.1.1.3 root 453: if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) {
454: if (address < hint->vme_end) {
455: *entry = hint;
1.1 root 456: return(TRUE);
1.1.1.3 root 457: } else {
458: vm_map_entry_t next = hint->vme_next;
459:
460: if ((next == vm_map_to_entry(map))
461: || (address < next->vme_start)) {
462: *entry = hint;
463: return(FALSE);
464: }
1.1 root 465: }
466: }
467:
468: /*
1.1.1.3 root 469: * If the hint didn't help, use the red-black tree.
1.1 root 470: */
471:
1.1.1.3 root 472: node = rbtree_lookup_nearest(&map->hdr.tree, address,
473: vm_map_entry_cmp_lookup, RBTREE_LEFT);
1.1 root 474:
1.1.1.3 root 475: if (node == NULL) {
476: *entry = vm_map_to_entry(map);
477: SAVE_HINT(map, *entry);
478: return(FALSE);
479: } else {
480: *entry = rbtree_entry(node, struct vm_map_entry, tree_node);
481: SAVE_HINT(map, *entry);
482: return((address < (*entry)->vme_end) ? TRUE : FALSE);
1.1 root 483: }
484: }
485:
486: /*
487: * Routine: invalid_user_access
488: *
489: * Verifies whether user access is valid.
490: */
491:
492: boolean_t
1.1.1.4 ! root 493: invalid_user_access(
! 494: vm_map_t map,
! 495: vm_offset_t start,
! 496: vm_offset_t end,
! 497: vm_prot_t prot)
1.1 root 498: {
499: vm_map_entry_t entry;
500:
501: return (map == VM_MAP_NULL || map == kernel_map ||
502: !vm_map_lookup_entry(map, start, &entry) ||
503: entry->vme_end < end ||
504: (prot & ~(entry->protection)));
505: }
506:
507:
508: /*
509: * Routine: vm_map_find_entry
510: * Purpose:
511: * Allocate a range in the specified virtual address map,
512: * returning the entry allocated for that range.
513: * Used by kmem_alloc, etc. Returns wired entries.
514: *
515: * The map must be locked.
516: *
517: * If an entry is allocated, the object/offset fields
518: * are initialized to zero. If an object is supplied,
519: * then an existing entry may be extended.
520: */
1.1.1.4 ! root 521: kern_return_t vm_map_find_entry(
! 522: vm_map_t map,
! 523: vm_offset_t *address, /* OUT */
! 524: vm_size_t size,
! 525: vm_offset_t mask,
! 526: vm_object_t object,
! 527: vm_map_entry_t *o_entry) /* OUT */
1.1 root 528: {
1.1.1.4 ! root 529: vm_map_entry_t entry, new_entry;
! 530: vm_offset_t start;
! 531: vm_offset_t end;
1.1 root 532:
533: /*
534: * Look for the first possible address;
535: * if there's already something at this
536: * address, we have to start after it.
537: */
538:
539: if ((entry = map->first_free) == vm_map_to_entry(map))
540: start = map->min_offset;
541: else
542: start = entry->vme_end;
543:
544: /*
545: * In any case, the "entry" always precedes
546: * the proposed new region throughout the loop:
547: */
548:
549: while (TRUE) {
1.1.1.4 ! root 550: vm_map_entry_t next;
1.1 root 551:
552: /*
553: * Find the end of the proposed new region.
554: * Be sure we didn't go beyond the end, or
555: * wrap around the address.
556: */
557:
1.1.1.3 root 558: if (((start + mask) & ~mask) < start) {
559: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 560: return(KERN_NO_SPACE);
1.1.1.3 root 561: }
1.1 root 562: start = ((start + mask) & ~mask);
563: end = start + size;
564:
1.1.1.3 root 565: if ((end > map->max_offset) || (end < start)) {
566: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 567: return(KERN_NO_SPACE);
1.1.1.3 root 568: }
1.1 root 569:
570: /*
571: * If there are no more entries, we must win.
572: */
573:
574: next = entry->vme_next;
575: if (next == vm_map_to_entry(map))
576: break;
577:
578: /*
579: * If there is another entry, it must be
580: * after the end of the potential new region.
581: */
582:
583: if (next->vme_start >= end)
584: break;
585:
586: /*
587: * Didn't fit -- move to the next entry.
588: */
589:
590: entry = next;
591: start = entry->vme_end;
592: }
593:
594: /*
595: * At this point,
596: * "start" and "end" should define the endpoints of the
597: * available new range, and
598: * "entry" should refer to the region before the new
599: * range, and
600: *
601: * the map should be locked.
602: */
603:
604: *address = start;
605:
606: /*
607: * See whether we can avoid creating a new entry by
608: * extending one of our neighbors. [So far, we only attempt to
609: * extend from below.]
610: */
611:
612: if ((object != VM_OBJECT_NULL) &&
613: (entry != vm_map_to_entry(map)) &&
614: (entry->vme_end == start) &&
615: (!entry->is_shared) &&
616: (!entry->is_sub_map) &&
617: (entry->object.vm_object == object) &&
618: (entry->needs_copy == FALSE) &&
619: (entry->inheritance == VM_INHERIT_DEFAULT) &&
620: (entry->protection == VM_PROT_DEFAULT) &&
621: (entry->max_protection == VM_PROT_ALL) &&
622: (entry->wired_count == 1) &&
623: (entry->user_wired_count == 0) &&
624: (entry->projected_on == 0)) {
625: /*
626: * Because this is a special case,
627: * we don't need to use vm_object_coalesce.
628: */
629:
630: entry->vme_end = end;
631: new_entry = entry;
632: } else {
633: new_entry = vm_map_entry_create(map);
634:
635: new_entry->vme_start = start;
636: new_entry->vme_end = end;
637:
638: new_entry->is_shared = FALSE;
639: new_entry->is_sub_map = FALSE;
640: new_entry->object.vm_object = VM_OBJECT_NULL;
641: new_entry->offset = (vm_offset_t) 0;
642:
643: new_entry->needs_copy = FALSE;
644:
645: new_entry->inheritance = VM_INHERIT_DEFAULT;
646: new_entry->protection = VM_PROT_DEFAULT;
647: new_entry->max_protection = VM_PROT_ALL;
648: new_entry->wired_count = 1;
649: new_entry->user_wired_count = 0;
650:
651: new_entry->in_transition = FALSE;
652: new_entry->needs_wakeup = FALSE;
653: new_entry->projected_on = 0;
654:
655: /*
656: * Insert the new entry into the list
657: */
658:
659: vm_map_entry_link(map, entry, new_entry);
660: }
661:
662: map->size += size;
663:
664: /*
665: * Update the free space hint and the lookup hint
666: */
667:
668: map->first_free = new_entry;
669: SAVE_HINT(map, new_entry);
670:
671: *o_entry = new_entry;
672: return(KERN_SUCCESS);
673: }
674:
1.1.1.4 ! root 675: boolean_t vm_map_pmap_enter_print = FALSE;
! 676: boolean_t vm_map_pmap_enter_enable = FALSE;
1.1 root 677:
678: /*
679: * Routine: vm_map_pmap_enter
680: *
681: * Description:
682: * Force pages from the specified object to be entered into
683: * the pmap at the specified address if they are present.
684: * As soon as a page not found in the object the scan ends.
685: *
686: * Returns:
1.1.1.2 root 687: * Nothing.
1.1 root 688: *
689: * In/out conditions:
690: * The source map should not be locked on entry.
691: */
692: void
1.1.1.4 ! root 693: vm_map_pmap_enter(
! 694: vm_map_t map,
! 695: vm_offset_t addr,
! 696: vm_offset_t end_addr,
! 697: vm_object_t object,
! 698: vm_offset_t offset,
! 699: vm_prot_t protection)
1.1 root 700: {
701: while (addr < end_addr) {
1.1.1.4 ! root 702: vm_page_t m;
1.1 root 703:
704: vm_object_lock(object);
705: vm_object_paging_begin(object);
706:
707: m = vm_page_lookup(object, offset);
708: if (m == VM_PAGE_NULL || m->absent) {
709: vm_object_paging_end(object);
710: vm_object_unlock(object);
711: return;
712: }
713:
714: if (vm_map_pmap_enter_print) {
715: printf("vm_map_pmap_enter:");
1.1.1.3 root 716: printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
1.1 root 717: map, addr, object, offset);
718: }
719:
720: m->busy = TRUE;
721: vm_object_unlock(object);
722:
723: PMAP_ENTER(map->pmap, addr, m,
724: protection, FALSE);
725:
726: vm_object_lock(object);
727: PAGE_WAKEUP_DONE(m);
728: vm_page_lock_queues();
729: if (!m->active && !m->inactive)
730: vm_page_activate(m);
731: vm_page_unlock_queues();
732: vm_object_paging_end(object);
733: vm_object_unlock(object);
734:
735: offset += PAGE_SIZE;
736: addr += PAGE_SIZE;
737: }
738: }
739:
740: /*
741: * Routine: vm_map_enter
742: *
743: * Description:
744: * Allocate a range in the specified virtual address map.
745: * The resulting range will refer to memory defined by
746: * the given memory object and offset into that object.
747: *
748: * Arguments are as defined in the vm_map call.
749: */
750: kern_return_t vm_map_enter(
1.1.1.4 ! root 751: vm_map_t map,
! 752: vm_offset_t *address, /* IN/OUT */
! 753: vm_size_t size,
! 754: vm_offset_t mask,
! 755: boolean_t anywhere,
! 756: vm_object_t object,
! 757: vm_offset_t offset,
! 758: boolean_t needs_copy,
! 759: vm_prot_t cur_protection,
! 760: vm_prot_t max_protection,
! 761: vm_inherit_t inheritance)
! 762: {
! 763: vm_map_entry_t entry;
! 764: vm_offset_t start;
! 765: vm_offset_t end;
! 766: kern_return_t result = KERN_SUCCESS;
1.1 root 767:
768: #define RETURN(value) { result = value; goto BailOut; }
769:
1.1.1.3 root 770: if (size == 0)
771: return KERN_INVALID_ARGUMENT;
772:
1.1 root 773: StartAgain: ;
774:
775: start = *address;
776:
777: if (anywhere) {
778: vm_map_lock(map);
779:
780: /*
781: * Calculate the first possible address.
782: */
783:
784: if (start < map->min_offset)
785: start = map->min_offset;
786: if (start > map->max_offset)
787: RETURN(KERN_NO_SPACE);
788:
789: /*
790: * Look for the first possible address;
791: * if there's already something at this
792: * address, we have to start after it.
793: */
794:
795: if (start == map->min_offset) {
796: if ((entry = map->first_free) != vm_map_to_entry(map))
797: start = entry->vme_end;
798: } else {
799: vm_map_entry_t tmp_entry;
800: if (vm_map_lookup_entry(map, start, &tmp_entry))
801: start = tmp_entry->vme_end;
802: entry = tmp_entry;
803: }
804:
805: /*
806: * In any case, the "entry" always precedes
807: * the proposed new region throughout the
808: * loop:
809: */
810:
811: while (TRUE) {
1.1.1.4 ! root 812: vm_map_entry_t next;
1.1 root 813:
814: /*
815: * Find the end of the proposed new region.
816: * Be sure we didn't go beyond the end, or
817: * wrap around the address.
818: */
819:
1.1.1.3 root 820: if (((start + mask) & ~mask) < start) {
821: printf_once("no more room for vm_map_enter in %p\n", map);
822: RETURN(KERN_NO_SPACE);
823: }
1.1 root 824: start = ((start + mask) & ~mask);
825: end = start + size;
826:
827: if ((end > map->max_offset) || (end < start)) {
828: if (map->wait_for_space) {
829: if (size <= (map->max_offset -
830: map->min_offset)) {
831: assert_wait((event_t) map, TRUE);
832: vm_map_unlock(map);
833: thread_block((void (*)()) 0);
834: goto StartAgain;
835: }
836: }
1.1.1.2 root 837:
1.1.1.3 root 838: printf_once("no more room for vm_map_enter in %p\n", map);
1.1 root 839: RETURN(KERN_NO_SPACE);
840: }
841:
842: /*
843: * If there are no more entries, we must win.
844: */
845:
846: next = entry->vme_next;
847: if (next == vm_map_to_entry(map))
848: break;
849:
850: /*
851: * If there is another entry, it must be
852: * after the end of the potential new region.
853: */
854:
855: if (next->vme_start >= end)
856: break;
857:
858: /*
859: * Didn't fit -- move to the next entry.
860: */
861:
862: entry = next;
863: start = entry->vme_end;
864: }
865: *address = start;
866: } else {
867: vm_map_entry_t temp_entry;
868:
869: /*
870: * Verify that:
871: * the address doesn't itself violate
872: * the mask requirement.
873: */
874:
875: if ((start & mask) != 0)
876: return(KERN_NO_SPACE);
877:
878: vm_map_lock(map);
879:
880: /*
881: * ... the address is within bounds
882: */
883:
884: end = start + size;
885:
886: if ((start < map->min_offset) ||
887: (end > map->max_offset) ||
888: (start >= end)) {
889: RETURN(KERN_INVALID_ADDRESS);
890: }
891:
892: /*
893: * ... the starting address isn't allocated
894: */
895:
896: if (vm_map_lookup_entry(map, start, &temp_entry))
897: RETURN(KERN_NO_SPACE);
898:
899: entry = temp_entry;
900:
901: /*
902: * ... the next region doesn't overlap the
903: * end point.
904: */
905:
906: if ((entry->vme_next != vm_map_to_entry(map)) &&
907: (entry->vme_next->vme_start < end))
908: RETURN(KERN_NO_SPACE);
909: }
910:
911: /*
912: * At this point,
913: * "start" and "end" should define the endpoints of the
914: * available new range, and
915: * "entry" should refer to the region before the new
916: * range, and
917: *
918: * the map should be locked.
919: */
920:
921: /*
922: * See whether we can avoid creating a new entry (and object) by
923: * extending one of our neighbors. [So far, we only attempt to
924: * extend from below.]
925: */
926:
927: if ((object == VM_OBJECT_NULL) &&
928: (entry != vm_map_to_entry(map)) &&
929: (entry->vme_end == start) &&
930: (!entry->is_shared) &&
931: (!entry->is_sub_map) &&
932: (entry->inheritance == inheritance) &&
933: (entry->protection == cur_protection) &&
934: (entry->max_protection == max_protection) &&
935: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
1.1.1.2 root 936: (entry->projected_on == 0)) {
1.1 root 937: if (vm_object_coalesce(entry->object.vm_object,
938: VM_OBJECT_NULL,
939: entry->offset,
940: (vm_offset_t) 0,
941: (vm_size_t)(entry->vme_end - entry->vme_start),
942: (vm_size_t)(end - entry->vme_end))) {
943:
944: /*
945: * Coalesced the two objects - can extend
946: * the previous map entry to include the
947: * new range.
948: */
949: map->size += (end - entry->vme_end);
950: entry->vme_end = end;
951: RETURN(KERN_SUCCESS);
952: }
953: }
954:
955: /*
956: * Create a new entry
957: */
958:
959: /**/ {
1.1.1.4 ! root 960: vm_map_entry_t new_entry;
1.1 root 961:
962: new_entry = vm_map_entry_create(map);
963:
964: new_entry->vme_start = start;
965: new_entry->vme_end = end;
966:
967: new_entry->is_shared = FALSE;
968: new_entry->is_sub_map = FALSE;
969: new_entry->object.vm_object = object;
970: new_entry->offset = offset;
971:
972: new_entry->needs_copy = needs_copy;
973:
974: new_entry->inheritance = inheritance;
975: new_entry->protection = cur_protection;
976: new_entry->max_protection = max_protection;
977: new_entry->wired_count = 0;
978: new_entry->user_wired_count = 0;
979:
980: new_entry->in_transition = FALSE;
981: new_entry->needs_wakeup = FALSE;
982: new_entry->projected_on = 0;
983:
984: /*
985: * Insert the new entry into the list
986: */
987:
988: vm_map_entry_link(map, entry, new_entry);
989: map->size += size;
990:
991: /*
992: * Update the free space hint and the lookup hint
993: */
994:
995: if ((map->first_free == entry) &&
996: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
997: >= new_entry->vme_start))
998: map->first_free = new_entry;
999:
1000: SAVE_HINT(map, new_entry);
1001:
1002: vm_map_unlock(map);
1003:
1004: if ((object != VM_OBJECT_NULL) &&
1005: (vm_map_pmap_enter_enable) &&
1006: (!anywhere) &&
1.1.1.2 root 1007: (!needs_copy) &&
1.1 root 1008: (size < (128*1024))) {
1.1.1.2 root 1009: vm_map_pmap_enter(map, start, end,
1.1 root 1010: object, offset, cur_protection);
1011: }
1012:
1013: return(result);
1014: /**/ }
1015:
1016: BailOut: ;
1017:
1018: vm_map_unlock(map);
1019: return(result);
1020:
1021: #undef RETURN
1022: }
1023:
1024: /*
1025: * vm_map_clip_start: [ internal use only ]
1026: *
1027: * Asserts that the given entry begins at or after
1028: * the specified address; if necessary,
1029: * it splits the entry into two.
1030: */
1031: #define vm_map_clip_start(map, entry, startaddr) \
1032: MACRO_BEGIN \
1033: if ((startaddr) > (entry)->vme_start) \
1034: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1035: MACRO_END
1036:
1037: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1038: MACRO_BEGIN \
1039: if ((startaddr) > (entry)->vme_start) \
1040: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1041: MACRO_END
1042:
1043: /*
1044: * This routine is called only when it is known that
1045: * the entry must be split.
1046: */
1.1.1.4 ! root 1047: void _vm_map_clip_start(
! 1048: struct vm_map_header *map_header,
! 1049: vm_map_entry_t entry,
! 1050: vm_offset_t start)
1.1 root 1051: {
1.1.1.4 ! root 1052: vm_map_entry_t new_entry;
1.1 root 1053:
1054: /*
1055: * Split off the front portion --
1056: * note that we must insert the new
1057: * entry BEFORE this one, so that
1058: * this entry has the specified starting
1059: * address.
1060: */
1061:
1062: new_entry = _vm_map_entry_create(map_header);
1063: vm_map_entry_copy_full(new_entry, entry);
1064:
1065: new_entry->vme_end = start;
1066: entry->offset += (start - entry->vme_start);
1067: entry->vme_start = start;
1068:
1069: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1070:
1071: if (entry->is_sub_map)
1072: vm_map_reference(new_entry->object.sub_map);
1073: else
1074: vm_object_reference(new_entry->object.vm_object);
1075: }
1076:
1077: /*
1078: * vm_map_clip_end: [ internal use only ]
1079: *
1080: * Asserts that the given entry ends at or before
1081: * the specified address; if necessary,
1082: * it splits the entry into two.
1083: */
1084: #define vm_map_clip_end(map, entry, endaddr) \
1085: MACRO_BEGIN \
1086: if ((endaddr) < (entry)->vme_end) \
1087: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1088: MACRO_END
1089:
1090: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1091: MACRO_BEGIN \
1092: if ((endaddr) < (entry)->vme_end) \
1093: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1094: MACRO_END
1095:
1096: /*
1097: * This routine is called only when it is known that
1098: * the entry must be split.
1099: */
1.1.1.4 ! root 1100: void _vm_map_clip_end(
! 1101: struct vm_map_header *map_header,
! 1102: vm_map_entry_t entry,
! 1103: vm_offset_t end)
1.1 root 1104: {
1.1.1.4 ! root 1105: vm_map_entry_t new_entry;
1.1 root 1106:
1107: /*
1108: * Create a new entry and insert it
1109: * AFTER the specified entry
1110: */
1111:
1112: new_entry = _vm_map_entry_create(map_header);
1113: vm_map_entry_copy_full(new_entry, entry);
1114:
1115: new_entry->vme_start = entry->vme_end = end;
1116: new_entry->offset += (end - entry->vme_start);
1117:
1118: _vm_map_entry_link(map_header, entry, new_entry);
1119:
1120: if (entry->is_sub_map)
1121: vm_map_reference(new_entry->object.sub_map);
1122: else
1123: vm_object_reference(new_entry->object.vm_object);
1124: }
1125:
1126: /*
1127: * VM_MAP_RANGE_CHECK: [ internal use only ]
1128: *
1129: * Asserts that the starting and ending region
1130: * addresses fall within the valid range of the map.
1131: */
1132: #define VM_MAP_RANGE_CHECK(map, start, end) \
1133: { \
1134: if (start < vm_map_min(map)) \
1135: start = vm_map_min(map); \
1136: if (end > vm_map_max(map)) \
1137: end = vm_map_max(map); \
1138: if (start > end) \
1139: start = end; \
1140: }
1141:
1142: /*
1143: * vm_map_submap: [ kernel use only ]
1144: *
1145: * Mark the given range as handled by a subordinate map.
1146: *
1147: * This range must have been created with vm_map_find using
1148: * the vm_submap_object, and no other operations may have been
1149: * performed on this range prior to calling vm_map_submap.
1150: *
1151: * Only a limited number of operations can be performed
1152: * within this rage after calling vm_map_submap:
1153: * vm_fault
1154: * [Don't try vm_map_copyin!]
1155: *
1156: * To remove a submapping, one must first remove the
1157: * range from the superior map, and then destroy the
1158: * submap (if desired). [Better yet, don't try it.]
1159: */
1.1.1.4 ! root 1160: kern_return_t vm_map_submap(
! 1161: vm_map_t map,
! 1162: vm_offset_t start,
! 1163: vm_offset_t end,
! 1164: vm_map_t submap)
1.1 root 1165: {
1166: vm_map_entry_t entry;
1.1.1.4 ! root 1167: kern_return_t result = KERN_INVALID_ARGUMENT;
! 1168: vm_object_t object;
1.1 root 1169:
1170: vm_map_lock(map);
1171:
1172: VM_MAP_RANGE_CHECK(map, start, end);
1173:
1174: if (vm_map_lookup_entry(map, start, &entry)) {
1175: vm_map_clip_start(map, entry, start);
1176: }
1177: else
1178: entry = entry->vme_next;
1179:
1180: vm_map_clip_end(map, entry, end);
1181:
1182: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1183: (!entry->is_sub_map) &&
1184: ((object = entry->object.vm_object) == vm_submap_object) &&
1185: (object->resident_page_count == 0) &&
1186: (object->copy == VM_OBJECT_NULL) &&
1187: (object->shadow == VM_OBJECT_NULL) &&
1188: (!object->pager_created)) {
1189: entry->object.vm_object = VM_OBJECT_NULL;
1190: vm_object_deallocate(object);
1191: entry->is_sub_map = TRUE;
1192: vm_map_reference(entry->object.sub_map = submap);
1193: result = KERN_SUCCESS;
1194: }
1195: vm_map_unlock(map);
1196:
1197: return(result);
1198: }
1199:
1200: /*
1201: * vm_map_protect:
1202: *
1203: * Sets the protection of the specified address
1204: * region in the target map. If "set_max" is
1205: * specified, the maximum protection is to be set;
1206: * otherwise, only the current protection is affected.
1207: */
1.1.1.4 ! root 1208: kern_return_t vm_map_protect(
! 1209: vm_map_t map,
! 1210: vm_offset_t start,
! 1211: vm_offset_t end,
! 1212: vm_prot_t new_prot,
! 1213: boolean_t set_max)
1.1 root 1214: {
1.1.1.4 ! root 1215: vm_map_entry_t current;
! 1216: vm_map_entry_t entry;
1.1 root 1217:
1218: vm_map_lock(map);
1219:
1220: VM_MAP_RANGE_CHECK(map, start, end);
1221:
1222: if (vm_map_lookup_entry(map, start, &entry)) {
1223: vm_map_clip_start(map, entry, start);
1224: }
1225: else
1226: entry = entry->vme_next;
1227:
1228: /*
1229: * Make a first pass to check for protection
1230: * violations.
1231: */
1232:
1233: current = entry;
1234: while ((current != vm_map_to_entry(map)) &&
1235: (current->vme_start < end)) {
1236:
1237: if (current->is_sub_map) {
1238: vm_map_unlock(map);
1239: return(KERN_INVALID_ARGUMENT);
1240: }
1241: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1242: != new_prot) {
1243: vm_map_unlock(map);
1244: return(KERN_PROTECTION_FAILURE);
1245: }
1246:
1247: current = current->vme_next;
1248: }
1249:
1250: /*
1251: * Go back and fix up protections.
1252: * [Note that clipping is not necessary the second time.]
1253: */
1254:
1255: current = entry;
1256:
1257: while ((current != vm_map_to_entry(map)) &&
1258: (current->vme_start < end)) {
1259:
1260: vm_prot_t old_prot;
1261:
1262: vm_map_clip_end(map, current, end);
1263:
1264: old_prot = current->protection;
1265: if (set_max)
1266: current->protection =
1267: (current->max_protection = new_prot) &
1268: old_prot;
1269: else
1270: current->protection = new_prot;
1271:
1272: /*
1273: * Update physical map if necessary.
1274: */
1275:
1276: if (current->protection != old_prot) {
1277: pmap_protect(map->pmap, current->vme_start,
1278: current->vme_end,
1279: current->protection);
1280: }
1281: current = current->vme_next;
1282: }
1283:
1284: vm_map_unlock(map);
1285: return(KERN_SUCCESS);
1286: }
1287:
1288: /*
1289: * vm_map_inherit:
1290: *
1291: * Sets the inheritance of the specified address
1292: * range in the target map. Inheritance
1293: * affects how the map will be shared with
1294: * child maps at the time of vm_map_fork.
1295: */
1.1.1.4 ! root 1296: kern_return_t vm_map_inherit(
! 1297: vm_map_t map,
! 1298: vm_offset_t start,
! 1299: vm_offset_t end,
! 1300: vm_inherit_t new_inheritance)
1.1 root 1301: {
1.1.1.4 ! root 1302: vm_map_entry_t entry;
1.1 root 1303: vm_map_entry_t temp_entry;
1304:
1305: vm_map_lock(map);
1306:
1307: VM_MAP_RANGE_CHECK(map, start, end);
1308:
1309: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1310: entry = temp_entry;
1311: vm_map_clip_start(map, entry, start);
1312: }
1313: else
1314: entry = temp_entry->vme_next;
1315:
1316: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1317: vm_map_clip_end(map, entry, end);
1318:
1319: entry->inheritance = new_inheritance;
1320:
1321: entry = entry->vme_next;
1322: }
1323:
1324: vm_map_unlock(map);
1325: return(KERN_SUCCESS);
1326: }
1327:
1328: /*
1329: * vm_map_pageable_common:
1330: *
1331: * Sets the pageability of the specified address
1332: * range in the target map. Regions specified
1333: * as not pageable require locked-down physical
1334: * memory and physical page maps. access_type indicates
1335: * types of accesses that must not generate page faults.
1336: * This is checked against protection of memory being locked-down.
1337: * access_type of VM_PROT_NONE makes memory pageable.
1338: *
1339: * The map must not be locked, but a reference
1340: * must remain to the map throughout the call.
1341: *
1342: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1343: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1344: */
1.1.1.4 ! root 1345: kern_return_t vm_map_pageable_common(
! 1346: vm_map_t map,
! 1347: vm_offset_t start,
! 1348: vm_offset_t end,
! 1349: vm_prot_t access_type,
! 1350: boolean_t user_wire)
1.1 root 1351: {
1.1.1.4 ! root 1352: vm_map_entry_t entry;
1.1 root 1353: vm_map_entry_t start_entry;
1354:
1355: vm_map_lock(map);
1356:
1357: VM_MAP_RANGE_CHECK(map, start, end);
1358:
1359: if (vm_map_lookup_entry(map, start, &start_entry)) {
1360: entry = start_entry;
1361: /*
1362: * vm_map_clip_start will be done later.
1363: */
1364: }
1365: else {
1366: /*
1367: * Start address is not in map; this is fatal.
1368: */
1369: vm_map_unlock(map);
1370: return(KERN_FAILURE);
1371: }
1372:
1373: /*
1374: * Actions are rather different for wiring and unwiring,
1375: * so we have two separate cases.
1376: */
1377:
1378: if (access_type == VM_PROT_NONE) {
1379:
1380: vm_map_clip_start(map, entry, start);
1381:
1382: /*
1383: * Unwiring. First ensure that the range to be
1384: * unwired is really wired down.
1385: */
1386: while ((entry != vm_map_to_entry(map)) &&
1387: (entry->vme_start < end)) {
1388:
1389: if ((entry->wired_count == 0) ||
1.1.1.2 root 1390: ((entry->vme_end < end) &&
1.1 root 1391: ((entry->vme_next == vm_map_to_entry(map)) ||
1392: (entry->vme_next->vme_start > entry->vme_end))) ||
1393: (user_wire && (entry->user_wired_count == 0))) {
1394: vm_map_unlock(map);
1395: return(KERN_INVALID_ARGUMENT);
1396: }
1397: entry = entry->vme_next;
1398: }
1399:
1400: /*
1401: * Now decrement the wiring count for each region.
1402: * If a region becomes completely unwired,
1403: * unwire its physical pages and mappings.
1404: */
1405: entry = start_entry;
1406: while ((entry != vm_map_to_entry(map)) &&
1407: (entry->vme_start < end)) {
1408: vm_map_clip_end(map, entry, end);
1409:
1410: if (user_wire) {
1411: if (--(entry->user_wired_count) == 0)
1412: entry->wired_count--;
1413: }
1414: else {
1415: entry->wired_count--;
1416: }
1.1.1.2 root 1417:
1.1 root 1418: if (entry->wired_count == 0)
1419: vm_fault_unwire(map, entry);
1420:
1421: entry = entry->vme_next;
1422: }
1423: }
1424:
1425: else {
1426: /*
1427: * Wiring. We must do this in two passes:
1428: *
1429: * 1. Holding the write lock, we create any shadow
1430: * or zero-fill objects that need to be created.
1431: * Then we clip each map entry to the region to be
1432: * wired and increment its wiring count. We
1433: * create objects before clipping the map entries
1434: * to avoid object proliferation.
1435: *
1436: * 2. We downgrade to a read lock, and call
1437: * vm_fault_wire to fault in the pages for any
1438: * newly wired area (wired_count is 1).
1439: *
1440: * Downgrading to a read lock for vm_fault_wire avoids
1441: * a possible deadlock with another thread that may have
1442: * faulted on one of the pages to be wired (it would mark
1443: * the page busy, blocking us, then in turn block on the
1444: * map lock that we hold). Because of problems in the
1445: * recursive lock package, we cannot upgrade to a write
1446: * lock in vm_map_lookup. Thus, any actions that require
1447: * the write lock must be done beforehand. Because we
1448: * keep the read lock on the map, the copy-on-write
1449: * status of the entries we modify here cannot change.
1450: */
1451:
1452: /*
1453: * Pass 1.
1454: */
1455: while ((entry != vm_map_to_entry(map)) &&
1456: (entry->vme_start < end)) {
1457: vm_map_clip_end(map, entry, end);
1458:
1459: if (entry->wired_count == 0) {
1460:
1461: /*
1462: * Perform actions of vm_map_lookup that need
1463: * the write lock on the map: create a shadow
1464: * object for a copy-on-write region, or an
1465: * object for a zero-fill region.
1466: */
1467: if (entry->needs_copy &&
1468: ((entry->protection & VM_PROT_WRITE) != 0)) {
1469:
1470: vm_object_shadow(&entry->object.vm_object,
1471: &entry->offset,
1472: (vm_size_t)(entry->vme_end
1473: - entry->vme_start));
1474: entry->needs_copy = FALSE;
1475: }
1476: if (entry->object.vm_object == VM_OBJECT_NULL) {
1477: entry->object.vm_object =
1478: vm_object_allocate(
1.1.1.2 root 1479: (vm_size_t)(entry->vme_end
1.1 root 1480: - entry->vme_start));
1481: entry->offset = (vm_offset_t)0;
1482: }
1483: }
1484: vm_map_clip_start(map, entry, start);
1485: vm_map_clip_end(map, entry, end);
1486:
1487: if (user_wire) {
1488: if ((entry->user_wired_count)++ == 0)
1489: entry->wired_count++;
1490: }
1491: else {
1492: entry->wired_count++;
1493: }
1494:
1495: /*
1496: * Check for holes and protection mismatch.
1497: * Holes: Next entry should be contiguous unless
1498: * this is the end of the region.
1499: * Protection: Access requested must be allowed.
1500: */
1.1.1.2 root 1501: if (((entry->vme_end < end) &&
1.1 root 1502: ((entry->vme_next == vm_map_to_entry(map)) ||
1503: (entry->vme_next->vme_start > entry->vme_end))) ||
1504: ((entry->protection & access_type) != access_type)) {
1505: /*
1506: * Found a hole or protection problem.
1507: * Object creation actions
1508: * do not need to be undone, but the
1509: * wired counts need to be restored.
1510: */
1511: while ((entry != vm_map_to_entry(map)) &&
1512: (entry->vme_end > start)) {
1513: if (user_wire) {
1514: if (--(entry->user_wired_count) == 0)
1515: entry->wired_count--;
1516: }
1517: else {
1518: entry->wired_count--;
1519: }
1520:
1521: entry = entry->vme_prev;
1522: }
1523:
1524: vm_map_unlock(map);
1525: return(KERN_FAILURE);
1526: }
1527: entry = entry->vme_next;
1528: }
1529:
1530: /*
1531: * Pass 2.
1532: */
1533:
1534: /*
1535: * HACK HACK HACK HACK
1536: *
1537: * If we are wiring in the kernel map or a submap of it,
1538: * unlock the map to avoid deadlocks. We trust that the
1539: * kernel threads are well-behaved, and therefore will
1540: * not do anything destructive to this region of the map
1541: * while we have it unlocked. We cannot trust user threads
1542: * to do the same.
1543: *
1544: * HACK HACK HACK HACK
1545: */
1546: if (vm_map_pmap(map) == kernel_pmap) {
1547: vm_map_unlock(map); /* trust me ... */
1548: }
1549: else {
1550: vm_map_lock_set_recursive(map);
1551: vm_map_lock_write_to_read(map);
1552: }
1553:
1554: entry = start_entry;
1555: while (entry != vm_map_to_entry(map) &&
1556: entry->vme_start < end) {
1557: /*
1558: * Wiring cases:
1559: * Kernel: wired == 1 && user_wired == 0
1560: * User: wired == 1 && user_wired == 1
1561: *
1562: * Don't need to wire if either is > 1. wired = 0 &&
1563: * user_wired == 1 can't happen.
1564: */
1565:
1566: /*
1567: * XXX This assumes that the faults always succeed.
1568: */
1569: if ((entry->wired_count == 1) &&
1570: (entry->user_wired_count <= 1)) {
1571: vm_fault_wire(map, entry);
1572: }
1573: entry = entry->vme_next;
1574: }
1575:
1576: if (vm_map_pmap(map) == kernel_pmap) {
1577: vm_map_lock(map);
1578: }
1579: else {
1580: vm_map_lock_clear_recursive(map);
1581: }
1582: }
1583:
1584: vm_map_unlock(map);
1585:
1586: return(KERN_SUCCESS);
1587: }
1588:
1589: /*
1590: * vm_map_entry_delete: [ internal use only ]
1591: *
1592: * Deallocate the given entry from the target map.
1.1.1.2 root 1593: */
1.1.1.4 ! root 1594: void vm_map_entry_delete(
! 1595: vm_map_t map,
! 1596: vm_map_entry_t entry)
1.1 root 1597: {
1.1.1.4 ! root 1598: vm_offset_t s, e;
! 1599: vm_object_t object;
1.1 root 1600: extern vm_object_t kernel_object;
1601:
1602: s = entry->vme_start;
1603: e = entry->vme_end;
1604:
1605: /*Check if projected buffer*/
1606: if (map != kernel_map && entry->projected_on != 0) {
1607: /*Check if projected kernel entry is persistent;
1608: may only manipulate directly if it is*/
1609: if (entry->projected_on->projected_on == 0)
1610: entry->wired_count = 0; /*Avoid unwire fault*/
1611: else
1612: return;
1613: }
1614:
1615: /*
1616: * Get the object. Null objects cannot have pmap entries.
1617: */
1618:
1619: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1620:
1621: /*
1622: * Unwire before removing addresses from the pmap;
1623: * otherwise, unwiring will put the entries back in
1624: * the pmap.
1625: */
1626:
1627: if (entry->wired_count != 0) {
1628: vm_fault_unwire(map, entry);
1629: entry->wired_count = 0;
1630: entry->user_wired_count = 0;
1631: }
1632:
1633: /*
1634: * If the object is shared, we must remove
1635: * *all* references to this data, since we can't
1636: * find all of the physical maps which are sharing
1637: * it.
1638: */
1639:
1640: if (object == kernel_object) {
1641: vm_object_lock(object);
1642: vm_object_page_remove(object, entry->offset,
1643: entry->offset + (e - s));
1644: vm_object_unlock(object);
1645: } else if (entry->is_shared) {
1646: vm_object_pmap_remove(object,
1647: entry->offset,
1648: entry->offset + (e - s));
1649: }
1650: else {
1651: pmap_remove(map->pmap, s, e);
1652: }
1653: }
1654:
1655: /*
1656: * Deallocate the object only after removing all
1657: * pmap entries pointing to its pages.
1658: */
1659:
1660: if (entry->is_sub_map)
1661: vm_map_deallocate(entry->object.sub_map);
1662: else
1663: vm_object_deallocate(entry->object.vm_object);
1664:
1665: vm_map_entry_unlink(map, entry);
1666: map->size -= e - s;
1667:
1668: vm_map_entry_dispose(map, entry);
1669: }
1670:
1671: /*
1672: * vm_map_delete: [ internal use only ]
1673: *
1674: * Deallocates the given address range from the target
1675: * map.
1676: */
1677:
1.1.1.4 ! root 1678: kern_return_t vm_map_delete(
! 1679: vm_map_t map,
! 1680: vm_offset_t start,
! 1681: vm_offset_t end)
1.1 root 1682: {
1683: vm_map_entry_t entry;
1684: vm_map_entry_t first_entry;
1685:
1686: /*
1687: * Find the start of the region, and clip it
1688: */
1689:
1690: if (!vm_map_lookup_entry(map, start, &first_entry))
1691: entry = first_entry->vme_next;
1692: else {
1693: entry = first_entry;
1694: vm_map_clip_start(map, entry, start);
1695:
1696: /*
1697: * Fix the lookup hint now, rather than each
1698: * time though the loop.
1699: */
1700:
1701: SAVE_HINT(map, entry->vme_prev);
1702: }
1703:
1704: /*
1705: * Save the free space hint
1706: */
1707:
1708: if (map->first_free->vme_start >= start)
1709: map->first_free = entry->vme_prev;
1710:
1711: /*
1712: * Step through all entries in this region
1713: */
1714:
1715: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1716: vm_map_entry_t next;
1717:
1718: vm_map_clip_end(map, entry, end);
1719:
1720: /*
1721: * If the entry is in transition, we must wait
1722: * for it to exit that state. It could be clipped
1723: * while we leave the map unlocked.
1724: */
1725: if(entry->in_transition) {
1726: /*
1727: * Say that we are waiting, and wait for entry.
1728: */
1729: entry->needs_wakeup = TRUE;
1730: vm_map_entry_wait(map, FALSE);
1731: vm_map_lock(map);
1732:
1733: /*
1734: * The entry could have been clipped or it
1735: * may not exist anymore. look it up again.
1736: */
1737: if(!vm_map_lookup_entry(map, start, &entry)) {
1738: entry = entry->vme_next;
1739: }
1740: continue;
1741: }
1742:
1743: next = entry->vme_next;
1744:
1745: vm_map_entry_delete(map, entry);
1746: entry = next;
1747: }
1748:
1749: if (map->wait_for_space)
1750: thread_wakeup((event_t) map);
1751:
1752: return(KERN_SUCCESS);
1753: }
1754:
1755: /*
1756: * vm_map_remove:
1757: *
1758: * Remove the given address range from the target map.
1759: * This is the exported form of vm_map_delete.
1760: */
1.1.1.4 ! root 1761: kern_return_t vm_map_remove(
! 1762: vm_map_t map,
! 1763: vm_offset_t start,
! 1764: vm_offset_t end)
1.1 root 1765: {
1.1.1.4 ! root 1766: kern_return_t result;
1.1 root 1767:
1768: vm_map_lock(map);
1769: VM_MAP_RANGE_CHECK(map, start, end);
1770: result = vm_map_delete(map, start, end);
1771: vm_map_unlock(map);
1772:
1773: return(result);
1774: }
1775:
1776:
1777: /*
1778: * vm_map_copy_steal_pages:
1779: *
1780: * Steal all the pages from a vm_map_copy page_list by copying ones
1781: * that have not already been stolen.
1782: */
1783: void
1.1.1.4 ! root 1784: vm_map_copy_steal_pages(vm_map_copy_t copy)
1.1 root 1785: {
1.1.1.4 ! root 1786: vm_page_t m, new_m;
! 1787: int i;
! 1788: vm_object_t object;
1.1 root 1789:
1790: for (i = 0; i < copy->cpy_npages; i++) {
1791:
1792: /*
1793: * If the page is not tabled, then it's already stolen.
1794: */
1795: m = copy->cpy_page_list[i];
1796: if (!m->tabled)
1797: continue;
1798:
1799: /*
1800: * Page was not stolen, get a new
1801: * one and do the copy now.
1802: */
1.1.1.2 root 1803: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) {
1.1 root 1804: VM_PAGE_WAIT((void(*)()) 0);
1805: }
1806:
1807: vm_page_copy(m, new_m);
1808:
1809: object = m->object;
1810: vm_object_lock(object);
1811: vm_page_lock_queues();
1812: if (!m->active && !m->inactive)
1813: vm_page_activate(m);
1814: vm_page_unlock_queues();
1815: PAGE_WAKEUP_DONE(m);
1816: vm_object_paging_end(object);
1817: vm_object_unlock(object);
1818:
1819: copy->cpy_page_list[i] = new_m;
1820: }
1821: }
1822:
1823: /*
1824: * vm_map_copy_page_discard:
1825: *
1826: * Get rid of the pages in a page_list copy. If the pages are
1827: * stolen, they are freed. If the pages are not stolen, they
1828: * are unbusied, and associated state is cleaned up.
1829: */
1.1.1.4 ! root 1830: void vm_map_copy_page_discard(vm_map_copy_t copy)
1.1 root 1831: {
1832: while (copy->cpy_npages > 0) {
1833: vm_page_t m;
1834:
1835: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1836: VM_PAGE_NULL) {
1837:
1838: /*
1839: * If it's not in the table, then it's
1840: * a stolen page that goes back
1841: * to the free list. Else it belongs
1842: * to some object, and we hold a
1843: * paging reference on that object.
1844: */
1845: if (!m->tabled) {
1846: VM_PAGE_FREE(m);
1847: }
1848: else {
1849: vm_object_t object;
1850:
1851: object = m->object;
1852:
1853: vm_object_lock(object);
1854: vm_page_lock_queues();
1855: if (!m->active && !m->inactive)
1856: vm_page_activate(m);
1857: vm_page_unlock_queues();
1858:
1859: PAGE_WAKEUP_DONE(m);
1860: vm_object_paging_end(object);
1861: vm_object_unlock(object);
1862: }
1863: }
1864: }
1865: }
1866:
1867: /*
1868: * Routine: vm_map_copy_discard
1869: *
1870: * Description:
1871: * Dispose of a map copy object (returned by
1872: * vm_map_copyin).
1873: */
1874: void
1.1.1.4 ! root 1875: vm_map_copy_discard(vm_map_copy_t copy)
1.1 root 1876: {
1877: free_next_copy:
1878: if (copy == VM_MAP_COPY_NULL)
1879: return;
1880:
1881: switch (copy->type) {
1882: case VM_MAP_COPY_ENTRY_LIST:
1883: while (vm_map_copy_first_entry(copy) !=
1884: vm_map_copy_to_entry(copy)) {
1885: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
1886:
1887: vm_map_copy_entry_unlink(copy, entry);
1888: vm_object_deallocate(entry->object.vm_object);
1889: vm_map_copy_entry_dispose(copy, entry);
1890: }
1891: break;
1892: case VM_MAP_COPY_OBJECT:
1893: vm_object_deallocate(copy->cpy_object);
1894: break;
1895: case VM_MAP_COPY_PAGE_LIST:
1896:
1897: /*
1898: * To clean this up, we have to unbusy all the pages
1899: * and release the paging references in their objects.
1900: */
1901: if (copy->cpy_npages > 0)
1902: vm_map_copy_page_discard(copy);
1903:
1904: /*
1905: * If there's a continuation, abort it. The
1906: * abort routine releases any storage.
1907: */
1908: if (vm_map_copy_has_cont(copy)) {
1909:
1910: /*
1911: * Special case: recognize
1912: * vm_map_copy_discard_cont and optimize
1913: * here to avoid tail recursion.
1914: */
1915: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1.1.1.4 ! root 1916: vm_map_copy_t new_copy;
1.1 root 1917:
1918: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1.1.1.3 root 1919: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1920: copy = new_copy;
1921: goto free_next_copy;
1922: }
1923: else {
1924: vm_map_copy_abort_cont(copy);
1925: }
1926: }
1927:
1928: break;
1929: }
1.1.1.3 root 1930: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1931: }
1932:
1933: /*
1934: * Routine: vm_map_copy_copy
1935: *
1936: * Description:
1937: * Move the information in a map copy object to
1938: * a new map copy object, leaving the old one
1939: * empty.
1940: *
1941: * This is used by kernel routines that need
1942: * to look at out-of-line data (in copyin form)
1943: * before deciding whether to return SUCCESS.
1944: * If the routine returns FAILURE, the original
1945: * copy object will be deallocated; therefore,
1946: * these routines must make a copy of the copy
1947: * object and leave the original empty so that
1948: * deallocation will not fail.
1949: */
1950: vm_map_copy_t
1.1.1.4 ! root 1951: vm_map_copy_copy(vm_map_copy_t copy)
1.1 root 1952: {
1953: vm_map_copy_t new_copy;
1954:
1955: if (copy == VM_MAP_COPY_NULL)
1956: return VM_MAP_COPY_NULL;
1957:
1958: /*
1959: * Allocate a new copy object, and copy the information
1960: * from the old one into it.
1961: */
1962:
1.1.1.3 root 1963: new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 1964: *new_copy = *copy;
1965:
1966: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
1967: /*
1968: * The links in the entry chain must be
1969: * changed to point to the new copy object.
1970: */
1971: vm_map_copy_first_entry(copy)->vme_prev
1972: = vm_map_copy_to_entry(new_copy);
1973: vm_map_copy_last_entry(copy)->vme_next
1974: = vm_map_copy_to_entry(new_copy);
1975: }
1976:
1977: /*
1978: * Change the old copy object into one that contains
1979: * nothing to be deallocated.
1980: */
1981: copy->type = VM_MAP_COPY_OBJECT;
1982: copy->cpy_object = VM_OBJECT_NULL;
1983:
1984: /*
1985: * Return the new object.
1986: */
1987: return new_copy;
1988: }
1989:
1990: /*
1991: * Routine: vm_map_copy_discard_cont
1992: *
1993: * Description:
1994: * A version of vm_map_copy_discard that can be called
1995: * as a continuation from a vm_map_copy page list.
1996: */
1.1.1.4 ! root 1997: kern_return_t vm_map_copy_discard_cont(
! 1998: vm_map_copyin_args_t cont_args,
! 1999: vm_map_copy_t *copy_result) /* OUT */
1.1 root 2000: {
2001: vm_map_copy_discard((vm_map_copy_t) cont_args);
2002: if (copy_result != (vm_map_copy_t *)0)
2003: *copy_result = VM_MAP_COPY_NULL;
2004: return(KERN_SUCCESS);
2005: }
2006:
2007: /*
2008: * Routine: vm_map_copy_overwrite
2009: *
2010: * Description:
2011: * Copy the memory described by the map copy
2012: * object (copy; returned by vm_map_copyin) onto
2013: * the specified destination region (dst_map, dst_addr).
2014: * The destination must be writeable.
2015: *
2016: * Unlike vm_map_copyout, this routine actually
2017: * writes over previously-mapped memory. If the
2018: * previous mapping was to a permanent (user-supplied)
2019: * memory object, it is preserved.
2020: *
2021: * The attributes (protection and inheritance) of the
2022: * destination region are preserved.
2023: *
2024: * If successful, consumes the copy object.
2025: * Otherwise, the caller is responsible for it.
2026: *
2027: * Implementation notes:
2028: * To overwrite temporary virtual memory, it is
2029: * sufficient to remove the previous mapping and insert
2030: * the new copy. This replacement is done either on
2031: * the whole region (if no permanent virtual memory
2032: * objects are embedded in the destination region) or
2033: * in individual map entries.
2034: *
2035: * To overwrite permanent virtual memory, it is
2036: * necessary to copy each page, as the external
2037: * memory management interface currently does not
2038: * provide any optimizations.
2039: *
2040: * Once a page of permanent memory has been overwritten,
2041: * it is impossible to interrupt this function; otherwise,
2042: * the call would be neither atomic nor location-independent.
2043: * The kernel-state portion of a user thread must be
2044: * interruptible.
2045: *
2046: * It may be expensive to forward all requests that might
2047: * overwrite permanent memory (vm_write, vm_copy) to
2048: * uninterruptible kernel threads. This routine may be
2049: * called by interruptible threads; however, success is
2050: * not guaranteed -- if the request cannot be performed
2051: * atomically and interruptibly, an error indication is
2052: * returned.
2053: */
1.1.1.4 ! root 2054: kern_return_t vm_map_copy_overwrite(
! 2055: vm_map_t dst_map,
! 2056: vm_offset_t dst_addr,
! 2057: vm_map_copy_t copy,
! 2058: boolean_t interruptible)
1.1 root 2059: {
2060: vm_size_t size;
2061: vm_offset_t start;
2062: vm_map_entry_t tmp_entry;
2063: vm_map_entry_t entry;
2064:
2065: boolean_t contains_permanent_objects = FALSE;
2066:
2067: interruptible = FALSE; /* XXX */
2068:
2069: /*
2070: * Check for null copy object.
2071: */
2072:
2073: if (copy == VM_MAP_COPY_NULL)
2074: return(KERN_SUCCESS);
2075:
2076: /*
2077: * Only works for entry lists at the moment. Will
2078: * support page lists LATER.
2079: */
2080:
2081: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2082:
2083: /*
2084: * Currently this routine only handles page-aligned
2085: * regions. Eventually, it should handle misalignments
2086: * by actually copying pages.
2087: */
2088:
2089: if (!page_aligned(copy->offset) ||
2090: !page_aligned(copy->size) ||
2091: !page_aligned(dst_addr))
2092: return(KERN_INVALID_ARGUMENT);
2093:
2094: size = copy->size;
2095:
2096: if (size == 0) {
2097: vm_map_copy_discard(copy);
2098: return(KERN_SUCCESS);
2099: }
2100:
2101: /*
2102: * Verify that the destination is all writeable
2103: * initially.
2104: */
2105: start_pass_1:
2106: vm_map_lock(dst_map);
2107: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2108: vm_map_unlock(dst_map);
2109: return(KERN_INVALID_ADDRESS);
2110: }
2111: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2112: for (entry = tmp_entry;;) {
2113: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2114: vm_map_entry_t next = entry->vme_next;
2115:
2116: if ( ! (entry->protection & VM_PROT_WRITE)) {
2117: vm_map_unlock(dst_map);
2118: return(KERN_PROTECTION_FAILURE);
2119: }
2120:
2121: /*
2122: * If the entry is in transition, we must wait
2123: * for it to exit that state. Anything could happen
2124: * when we unlock the map, so start over.
2125: */
2126: if (entry->in_transition) {
2127:
2128: /*
2129: * Say that we are waiting, and wait for entry.
2130: */
2131: entry->needs_wakeup = TRUE;
2132: vm_map_entry_wait(dst_map, FALSE);
2133:
2134: goto start_pass_1;
2135: }
2136:
2137: if (size <= sub_size)
2138: break;
2139:
2140: if ((next == vm_map_to_entry(dst_map)) ||
2141: (next->vme_start != entry->vme_end)) {
2142: vm_map_unlock(dst_map);
2143: return(KERN_INVALID_ADDRESS);
2144: }
2145:
2146:
2147: /*
2148: * Check for permanent objects in the destination.
2149: */
2150:
2151: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2152: !entry->object.vm_object->temporary)
2153: contains_permanent_objects = TRUE;
2154:
2155: size -= sub_size;
2156: entry = next;
2157: }
2158:
2159: /*
2160: * If there are permanent objects in the destination, then
2161: * the copy cannot be interrupted.
2162: */
2163:
1.1.1.3 root 2164: if (interruptible && contains_permanent_objects) {
2165: vm_map_unlock(dst_map);
1.1 root 2166: return(KERN_FAILURE); /* XXX */
1.1.1.3 root 2167: }
1.1 root 2168:
2169: /*
2170: * XXXO If there are no permanent objects in the destination,
1.1.1.3 root 2171: * XXXO and the source and destination map entry caches match,
1.1.1.2 root 2172: * XXXO and the destination map entry is not shared,
1.1 root 2173: * XXXO then the map entries can be deleted and replaced
2174: * XXXO with those from the copy. The following code is the
2175: * XXXO basic idea of what to do, but there are lots of annoying
2176: * XXXO little details about getting protection and inheritance
2177: * XXXO right. Should add protection, inheritance, and sharing checks
2178: * XXXO to the above pass and make sure that no wiring is involved.
2179: */
2180: /*
2181: * if (!contains_permanent_objects &&
2182: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2183: *
2184: * *
2185: * * Run over copy and adjust entries. Steal code
2186: * * from vm_map_copyout() to do this.
2187: * *
2188: *
2189: * tmp_entry = tmp_entry->vme_prev;
2190: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2191: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2192: *
2193: * vm_map_unlock(dst_map);
2194: * vm_map_copy_discard(copy);
2195: * }
2196: */
2197: /*
2198: *
2199: * Make a second pass, overwriting the data
2200: * At the beginning of each loop iteration,
2201: * the next entry to be overwritten is "tmp_entry"
2202: * (initially, the value returned from the lookup above),
2203: * and the starting address expected in that entry
2204: * is "start".
2205: */
2206:
2207: start = dst_addr;
2208:
2209: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2210: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2211: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2212: vm_object_t object;
1.1.1.2 root 2213:
1.1 root 2214: entry = tmp_entry;
2215: size = (entry->vme_end - entry->vme_start);
2216: /*
2217: * Make sure that no holes popped up in the
2218: * address map, and that the protection is
2219: * still valid, in case the map was unlocked
2220: * earlier.
2221: */
2222:
2223: if (entry->vme_start != start) {
2224: vm_map_unlock(dst_map);
2225: return(KERN_INVALID_ADDRESS);
2226: }
2227: assert(entry != vm_map_to_entry(dst_map));
2228:
2229: /*
2230: * Check protection again
2231: */
2232:
2233: if ( ! (entry->protection & VM_PROT_WRITE)) {
2234: vm_map_unlock(dst_map);
2235: return(KERN_PROTECTION_FAILURE);
2236: }
2237:
2238: /*
2239: * Adjust to source size first
2240: */
2241:
2242: if (copy_size < size) {
2243: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2244: size = copy_size;
2245: }
2246:
2247: /*
2248: * Adjust to destination size
2249: */
2250:
2251: if (size < copy_size) {
2252: vm_map_copy_clip_end(copy, copy_entry,
2253: copy_entry->vme_start + size);
2254: copy_size = size;
2255: }
2256:
2257: assert((entry->vme_end - entry->vme_start) == size);
2258: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2259: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2260:
2261: /*
2262: * If the destination contains temporary unshared memory,
2263: * we can perform the copy by throwing it away and
2264: * installing the source data.
2265: */
2266:
2267: object = entry->object.vm_object;
2268: if (!entry->is_shared &&
2269: ((object == VM_OBJECT_NULL) || object->temporary)) {
2270: vm_object_t old_object = entry->object.vm_object;
2271: vm_offset_t old_offset = entry->offset;
2272:
2273: entry->object = copy_entry->object;
2274: entry->offset = copy_entry->offset;
2275: entry->needs_copy = copy_entry->needs_copy;
2276: entry->wired_count = 0;
2277: entry->user_wired_count = 0;
2278:
2279: vm_map_copy_entry_unlink(copy, copy_entry);
2280: vm_map_copy_entry_dispose(copy, copy_entry);
2281:
2282: vm_object_pmap_protect(
2283: old_object,
2284: old_offset,
2285: size,
2286: dst_map->pmap,
2287: tmp_entry->vme_start,
2288: VM_PROT_NONE);
2289:
2290: vm_object_deallocate(old_object);
2291:
2292: /*
2293: * Set up for the next iteration. The map
2294: * has not been unlocked, so the next
2295: * address should be at the end of this
2296: * entry, and the next map entry should be
2297: * the one following it.
2298: */
2299:
2300: start = tmp_entry->vme_end;
2301: tmp_entry = tmp_entry->vme_next;
2302: } else {
2303: vm_map_version_t version;
2304: vm_object_t dst_object = entry->object.vm_object;
2305: vm_offset_t dst_offset = entry->offset;
2306: kern_return_t r;
2307:
2308: /*
2309: * Take an object reference, and record
2310: * the map version information so that the
2311: * map can be safely unlocked.
2312: */
2313:
2314: vm_object_reference(dst_object);
2315:
2316: version.main_timestamp = dst_map->timestamp;
2317:
2318: vm_map_unlock(dst_map);
2319:
2320: /*
2321: * Copy as much as possible in one pass
2322: */
2323:
2324: copy_size = size;
2325: r = vm_fault_copy(
2326: copy_entry->object.vm_object,
2327: copy_entry->offset,
2328: ©_size,
2329: dst_object,
2330: dst_offset,
2331: dst_map,
2332: &version,
2333: FALSE /* XXX interruptible */ );
2334:
2335: /*
2336: * Release the object reference
2337: */
2338:
2339: vm_object_deallocate(dst_object);
2340:
2341: /*
2342: * If a hard error occurred, return it now
2343: */
2344:
2345: if (r != KERN_SUCCESS)
2346: return(r);
2347:
2348: if (copy_size != 0) {
2349: /*
2350: * Dispose of the copied region
2351: */
2352:
2353: vm_map_copy_clip_end(copy, copy_entry,
2354: copy_entry->vme_start + copy_size);
2355: vm_map_copy_entry_unlink(copy, copy_entry);
2356: vm_object_deallocate(copy_entry->object.vm_object);
2357: vm_map_copy_entry_dispose(copy, copy_entry);
2358: }
2359:
2360: /*
2361: * Pick up in the destination map where we left off.
2362: *
2363: * Use the version information to avoid a lookup
2364: * in the normal case.
2365: */
2366:
2367: start += copy_size;
2368: vm_map_lock(dst_map);
2369: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2370: /* We can safely use saved tmp_entry value */
2371:
2372: vm_map_clip_end(dst_map, tmp_entry, start);
2373: tmp_entry = tmp_entry->vme_next;
2374: } else {
2375: /* Must do lookup of tmp_entry */
2376:
2377: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2378: vm_map_unlock(dst_map);
2379: return(KERN_INVALID_ADDRESS);
2380: }
2381: vm_map_clip_start(dst_map, tmp_entry, start);
2382: }
2383: }
2384:
2385: }
2386: vm_map_unlock(dst_map);
2387:
2388: /*
2389: * Throw away the vm_map_copy object
2390: */
2391: vm_map_copy_discard(copy);
2392:
2393: return(KERN_SUCCESS);
2394: }
2395:
2396: /*
2397: * Macro: vm_map_copy_insert
1.1.1.2 root 2398: *
1.1 root 2399: * Description:
2400: * Link a copy chain ("copy") into a map at the
2401: * specified location (after "where").
2402: * Side effects:
2403: * The copy chain is destroyed.
2404: * Warning:
2405: * The arguments are evaluated multiple times.
2406: */
2407: #define vm_map_copy_insert(map, where, copy) \
2408: MACRO_BEGIN \
1.1.1.3 root 2409: struct rbtree_node *node, *tmp; \
2410: rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \
2411: rbtree_insert(&(map)->hdr.tree, node, \
2412: vm_map_entry_cmp_insert); \
1.1 root 2413: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \
2414: ->vme_next = ((where)->vme_next); \
2415: ((where)->vme_next = vm_map_copy_first_entry(copy)) \
2416: ->vme_prev = (where); \
2417: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \
1.1.1.3 root 2418: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
1.1 root 2419: MACRO_END
2420:
2421: /*
2422: * Routine: vm_map_copyout
2423: *
2424: * Description:
2425: * Copy out a copy chain ("copy") into newly-allocated
2426: * space in the destination map.
2427: *
2428: * If successful, consumes the copy object.
2429: * Otherwise, the caller is responsible for it.
2430: */
1.1.1.4 ! root 2431: kern_return_t vm_map_copyout(
! 2432: vm_map_t dst_map,
! 2433: vm_offset_t *dst_addr, /* OUT */
! 2434: vm_map_copy_t copy)
1.1 root 2435: {
2436: vm_size_t size;
2437: vm_size_t adjustment;
2438: vm_offset_t start;
2439: vm_offset_t vm_copy_start;
2440: vm_map_entry_t last;
2441: vm_map_entry_t entry;
2442:
2443: /*
2444: * Check for null copy object.
2445: */
2446:
2447: if (copy == VM_MAP_COPY_NULL) {
2448: *dst_addr = 0;
2449: return(KERN_SUCCESS);
2450: }
2451:
2452: /*
2453: * Check for special copy object, created
2454: * by vm_map_copyin_object.
2455: */
2456:
2457: if (copy->type == VM_MAP_COPY_OBJECT) {
2458: vm_object_t object = copy->cpy_object;
2459: vm_size_t offset = copy->offset;
2460: vm_size_t tmp_size = copy->size;
2461: kern_return_t kr;
2462:
2463: *dst_addr = 0;
2464: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2465: (vm_offset_t) 0, TRUE,
2466: object, offset, FALSE,
2467: VM_PROT_DEFAULT, VM_PROT_ALL,
2468: VM_INHERIT_DEFAULT);
2469: if (kr != KERN_SUCCESS)
2470: return(kr);
1.1.1.3 root 2471: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2472: return(KERN_SUCCESS);
2473: }
2474:
2475: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2476: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2477:
2478: /*
2479: * Find space for the data
2480: */
2481:
2482: vm_copy_start = trunc_page(copy->offset);
2483: size = round_page(copy->offset + copy->size) - vm_copy_start;
2484:
2485: StartAgain: ;
2486:
2487: vm_map_lock(dst_map);
2488: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
2489: vm_map_min(dst_map) : last->vme_end;
2490:
2491: while (TRUE) {
2492: vm_map_entry_t next = last->vme_next;
2493: vm_offset_t end = start + size;
2494:
2495: if ((end > dst_map->max_offset) || (end < start)) {
2496: if (dst_map->wait_for_space) {
2497: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
2498: assert_wait((event_t) dst_map, TRUE);
2499: vm_map_unlock(dst_map);
2500: thread_block((void (*)()) 0);
2501: goto StartAgain;
2502: }
2503: }
2504: vm_map_unlock(dst_map);
1.1.1.3 root 2505: printf_once("no more room for vm_map_copyout in %p\n", dst_map);
1.1 root 2506: return(KERN_NO_SPACE);
2507: }
2508:
2509: if ((next == vm_map_to_entry(dst_map)) ||
2510: (next->vme_start >= end))
2511: break;
2512:
2513: last = next;
2514: start = last->vme_end;
2515: }
2516:
2517: /*
2518: * Since we're going to just drop the map
2519: * entries from the copy into the destination
2520: * map, they must come from the same pool.
2521: */
2522:
2523: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2524: /*
2525: * Mismatches occur when dealing with the default
2526: * pager.
2527: */
1.1.1.3 root 2528: kmem_cache_t old_cache;
1.1 root 2529: vm_map_entry_t next, new;
2530:
2531: /*
1.1.1.3 root 2532: * Find the cache that the copies were allocated from
1.1 root 2533: */
1.1.1.3 root 2534: old_cache = (copy->cpy_hdr.entries_pageable)
2535: ? &vm_map_entry_cache
2536: : &vm_map_kentry_cache;
1.1 root 2537: entry = vm_map_copy_first_entry(copy);
2538:
2539: /*
2540: * Reinitialize the copy so that vm_map_copy_entry_link
2541: * will work.
2542: */
2543: copy->cpy_hdr.nentries = 0;
2544: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
2545: vm_map_copy_first_entry(copy) =
2546: vm_map_copy_last_entry(copy) =
2547: vm_map_copy_to_entry(copy);
2548:
2549: /*
2550: * Copy each entry.
2551: */
2552: while (entry != vm_map_copy_to_entry(copy)) {
2553: new = vm_map_copy_entry_create(copy);
2554: vm_map_entry_copy_full(new, entry);
2555: vm_map_copy_entry_link(copy,
2556: vm_map_copy_last_entry(copy),
2557: new);
2558: next = entry->vme_next;
1.1.1.3 root 2559: kmem_cache_free(old_cache, (vm_offset_t) entry);
1.1 root 2560: entry = next;
2561: }
2562: }
2563:
2564: /*
2565: * Adjust the addresses in the copy chain, and
2566: * reset the region attributes.
2567: */
2568:
2569: adjustment = start - vm_copy_start;
2570: for (entry = vm_map_copy_first_entry(copy);
2571: entry != vm_map_copy_to_entry(copy);
2572: entry = entry->vme_next) {
2573: entry->vme_start += adjustment;
2574: entry->vme_end += adjustment;
2575:
2576: entry->inheritance = VM_INHERIT_DEFAULT;
2577: entry->protection = VM_PROT_DEFAULT;
2578: entry->max_protection = VM_PROT_ALL;
2579: entry->projected_on = 0;
2580:
2581: /*
2582: * If the entry is now wired,
2583: * map the pages into the destination map.
2584: */
2585: if (entry->wired_count != 0) {
1.1.1.4 ! root 2586: vm_offset_t va;
! 2587: vm_offset_t offset;
! 2588: vm_object_t object;
1.1 root 2589:
2590: object = entry->object.vm_object;
2591: offset = entry->offset;
2592: va = entry->vme_start;
2593:
2594: pmap_pageable(dst_map->pmap,
2595: entry->vme_start,
2596: entry->vme_end,
2597: TRUE);
2598:
2599: while (va < entry->vme_end) {
1.1.1.4 ! root 2600: vm_page_t m;
1.1 root 2601:
2602: /*
2603: * Look up the page in the object.
2604: * Assert that the page will be found in the
2605: * top object:
2606: * either
2607: * the object was newly created by
2608: * vm_object_copy_slowly, and has
2609: * copies of all of the pages from
2610: * the source object
2611: * or
2612: * the object was moved from the old
2613: * map entry; because the old map
2614: * entry was wired, all of the pages
2615: * were in the top-level object.
2616: * (XXX not true if we wire pages for
2617: * reading)
2618: */
2619: vm_object_lock(object);
2620: vm_object_paging_begin(object);
2621:
2622: m = vm_page_lookup(object, offset);
2623: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2624: m->absent)
2625: panic("vm_map_copyout: wiring 0x%x", m);
2626:
2627: m->busy = TRUE;
2628: vm_object_unlock(object);
2629:
2630: PMAP_ENTER(dst_map->pmap, va, m,
2631: entry->protection, TRUE);
2632:
2633: vm_object_lock(object);
2634: PAGE_WAKEUP_DONE(m);
2635: /* the page is wired, so we don't have to activate */
2636: vm_object_paging_end(object);
2637: vm_object_unlock(object);
2638:
2639: offset += PAGE_SIZE;
2640: va += PAGE_SIZE;
2641: }
2642: }
2643:
2644:
2645: }
2646:
2647: /*
2648: * Correct the page alignment for the result
2649: */
2650:
2651: *dst_addr = start + (copy->offset - vm_copy_start);
2652:
2653: /*
2654: * Update the hints and the map size
2655: */
2656:
2657: if (dst_map->first_free == last)
2658: dst_map->first_free = vm_map_copy_last_entry(copy);
2659: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2660:
2661: dst_map->size += size;
2662:
2663: /*
2664: * Link in the copy
2665: */
2666:
2667: vm_map_copy_insert(dst_map, last, copy);
2668:
2669: vm_map_unlock(dst_map);
2670:
2671: /*
2672: * XXX If wiring_required, call vm_map_pageable
2673: */
2674:
2675: return(KERN_SUCCESS);
2676: }
2677:
2678: /*
2679: *
2680: * vm_map_copyout_page_list:
2681: *
2682: * Version of vm_map_copyout() for page list vm map copies.
2683: *
2684: */
1.1.1.4 ! root 2685: kern_return_t vm_map_copyout_page_list(
! 2686: vm_map_t dst_map,
! 2687: vm_offset_t *dst_addr, /* OUT */
! 2688: vm_map_copy_t copy)
1.1 root 2689: {
2690: vm_size_t size;
2691: vm_offset_t start;
2692: vm_offset_t end;
2693: vm_offset_t offset;
2694: vm_map_entry_t last;
2695: vm_object_t object;
2696: vm_page_t *page_list, m;
2697: vm_map_entry_t entry;
2698: vm_offset_t old_last_offset;
2699: boolean_t cont_invoked, needs_wakeup = FALSE;
2700: kern_return_t result = KERN_SUCCESS;
2701: vm_map_copy_t orig_copy;
2702: vm_offset_t dst_offset;
2703: boolean_t must_wire;
2704:
2705: /*
2706: * Make sure the pages are stolen, because we are
2707: * going to put them in a new object. Assume that
2708: * all pages are identical to first in this regard.
2709: */
2710:
2711: page_list = ©->cpy_page_list[0];
2712: if ((*page_list)->tabled)
2713: vm_map_copy_steal_pages(copy);
2714:
2715: /*
2716: * Find space for the data
2717: */
2718:
2719: size = round_page(copy->offset + copy->size) -
2720: trunc_page(copy->offset);
2721: StartAgain:
2722: vm_map_lock(dst_map);
2723: must_wire = dst_map->wiring_required;
2724:
2725: last = dst_map->first_free;
2726: if (last == vm_map_to_entry(dst_map)) {
2727: start = vm_map_min(dst_map);
2728: } else {
2729: start = last->vme_end;
2730: }
2731:
2732: while (TRUE) {
2733: vm_map_entry_t next = last->vme_next;
2734: end = start + size;
2735:
2736: if ((end > dst_map->max_offset) || (end < start)) {
2737: if (dst_map->wait_for_space) {
2738: if (size <= (dst_map->max_offset -
2739: dst_map->min_offset)) {
2740: assert_wait((event_t) dst_map, TRUE);
2741: vm_map_unlock(dst_map);
2742: thread_block((void (*)()) 0);
2743: goto StartAgain;
2744: }
2745: }
2746: vm_map_unlock(dst_map);
1.1.1.3 root 2747: printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map);
1.1 root 2748: return(KERN_NO_SPACE);
2749: }
2750:
2751: if ((next == vm_map_to_entry(dst_map)) ||
2752: (next->vme_start >= end)) {
2753: break;
2754: }
2755:
2756: last = next;
2757: start = last->vme_end;
2758: }
2759:
2760: /*
2761: * See whether we can avoid creating a new entry (and object) by
2762: * extending one of our neighbors. [So far, we only attempt to
2763: * extend from below.]
2764: *
2765: * The code path below here is a bit twisted. If any of the
2766: * extension checks fails, we branch to create_object. If
2767: * it all works, we fall out the bottom and goto insert_pages.
2768: */
2769: if (last == vm_map_to_entry(dst_map) ||
2770: last->vme_end != start ||
2771: last->is_shared != FALSE ||
2772: last->is_sub_map != FALSE ||
2773: last->inheritance != VM_INHERIT_DEFAULT ||
2774: last->protection != VM_PROT_DEFAULT ||
2775: last->max_protection != VM_PROT_ALL ||
2776: (must_wire ? (last->wired_count != 1 ||
2777: last->user_wired_count != 1) :
2778: (last->wired_count != 0))) {
2779: goto create_object;
2780: }
1.1.1.2 root 2781:
1.1 root 2782: /*
2783: * If this entry needs an object, make one.
2784: */
2785: if (last->object.vm_object == VM_OBJECT_NULL) {
2786: object = vm_object_allocate(
2787: (vm_size_t)(last->vme_end - last->vme_start + size));
2788: last->object.vm_object = object;
2789: last->offset = 0;
2790: vm_object_lock(object);
2791: }
2792: else {
2793: vm_offset_t prev_offset = last->offset;
2794: vm_size_t prev_size = start - last->vme_start;
2795: vm_size_t new_size;
2796:
2797: /*
2798: * This is basically vm_object_coalesce.
2799: */
2800:
2801: object = last->object.vm_object;
2802: vm_object_lock(object);
2803:
2804: /*
2805: * Try to collapse the object first
2806: */
2807: vm_object_collapse(object);
2808:
2809: /*
2810: * Can't coalesce if pages not mapped to
2811: * last may be in use anyway:
2812: * . more than one reference
2813: * . paged out
2814: * . shadows another object
2815: * . has a copy elsewhere
2816: * . paging references (pages might be in page-list)
2817: */
2818:
2819: if ((object->ref_count > 1) ||
2820: object->pager_created ||
2821: (object->shadow != VM_OBJECT_NULL) ||
2822: (object->copy != VM_OBJECT_NULL) ||
2823: (object->paging_in_progress != 0)) {
2824: vm_object_unlock(object);
2825: goto create_object;
2826: }
2827:
2828: /*
2829: * Extend the object if necessary. Don't have to call
2830: * vm_object_page_remove because the pages aren't mapped,
2831: * and vm_page_replace will free up any old ones it encounters.
2832: */
2833: new_size = prev_offset + prev_size + size;
2834: if (new_size > object->size)
2835: object->size = new_size;
2836: }
2837:
2838: /*
2839: * Coalesced the two objects - can extend
2840: * the previous map entry to include the
2841: * new range.
2842: */
2843: dst_map->size += size;
2844: last->vme_end = end;
2845:
2846: SAVE_HINT(dst_map, last);
2847:
2848: goto insert_pages;
2849:
2850: create_object:
2851:
2852: /*
2853: * Create object
2854: */
2855: object = vm_object_allocate(size);
2856:
2857: /*
2858: * Create entry
2859: */
2860:
2861: entry = vm_map_entry_create(dst_map);
2862:
2863: entry->object.vm_object = object;
2864: entry->offset = 0;
2865:
2866: entry->is_shared = FALSE;
2867: entry->is_sub_map = FALSE;
2868: entry->needs_copy = FALSE;
2869:
2870: if (must_wire) {
2871: entry->wired_count = 1;
2872: entry->user_wired_count = 1;
2873: } else {
2874: entry->wired_count = 0;
2875: entry->user_wired_count = 0;
2876: }
2877:
2878: entry->in_transition = TRUE;
2879: entry->needs_wakeup = FALSE;
2880:
2881: entry->vme_start = start;
2882: entry->vme_end = start + size;
1.1.1.2 root 2883:
1.1 root 2884: entry->inheritance = VM_INHERIT_DEFAULT;
2885: entry->protection = VM_PROT_DEFAULT;
2886: entry->max_protection = VM_PROT_ALL;
2887: entry->projected_on = 0;
2888:
2889: vm_object_lock(object);
2890:
2891: /*
2892: * Update the hints and the map size
2893: */
2894: if (dst_map->first_free == last) {
2895: dst_map->first_free = entry;
2896: }
2897: SAVE_HINT(dst_map, entry);
2898: dst_map->size += size;
2899:
2900: /*
2901: * Link in the entry
2902: */
2903: vm_map_entry_link(dst_map, last, entry);
2904: last = entry;
2905:
2906: /*
1.1.1.2 root 2907: * Transfer pages into new object.
1.1 root 2908: * Scan page list in vm_map_copy.
2909: */
2910: insert_pages:
2911: dst_offset = copy->offset & PAGE_MASK;
2912: cont_invoked = FALSE;
2913: orig_copy = copy;
2914: last->in_transition = TRUE;
2915: old_last_offset = last->offset
2916: + (start - last->vme_start);
2917:
2918: vm_page_lock_queues();
2919:
2920: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2921: m = *page_list;
2922: assert(m && !m->tabled);
2923:
2924: /*
2925: * Must clear busy bit in page before inserting it.
2926: * Ok to skip wakeup logic because nobody else
2927: * can possibly know about this page.
2928: * The page is dirty in its new object.
2929: */
2930:
2931: assert(!m->wanted);
2932:
2933: m->busy = FALSE;
2934: m->dirty = TRUE;
2935: vm_page_replace(m, object, old_last_offset + offset);
2936: if (must_wire) {
2937: vm_page_wire(m);
2938: PMAP_ENTER(dst_map->pmap,
2939: last->vme_start + m->offset - last->offset,
2940: m, last->protection, TRUE);
2941: } else {
2942: vm_page_activate(m);
2943: }
2944:
2945: *page_list++ = VM_PAGE_NULL;
2946: if (--(copy->cpy_npages) == 0 &&
2947: vm_map_copy_has_cont(copy)) {
2948: vm_map_copy_t new_copy;
2949:
2950: /*
2951: * Ok to unlock map because entry is
2952: * marked in_transition.
2953: */
2954: cont_invoked = TRUE;
2955: vm_page_unlock_queues();
2956: vm_object_unlock(object);
2957: vm_map_unlock(dst_map);
2958: vm_map_copy_invoke_cont(copy, &new_copy, &result);
2959:
2960: if (result == KERN_SUCCESS) {
2961:
2962: /*
2963: * If we got back a copy with real pages,
2964: * steal them now. Either all of the
2965: * pages in the list are tabled or none
2966: * of them are; mixtures are not possible.
2967: *
2968: * Save original copy for consume on
2969: * success logic at end of routine.
2970: */
2971: if (copy != orig_copy)
2972: vm_map_copy_discard(copy);
2973:
2974: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
2975: page_list = ©->cpy_page_list[0];
2976: if ((*page_list)->tabled)
2977: vm_map_copy_steal_pages(copy);
2978: }
2979: }
2980: else {
2981: /*
2982: * Continuation failed.
2983: */
2984: vm_map_lock(dst_map);
2985: goto error;
2986: }
2987:
2988: vm_map_lock(dst_map);
2989: vm_object_lock(object);
2990: vm_page_lock_queues();
2991: }
2992: }
2993:
2994: vm_page_unlock_queues();
2995: vm_object_unlock(object);
2996:
2997: *dst_addr = start + dst_offset;
1.1.1.2 root 2998:
1.1 root 2999: /*
3000: * Clear the in transition bits. This is easy if we
3001: * didn't have a continuation.
3002: */
3003: error:
3004: if (!cont_invoked) {
3005: /*
3006: * We didn't unlock the map, so nobody could
3007: * be waiting.
3008: */
3009: last->in_transition = FALSE;
3010: assert(!last->needs_wakeup);
3011: needs_wakeup = FALSE;
3012: }
3013: else {
3014: if (!vm_map_lookup_entry(dst_map, start, &entry))
3015: panic("vm_map_copyout_page_list: missing entry");
3016:
3017: /*
3018: * Clear transition bit for all constituent entries that
3019: * were in the original entry. Also check for waiters.
3020: */
3021: while((entry != vm_map_to_entry(dst_map)) &&
3022: (entry->vme_start < end)) {
3023: assert(entry->in_transition);
3024: entry->in_transition = FALSE;
3025: if(entry->needs_wakeup) {
3026: entry->needs_wakeup = FALSE;
3027: needs_wakeup = TRUE;
3028: }
3029: entry = entry->vme_next;
3030: }
3031: }
1.1.1.2 root 3032:
1.1 root 3033: if (result != KERN_SUCCESS)
3034: vm_map_delete(dst_map, start, end);
3035:
3036: vm_map_unlock(dst_map);
3037:
3038: if (needs_wakeup)
3039: vm_map_entry_wakeup(dst_map);
3040:
3041: /*
3042: * Consume on success logic.
3043: */
3044: if (copy != orig_copy) {
1.1.1.3 root 3045: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 3046: }
3047: if (result == KERN_SUCCESS) {
1.1.1.3 root 3048: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
1.1 root 3049: }
1.1.1.2 root 3050:
1.1 root 3051: return(result);
3052: }
3053:
3054: /*
3055: * Routine: vm_map_copyin
3056: *
3057: * Description:
3058: * Copy the specified region (src_addr, len) from the
3059: * source address space (src_map), possibly removing
3060: * the region from the source address space (src_destroy).
3061: *
3062: * Returns:
3063: * A vm_map_copy_t object (copy_result), suitable for
3064: * insertion into another address space (using vm_map_copyout),
3065: * copying over another address space region (using
3066: * vm_map_copy_overwrite). If the copy is unused, it
3067: * should be destroyed (using vm_map_copy_discard).
3068: *
3069: * In/out conditions:
3070: * The source map should not be locked on entry.
3071: */
1.1.1.4 ! root 3072: kern_return_t vm_map_copyin(
! 3073: vm_map_t src_map,
! 3074: vm_offset_t src_addr,
! 3075: vm_size_t len,
! 3076: boolean_t src_destroy,
! 3077: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3078: {
3079: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3080: * in multi-level lookup, this
3081: * entry contains the actual
3082: * vm_object/offset.
3083: */
3084:
3085: vm_offset_t src_start; /* Start of current entry --
3086: * where copy is taking place now
3087: */
3088: vm_offset_t src_end; /* End of entire region to be
3089: * copied */
3090:
3091: vm_map_copy_t copy; /* Resulting copy */
3092:
3093: /*
3094: * Check for copies of zero bytes.
3095: */
3096:
3097: if (len == 0) {
3098: *copy_result = VM_MAP_COPY_NULL;
3099: return(KERN_SUCCESS);
3100: }
3101:
3102: /*
3103: * Compute start and end of region
3104: */
3105:
3106: src_start = trunc_page(src_addr);
3107: src_end = round_page(src_addr + len);
3108:
3109: /*
3110: * Check that the end address doesn't overflow
3111: */
3112:
3113: if (src_end <= src_start)
3114: if ((src_end < src_start) || (src_start != 0))
3115: return(KERN_INVALID_ADDRESS);
3116:
3117: /*
3118: * Allocate a header element for the list.
3119: *
1.1.1.2 root 3120: * Use the start and end in the header to
1.1 root 3121: * remember the endpoints prior to rounding.
3122: */
3123:
1.1.1.3 root 3124: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3125: vm_map_copy_first_entry(copy) =
3126: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3127: copy->type = VM_MAP_COPY_ENTRY_LIST;
3128: copy->cpy_hdr.nentries = 0;
3129: copy->cpy_hdr.entries_pageable = TRUE;
1.1.1.3 root 3130: rbtree_init(©->cpy_hdr.tree);
1.1 root 3131:
3132: copy->offset = src_addr;
3133: copy->size = len;
1.1.1.2 root 3134:
1.1 root 3135: #define RETURN(x) \
3136: MACRO_BEGIN \
3137: vm_map_unlock(src_map); \
3138: vm_map_copy_discard(copy); \
3139: MACRO_RETURN(x); \
3140: MACRO_END
3141:
3142: /*
3143: * Find the beginning of the region.
3144: */
3145:
3146: vm_map_lock(src_map);
3147:
3148: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3149: RETURN(KERN_INVALID_ADDRESS);
3150: vm_map_clip_start(src_map, tmp_entry, src_start);
3151:
3152: /*
3153: * Go through entries until we get to the end.
3154: */
3155:
3156: while (TRUE) {
3157: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3158: vm_size_t src_size; /* Size of source
3159: * map entry (in both
3160: * maps)
3161: */
3162:
3163: vm_object_t src_object; /* Object to copy */
3164: vm_offset_t src_offset;
3165:
3166: boolean_t src_needs_copy; /* Should source map
3167: * be made read-only
3168: * for copy-on-write?
3169: */
3170:
3171: vm_map_entry_t new_entry; /* Map entry for copy */
3172: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3173:
3174: boolean_t was_wired; /* Was source wired? */
3175: vm_map_version_t version; /* Version before locks
3176: * dropped to make copy
3177: */
3178:
3179: /*
3180: * Verify that the region can be read.
3181: */
3182:
3183: if (! (src_entry->protection & VM_PROT_READ))
3184: RETURN(KERN_PROTECTION_FAILURE);
3185:
3186: /*
3187: * Clip against the endpoints of the entire region.
3188: */
3189:
3190: vm_map_clip_end(src_map, src_entry, src_end);
3191:
3192: src_size = src_entry->vme_end - src_start;
3193: src_object = src_entry->object.vm_object;
3194: src_offset = src_entry->offset;
3195: was_wired = (src_entry->wired_count != 0);
3196:
3197: /*
3198: * Create a new address map entry to
3199: * hold the result. Fill in the fields from
3200: * the appropriate source entries.
3201: */
3202:
3203: new_entry = vm_map_copy_entry_create(copy);
3204: vm_map_entry_copy(new_entry, src_entry);
3205:
3206: /*
3207: * Attempt non-blocking copy-on-write optimizations.
3208: */
3209:
3210: if (src_destroy &&
3211: (src_object == VM_OBJECT_NULL ||
3212: (src_object->temporary && !src_object->use_shared_copy)))
3213: {
3214: /*
3215: * If we are destroying the source, and the object
3216: * is temporary, and not shared writable,
3217: * we can move the object reference
3218: * from the source to the copy. The copy is
3219: * copy-on-write only if the source is.
3220: * We make another reference to the object, because
3221: * destroying the source entry will deallocate it.
3222: */
3223: vm_object_reference(src_object);
3224:
3225: /*
3226: * Copy is always unwired. vm_map_copy_entry
3227: * set its wired count to zero.
3228: */
3229:
3230: goto CopySuccessful;
3231: }
3232:
3233: if (!was_wired &&
3234: vm_object_copy_temporary(
3235: &new_entry->object.vm_object,
3236: &new_entry->offset,
3237: &src_needs_copy,
3238: &new_entry_needs_copy)) {
3239:
3240: new_entry->needs_copy = new_entry_needs_copy;
3241:
3242: /*
3243: * Handle copy-on-write obligations
3244: */
3245:
3246: if (src_needs_copy && !tmp_entry->needs_copy) {
3247: vm_object_pmap_protect(
3248: src_object,
3249: src_offset,
3250: src_size,
3251: (src_entry->is_shared ? PMAP_NULL
3252: : src_map->pmap),
3253: src_entry->vme_start,
3254: src_entry->protection &
3255: ~VM_PROT_WRITE);
3256:
3257: tmp_entry->needs_copy = TRUE;
3258: }
3259:
3260: /*
3261: * The map has never been unlocked, so it's safe to
3262: * move to the next entry rather than doing another
3263: * lookup.
3264: */
3265:
3266: goto CopySuccessful;
3267: }
3268:
3269: new_entry->needs_copy = FALSE;
3270:
3271: /*
3272: * Take an object reference, so that we may
3273: * release the map lock(s).
3274: */
3275:
3276: assert(src_object != VM_OBJECT_NULL);
3277: vm_object_reference(src_object);
3278:
3279: /*
3280: * Record the timestamp for later verification.
3281: * Unlock the map.
3282: */
3283:
3284: version.main_timestamp = src_map->timestamp;
3285: vm_map_unlock(src_map);
3286:
3287: /*
3288: * Perform the copy
3289: */
3290:
3291: if (was_wired) {
3292: vm_object_lock(src_object);
3293: (void) vm_object_copy_slowly(
3294: src_object,
3295: src_offset,
3296: src_size,
3297: FALSE,
3298: &new_entry->object.vm_object);
3299: new_entry->offset = 0;
3300: new_entry->needs_copy = FALSE;
3301: } else {
3302: kern_return_t result;
3303:
3304: result = vm_object_copy_strategically(src_object,
3305: src_offset,
3306: src_size,
3307: &new_entry->object.vm_object,
3308: &new_entry->offset,
3309: &new_entry_needs_copy);
3310:
3311: new_entry->needs_copy = new_entry_needs_copy;
1.1.1.2 root 3312:
1.1 root 3313:
3314: if (result != KERN_SUCCESS) {
3315: vm_map_copy_entry_dispose(copy, new_entry);
3316:
3317: vm_map_lock(src_map);
3318: RETURN(result);
3319: }
3320:
3321: }
3322:
3323: /*
3324: * Throw away the extra reference
3325: */
3326:
3327: vm_object_deallocate(src_object);
3328:
3329: /*
3330: * Verify that the map has not substantially
3331: * changed while the copy was being made.
3332: */
3333:
3334: vm_map_lock(src_map); /* Increments timestamp once! */
3335:
3336: if ((version.main_timestamp + 1) == src_map->timestamp)
3337: goto CopySuccessful;
3338:
3339: /*
3340: * Simple version comparison failed.
3341: *
3342: * Retry the lookup and verify that the
3343: * same object/offset are still present.
3344: *
3345: * [Note: a memory manager that colludes with
3346: * the calling task can detect that we have
3347: * cheated. While the map was unlocked, the
3348: * mapping could have been changed and restored.]
3349: */
3350:
3351: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3352: vm_map_copy_entry_dispose(copy, new_entry);
3353: RETURN(KERN_INVALID_ADDRESS);
3354: }
3355:
3356: src_entry = tmp_entry;
3357: vm_map_clip_start(src_map, src_entry, src_start);
3358:
3359: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3360: goto VerificationFailed;
3361:
3362: if (src_entry->vme_end < new_entry->vme_end)
3363: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3364:
3365: if ((src_entry->object.vm_object != src_object) ||
3366: (src_entry->offset != src_offset) ) {
3367:
3368: /*
3369: * Verification failed.
3370: *
3371: * Start over with this top-level entry.
3372: */
3373:
3374: VerificationFailed: ;
3375:
3376: vm_object_deallocate(new_entry->object.vm_object);
3377: vm_map_copy_entry_dispose(copy, new_entry);
3378: tmp_entry = src_entry;
3379: continue;
3380: }
3381:
3382: /*
3383: * Verification succeeded.
3384: */
3385:
3386: CopySuccessful: ;
3387:
3388: /*
3389: * Link in the new copy entry.
3390: */
3391:
3392: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3393: new_entry);
1.1.1.2 root 3394:
1.1 root 3395: /*
3396: * Determine whether the entire region
3397: * has been copied.
3398: */
3399: src_start = new_entry->vme_end;
3400: if ((src_start >= src_end) && (src_end != 0))
3401: break;
3402:
3403: /*
3404: * Verify that there are no gaps in the region
3405: */
3406:
3407: tmp_entry = src_entry->vme_next;
3408: if (tmp_entry->vme_start != src_start)
3409: RETURN(KERN_INVALID_ADDRESS);
3410: }
3411:
3412: /*
3413: * If the source should be destroyed, do it now, since the
1.1.1.2 root 3414: * copy was successful.
1.1 root 3415: */
3416: if (src_destroy)
3417: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3418:
3419: vm_map_unlock(src_map);
3420:
3421: *copy_result = copy;
3422: return(KERN_SUCCESS);
3423:
3424: #undef RETURN
3425: }
3426:
3427: /*
3428: * vm_map_copyin_object:
3429: *
3430: * Create a copy object from an object.
3431: * Our caller donates an object reference.
3432: */
3433:
1.1.1.4 ! root 3434: kern_return_t vm_map_copyin_object(
! 3435: vm_object_t object,
! 3436: vm_offset_t offset, /* offset of region in object */
! 3437: vm_size_t size, /* size of region in object */
! 3438: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3439: {
3440: vm_map_copy_t copy; /* Resulting copy */
3441:
3442: /*
3443: * We drop the object into a special copy object
3444: * that contains the object directly. These copy objects
3445: * are distinguished by entries_pageable == FALSE
3446: * and null links.
3447: */
3448:
1.1.1.3 root 3449: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3450: vm_map_copy_first_entry(copy) =
3451: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3452: copy->type = VM_MAP_COPY_OBJECT;
3453: copy->cpy_object = object;
3454: copy->offset = offset;
3455: copy->size = size;
3456:
3457: *copy_result = copy;
3458: return(KERN_SUCCESS);
3459: }
3460:
3461: /*
3462: * vm_map_copyin_page_list_cont:
3463: *
3464: * Continuation routine for vm_map_copyin_page_list.
1.1.1.2 root 3465: *
1.1 root 3466: * If vm_map_copyin_page_list can't fit the entire vm range
3467: * into a single page list object, it creates a continuation.
3468: * When the target of the operation has used the pages in the
3469: * initial page list, it invokes the continuation, which calls
3470: * this routine. If an error happens, the continuation is aborted
3471: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3472: * pages are discarded from the initial page list before invoking
3473: * the continuation.
3474: *
3475: * NOTE: This is not the same sort of continuation used by
3476: * the scheduler.
3477: */
3478:
1.1.1.4 ! root 3479: kern_return_t vm_map_copyin_page_list_cont(
! 3480: vm_map_copyin_args_t cont_args,
! 3481: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3482: {
3483: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
1.1.1.4 ! root 3484: boolean_t do_abort, src_destroy, src_destroy_only;
1.1 root 3485:
3486: /*
3487: * Check for cases that only require memory destruction.
3488: */
3489: do_abort = (copy_result == (vm_map_copy_t *) 0);
3490: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3491: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3492:
3493: if (do_abort || src_destroy_only) {
1.1.1.2 root 3494: if (src_destroy)
1.1 root 3495: result = vm_map_remove(cont_args->map,
3496: cont_args->destroy_addr,
3497: cont_args->destroy_addr + cont_args->destroy_len);
3498: if (!do_abort)
3499: *copy_result = VM_MAP_COPY_NULL;
3500: }
3501: else {
3502: result = vm_map_copyin_page_list(cont_args->map,
3503: cont_args->src_addr, cont_args->src_len, src_destroy,
3504: cont_args->steal_pages, copy_result, TRUE);
3505:
3506: if (src_destroy && !cont_args->steal_pages &&
3507: vm_map_copy_has_cont(*copy_result)) {
3508: vm_map_copyin_args_t new_args;
3509: /*
3510: * Transfer old destroy info.
3511: */
3512: new_args = (vm_map_copyin_args_t)
3513: (*copy_result)->cpy_cont_args;
3514: new_args->destroy_addr = cont_args->destroy_addr;
3515: new_args->destroy_len = cont_args->destroy_len;
3516: }
3517: }
1.1.1.2 root 3518:
1.1 root 3519: vm_map_deallocate(cont_args->map);
3520: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3521:
3522: return(result);
3523: }
3524:
3525: /*
3526: * vm_map_copyin_page_list:
3527: *
3528: * This is a variant of vm_map_copyin that copies in a list of pages.
3529: * If steal_pages is TRUE, the pages are only in the returned list.
3530: * If steal_pages is FALSE, the pages are busy and still in their
3531: * objects. A continuation may be returned if not all the pages fit:
3532: * the recipient of this copy_result must be prepared to deal with it.
3533: */
3534:
1.1.1.4 ! root 3535: kern_return_t vm_map_copyin_page_list(
! 3536: vm_map_t src_map,
! 3537: vm_offset_t src_addr,
! 3538: vm_size_t len,
! 3539: boolean_t src_destroy,
! 3540: boolean_t steal_pages,
! 3541: vm_map_copy_t *copy_result, /* OUT */
! 3542: boolean_t is_cont)
1.1 root 3543: {
3544: vm_map_entry_t src_entry;
3545: vm_page_t m;
3546: vm_offset_t src_start;
3547: vm_offset_t src_end;
3548: vm_size_t src_size;
3549: vm_object_t src_object;
3550: vm_offset_t src_offset;
3551: vm_offset_t src_last_offset;
3552: vm_map_copy_t copy; /* Resulting copy */
3553: kern_return_t result = KERN_SUCCESS;
3554: boolean_t need_map_lookup;
3555: vm_map_copyin_args_t cont_args;
3556:
3557: /*
3558: * If steal_pages is FALSE, this leaves busy pages in
3559: * the object. A continuation must be used if src_destroy
3560: * is true in this case (!steal_pages && src_destroy).
3561: *
3562: * XXX Still have a more general problem of what happens
3563: * XXX if the same page occurs twice in a list. Deadlock
3564: * XXX can happen if vm_fault_page was called. A
3565: * XXX possible solution is to use a continuation if vm_fault_page
3566: * XXX is called and we cross a map entry boundary.
3567: */
3568:
3569: /*
3570: * Check for copies of zero bytes.
3571: */
3572:
3573: if (len == 0) {
3574: *copy_result = VM_MAP_COPY_NULL;
3575: return(KERN_SUCCESS);
3576: }
3577:
3578: /*
3579: * Compute start and end of region
3580: */
3581:
3582: src_start = trunc_page(src_addr);
3583: src_end = round_page(src_addr + len);
3584:
3585: /*
3586: * Check that the end address doesn't overflow
3587: */
3588:
3589: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3590: return KERN_INVALID_ADDRESS;
3591: }
3592:
3593: /*
3594: * Allocate a header element for the page list.
3595: *
3596: * Record original offset and size, as caller may not
3597: * be page-aligned.
3598: */
3599:
1.1.1.3 root 3600: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3601: copy->type = VM_MAP_COPY_PAGE_LIST;
3602: copy->cpy_npages = 0;
3603: copy->offset = src_addr;
3604: copy->size = len;
3605: copy->cpy_cont = ((kern_return_t (*)()) 0);
3606: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
1.1.1.2 root 3607:
1.1 root 3608: /*
3609: * Find the beginning of the region.
3610: */
3611:
3612: do_map_lookup:
3613:
3614: vm_map_lock(src_map);
3615:
3616: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3617: result = KERN_INVALID_ADDRESS;
3618: goto error;
3619: }
3620: need_map_lookup = FALSE;
3621:
3622: /*
3623: * Go through entries until we get to the end.
3624: */
3625:
3626: while (TRUE) {
3627:
3628: if (! (src_entry->protection & VM_PROT_READ)) {
3629: result = KERN_PROTECTION_FAILURE;
3630: goto error;
3631: }
3632:
3633: if (src_end > src_entry->vme_end)
3634: src_size = src_entry->vme_end - src_start;
3635: else
3636: src_size = src_end - src_start;
3637:
3638: src_object = src_entry->object.vm_object;
3639: src_offset = src_entry->offset +
3640: (src_start - src_entry->vme_start);
3641:
3642: /*
3643: * If src_object is NULL, allocate it now;
3644: * we're going to fault on it shortly.
3645: */
3646: if (src_object == VM_OBJECT_NULL) {
3647: src_object = vm_object_allocate((vm_size_t)
3648: src_entry->vme_end -
3649: src_entry->vme_start);
3650: src_entry->object.vm_object = src_object;
3651: }
3652:
3653: /*
3654: * Iterate over pages. Fault in ones that aren't present.
3655: */
3656: src_last_offset = src_offset + src_size;
3657: for (; (src_offset < src_last_offset && !need_map_lookup);
3658: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3659:
3660: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3661: make_continuation:
3662: /*
3663: * At this point we have the max number of
3664: * pages busy for this thread that we're
3665: * willing to allow. Stop here and record
3666: * arguments for the remainder. Note:
3667: * this means that this routine isn't atomic,
3668: * but that's the breaks. Note that only
3669: * the first vm_map_copy_t that comes back
3670: * from this routine has the right offset
3671: * and size; those from continuations are
3672: * page rounded, and short by the amount
3673: * already done.
3674: *
3675: * Reset src_end so the src_destroy
3676: * code at the bottom doesn't do
3677: * something stupid.
3678: */
3679:
1.1.1.2 root 3680: cont_args = (vm_map_copyin_args_t)
1.1 root 3681: kalloc(sizeof(vm_map_copyin_args_data_t));
3682: cont_args->map = src_map;
3683: vm_map_reference(src_map);
3684: cont_args->src_addr = src_start;
3685: cont_args->src_len = len - (src_start - src_addr);
3686: if (src_destroy) {
3687: cont_args->destroy_addr = cont_args->src_addr;
3688: cont_args->destroy_len = cont_args->src_len;
3689: }
3690: else {
3691: cont_args->destroy_addr = (vm_offset_t) 0;
3692: cont_args->destroy_len = (vm_offset_t) 0;
3693: }
3694: cont_args->steal_pages = steal_pages;
3695:
3696: copy->cpy_cont_args = (char *) cont_args;
3697: copy->cpy_cont = vm_map_copyin_page_list_cont;
3698:
3699: src_end = src_start;
3700: vm_map_clip_end(src_map, src_entry, src_end);
3701: break;
3702: }
3703:
3704: /*
3705: * Try to find the page of data.
3706: */
3707: vm_object_lock(src_object);
3708: vm_object_paging_begin(src_object);
3709: if (((m = vm_page_lookup(src_object, src_offset)) !=
3710: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3711: !m->absent && !m->error) {
3712:
3713: /*
3714: * This is the page. Mark it busy
3715: * and keep the paging reference on
3716: * the object whilst we do our thing.
3717: */
3718: m->busy = TRUE;
3719:
3720: /*
3721: * Also write-protect the page, so
3722: * that the map`s owner cannot change
3723: * the data. The busy bit will prevent
3724: * faults on the page from succeeding
3725: * until the copy is released; after
3726: * that, the page can be re-entered
3727: * as writable, since we didn`t alter
3728: * the map entry. This scheme is a
3729: * cheap copy-on-write.
3730: *
3731: * Don`t forget the protection and
3732: * the page_lock value!
3733: *
3734: * If the source is being destroyed
3735: * AND not shared writable, we don`t
3736: * have to protect the page, since
3737: * we will destroy the (only)
3738: * writable mapping later.
3739: */
3740: if (!src_destroy ||
3741: src_object->use_shared_copy)
3742: {
3743: pmap_page_protect(m->phys_addr,
3744: src_entry->protection
3745: & ~m->page_lock
3746: & ~VM_PROT_WRITE);
3747: }
3748:
3749: }
3750: else {
3751: vm_prot_t result_prot;
3752: vm_page_t top_page;
3753: kern_return_t kr;
1.1.1.2 root 3754:
1.1 root 3755: /*
3756: * Have to fault the page in; must
3757: * unlock the map to do so. While
3758: * the map is unlocked, anything
3759: * can happen, we must lookup the
3760: * map entry before continuing.
3761: */
3762: vm_map_unlock(src_map);
3763: need_map_lookup = TRUE;
3764: retry:
3765: result_prot = VM_PROT_READ;
1.1.1.2 root 3766:
1.1 root 3767: kr = vm_fault_page(src_object, src_offset,
3768: VM_PROT_READ, FALSE, FALSE,
3769: &result_prot, &m, &top_page,
3770: FALSE, (void (*)()) 0);
3771: /*
3772: * Cope with what happened.
3773: */
3774: switch (kr) {
3775: case VM_FAULT_SUCCESS:
3776: break;
3777: case VM_FAULT_INTERRUPTED: /* ??? */
3778: case VM_FAULT_RETRY:
3779: vm_object_lock(src_object);
3780: vm_object_paging_begin(src_object);
3781: goto retry;
3782: case VM_FAULT_MEMORY_SHORTAGE:
3783: VM_PAGE_WAIT((void (*)()) 0);
3784: vm_object_lock(src_object);
3785: vm_object_paging_begin(src_object);
3786: goto retry;
3787: case VM_FAULT_FICTITIOUS_SHORTAGE:
3788: vm_page_more_fictitious();
3789: vm_object_lock(src_object);
3790: vm_object_paging_begin(src_object);
3791: goto retry;
3792: case VM_FAULT_MEMORY_ERROR:
3793: /*
3794: * Something broke. If this
3795: * is a continuation, return
3796: * a partial result if possible,
3797: * else fail the whole thing.
3798: * In the continuation case, the
3799: * next continuation call will
3800: * get this error if it persists.
3801: */
3802: vm_map_lock(src_map);
3803: if (is_cont &&
3804: copy->cpy_npages != 0)
3805: goto make_continuation;
3806:
3807: result = KERN_MEMORY_ERROR;
3808: goto error;
3809: }
1.1.1.2 root 3810:
1.1 root 3811: if (top_page != VM_PAGE_NULL) {
3812: vm_object_lock(src_object);
3813: VM_PAGE_FREE(top_page);
3814: vm_object_paging_end(src_object);
3815: vm_object_unlock(src_object);
3816: }
3817:
3818: /*
3819: * We do not need to write-protect
3820: * the page, since it cannot have
3821: * been in the pmap (and we did not
3822: * enter it above). The busy bit
3823: * will protect the page from being
3824: * entered as writable until it is
3825: * unlocked.
3826: */
3827:
3828: }
3829:
3830: /*
3831: * The page is busy, its object is locked, and
3832: * we have a paging reference on it. Either
3833: * the map is locked, or need_map_lookup is
3834: * TRUE.
1.1.1.2 root 3835: *
1.1 root 3836: * Put the page in the page list.
3837: */
3838: copy->cpy_page_list[copy->cpy_npages++] = m;
3839: vm_object_unlock(m->object);
3840: }
1.1.1.2 root 3841:
1.1 root 3842: /*
3843: * DETERMINE whether the entire region
3844: * has been copied.
3845: */
3846: if (src_start >= src_end && src_end != 0) {
3847: if (need_map_lookup)
3848: vm_map_lock(src_map);
3849: break;
3850: }
3851:
3852: /*
3853: * If need_map_lookup is TRUE, have to start over with
3854: * another map lookup. Note that we dropped the map
3855: * lock (to call vm_fault_page) above only in this case.
3856: */
3857: if (need_map_lookup)
3858: goto do_map_lookup;
3859:
3860: /*
3861: * Verify that there are no gaps in the region
3862: */
3863:
3864: src_start = src_entry->vme_end;
3865: src_entry = src_entry->vme_next;
3866: if (src_entry->vme_start != src_start) {
3867: result = KERN_INVALID_ADDRESS;
3868: goto error;
3869: }
3870: }
3871:
3872: /*
3873: * If steal_pages is true, make sure all
3874: * pages in the copy are not in any object
3875: * We try to remove them from the original
3876: * object, but we may have to copy them.
3877: *
3878: * At this point every page in the list is busy
3879: * and holds a paging reference to its object.
3880: * When we're done stealing, every page is busy,
3881: * and in no object (m->tabled == FALSE).
3882: */
3883: src_start = trunc_page(src_addr);
3884: if (steal_pages) {
1.1.1.4 ! root 3885: int i;
1.1 root 3886: vm_offset_t unwire_end;
3887:
3888: unwire_end = src_start;
3889: for (i = 0; i < copy->cpy_npages; i++) {
3890:
3891: /*
3892: * Remove the page from its object if it
3893: * can be stolen. It can be stolen if:
3894: *
1.1.1.2 root 3895: * (1) The source is being destroyed,
1.1 root 3896: * the object is temporary, and
3897: * not shared.
3898: * (2) The page is not precious.
3899: *
3900: * The not shared check consists of two
3901: * parts: (a) there are no objects that
3902: * shadow this object. (b) it is not the
3903: * object in any shared map entries (i.e.,
3904: * use_shared_copy is not set).
3905: *
3906: * The first check (a) means that we can't
3907: * steal pages from objects that are not
3908: * at the top of their shadow chains. This
3909: * should not be a frequent occurrence.
3910: *
3911: * Stealing wired pages requires telling the
3912: * pmap module to let go of them.
1.1.1.2 root 3913: *
1.1 root 3914: * NOTE: stealing clean pages from objects
3915: * whose mappings survive requires a call to
3916: * the pmap module. Maybe later.
3917: */
3918: m = copy->cpy_page_list[i];
3919: src_object = m->object;
3920: vm_object_lock(src_object);
3921:
3922: if (src_destroy &&
3923: src_object->temporary &&
3924: (!src_object->shadowed) &&
3925: (!src_object->use_shared_copy) &&
3926: !m->precious) {
3927: vm_offset_t page_vaddr;
1.1.1.2 root 3928:
1.1 root 3929: page_vaddr = src_start + (i * PAGE_SIZE);
3930: if (m->wire_count > 0) {
3931:
3932: assert(m->wire_count == 1);
3933: /*
3934: * In order to steal a wired
3935: * page, we have to unwire it
3936: * first. We do this inline
3937: * here because we have the page.
3938: *
3939: * Step 1: Unwire the map entry.
3940: * Also tell the pmap module
3941: * that this piece of the
3942: * pmap is pageable.
3943: */
3944: vm_object_unlock(src_object);
3945: if (page_vaddr >= unwire_end) {
3946: if (!vm_map_lookup_entry(src_map,
3947: page_vaddr, &src_entry))
3948: panic("vm_map_copyin_page_list: missing wired map entry");
3949:
3950: vm_map_clip_start(src_map, src_entry,
3951: page_vaddr);
3952: vm_map_clip_end(src_map, src_entry,
3953: src_start + src_size);
3954:
3955: assert(src_entry->wired_count > 0);
3956: src_entry->wired_count = 0;
3957: src_entry->user_wired_count = 0;
3958: unwire_end = src_entry->vme_end;
3959: pmap_pageable(vm_map_pmap(src_map),
3960: page_vaddr, unwire_end, TRUE);
3961: }
3962:
3963: /*
3964: * Step 2: Unwire the page.
3965: * pmap_remove handles this for us.
3966: */
3967: vm_object_lock(src_object);
3968: }
3969:
3970: /*
3971: * Don't need to remove the mapping;
3972: * vm_map_delete will handle it.
3973: *
3974: * Steal the page. Setting the wire count
3975: * to zero is vm_page_unwire without
3976: * activating the page.
3977: */
3978: vm_page_lock_queues();
3979: vm_page_remove(m);
3980: if (m->wire_count > 0) {
3981: m->wire_count = 0;
3982: vm_page_wire_count--;
3983: } else {
3984: VM_PAGE_QUEUES_REMOVE(m);
3985: }
3986: vm_page_unlock_queues();
3987: }
3988: else {
3989: /*
3990: * Have to copy this page. Have to
3991: * unlock the map while copying,
3992: * hence no further page stealing.
3993: * Hence just copy all the pages.
3994: * Unlock the map while copying;
3995: * This means no further page stealing.
3996: */
3997: vm_object_unlock(src_object);
3998: vm_map_unlock(src_map);
3999:
4000: vm_map_copy_steal_pages(copy);
4001:
4002: vm_map_lock(src_map);
4003: break;
4004: }
4005:
4006: vm_object_paging_end(src_object);
4007: vm_object_unlock(src_object);
4008: }
4009:
4010: /*
4011: * If the source should be destroyed, do it now, since the
4012: * copy was successful.
4013: */
4014:
4015: if (src_destroy) {
4016: (void) vm_map_delete(src_map, src_start, src_end);
4017: }
4018: }
4019: else {
4020: /*
4021: * !steal_pages leaves busy pages in the map.
4022: * This will cause src_destroy to hang. Use
4023: * a continuation to prevent this.
4024: */
4025: if (src_destroy && !vm_map_copy_has_cont(copy)) {
1.1.1.2 root 4026: cont_args = (vm_map_copyin_args_t)
1.1 root 4027: kalloc(sizeof(vm_map_copyin_args_data_t));
4028: vm_map_reference(src_map);
4029: cont_args->map = src_map;
4030: cont_args->src_addr = (vm_offset_t) 0;
4031: cont_args->src_len = (vm_size_t) 0;
4032: cont_args->destroy_addr = src_start;
4033: cont_args->destroy_len = src_end - src_start;
4034: cont_args->steal_pages = FALSE;
4035:
4036: copy->cpy_cont_args = (char *) cont_args;
4037: copy->cpy_cont = vm_map_copyin_page_list_cont;
4038: }
1.1.1.2 root 4039:
1.1 root 4040: }
4041:
4042: vm_map_unlock(src_map);
4043:
4044: *copy_result = copy;
4045: return(result);
4046:
4047: error:
4048: vm_map_unlock(src_map);
4049: vm_map_copy_discard(copy);
4050: return(result);
4051: }
4052:
4053: /*
4054: * vm_map_fork:
4055: *
4056: * Create and return a new map based on the old
4057: * map, according to the inheritance values on the
4058: * regions in that map.
4059: *
4060: * The source map must not be locked.
4061: */
1.1.1.4 ! root 4062: vm_map_t vm_map_fork(vm_map_t old_map)
1.1 root 4063: {
4064: vm_map_t new_map;
4065: vm_map_entry_t old_entry;
4066: vm_map_entry_t new_entry;
4067: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4068: vm_size_t new_size = 0;
4069: vm_size_t entry_size;
4070: vm_object_t object;
4071:
4072: vm_map_lock(old_map);
4073:
4074: new_map = vm_map_create(new_pmap,
4075: old_map->min_offset,
4076: old_map->max_offset,
4077: old_map->hdr.entries_pageable);
4078:
4079: for (
4080: old_entry = vm_map_first_entry(old_map);
4081: old_entry != vm_map_to_entry(old_map);
4082: ) {
4083: if (old_entry->is_sub_map)
4084: panic("vm_map_fork: encountered a submap");
4085:
4086: entry_size = (old_entry->vme_end - old_entry->vme_start);
4087:
4088: switch (old_entry->inheritance) {
4089: case VM_INHERIT_NONE:
4090: break;
4091:
4092: case VM_INHERIT_SHARE:
4093: /*
4094: * New sharing code. New map entry
4095: * references original object. Temporary
4096: * objects use asynchronous copy algorithm for
4097: * future copies. First make sure we have
4098: * the right object. If we need a shadow,
4099: * or someone else already has one, then
4100: * make a new shadow and share it.
4101: */
4102:
4103: object = old_entry->object.vm_object;
4104: if (object == VM_OBJECT_NULL) {
4105: object = vm_object_allocate(
4106: (vm_size_t)(old_entry->vme_end -
4107: old_entry->vme_start));
4108: old_entry->offset = 0;
4109: old_entry->object.vm_object = object;
4110: assert(!old_entry->needs_copy);
4111: }
4112: else if (old_entry->needs_copy || object->shadowed ||
4113: (object->temporary && !old_entry->is_shared &&
4114: object->size > (vm_size_t)(old_entry->vme_end -
4115: old_entry->vme_start))) {
4116:
4117: assert(object->temporary);
4118: assert(!(object->shadowed && old_entry->is_shared));
4119: vm_object_shadow(
4120: &old_entry->object.vm_object,
4121: &old_entry->offset,
4122: (vm_size_t) (old_entry->vme_end -
4123: old_entry->vme_start));
1.1.1.2 root 4124:
1.1 root 4125: /*
4126: * If we're making a shadow for other than
4127: * copy on write reasons, then we have
4128: * to remove write permission.
4129: */
4130:
4131: if (!old_entry->needs_copy &&
4132: (old_entry->protection & VM_PROT_WRITE)) {
4133: pmap_protect(vm_map_pmap(old_map),
4134: old_entry->vme_start,
4135: old_entry->vme_end,
4136: old_entry->protection &
4137: ~VM_PROT_WRITE);
4138: }
4139: old_entry->needs_copy = FALSE;
4140: object = old_entry->object.vm_object;
4141: }
4142:
4143: /*
4144: * Set use_shared_copy to indicate that
4145: * object must use shared (delayed) copy-on
4146: * write. This is ignored for permanent objects.
4147: * Bump the reference count for the new entry
4148: */
4149:
4150: vm_object_lock(object);
4151: object->use_shared_copy = TRUE;
4152: object->ref_count++;
4153: vm_object_unlock(object);
4154:
1.1.1.3 root 4155: new_entry = vm_map_entry_create(new_map);
4156:
1.1 root 4157: if (old_entry->projected_on != 0) {
4158: /*
4159: * If entry is projected buffer, clone the
4160: * entry exactly.
4161: */
4162:
4163: vm_map_entry_copy_full(new_entry, old_entry);
4164:
4165: } else {
4166: /*
4167: * Clone the entry, using object ref from above.
4168: * Mark both entries as shared.
4169: */
4170:
4171: vm_map_entry_copy(new_entry, old_entry);
4172: old_entry->is_shared = TRUE;
4173: new_entry->is_shared = TRUE;
4174: }
4175:
4176: /*
4177: * Insert the entry into the new map -- we
4178: * know we're inserting at the end of the new
4179: * map.
4180: */
4181:
4182: vm_map_entry_link(
4183: new_map,
4184: vm_map_last_entry(new_map),
4185: new_entry);
4186:
4187: /*
4188: * Update the physical map
4189: */
4190:
4191: pmap_copy(new_map->pmap, old_map->pmap,
4192: new_entry->vme_start,
4193: entry_size,
4194: old_entry->vme_start);
4195:
4196: new_size += entry_size;
4197: break;
4198:
4199: case VM_INHERIT_COPY:
4200: if (old_entry->wired_count == 0) {
4201: boolean_t src_needs_copy;
4202: boolean_t new_entry_needs_copy;
4203:
4204: new_entry = vm_map_entry_create(new_map);
4205: vm_map_entry_copy(new_entry, old_entry);
4206:
4207: if (vm_object_copy_temporary(
4208: &new_entry->object.vm_object,
4209: &new_entry->offset,
4210: &src_needs_copy,
4211: &new_entry_needs_copy)) {
4212:
4213: /*
4214: * Handle copy-on-write obligations
4215: */
4216:
4217: if (src_needs_copy && !old_entry->needs_copy) {
4218: vm_object_pmap_protect(
4219: old_entry->object.vm_object,
4220: old_entry->offset,
4221: entry_size,
4222: (old_entry->is_shared ?
4223: PMAP_NULL :
4224: old_map->pmap),
4225: old_entry->vme_start,
4226: old_entry->protection &
4227: ~VM_PROT_WRITE);
4228:
4229: old_entry->needs_copy = TRUE;
4230: }
4231:
4232: new_entry->needs_copy = new_entry_needs_copy;
4233:
4234: /*
4235: * Insert the entry at the end
4236: * of the map.
4237: */
4238:
4239: vm_map_entry_link(new_map,
4240: vm_map_last_entry(new_map),
4241: new_entry);
4242:
4243:
4244: new_size += entry_size;
4245: break;
4246: }
4247:
4248: vm_map_entry_dispose(new_map, new_entry);
4249: }
4250:
4251: /* INNER BLOCK (copy cannot be optimized) */ {
4252:
4253: vm_offset_t start = old_entry->vme_start;
4254: vm_map_copy_t copy;
4255: vm_map_entry_t last = vm_map_last_entry(new_map);
4256:
4257: vm_map_unlock(old_map);
4258: if (vm_map_copyin(old_map,
4259: start,
4260: entry_size,
4261: FALSE,
1.1.1.2 root 4262: ©)
1.1 root 4263: != KERN_SUCCESS) {
4264: vm_map_lock(old_map);
4265: if (!vm_map_lookup_entry(old_map, start, &last))
4266: last = last->vme_next;
4267: old_entry = last;
4268: /*
4269: * For some error returns, want to
4270: * skip to the next element.
4271: */
4272:
4273: continue;
4274: }
4275:
4276: /*
4277: * Insert the copy into the new map
4278: */
4279:
4280: vm_map_copy_insert(new_map, last, copy);
4281: new_size += entry_size;
4282:
4283: /*
4284: * Pick up the traversal at the end of
4285: * the copied region.
4286: */
4287:
4288: vm_map_lock(old_map);
4289: start += entry_size;
4290: if (!vm_map_lookup_entry(old_map, start, &last))
4291: last = last->vme_next;
4292: else
4293: vm_map_clip_start(old_map, last, start);
4294: old_entry = last;
4295:
4296: continue;
4297: /* INNER BLOCK (copy cannot be optimized) */ }
4298: }
4299: old_entry = old_entry->vme_next;
4300: }
4301:
4302: new_map->size = new_size;
4303: vm_map_unlock(old_map);
4304:
4305: return(new_map);
4306: }
4307:
4308: /*
4309: * vm_map_lookup:
4310: *
4311: * Finds the VM object, offset, and
4312: * protection for a given virtual address in the
4313: * specified map, assuming a page fault of the
4314: * type specified.
4315: *
4316: * Returns the (object, offset, protection) for
4317: * this address, whether it is wired down, and whether
4318: * this map has the only reference to the data in question.
4319: * In order to later verify this lookup, a "version"
4320: * is returned.
4321: *
4322: * The map should not be locked; it will not be
4323: * locked on exit. In order to guarantee the
4324: * existence of the returned object, it is returned
4325: * locked.
4326: *
4327: * If a lookup is requested with "write protection"
4328: * specified, the map may be changed to perform virtual
4329: * copying operations, although the data referenced will
4330: * remain the same.
4331: */
1.1.1.4 ! root 4332: kern_return_t vm_map_lookup(
! 4333: vm_map_t *var_map, /* IN/OUT */
! 4334: vm_offset_t vaddr,
! 4335: vm_prot_t fault_type,
! 4336:
! 4337: vm_map_version_t *out_version, /* OUT */
! 4338: vm_object_t *object, /* OUT */
! 4339: vm_offset_t *offset, /* OUT */
! 4340: vm_prot_t *out_prot, /* OUT */
! 4341: boolean_t *wired) /* OUT */
! 4342: {
! 4343: vm_map_entry_t entry;
! 4344: vm_map_t map = *var_map;
! 4345: vm_prot_t prot;
1.1 root 4346:
4347: RetryLookup: ;
4348:
4349: /*
4350: * Lookup the faulting address.
4351: */
4352:
4353: vm_map_lock_read(map);
4354:
4355: #define RETURN(why) \
4356: { \
4357: vm_map_unlock_read(map); \
4358: return(why); \
4359: }
4360:
4361: /*
4362: * If the map has an interesting hint, try it before calling
4363: * full blown lookup routine.
4364: */
4365:
4366: simple_lock(&map->hint_lock);
4367: entry = map->hint;
4368: simple_unlock(&map->hint_lock);
4369:
4370: if ((entry == vm_map_to_entry(map)) ||
4371: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4372: vm_map_entry_t tmp_entry;
4373:
4374: /*
4375: * Entry was either not a valid hint, or the vaddr
4376: * was not contained in the entry, so do a full lookup.
4377: */
4378: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4379: RETURN(KERN_INVALID_ADDRESS);
4380:
4381: entry = tmp_entry;
4382: }
4383:
4384: /*
4385: * Handle submaps.
4386: */
4387:
4388: if (entry->is_sub_map) {
4389: vm_map_t old_map = map;
4390:
4391: *var_map = map = entry->object.sub_map;
4392: vm_map_unlock_read(old_map);
4393: goto RetryLookup;
4394: }
1.1.1.2 root 4395:
1.1 root 4396: /*
4397: * Check whether this task is allowed to have
4398: * this page.
4399: */
4400:
4401: prot = entry->protection;
4402:
1.1.1.3 root 4403: if ((fault_type & (prot)) != fault_type) {
1.1 root 4404: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4405: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4406: } else {
4407: RETURN(KERN_PROTECTION_FAILURE);
4408: }
1.1.1.3 root 4409: }
1.1 root 4410:
4411: /*
4412: * If this page is not pageable, we have to get
4413: * it for all possible accesses.
4414: */
4415:
1.1.1.3 root 4416: if ((*wired = (entry->wired_count != 0)))
1.1 root 4417: prot = fault_type = entry->protection;
4418:
4419: /*
4420: * If the entry was copy-on-write, we either ...
4421: */
4422:
4423: if (entry->needs_copy) {
4424: /*
4425: * If we want to write the page, we may as well
4426: * handle that now since we've got the map locked.
4427: *
4428: * If we don't need to write the page, we just
4429: * demote the permissions allowed.
4430: */
4431:
4432: if (fault_type & VM_PROT_WRITE) {
4433: /*
4434: * Make a new object, and place it in the
4435: * object chain. Note that no new references
4436: * have appeared -- one just moved from the
4437: * map to the new object.
4438: */
4439:
4440: if (vm_map_lock_read_to_write(map)) {
4441: goto RetryLookup;
4442: }
4443: map->timestamp++;
4444:
4445: vm_object_shadow(
4446: &entry->object.vm_object,
4447: &entry->offset,
4448: (vm_size_t) (entry->vme_end - entry->vme_start));
1.1.1.2 root 4449:
1.1 root 4450: entry->needs_copy = FALSE;
1.1.1.2 root 4451:
1.1 root 4452: vm_map_lock_write_to_read(map);
4453: }
4454: else {
4455: /*
4456: * We're attempting to read a copy-on-write
4457: * page -- don't allow writes.
4458: */
4459:
4460: prot &= (~VM_PROT_WRITE);
4461: }
4462: }
4463:
4464: /*
4465: * Create an object if necessary.
4466: */
4467: if (entry->object.vm_object == VM_OBJECT_NULL) {
4468:
4469: if (vm_map_lock_read_to_write(map)) {
4470: goto RetryLookup;
4471: }
4472:
4473: entry->object.vm_object = vm_object_allocate(
4474: (vm_size_t)(entry->vme_end - entry->vme_start));
4475: entry->offset = 0;
4476: vm_map_lock_write_to_read(map);
4477: }
4478:
4479: /*
4480: * Return the object/offset from this entry. If the entry
4481: * was copy-on-write or empty, it has been fixed up. Also
4482: * return the protection.
4483: */
4484:
4485: *offset = (vaddr - entry->vme_start) + entry->offset;
4486: *object = entry->object.vm_object;
4487: *out_prot = prot;
4488:
4489: /*
4490: * Lock the object to prevent it from disappearing
4491: */
4492:
4493: vm_object_lock(*object);
4494:
4495: /*
4496: * Save the version number and unlock the map.
4497: */
4498:
4499: out_version->main_timestamp = map->timestamp;
4500:
4501: RETURN(KERN_SUCCESS);
1.1.1.2 root 4502:
1.1 root 4503: #undef RETURN
4504: }
4505:
4506: /*
4507: * vm_map_verify:
4508: *
4509: * Verifies that the map in question has not changed
4510: * since the given version. If successful, the map
4511: * will not change until vm_map_verify_done() is called.
4512: */
1.1.1.4 ! root 4513: boolean_t vm_map_verify(
! 4514: vm_map_t map,
! 4515: vm_map_version_t *version) /* REF */
1.1 root 4516: {
4517: boolean_t result;
4518:
4519: vm_map_lock_read(map);
4520: result = (map->timestamp == version->main_timestamp);
4521:
4522: if (!result)
4523: vm_map_unlock_read(map);
4524:
4525: return(result);
4526: }
4527:
4528: /*
4529: * vm_map_verify_done:
4530: *
4531: * Releases locks acquired by a vm_map_verify.
4532: *
4533: * This is now a macro in vm/vm_map.h. It does a
4534: * vm_map_unlock_read on the map.
4535: */
4536:
4537: /*
4538: * vm_region:
4539: *
4540: * User call to obtain information about a region in
4541: * a task's address map.
4542: */
4543:
1.1.1.4 ! root 4544: kern_return_t vm_region(
! 4545: vm_map_t map,
! 4546: vm_offset_t *address, /* IN/OUT */
! 4547: vm_size_t *size, /* OUT */
! 4548: vm_prot_t *protection, /* OUT */
! 4549: vm_prot_t *max_protection, /* OUT */
! 4550: vm_inherit_t *inheritance, /* OUT */
! 4551: boolean_t *is_shared, /* OUT */
! 4552: ipc_port_t *object_name, /* OUT */
! 4553: vm_offset_t *offset_in_object) /* OUT */
1.1 root 4554: {
4555: vm_map_entry_t tmp_entry;
4556: vm_map_entry_t entry;
4557: vm_offset_t tmp_offset;
4558: vm_offset_t start;
4559:
4560: if (map == VM_MAP_NULL)
4561: return(KERN_INVALID_ARGUMENT);
4562:
4563: start = *address;
4564:
4565: vm_map_lock_read(map);
4566: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4567: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4568: vm_map_unlock_read(map);
4569: return(KERN_NO_SPACE);
4570: }
4571: } else {
4572: entry = tmp_entry;
4573: }
4574:
4575: start = entry->vme_start;
4576: *protection = entry->protection;
4577: *max_protection = entry->max_protection;
4578: *inheritance = entry->inheritance;
4579: *address = start;
4580: *size = (entry->vme_end - start);
4581:
4582: tmp_offset = entry->offset;
4583:
4584:
4585: if (entry->is_sub_map) {
4586: *is_shared = FALSE;
4587: *object_name = IP_NULL;
4588: *offset_in_object = tmp_offset;
4589: } else {
4590: *is_shared = entry->is_shared;
4591: *object_name = vm_object_name(entry->object.vm_object);
4592: *offset_in_object = tmp_offset;
4593: }
4594:
4595: vm_map_unlock_read(map);
4596:
4597: return(KERN_SUCCESS);
4598: }
4599:
4600: /*
4601: * Routine: vm_map_simplify
4602: *
4603: * Description:
4604: * Attempt to simplify the map representation in
4605: * the vicinity of the given starting address.
4606: * Note:
4607: * This routine is intended primarily to keep the
4608: * kernel maps more compact -- they generally don't
4609: * benefit from the "expand a map entry" technology
4610: * at allocation time because the adjacent entry
4611: * is often wired down.
4612: */
1.1.1.4 ! root 4613: void vm_map_simplify(
! 4614: vm_map_t map,
! 4615: vm_offset_t start)
1.1 root 4616: {
4617: vm_map_entry_t this_entry;
4618: vm_map_entry_t prev_entry;
4619:
4620: vm_map_lock(map);
4621: if (
4622: (vm_map_lookup_entry(map, start, &this_entry)) &&
4623: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4624:
4625: (prev_entry->vme_end == start) &&
4626:
4627: (prev_entry->is_shared == FALSE) &&
4628: (prev_entry->is_sub_map == FALSE) &&
4629:
4630: (this_entry->is_shared == FALSE) &&
4631: (this_entry->is_sub_map == FALSE) &&
4632:
4633: (prev_entry->inheritance == this_entry->inheritance) &&
4634: (prev_entry->protection == this_entry->protection) &&
4635: (prev_entry->max_protection == this_entry->max_protection) &&
4636: (prev_entry->wired_count == this_entry->wired_count) &&
4637: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4638:
4639: (prev_entry->needs_copy == this_entry->needs_copy) &&
4640:
4641: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4642: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4643: == this_entry->offset) &&
4644: (prev_entry->projected_on == 0) &&
1.1.1.2 root 4645: (this_entry->projected_on == 0)
1.1 root 4646: ) {
4647: if (map->first_free == this_entry)
4648: map->first_free = prev_entry;
4649:
4650: SAVE_HINT(map, prev_entry);
4651: vm_map_entry_unlink(map, this_entry);
4652: prev_entry->vme_end = this_entry->vme_end;
4653: vm_object_deallocate(this_entry->object.vm_object);
4654: vm_map_entry_dispose(map, this_entry);
4655: }
4656: vm_map_unlock(map);
4657: }
4658:
4659:
4660: /*
4661: * Routine: vm_map_machine_attribute
4662: * Purpose:
4663: * Provide machine-specific attributes to mappings,
4664: * such as cachability etc. for machines that provide
4665: * them. NUMA architectures and machines with big/strange
4666: * caches will use this.
4667: * Note:
4668: * Responsibilities for locking and checking are handled here,
4669: * everything else in the pmap module. If any non-volatile
4670: * information must be kept, the pmap module should handle
4671: * it itself. [This assumes that attributes do not
4672: * need to be inherited, which seems ok to me]
4673: */
1.1.1.4 ! root 4674: kern_return_t vm_map_machine_attribute(
! 4675: vm_map_t map,
! 4676: vm_offset_t address,
! 4677: vm_size_t size,
! 4678: vm_machine_attribute_t attribute,
! 4679: vm_machine_attribute_val_t* value) /* IN/OUT */
1.1 root 4680: {
4681: kern_return_t ret;
4682:
4683: if (address < vm_map_min(map) ||
4684: (address + size) > vm_map_max(map))
4685: return KERN_INVALID_ARGUMENT;
4686:
4687: vm_map_lock(map);
4688:
4689: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4690:
4691: vm_map_unlock(map);
4692:
4693: return ret;
4694: }
4695:
4696:
4697: #if MACH_KDB
4698:
4699: #define printf kdbprintf
4700:
4701: /*
4702: * vm_map_print: [ debug ]
4703: */
1.1.1.4 ! root 4704: void vm_map_print(vm_map_t map)
1.1 root 4705: {
1.1.1.4 ! root 4706: vm_map_entry_t entry;
1.1 root 4707:
4708: iprintf("Task map 0x%X: pmap=0x%X,",
4709: (vm_offset_t) map, (vm_offset_t) (map->pmap));
4710: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4711: printf("version=%d\n", map->timestamp);
4712: indent += 2;
4713: for (entry = vm_map_first_entry(map);
4714: entry != vm_map_to_entry(map);
4715: entry = entry->vme_next) {
4716: static char *inheritance_name[3] = { "share", "copy", "none"};
4717:
4718: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4719: printf("start=0x%X, end=0x%X, ",
4720: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
4721: printf("prot=%X/%X/%s, ",
4722: entry->protection,
4723: entry->max_protection,
4724: inheritance_name[entry->inheritance]);
4725: if (entry->wired_count != 0) {
4726: printf("wired(");
4727: if (entry->user_wired_count != 0)
4728: printf("u");
4729: if (entry->wired_count >
4730: ((entry->user_wired_count == 0) ? 0 : 1))
4731: printf("k");
4732: printf(") ");
4733: }
4734: if (entry->in_transition) {
4735: printf("in transition");
4736: if (entry->needs_wakeup)
4737: printf("(wake request)");
4738: printf(", ");
4739: }
4740: if (entry->is_sub_map) {
4741: printf("submap=0x%X, offset=0x%X\n",
4742: (vm_offset_t) entry->object.sub_map,
4743: (vm_offset_t) entry->offset);
4744: } else {
4745: printf("object=0x%X, offset=0x%X",
4746: (vm_offset_t) entry->object.vm_object,
4747: (vm_offset_t) entry->offset);
4748: if (entry->is_shared)
4749: printf(", shared");
4750: if (entry->needs_copy)
4751: printf(", copy needed");
4752: printf("\n");
4753:
4754: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4755: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
4756: indent += 2;
4757: vm_object_print(entry->object.vm_object);
4758: indent -= 2;
4759: }
4760: }
4761: }
4762: indent -= 2;
4763: }
4764:
4765: /*
4766: * Routine: vm_map_copy_print
4767: * Purpose:
4768: * Pretty-print a copy object for ddb.
4769: */
4770:
4771: void vm_map_copy_print(copy)
1.1.1.4 ! root 4772: const vm_map_copy_t copy;
1.1 root 4773: {
4774: int i, npages;
4775:
4776: printf("copy object 0x%x\n", copy);
4777:
4778: indent += 2;
4779:
4780: iprintf("type=%d", copy->type);
4781: switch (copy->type) {
4782: case VM_MAP_COPY_ENTRY_LIST:
4783: printf("[entry_list]");
4784: break;
1.1.1.2 root 4785:
1.1 root 4786: case VM_MAP_COPY_OBJECT:
4787: printf("[object]");
4788: break;
1.1.1.2 root 4789:
1.1 root 4790: case VM_MAP_COPY_PAGE_LIST:
4791: printf("[page_list]");
4792: break;
4793:
4794: default:
4795: printf("[bad type]");
4796: break;
4797: }
4798: printf(", offset=0x%x", copy->offset);
4799: printf(", size=0x%x\n", copy->size);
4800:
4801: switch (copy->type) {
4802: case VM_MAP_COPY_ENTRY_LIST:
4803: /* XXX add stuff here */
4804: break;
4805:
4806: case VM_MAP_COPY_OBJECT:
4807: iprintf("object=0x%x\n", copy->cpy_object);
4808: break;
4809:
4810: case VM_MAP_COPY_PAGE_LIST:
4811: iprintf("npages=%d", copy->cpy_npages);
4812: printf(", cont=%x", copy->cpy_cont);
4813: printf(", cont_args=%x\n", copy->cpy_cont_args);
4814: if (copy->cpy_npages < 0) {
4815: npages = 0;
4816: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4817: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4818: } else {
4819: npages = copy->cpy_npages;
4820: }
4821: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4822: for (i = 0; i < npages - 1; i++) {
4823: printf("0x%x, ", copy->cpy_page_list[i]);
4824: }
4825: if (npages > 0) {
4826: printf("0x%x", copy->cpy_page_list[npages - 1]);
4827: }
4828: printf("}\n");
4829: break;
4830: }
4831:
1.1.1.4 ! root 4832: indent -= 2;
1.1 root 4833: }
1.1.1.2 root 4834: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.