![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
1.1.1.3 root 37: #include <kern/printf.h>
1.1 root 38: #include <mach/kern_return.h>
39: #include <mach/port.h>
40: #include <mach/vm_attributes.h>
41: #include <mach/vm_param.h>
42: #include <kern/assert.h>
1.1.1.3 root 43: #include <kern/debug.h>
44: #include <kern/kalloc.h>
45: #include <kern/rbtree.h>
46: #include <kern/slab.h>
47: #include <vm/pmap.h>
1.1 root 48: #include <vm/vm_fault.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
1.1.1.3 root 52: #include <vm/vm_resident.h>
1.1 root 53: #include <vm/vm_kern.h>
54: #include <ipc/ipc_port.h>
55:
1.1.1.3 root 56: #if MACH_KDB
57: #include <ddb/db_output.h>
58: #include <vm/vm_print.h>
59: #endif /* MACH_KDB */
60:
1.1 root 61: /*
62: * Macros to copy a vm_map_entry. We must be careful to correctly
63: * manage the wired page count. vm_map_entry_copy() creates a new
64: * map entry to the same memory - the wired count in the new entry
65: * must be set to zero. vm_map_entry_copy_full() creates a new
66: * entry that is identical to the old entry. This preserves the
1.1.1.3 root 67: * wire count; it's used for map splitting and cache changing in
1.1 root 68: * vm_map_copyout.
69: */
70: #define vm_map_entry_copy(NEW,OLD) \
71: MACRO_BEGIN \
72: *(NEW) = *(OLD); \
73: (NEW)->is_shared = FALSE; \
74: (NEW)->needs_wakeup = FALSE; \
75: (NEW)->in_transition = FALSE; \
76: (NEW)->wired_count = 0; \
77: (NEW)->user_wired_count = 0; \
78: MACRO_END
79:
80: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
81:
82: /*
83: * Virtual memory maps provide for the mapping, protection,
84: * and sharing of virtual memory objects. In addition,
85: * this module provides for an efficient virtual copy of
86: * memory from one map to another.
87: *
88: * Synchronization is required prior to most operations.
89: *
90: * Maps consist of an ordered doubly-linked list of simple
1.1.1.3 root 91: * entries; a hint and a red-black tree are used to speed up lookups.
1.1 root 92: *
93: * Sharing maps have been deleted from this version of Mach.
94: * All shared objects are now mapped directly into the respective
95: * maps. This requires a change in the copy on write strategy;
96: * the asymmetric (delayed) strategy is used for shared temporary
97: * objects instead of the symmetric (shadow) strategy. This is
98: * selected by the (new) use_shared_copy bit in the object. See
99: * vm_object_copy_temporary in vm_object.c for details. All maps
100: * are now "top level" maps (either task map, kernel map or submap
1.1.1.2 root 101: * of the kernel map).
1.1 root 102: *
103: * Since portions of maps are specified by start/end addreses,
104: * which may not align with existing map entries, all
105: * routines merely "clip" entries to these start/end values.
106: * [That is, an entry is split into two, bordering at a
107: * start or end value.] Note that these clippings may not
108: * always be necessary (as the two resulting entries are then
109: * not changed); however, the clipping is done for convenience.
110: * No attempt is currently made to "glue back together" two
111: * abutting entries.
112: *
113: * The symmetric (shadow) copy strategy implements virtual copy
114: * by copying VM object references from one map to
115: * another, and then marking both regions as copy-on-write.
116: * It is important to note that only one writeable reference
117: * to a VM object region exists in any map when this strategy
118: * is used -- this means that shadow object creation can be
119: * delayed until a write operation occurs. The asymmetric (delayed)
120: * strategy allows multiple maps to have writeable references to
121: * the same region of a vm object, and hence cannot delay creating
122: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
123: * Copying of permanent objects is completely different; see
124: * vm_object_copy_strategically() in vm_object.c.
125: */
126:
1.1.1.3 root 127: struct kmem_cache vm_map_cache; /* cache for vm_map structures */
128: struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
129: struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
1.1 root 130:
131: /*
132: * Placeholder object for submap operations. This object is dropped
133: * into the range by a call to vm_map_find, and removed when
134: * vm_map_submap creates the submap.
135: */
136:
1.1.1.3 root 137: static struct vm_object vm_submap_object_store;
138: vm_object_t vm_submap_object = &vm_submap_object_store;
1.1 root 139:
140: /*
141: * vm_map_init:
142: *
143: * Initialize the vm_map module. Must be called before
144: * any other vm_map routines.
145: *
1.1.1.3 root 146: * Map and entry structures are allocated from caches -- we must
147: * initialize those caches.
1.1 root 148: *
1.1.1.6 ! root 149: * There are two caches of interest:
1.1 root 150: *
1.1.1.3 root 151: * vm_map_cache: used to allocate maps.
152: * vm_map_entry_cache: used to allocate map entries.
153: *
1.1.1.6 ! root 154: * We make sure the map entry cache allocates memory directly from the
! 155: * physical allocator to avoid recursion with this module.
1.1 root 156: */
157:
1.1.1.3 root 158: void vm_map_init(void)
159: {
160: kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
1.1.1.6 ! root 161: NULL, 0);
1.1.1.3 root 162: kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
1.1.1.6 ! root 163: sizeof(struct vm_map_entry), 0, NULL,
! 164: KMEM_CACHE_NOOFFSLAB | KMEM_CACHE_PHYSMEM);
1.1.1.3 root 165: kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
1.1.1.6 ! root 166: sizeof(struct vm_map_copy), 0, NULL, 0);
1.1 root 167:
168: /*
169: * Submap object is initialized by vm_object_init.
170: */
171: }
172:
1.1.1.4 root 173: void vm_map_setup(
174: vm_map_t map,
175: pmap_t pmap,
176: vm_offset_t min,
177: vm_offset_t max,
178: boolean_t pageable)
1.1.1.3 root 179: {
180: vm_map_first_entry(map) = vm_map_to_entry(map);
181: vm_map_last_entry(map) = vm_map_to_entry(map);
182: map->hdr.nentries = 0;
183: map->hdr.entries_pageable = pageable;
184: rbtree_init(&map->hdr.tree);
185:
186: map->size = 0;
1.1.1.5 root 187: map->user_wired = 0;
1.1.1.3 root 188: map->ref_count = 1;
189: map->pmap = pmap;
190: map->min_offset = min;
191: map->max_offset = max;
192: map->wiring_required = FALSE;
193: map->wait_for_space = FALSE;
194: map->first_free = vm_map_to_entry(map);
195: map->hint = vm_map_to_entry(map);
196: vm_map_lock_init(map);
197: simple_lock_init(&map->ref_lock);
198: simple_lock_init(&map->hint_lock);
199: }
200:
1.1 root 201: /*
202: * vm_map_create:
203: *
204: * Creates and returns a new empty VM map with
205: * the given physical map structure, and having
206: * the given lower and upper address bounds.
207: */
1.1.1.4 root 208: vm_map_t vm_map_create(
209: pmap_t pmap,
210: vm_offset_t min,
211: vm_offset_t max,
212: boolean_t pageable)
1.1 root 213: {
1.1.1.4 root 214: vm_map_t result;
1.1 root 215:
1.1.1.3 root 216: result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
1.1 root 217: if (result == VM_MAP_NULL)
218: panic("vm_map_create");
219:
1.1.1.3 root 220: vm_map_setup(result, pmap, min, max, pageable);
1.1 root 221:
222: return(result);
223: }
224:
225: /*
226: * vm_map_entry_create: [ internal use only ]
227: *
228: * Allocates a VM map entry for insertion in the
229: * given map (or map copy). No fields are filled.
230: */
231: #define vm_map_entry_create(map) \
232: _vm_map_entry_create(&(map)->hdr)
233:
234: #define vm_map_copy_entry_create(copy) \
235: _vm_map_entry_create(&(copy)->cpy_hdr)
236:
237: vm_map_entry_t _vm_map_entry_create(map_header)
1.1.1.4 root 238: const struct vm_map_header *map_header;
1.1 root 239: {
1.1.1.4 root 240: vm_map_entry_t entry;
1.1 root 241:
1.1.1.6 ! root 242: entry = (vm_map_entry_t) kmem_cache_alloc(&vm_map_entry_cache);
1.1 root 243: if (entry == VM_MAP_ENTRY_NULL)
244: panic("vm_map_entry_create");
245:
246: return(entry);
247: }
248:
249: /*
250: * vm_map_entry_dispose: [ internal use only ]
251: *
252: * Inverse of vm_map_entry_create.
253: */
254: #define vm_map_entry_dispose(map, entry) \
255: _vm_map_entry_dispose(&(map)->hdr, (entry))
256:
257: #define vm_map_copy_entry_dispose(map, entry) \
258: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
259:
260: void _vm_map_entry_dispose(map_header, entry)
1.1.1.4 root 261: const struct vm_map_header *map_header;
262: vm_map_entry_t entry;
1.1 root 263: {
1.1.1.6 ! root 264: (void)map_header;
1.1 root 265:
1.1.1.6 ! root 266: kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry);
1.1.1.3 root 267: }
268:
269: /*
270: * Red-black tree lookup/insert comparison functions
271: */
272: static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
273: const struct rbtree_node *node)
274: {
275: struct vm_map_entry *entry;
276:
277: entry = rbtree_entry(node, struct vm_map_entry, tree_node);
278:
279: if (addr < entry->vme_start)
280: return -1;
281: else if (addr < entry->vme_end)
282: return 0;
1.1 root 283: else
1.1.1.3 root 284: return 1;
285: }
286:
287: static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
288: const struct rbtree_node *b)
289: {
290: struct vm_map_entry *entry;
1.1 root 291:
1.1.1.3 root 292: entry = rbtree_entry(a, struct vm_map_entry, tree_node);
293: return vm_map_entry_cmp_lookup(entry->vme_start, b);
1.1 root 294: }
295:
296: /*
297: * vm_map_entry_{un,}link:
298: *
299: * Insert/remove entries from maps (or map copies).
1.1.1.3 root 300: *
301: * The start and end addresses of the entries must be properly set
302: * before using these macros.
1.1 root 303: */
304: #define vm_map_entry_link(map, after_where, entry) \
305: _vm_map_entry_link(&(map)->hdr, after_where, entry)
306:
307: #define vm_map_copy_entry_link(copy, after_where, entry) \
308: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
309:
310: #define _vm_map_entry_link(hdr, after_where, entry) \
311: MACRO_BEGIN \
312: (hdr)->nentries++; \
313: (entry)->vme_prev = (after_where); \
314: (entry)->vme_next = (after_where)->vme_next; \
315: (entry)->vme_prev->vme_next = \
316: (entry)->vme_next->vme_prev = (entry); \
1.1.1.3 root 317: rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \
318: vm_map_entry_cmp_insert); \
1.1 root 319: MACRO_END
320:
321: #define vm_map_entry_unlink(map, entry) \
322: _vm_map_entry_unlink(&(map)->hdr, entry)
323:
324: #define vm_map_copy_entry_unlink(copy, entry) \
325: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
326:
327: #define _vm_map_entry_unlink(hdr, entry) \
328: MACRO_BEGIN \
329: (hdr)->nentries--; \
330: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
331: (entry)->vme_prev->vme_next = (entry)->vme_next; \
1.1.1.3 root 332: rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
1.1 root 333: MACRO_END
334:
335: /*
336: * vm_map_reference:
337: *
338: * Creates another valid reference to the given map.
339: *
340: */
1.1.1.4 root 341: void vm_map_reference(vm_map_t map)
1.1 root 342: {
343: if (map == VM_MAP_NULL)
344: return;
345:
346: simple_lock(&map->ref_lock);
347: map->ref_count++;
348: simple_unlock(&map->ref_lock);
349: }
350:
351: /*
352: * vm_map_deallocate:
353: *
354: * Removes a reference from the specified map,
355: * destroying it if no references remain.
356: * The map should not be locked.
357: */
1.1.1.4 root 358: void vm_map_deallocate(vm_map_t map)
1.1 root 359: {
1.1.1.4 root 360: int c;
1.1 root 361:
362: if (map == VM_MAP_NULL)
363: return;
364:
365: simple_lock(&map->ref_lock);
366: c = --map->ref_count;
367: simple_unlock(&map->ref_lock);
368:
369: if (c > 0) {
370: return;
371: }
372:
373: projected_buffer_collect(map);
374: (void) vm_map_delete(map, map->min_offset, map->max_offset);
375:
376: pmap_destroy(map->pmap);
377:
1.1.1.3 root 378: kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
1.1 root 379: }
380:
381: /*
382: * SAVE_HINT:
383: *
384: * Saves the specified entry as the hint for
385: * future lookups. Performs necessary interlocks.
386: */
387: #define SAVE_HINT(map,value) \
388: simple_lock(&(map)->hint_lock); \
389: (map)->hint = (value); \
390: simple_unlock(&(map)->hint_lock);
391:
392: /*
393: * vm_map_lookup_entry: [ internal use only ]
394: *
395: * Finds the map entry containing (or
396: * immediately preceding) the specified address
397: * in the given map; the entry is returned
398: * in the "entry" parameter. The boolean
399: * result indicates whether the address is
400: * actually contained in the map.
401: */
1.1.1.4 root 402: boolean_t vm_map_lookup_entry(
403: vm_map_t map,
404: vm_offset_t address,
405: vm_map_entry_t *entry) /* OUT */
1.1 root 406: {
1.1.1.4 root 407: struct rbtree_node *node;
408: vm_map_entry_t hint;
1.1 root 409:
410: /*
1.1.1.3 root 411: * First, make a quick check to see if we are already
412: * looking at the entry we want (which is often the case).
1.1 root 413: */
414:
415: simple_lock(&map->hint_lock);
1.1.1.3 root 416: hint = map->hint;
1.1 root 417: simple_unlock(&map->hint_lock);
418:
1.1.1.3 root 419: if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) {
420: if (address < hint->vme_end) {
421: *entry = hint;
1.1 root 422: return(TRUE);
1.1.1.3 root 423: } else {
424: vm_map_entry_t next = hint->vme_next;
425:
426: if ((next == vm_map_to_entry(map))
427: || (address < next->vme_start)) {
428: *entry = hint;
429: return(FALSE);
430: }
1.1 root 431: }
432: }
433:
434: /*
1.1.1.3 root 435: * If the hint didn't help, use the red-black tree.
1.1 root 436: */
437:
1.1.1.3 root 438: node = rbtree_lookup_nearest(&map->hdr.tree, address,
439: vm_map_entry_cmp_lookup, RBTREE_LEFT);
1.1 root 440:
1.1.1.3 root 441: if (node == NULL) {
442: *entry = vm_map_to_entry(map);
443: SAVE_HINT(map, *entry);
444: return(FALSE);
445: } else {
446: *entry = rbtree_entry(node, struct vm_map_entry, tree_node);
447: SAVE_HINT(map, *entry);
448: return((address < (*entry)->vme_end) ? TRUE : FALSE);
1.1 root 449: }
450: }
451:
452: /*
453: * Routine: invalid_user_access
454: *
455: * Verifies whether user access is valid.
456: */
457:
458: boolean_t
1.1.1.4 root 459: invalid_user_access(
460: vm_map_t map,
461: vm_offset_t start,
462: vm_offset_t end,
463: vm_prot_t prot)
1.1 root 464: {
465: vm_map_entry_t entry;
466:
467: return (map == VM_MAP_NULL || map == kernel_map ||
468: !vm_map_lookup_entry(map, start, &entry) ||
469: entry->vme_end < end ||
470: (prot & ~(entry->protection)));
471: }
472:
473:
474: /*
475: * Routine: vm_map_find_entry
476: * Purpose:
477: * Allocate a range in the specified virtual address map,
478: * returning the entry allocated for that range.
479: * Used by kmem_alloc, etc. Returns wired entries.
480: *
481: * The map must be locked.
482: *
483: * If an entry is allocated, the object/offset fields
484: * are initialized to zero. If an object is supplied,
485: * then an existing entry may be extended.
486: */
1.1.1.4 root 487: kern_return_t vm_map_find_entry(
488: vm_map_t map,
489: vm_offset_t *address, /* OUT */
490: vm_size_t size,
491: vm_offset_t mask,
492: vm_object_t object,
493: vm_map_entry_t *o_entry) /* OUT */
1.1 root 494: {
1.1.1.4 root 495: vm_map_entry_t entry, new_entry;
496: vm_offset_t start;
497: vm_offset_t end;
1.1 root 498:
499: /*
500: * Look for the first possible address;
501: * if there's already something at this
502: * address, we have to start after it.
503: */
504:
505: if ((entry = map->first_free) == vm_map_to_entry(map))
506: start = map->min_offset;
507: else
508: start = entry->vme_end;
509:
510: /*
511: * In any case, the "entry" always precedes
512: * the proposed new region throughout the loop:
513: */
514:
515: while (TRUE) {
1.1.1.4 root 516: vm_map_entry_t next;
1.1 root 517:
518: /*
519: * Find the end of the proposed new region.
520: * Be sure we didn't go beyond the end, or
521: * wrap around the address.
522: */
523:
1.1.1.3 root 524: if (((start + mask) & ~mask) < start) {
525: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 526: return(KERN_NO_SPACE);
1.1.1.3 root 527: }
1.1 root 528: start = ((start + mask) & ~mask);
529: end = start + size;
530:
1.1.1.3 root 531: if ((end > map->max_offset) || (end < start)) {
532: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 533: return(KERN_NO_SPACE);
1.1.1.3 root 534: }
1.1 root 535:
536: /*
537: * If there are no more entries, we must win.
538: */
539:
540: next = entry->vme_next;
541: if (next == vm_map_to_entry(map))
542: break;
543:
544: /*
545: * If there is another entry, it must be
546: * after the end of the potential new region.
547: */
548:
549: if (next->vme_start >= end)
550: break;
551:
552: /*
553: * Didn't fit -- move to the next entry.
554: */
555:
556: entry = next;
557: start = entry->vme_end;
558: }
559:
560: /*
561: * At this point,
562: * "start" and "end" should define the endpoints of the
563: * available new range, and
564: * "entry" should refer to the region before the new
565: * range, and
566: *
567: * the map should be locked.
568: */
569:
570: *address = start;
571:
572: /*
573: * See whether we can avoid creating a new entry by
574: * extending one of our neighbors. [So far, we only attempt to
575: * extend from below.]
576: */
577:
578: if ((object != VM_OBJECT_NULL) &&
579: (entry != vm_map_to_entry(map)) &&
580: (entry->vme_end == start) &&
581: (!entry->is_shared) &&
582: (!entry->is_sub_map) &&
583: (entry->object.vm_object == object) &&
584: (entry->needs_copy == FALSE) &&
585: (entry->inheritance == VM_INHERIT_DEFAULT) &&
586: (entry->protection == VM_PROT_DEFAULT) &&
587: (entry->max_protection == VM_PROT_ALL) &&
588: (entry->wired_count == 1) &&
589: (entry->user_wired_count == 0) &&
590: (entry->projected_on == 0)) {
591: /*
592: * Because this is a special case,
593: * we don't need to use vm_object_coalesce.
594: */
595:
596: entry->vme_end = end;
597: new_entry = entry;
598: } else {
599: new_entry = vm_map_entry_create(map);
600:
601: new_entry->vme_start = start;
602: new_entry->vme_end = end;
603:
604: new_entry->is_shared = FALSE;
605: new_entry->is_sub_map = FALSE;
606: new_entry->object.vm_object = VM_OBJECT_NULL;
607: new_entry->offset = (vm_offset_t) 0;
608:
609: new_entry->needs_copy = FALSE;
610:
611: new_entry->inheritance = VM_INHERIT_DEFAULT;
612: new_entry->protection = VM_PROT_DEFAULT;
613: new_entry->max_protection = VM_PROT_ALL;
614: new_entry->wired_count = 1;
615: new_entry->user_wired_count = 0;
616:
617: new_entry->in_transition = FALSE;
618: new_entry->needs_wakeup = FALSE;
619: new_entry->projected_on = 0;
620:
621: /*
622: * Insert the new entry into the list
623: */
624:
625: vm_map_entry_link(map, entry, new_entry);
626: }
627:
628: map->size += size;
629:
630: /*
631: * Update the free space hint and the lookup hint
632: */
633:
634: map->first_free = new_entry;
635: SAVE_HINT(map, new_entry);
636:
637: *o_entry = new_entry;
638: return(KERN_SUCCESS);
639: }
640:
1.1.1.4 root 641: boolean_t vm_map_pmap_enter_print = FALSE;
642: boolean_t vm_map_pmap_enter_enable = FALSE;
1.1 root 643:
644: /*
645: * Routine: vm_map_pmap_enter
646: *
647: * Description:
648: * Force pages from the specified object to be entered into
649: * the pmap at the specified address if they are present.
650: * As soon as a page not found in the object the scan ends.
651: *
652: * Returns:
1.1.1.2 root 653: * Nothing.
1.1 root 654: *
655: * In/out conditions:
656: * The source map should not be locked on entry.
657: */
658: void
1.1.1.4 root 659: vm_map_pmap_enter(
660: vm_map_t map,
661: vm_offset_t addr,
662: vm_offset_t end_addr,
663: vm_object_t object,
664: vm_offset_t offset,
665: vm_prot_t protection)
1.1 root 666: {
667: while (addr < end_addr) {
1.1.1.4 root 668: vm_page_t m;
1.1 root 669:
670: vm_object_lock(object);
671: vm_object_paging_begin(object);
672:
673: m = vm_page_lookup(object, offset);
674: if (m == VM_PAGE_NULL || m->absent) {
675: vm_object_paging_end(object);
676: vm_object_unlock(object);
677: return;
678: }
679:
680: if (vm_map_pmap_enter_print) {
681: printf("vm_map_pmap_enter:");
1.1.1.3 root 682: printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
1.1 root 683: map, addr, object, offset);
684: }
685:
686: m->busy = TRUE;
687: vm_object_unlock(object);
688:
689: PMAP_ENTER(map->pmap, addr, m,
690: protection, FALSE);
691:
692: vm_object_lock(object);
693: PAGE_WAKEUP_DONE(m);
694: vm_page_lock_queues();
695: if (!m->active && !m->inactive)
696: vm_page_activate(m);
697: vm_page_unlock_queues();
698: vm_object_paging_end(object);
699: vm_object_unlock(object);
700:
701: offset += PAGE_SIZE;
702: addr += PAGE_SIZE;
703: }
704: }
705:
706: /*
707: * Routine: vm_map_enter
708: *
709: * Description:
710: * Allocate a range in the specified virtual address map.
711: * The resulting range will refer to memory defined by
712: * the given memory object and offset into that object.
713: *
714: * Arguments are as defined in the vm_map call.
715: */
716: kern_return_t vm_map_enter(
1.1.1.4 root 717: vm_map_t map,
718: vm_offset_t *address, /* IN/OUT */
719: vm_size_t size,
720: vm_offset_t mask,
721: boolean_t anywhere,
722: vm_object_t object,
723: vm_offset_t offset,
724: boolean_t needs_copy,
725: vm_prot_t cur_protection,
726: vm_prot_t max_protection,
727: vm_inherit_t inheritance)
728: {
729: vm_map_entry_t entry;
730: vm_offset_t start;
731: vm_offset_t end;
732: kern_return_t result = KERN_SUCCESS;
1.1 root 733:
734: #define RETURN(value) { result = value; goto BailOut; }
735:
1.1.1.3 root 736: if (size == 0)
737: return KERN_INVALID_ARGUMENT;
738:
1.1 root 739: StartAgain: ;
740:
741: start = *address;
742:
743: if (anywhere) {
744: vm_map_lock(map);
745:
746: /*
747: * Calculate the first possible address.
748: */
749:
750: if (start < map->min_offset)
751: start = map->min_offset;
752: if (start > map->max_offset)
753: RETURN(KERN_NO_SPACE);
754:
755: /*
756: * Look for the first possible address;
757: * if there's already something at this
758: * address, we have to start after it.
759: */
760:
761: if (start == map->min_offset) {
762: if ((entry = map->first_free) != vm_map_to_entry(map))
763: start = entry->vme_end;
764: } else {
765: vm_map_entry_t tmp_entry;
766: if (vm_map_lookup_entry(map, start, &tmp_entry))
767: start = tmp_entry->vme_end;
768: entry = tmp_entry;
769: }
770:
771: /*
772: * In any case, the "entry" always precedes
773: * the proposed new region throughout the
774: * loop:
775: */
776:
777: while (TRUE) {
1.1.1.4 root 778: vm_map_entry_t next;
1.1 root 779:
780: /*
781: * Find the end of the proposed new region.
782: * Be sure we didn't go beyond the end, or
783: * wrap around the address.
784: */
785:
1.1.1.3 root 786: if (((start + mask) & ~mask) < start) {
787: printf_once("no more room for vm_map_enter in %p\n", map);
788: RETURN(KERN_NO_SPACE);
789: }
1.1 root 790: start = ((start + mask) & ~mask);
791: end = start + size;
792:
793: if ((end > map->max_offset) || (end < start)) {
794: if (map->wait_for_space) {
795: if (size <= (map->max_offset -
796: map->min_offset)) {
797: assert_wait((event_t) map, TRUE);
798: vm_map_unlock(map);
799: thread_block((void (*)()) 0);
800: goto StartAgain;
801: }
802: }
1.1.1.2 root 803:
1.1.1.3 root 804: printf_once("no more room for vm_map_enter in %p\n", map);
1.1 root 805: RETURN(KERN_NO_SPACE);
806: }
807:
808: /*
809: * If there are no more entries, we must win.
810: */
811:
812: next = entry->vme_next;
813: if (next == vm_map_to_entry(map))
814: break;
815:
816: /*
817: * If there is another entry, it must be
818: * after the end of the potential new region.
819: */
820:
821: if (next->vme_start >= end)
822: break;
823:
824: /*
825: * Didn't fit -- move to the next entry.
826: */
827:
828: entry = next;
829: start = entry->vme_end;
830: }
831: *address = start;
832: } else {
833: vm_map_entry_t temp_entry;
834:
835: /*
836: * Verify that:
837: * the address doesn't itself violate
838: * the mask requirement.
839: */
840:
841: if ((start & mask) != 0)
842: return(KERN_NO_SPACE);
843:
844: vm_map_lock(map);
845:
846: /*
847: * ... the address is within bounds
848: */
849:
850: end = start + size;
851:
852: if ((start < map->min_offset) ||
853: (end > map->max_offset) ||
854: (start >= end)) {
855: RETURN(KERN_INVALID_ADDRESS);
856: }
857:
858: /*
859: * ... the starting address isn't allocated
860: */
861:
862: if (vm_map_lookup_entry(map, start, &temp_entry))
863: RETURN(KERN_NO_SPACE);
864:
865: entry = temp_entry;
866:
867: /*
868: * ... the next region doesn't overlap the
869: * end point.
870: */
871:
872: if ((entry->vme_next != vm_map_to_entry(map)) &&
873: (entry->vme_next->vme_start < end))
874: RETURN(KERN_NO_SPACE);
875: }
876:
877: /*
878: * At this point,
879: * "start" and "end" should define the endpoints of the
880: * available new range, and
881: * "entry" should refer to the region before the new
882: * range, and
883: *
884: * the map should be locked.
885: */
886:
887: /*
888: * See whether we can avoid creating a new entry (and object) by
889: * extending one of our neighbors. [So far, we only attempt to
890: * extend from below.]
891: */
892:
893: if ((object == VM_OBJECT_NULL) &&
894: (entry != vm_map_to_entry(map)) &&
895: (entry->vme_end == start) &&
896: (!entry->is_shared) &&
897: (!entry->is_sub_map) &&
898: (entry->inheritance == inheritance) &&
899: (entry->protection == cur_protection) &&
900: (entry->max_protection == max_protection) &&
901: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
1.1.1.2 root 902: (entry->projected_on == 0)) {
1.1 root 903: if (vm_object_coalesce(entry->object.vm_object,
904: VM_OBJECT_NULL,
905: entry->offset,
906: (vm_offset_t) 0,
907: (vm_size_t)(entry->vme_end - entry->vme_start),
908: (vm_size_t)(end - entry->vme_end))) {
909:
910: /*
911: * Coalesced the two objects - can extend
912: * the previous map entry to include the
913: * new range.
914: */
915: map->size += (end - entry->vme_end);
916: entry->vme_end = end;
917: RETURN(KERN_SUCCESS);
918: }
919: }
920:
921: /*
922: * Create a new entry
923: */
924:
925: /**/ {
1.1.1.4 root 926: vm_map_entry_t new_entry;
1.1 root 927:
928: new_entry = vm_map_entry_create(map);
929:
930: new_entry->vme_start = start;
931: new_entry->vme_end = end;
932:
933: new_entry->is_shared = FALSE;
934: new_entry->is_sub_map = FALSE;
935: new_entry->object.vm_object = object;
936: new_entry->offset = offset;
937:
938: new_entry->needs_copy = needs_copy;
939:
940: new_entry->inheritance = inheritance;
941: new_entry->protection = cur_protection;
942: new_entry->max_protection = max_protection;
943: new_entry->wired_count = 0;
944: new_entry->user_wired_count = 0;
945:
946: new_entry->in_transition = FALSE;
947: new_entry->needs_wakeup = FALSE;
948: new_entry->projected_on = 0;
949:
950: /*
951: * Insert the new entry into the list
952: */
953:
954: vm_map_entry_link(map, entry, new_entry);
955: map->size += size;
956:
957: /*
958: * Update the free space hint and the lookup hint
959: */
960:
961: if ((map->first_free == entry) &&
962: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
963: >= new_entry->vme_start))
964: map->first_free = new_entry;
965:
966: SAVE_HINT(map, new_entry);
967:
968: vm_map_unlock(map);
969:
970: if ((object != VM_OBJECT_NULL) &&
971: (vm_map_pmap_enter_enable) &&
972: (!anywhere) &&
1.1.1.2 root 973: (!needs_copy) &&
1.1 root 974: (size < (128*1024))) {
1.1.1.2 root 975: vm_map_pmap_enter(map, start, end,
1.1 root 976: object, offset, cur_protection);
977: }
978:
979: return(result);
980: /**/ }
981:
982: BailOut: ;
983:
984: vm_map_unlock(map);
985: return(result);
986:
987: #undef RETURN
988: }
989:
990: /*
991: * vm_map_clip_start: [ internal use only ]
992: *
993: * Asserts that the given entry begins at or after
994: * the specified address; if necessary,
995: * it splits the entry into two.
996: */
997: #define vm_map_clip_start(map, entry, startaddr) \
998: MACRO_BEGIN \
999: if ((startaddr) > (entry)->vme_start) \
1000: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1001: MACRO_END
1002:
1003: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1004: MACRO_BEGIN \
1005: if ((startaddr) > (entry)->vme_start) \
1006: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1007: MACRO_END
1008:
1009: /*
1010: * This routine is called only when it is known that
1011: * the entry must be split.
1012: */
1.1.1.4 root 1013: void _vm_map_clip_start(
1014: struct vm_map_header *map_header,
1015: vm_map_entry_t entry,
1016: vm_offset_t start)
1.1 root 1017: {
1.1.1.4 root 1018: vm_map_entry_t new_entry;
1.1 root 1019:
1020: /*
1021: * Split off the front portion --
1022: * note that we must insert the new
1023: * entry BEFORE this one, so that
1024: * this entry has the specified starting
1025: * address.
1026: */
1027:
1028: new_entry = _vm_map_entry_create(map_header);
1029: vm_map_entry_copy_full(new_entry, entry);
1030:
1031: new_entry->vme_end = start;
1032: entry->offset += (start - entry->vme_start);
1033: entry->vme_start = start;
1034:
1035: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1036:
1037: if (entry->is_sub_map)
1038: vm_map_reference(new_entry->object.sub_map);
1039: else
1040: vm_object_reference(new_entry->object.vm_object);
1041: }
1042:
1043: /*
1044: * vm_map_clip_end: [ internal use only ]
1045: *
1046: * Asserts that the given entry ends at or before
1047: * the specified address; if necessary,
1048: * it splits the entry into two.
1049: */
1050: #define vm_map_clip_end(map, entry, endaddr) \
1051: MACRO_BEGIN \
1052: if ((endaddr) < (entry)->vme_end) \
1053: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1054: MACRO_END
1055:
1056: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1057: MACRO_BEGIN \
1058: if ((endaddr) < (entry)->vme_end) \
1059: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1060: MACRO_END
1061:
1062: /*
1063: * This routine is called only when it is known that
1064: * the entry must be split.
1065: */
1.1.1.4 root 1066: void _vm_map_clip_end(
1067: struct vm_map_header *map_header,
1068: vm_map_entry_t entry,
1069: vm_offset_t end)
1.1 root 1070: {
1.1.1.4 root 1071: vm_map_entry_t new_entry;
1.1 root 1072:
1073: /*
1074: * Create a new entry and insert it
1075: * AFTER the specified entry
1076: */
1077:
1078: new_entry = _vm_map_entry_create(map_header);
1079: vm_map_entry_copy_full(new_entry, entry);
1080:
1081: new_entry->vme_start = entry->vme_end = end;
1082: new_entry->offset += (end - entry->vme_start);
1083:
1084: _vm_map_entry_link(map_header, entry, new_entry);
1085:
1086: if (entry->is_sub_map)
1087: vm_map_reference(new_entry->object.sub_map);
1088: else
1089: vm_object_reference(new_entry->object.vm_object);
1090: }
1091:
1092: /*
1093: * VM_MAP_RANGE_CHECK: [ internal use only ]
1094: *
1095: * Asserts that the starting and ending region
1096: * addresses fall within the valid range of the map.
1097: */
1098: #define VM_MAP_RANGE_CHECK(map, start, end) \
1099: { \
1100: if (start < vm_map_min(map)) \
1101: start = vm_map_min(map); \
1102: if (end > vm_map_max(map)) \
1103: end = vm_map_max(map); \
1104: if (start > end) \
1105: start = end; \
1106: }
1107:
1108: /*
1109: * vm_map_submap: [ kernel use only ]
1110: *
1111: * Mark the given range as handled by a subordinate map.
1112: *
1113: * This range must have been created with vm_map_find using
1114: * the vm_submap_object, and no other operations may have been
1115: * performed on this range prior to calling vm_map_submap.
1116: *
1117: * Only a limited number of operations can be performed
1118: * within this rage after calling vm_map_submap:
1119: * vm_fault
1120: * [Don't try vm_map_copyin!]
1121: *
1122: * To remove a submapping, one must first remove the
1123: * range from the superior map, and then destroy the
1124: * submap (if desired). [Better yet, don't try it.]
1125: */
1.1.1.4 root 1126: kern_return_t vm_map_submap(
1127: vm_map_t map,
1128: vm_offset_t start,
1129: vm_offset_t end,
1130: vm_map_t submap)
1.1 root 1131: {
1132: vm_map_entry_t entry;
1.1.1.4 root 1133: kern_return_t result = KERN_INVALID_ARGUMENT;
1134: vm_object_t object;
1.1 root 1135:
1136: vm_map_lock(map);
1137:
1138: VM_MAP_RANGE_CHECK(map, start, end);
1139:
1140: if (vm_map_lookup_entry(map, start, &entry)) {
1141: vm_map_clip_start(map, entry, start);
1142: }
1143: else
1144: entry = entry->vme_next;
1145:
1146: vm_map_clip_end(map, entry, end);
1147:
1148: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1149: (!entry->is_sub_map) &&
1150: ((object = entry->object.vm_object) == vm_submap_object) &&
1151: (object->resident_page_count == 0) &&
1152: (object->copy == VM_OBJECT_NULL) &&
1153: (object->shadow == VM_OBJECT_NULL) &&
1154: (!object->pager_created)) {
1155: entry->object.vm_object = VM_OBJECT_NULL;
1156: vm_object_deallocate(object);
1157: entry->is_sub_map = TRUE;
1158: vm_map_reference(entry->object.sub_map = submap);
1159: result = KERN_SUCCESS;
1160: }
1161: vm_map_unlock(map);
1162:
1163: return(result);
1164: }
1165:
1166: /*
1167: * vm_map_protect:
1168: *
1169: * Sets the protection of the specified address
1170: * region in the target map. If "set_max" is
1171: * specified, the maximum protection is to be set;
1172: * otherwise, only the current protection is affected.
1173: */
1.1.1.4 root 1174: kern_return_t vm_map_protect(
1175: vm_map_t map,
1176: vm_offset_t start,
1177: vm_offset_t end,
1178: vm_prot_t new_prot,
1179: boolean_t set_max)
1.1 root 1180: {
1.1.1.4 root 1181: vm_map_entry_t current;
1182: vm_map_entry_t entry;
1.1 root 1183:
1184: vm_map_lock(map);
1185:
1186: VM_MAP_RANGE_CHECK(map, start, end);
1187:
1188: if (vm_map_lookup_entry(map, start, &entry)) {
1189: vm_map_clip_start(map, entry, start);
1190: }
1191: else
1192: entry = entry->vme_next;
1193:
1194: /*
1195: * Make a first pass to check for protection
1196: * violations.
1197: */
1198:
1199: current = entry;
1200: while ((current != vm_map_to_entry(map)) &&
1201: (current->vme_start < end)) {
1202:
1203: if (current->is_sub_map) {
1204: vm_map_unlock(map);
1205: return(KERN_INVALID_ARGUMENT);
1206: }
1207: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1208: != new_prot) {
1209: vm_map_unlock(map);
1210: return(KERN_PROTECTION_FAILURE);
1211: }
1212:
1213: current = current->vme_next;
1214: }
1215:
1216: /*
1217: * Go back and fix up protections.
1218: * [Note that clipping is not necessary the second time.]
1219: */
1220:
1221: current = entry;
1222:
1223: while ((current != vm_map_to_entry(map)) &&
1224: (current->vme_start < end)) {
1225:
1226: vm_prot_t old_prot;
1227:
1228: vm_map_clip_end(map, current, end);
1229:
1230: old_prot = current->protection;
1231: if (set_max)
1232: current->protection =
1233: (current->max_protection = new_prot) &
1234: old_prot;
1235: else
1236: current->protection = new_prot;
1237:
1238: /*
1239: * Update physical map if necessary.
1240: */
1241:
1242: if (current->protection != old_prot) {
1243: pmap_protect(map->pmap, current->vme_start,
1244: current->vme_end,
1245: current->protection);
1246: }
1247: current = current->vme_next;
1248: }
1249:
1250: vm_map_unlock(map);
1251: return(KERN_SUCCESS);
1252: }
1253:
1254: /*
1255: * vm_map_inherit:
1256: *
1257: * Sets the inheritance of the specified address
1258: * range in the target map. Inheritance
1259: * affects how the map will be shared with
1260: * child maps at the time of vm_map_fork.
1261: */
1.1.1.4 root 1262: kern_return_t vm_map_inherit(
1263: vm_map_t map,
1264: vm_offset_t start,
1265: vm_offset_t end,
1266: vm_inherit_t new_inheritance)
1.1 root 1267: {
1.1.1.4 root 1268: vm_map_entry_t entry;
1.1 root 1269: vm_map_entry_t temp_entry;
1270:
1271: vm_map_lock(map);
1272:
1273: VM_MAP_RANGE_CHECK(map, start, end);
1274:
1275: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1276: entry = temp_entry;
1277: vm_map_clip_start(map, entry, start);
1278: }
1279: else
1280: entry = temp_entry->vme_next;
1281:
1282: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1283: vm_map_clip_end(map, entry, end);
1284:
1285: entry->inheritance = new_inheritance;
1286:
1287: entry = entry->vme_next;
1288: }
1289:
1290: vm_map_unlock(map);
1291: return(KERN_SUCCESS);
1292: }
1293:
1294: /*
1295: * vm_map_pageable_common:
1296: *
1297: * Sets the pageability of the specified address
1298: * range in the target map. Regions specified
1299: * as not pageable require locked-down physical
1300: * memory and physical page maps. access_type indicates
1301: * types of accesses that must not generate page faults.
1302: * This is checked against protection of memory being locked-down.
1303: * access_type of VM_PROT_NONE makes memory pageable.
1304: *
1305: * The map must not be locked, but a reference
1306: * must remain to the map throughout the call.
1307: *
1308: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1309: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1310: */
1.1.1.4 root 1311: kern_return_t vm_map_pageable_common(
1312: vm_map_t map,
1313: vm_offset_t start,
1314: vm_offset_t end,
1315: vm_prot_t access_type,
1316: boolean_t user_wire)
1.1 root 1317: {
1.1.1.4 root 1318: vm_map_entry_t entry;
1.1 root 1319: vm_map_entry_t start_entry;
1320:
1321: vm_map_lock(map);
1322:
1323: VM_MAP_RANGE_CHECK(map, start, end);
1324:
1325: if (vm_map_lookup_entry(map, start, &start_entry)) {
1326: entry = start_entry;
1327: /*
1328: * vm_map_clip_start will be done later.
1329: */
1330: }
1331: else {
1332: /*
1333: * Start address is not in map; this is fatal.
1334: */
1335: vm_map_unlock(map);
1336: return(KERN_FAILURE);
1337: }
1338:
1339: /*
1340: * Actions are rather different for wiring and unwiring,
1341: * so we have two separate cases.
1342: */
1343:
1344: if (access_type == VM_PROT_NONE) {
1345:
1346: vm_map_clip_start(map, entry, start);
1347:
1348: /*
1349: * Unwiring. First ensure that the range to be
1350: * unwired is really wired down.
1351: */
1352: while ((entry != vm_map_to_entry(map)) &&
1353: (entry->vme_start < end)) {
1354:
1355: if ((entry->wired_count == 0) ||
1.1.1.2 root 1356: ((entry->vme_end < end) &&
1.1 root 1357: ((entry->vme_next == vm_map_to_entry(map)) ||
1358: (entry->vme_next->vme_start > entry->vme_end))) ||
1359: (user_wire && (entry->user_wired_count == 0))) {
1360: vm_map_unlock(map);
1361: return(KERN_INVALID_ARGUMENT);
1362: }
1363: entry = entry->vme_next;
1364: }
1365:
1366: /*
1367: * Now decrement the wiring count for each region.
1368: * If a region becomes completely unwired,
1369: * unwire its physical pages and mappings.
1370: */
1371: entry = start_entry;
1372: while ((entry != vm_map_to_entry(map)) &&
1373: (entry->vme_start < end)) {
1374: vm_map_clip_end(map, entry, end);
1375:
1376: if (user_wire) {
1377: if (--(entry->user_wired_count) == 0)
1.1.1.5 root 1378: {
1379: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1380: entry->wired_count--;
1.1.1.5 root 1381: }
1.1 root 1382: }
1383: else {
1384: entry->wired_count--;
1385: }
1.1.1.2 root 1386:
1.1 root 1387: if (entry->wired_count == 0)
1388: vm_fault_unwire(map, entry);
1389:
1390: entry = entry->vme_next;
1391: }
1392: }
1393:
1394: else {
1395: /*
1396: * Wiring. We must do this in two passes:
1397: *
1398: * 1. Holding the write lock, we create any shadow
1399: * or zero-fill objects that need to be created.
1400: * Then we clip each map entry to the region to be
1401: * wired and increment its wiring count. We
1402: * create objects before clipping the map entries
1403: * to avoid object proliferation.
1404: *
1405: * 2. We downgrade to a read lock, and call
1406: * vm_fault_wire to fault in the pages for any
1407: * newly wired area (wired_count is 1).
1408: *
1409: * Downgrading to a read lock for vm_fault_wire avoids
1410: * a possible deadlock with another thread that may have
1411: * faulted on one of the pages to be wired (it would mark
1412: * the page busy, blocking us, then in turn block on the
1413: * map lock that we hold). Because of problems in the
1414: * recursive lock package, we cannot upgrade to a write
1415: * lock in vm_map_lookup. Thus, any actions that require
1416: * the write lock must be done beforehand. Because we
1417: * keep the read lock on the map, the copy-on-write
1418: * status of the entries we modify here cannot change.
1419: */
1420:
1421: /*
1422: * Pass 1.
1423: */
1424: while ((entry != vm_map_to_entry(map)) &&
1425: (entry->vme_start < end)) {
1426: vm_map_clip_end(map, entry, end);
1427:
1428: if (entry->wired_count == 0) {
1429:
1430: /*
1431: * Perform actions of vm_map_lookup that need
1432: * the write lock on the map: create a shadow
1433: * object for a copy-on-write region, or an
1434: * object for a zero-fill region.
1435: */
1436: if (entry->needs_copy &&
1437: ((entry->protection & VM_PROT_WRITE) != 0)) {
1438:
1439: vm_object_shadow(&entry->object.vm_object,
1440: &entry->offset,
1441: (vm_size_t)(entry->vme_end
1442: - entry->vme_start));
1443: entry->needs_copy = FALSE;
1444: }
1445: if (entry->object.vm_object == VM_OBJECT_NULL) {
1446: entry->object.vm_object =
1447: vm_object_allocate(
1.1.1.2 root 1448: (vm_size_t)(entry->vme_end
1.1 root 1449: - entry->vme_start));
1450: entry->offset = (vm_offset_t)0;
1451: }
1452: }
1453: vm_map_clip_start(map, entry, start);
1454: vm_map_clip_end(map, entry, end);
1455:
1456: if (user_wire) {
1457: if ((entry->user_wired_count)++ == 0)
1.1.1.5 root 1458: {
1459: map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 1460: entry->wired_count++;
1.1.1.5 root 1461: }
1.1 root 1462: }
1463: else {
1464: entry->wired_count++;
1465: }
1466:
1467: /*
1468: * Check for holes and protection mismatch.
1469: * Holes: Next entry should be contiguous unless
1470: * this is the end of the region.
1471: * Protection: Access requested must be allowed.
1472: */
1.1.1.2 root 1473: if (((entry->vme_end < end) &&
1.1 root 1474: ((entry->vme_next == vm_map_to_entry(map)) ||
1475: (entry->vme_next->vme_start > entry->vme_end))) ||
1476: ((entry->protection & access_type) != access_type)) {
1477: /*
1478: * Found a hole or protection problem.
1479: * Object creation actions
1480: * do not need to be undone, but the
1481: * wired counts need to be restored.
1482: */
1483: while ((entry != vm_map_to_entry(map)) &&
1484: (entry->vme_end > start)) {
1485: if (user_wire) {
1486: if (--(entry->user_wired_count) == 0)
1.1.1.6 ! root 1487: {
1.1.1.5 root 1488: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1489: entry->wired_count--;
1.1.1.6 ! root 1490: }
1.1 root 1491: }
1492: else {
1493: entry->wired_count--;
1494: }
1495:
1496: entry = entry->vme_prev;
1497: }
1498:
1499: vm_map_unlock(map);
1500: return(KERN_FAILURE);
1501: }
1502: entry = entry->vme_next;
1503: }
1504:
1505: /*
1506: * Pass 2.
1507: */
1508:
1509: /*
1510: * HACK HACK HACK HACK
1511: *
1512: * If we are wiring in the kernel map or a submap of it,
1513: * unlock the map to avoid deadlocks. We trust that the
1514: * kernel threads are well-behaved, and therefore will
1515: * not do anything destructive to this region of the map
1516: * while we have it unlocked. We cannot trust user threads
1517: * to do the same.
1518: *
1519: * HACK HACK HACK HACK
1520: */
1521: if (vm_map_pmap(map) == kernel_pmap) {
1522: vm_map_unlock(map); /* trust me ... */
1523: }
1524: else {
1525: vm_map_lock_set_recursive(map);
1526: vm_map_lock_write_to_read(map);
1527: }
1528:
1529: entry = start_entry;
1530: while (entry != vm_map_to_entry(map) &&
1531: entry->vme_start < end) {
1532: /*
1533: * Wiring cases:
1534: * Kernel: wired == 1 && user_wired == 0
1535: * User: wired == 1 && user_wired == 1
1536: *
1537: * Don't need to wire if either is > 1. wired = 0 &&
1538: * user_wired == 1 can't happen.
1539: */
1540:
1541: /*
1542: * XXX This assumes that the faults always succeed.
1543: */
1544: if ((entry->wired_count == 1) &&
1545: (entry->user_wired_count <= 1)) {
1546: vm_fault_wire(map, entry);
1547: }
1548: entry = entry->vme_next;
1549: }
1550:
1551: if (vm_map_pmap(map) == kernel_pmap) {
1552: vm_map_lock(map);
1553: }
1554: else {
1555: vm_map_lock_clear_recursive(map);
1556: }
1557: }
1558:
1559: vm_map_unlock(map);
1560:
1561: return(KERN_SUCCESS);
1562: }
1563:
1564: /*
1565: * vm_map_entry_delete: [ internal use only ]
1566: *
1567: * Deallocate the given entry from the target map.
1.1.1.2 root 1568: */
1.1.1.4 root 1569: void vm_map_entry_delete(
1570: vm_map_t map,
1571: vm_map_entry_t entry)
1.1 root 1572: {
1.1.1.4 root 1573: vm_offset_t s, e;
1574: vm_object_t object;
1.1 root 1575: extern vm_object_t kernel_object;
1576:
1577: s = entry->vme_start;
1578: e = entry->vme_end;
1579:
1580: /*Check if projected buffer*/
1581: if (map != kernel_map && entry->projected_on != 0) {
1582: /*Check if projected kernel entry is persistent;
1583: may only manipulate directly if it is*/
1584: if (entry->projected_on->projected_on == 0)
1585: entry->wired_count = 0; /*Avoid unwire fault*/
1586: else
1587: return;
1588: }
1589:
1590: /*
1591: * Get the object. Null objects cannot have pmap entries.
1592: */
1593:
1594: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1595:
1596: /*
1597: * Unwire before removing addresses from the pmap;
1598: * otherwise, unwiring will put the entries back in
1599: * the pmap.
1600: */
1601:
1602: if (entry->wired_count != 0) {
1603: vm_fault_unwire(map, entry);
1604: entry->wired_count = 0;
1.1.1.5 root 1605: if (entry->user_wired_count)
1606: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1607: entry->user_wired_count = 0;
1608: }
1609:
1610: /*
1611: * If the object is shared, we must remove
1612: * *all* references to this data, since we can't
1613: * find all of the physical maps which are sharing
1614: * it.
1615: */
1616:
1617: if (object == kernel_object) {
1618: vm_object_lock(object);
1619: vm_object_page_remove(object, entry->offset,
1620: entry->offset + (e - s));
1621: vm_object_unlock(object);
1622: } else if (entry->is_shared) {
1623: vm_object_pmap_remove(object,
1624: entry->offset,
1625: entry->offset + (e - s));
1626: }
1627: else {
1628: pmap_remove(map->pmap, s, e);
1629: }
1630: }
1631:
1632: /*
1633: * Deallocate the object only after removing all
1634: * pmap entries pointing to its pages.
1635: */
1636:
1637: if (entry->is_sub_map)
1638: vm_map_deallocate(entry->object.sub_map);
1639: else
1640: vm_object_deallocate(entry->object.vm_object);
1641:
1642: vm_map_entry_unlink(map, entry);
1643: map->size -= e - s;
1644:
1645: vm_map_entry_dispose(map, entry);
1646: }
1647:
1648: /*
1649: * vm_map_delete: [ internal use only ]
1650: *
1651: * Deallocates the given address range from the target
1652: * map.
1653: */
1654:
1.1.1.4 root 1655: kern_return_t vm_map_delete(
1656: vm_map_t map,
1657: vm_offset_t start,
1658: vm_offset_t end)
1.1 root 1659: {
1660: vm_map_entry_t entry;
1661: vm_map_entry_t first_entry;
1662:
1663: /*
1664: * Find the start of the region, and clip it
1665: */
1666:
1667: if (!vm_map_lookup_entry(map, start, &first_entry))
1668: entry = first_entry->vme_next;
1669: else {
1670: entry = first_entry;
1671: vm_map_clip_start(map, entry, start);
1672:
1673: /*
1674: * Fix the lookup hint now, rather than each
1675: * time though the loop.
1676: */
1677:
1678: SAVE_HINT(map, entry->vme_prev);
1679: }
1680:
1681: /*
1682: * Save the free space hint
1683: */
1684:
1685: if (map->first_free->vme_start >= start)
1686: map->first_free = entry->vme_prev;
1687:
1688: /*
1689: * Step through all entries in this region
1690: */
1691:
1692: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1693: vm_map_entry_t next;
1694:
1695: vm_map_clip_end(map, entry, end);
1696:
1697: /*
1698: * If the entry is in transition, we must wait
1699: * for it to exit that state. It could be clipped
1700: * while we leave the map unlocked.
1701: */
1702: if(entry->in_transition) {
1703: /*
1704: * Say that we are waiting, and wait for entry.
1705: */
1706: entry->needs_wakeup = TRUE;
1707: vm_map_entry_wait(map, FALSE);
1708: vm_map_lock(map);
1709:
1710: /*
1711: * The entry could have been clipped or it
1712: * may not exist anymore. look it up again.
1713: */
1714: if(!vm_map_lookup_entry(map, start, &entry)) {
1715: entry = entry->vme_next;
1716: }
1717: continue;
1718: }
1719:
1720: next = entry->vme_next;
1721:
1722: vm_map_entry_delete(map, entry);
1723: entry = next;
1724: }
1725:
1726: if (map->wait_for_space)
1727: thread_wakeup((event_t) map);
1728:
1729: return(KERN_SUCCESS);
1730: }
1731:
1732: /*
1733: * vm_map_remove:
1734: *
1735: * Remove the given address range from the target map.
1736: * This is the exported form of vm_map_delete.
1737: */
1.1.1.4 root 1738: kern_return_t vm_map_remove(
1739: vm_map_t map,
1740: vm_offset_t start,
1741: vm_offset_t end)
1.1 root 1742: {
1.1.1.4 root 1743: kern_return_t result;
1.1 root 1744:
1745: vm_map_lock(map);
1746: VM_MAP_RANGE_CHECK(map, start, end);
1747: result = vm_map_delete(map, start, end);
1748: vm_map_unlock(map);
1749:
1750: return(result);
1751: }
1752:
1753:
1754: /*
1755: * vm_map_copy_steal_pages:
1756: *
1757: * Steal all the pages from a vm_map_copy page_list by copying ones
1758: * that have not already been stolen.
1759: */
1760: void
1.1.1.4 root 1761: vm_map_copy_steal_pages(vm_map_copy_t copy)
1.1 root 1762: {
1.1.1.4 root 1763: vm_page_t m, new_m;
1764: int i;
1765: vm_object_t object;
1.1 root 1766:
1767: for (i = 0; i < copy->cpy_npages; i++) {
1768:
1769: /*
1770: * If the page is not tabled, then it's already stolen.
1771: */
1772: m = copy->cpy_page_list[i];
1773: if (!m->tabled)
1774: continue;
1775:
1776: /*
1777: * Page was not stolen, get a new
1778: * one and do the copy now.
1779: */
1.1.1.2 root 1780: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) {
1.1 root 1781: VM_PAGE_WAIT((void(*)()) 0);
1782: }
1783:
1784: vm_page_copy(m, new_m);
1785:
1786: object = m->object;
1787: vm_object_lock(object);
1788: vm_page_lock_queues();
1789: if (!m->active && !m->inactive)
1790: vm_page_activate(m);
1791: vm_page_unlock_queues();
1792: PAGE_WAKEUP_DONE(m);
1793: vm_object_paging_end(object);
1794: vm_object_unlock(object);
1795:
1796: copy->cpy_page_list[i] = new_m;
1797: }
1798: }
1799:
1800: /*
1801: * vm_map_copy_page_discard:
1802: *
1803: * Get rid of the pages in a page_list copy. If the pages are
1804: * stolen, they are freed. If the pages are not stolen, they
1805: * are unbusied, and associated state is cleaned up.
1806: */
1.1.1.4 root 1807: void vm_map_copy_page_discard(vm_map_copy_t copy)
1.1 root 1808: {
1809: while (copy->cpy_npages > 0) {
1810: vm_page_t m;
1811:
1812: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1813: VM_PAGE_NULL) {
1814:
1815: /*
1816: * If it's not in the table, then it's
1817: * a stolen page that goes back
1818: * to the free list. Else it belongs
1819: * to some object, and we hold a
1820: * paging reference on that object.
1821: */
1822: if (!m->tabled) {
1823: VM_PAGE_FREE(m);
1824: }
1825: else {
1826: vm_object_t object;
1827:
1828: object = m->object;
1829:
1830: vm_object_lock(object);
1831: vm_page_lock_queues();
1832: if (!m->active && !m->inactive)
1833: vm_page_activate(m);
1834: vm_page_unlock_queues();
1835:
1836: PAGE_WAKEUP_DONE(m);
1837: vm_object_paging_end(object);
1838: vm_object_unlock(object);
1839: }
1840: }
1841: }
1842: }
1843:
1844: /*
1845: * Routine: vm_map_copy_discard
1846: *
1847: * Description:
1848: * Dispose of a map copy object (returned by
1849: * vm_map_copyin).
1850: */
1851: void
1.1.1.4 root 1852: vm_map_copy_discard(vm_map_copy_t copy)
1.1 root 1853: {
1854: free_next_copy:
1855: if (copy == VM_MAP_COPY_NULL)
1856: return;
1857:
1858: switch (copy->type) {
1859: case VM_MAP_COPY_ENTRY_LIST:
1860: while (vm_map_copy_first_entry(copy) !=
1861: vm_map_copy_to_entry(copy)) {
1862: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
1863:
1864: vm_map_copy_entry_unlink(copy, entry);
1865: vm_object_deallocate(entry->object.vm_object);
1866: vm_map_copy_entry_dispose(copy, entry);
1867: }
1868: break;
1869: case VM_MAP_COPY_OBJECT:
1870: vm_object_deallocate(copy->cpy_object);
1871: break;
1872: case VM_MAP_COPY_PAGE_LIST:
1873:
1874: /*
1875: * To clean this up, we have to unbusy all the pages
1876: * and release the paging references in their objects.
1877: */
1878: if (copy->cpy_npages > 0)
1879: vm_map_copy_page_discard(copy);
1880:
1881: /*
1882: * If there's a continuation, abort it. The
1883: * abort routine releases any storage.
1884: */
1885: if (vm_map_copy_has_cont(copy)) {
1886:
1887: /*
1888: * Special case: recognize
1889: * vm_map_copy_discard_cont and optimize
1890: * here to avoid tail recursion.
1891: */
1892: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1.1.1.4 root 1893: vm_map_copy_t new_copy;
1.1 root 1894:
1895: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1.1.1.3 root 1896: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1897: copy = new_copy;
1898: goto free_next_copy;
1899: }
1900: else {
1901: vm_map_copy_abort_cont(copy);
1902: }
1903: }
1904:
1905: break;
1906: }
1.1.1.3 root 1907: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1908: }
1909:
1910: /*
1911: * Routine: vm_map_copy_copy
1912: *
1913: * Description:
1914: * Move the information in a map copy object to
1915: * a new map copy object, leaving the old one
1916: * empty.
1917: *
1918: * This is used by kernel routines that need
1919: * to look at out-of-line data (in copyin form)
1920: * before deciding whether to return SUCCESS.
1921: * If the routine returns FAILURE, the original
1922: * copy object will be deallocated; therefore,
1923: * these routines must make a copy of the copy
1924: * object and leave the original empty so that
1925: * deallocation will not fail.
1926: */
1927: vm_map_copy_t
1.1.1.4 root 1928: vm_map_copy_copy(vm_map_copy_t copy)
1.1 root 1929: {
1930: vm_map_copy_t new_copy;
1931:
1932: if (copy == VM_MAP_COPY_NULL)
1933: return VM_MAP_COPY_NULL;
1934:
1935: /*
1936: * Allocate a new copy object, and copy the information
1937: * from the old one into it.
1938: */
1939:
1.1.1.3 root 1940: new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 1941: *new_copy = *copy;
1942:
1943: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
1944: /*
1945: * The links in the entry chain must be
1946: * changed to point to the new copy object.
1947: */
1948: vm_map_copy_first_entry(copy)->vme_prev
1949: = vm_map_copy_to_entry(new_copy);
1950: vm_map_copy_last_entry(copy)->vme_next
1951: = vm_map_copy_to_entry(new_copy);
1952: }
1953:
1954: /*
1955: * Change the old copy object into one that contains
1956: * nothing to be deallocated.
1957: */
1958: copy->type = VM_MAP_COPY_OBJECT;
1959: copy->cpy_object = VM_OBJECT_NULL;
1960:
1961: /*
1962: * Return the new object.
1963: */
1964: return new_copy;
1965: }
1966:
1967: /*
1968: * Routine: vm_map_copy_discard_cont
1969: *
1970: * Description:
1971: * A version of vm_map_copy_discard that can be called
1972: * as a continuation from a vm_map_copy page list.
1973: */
1.1.1.4 root 1974: kern_return_t vm_map_copy_discard_cont(
1975: vm_map_copyin_args_t cont_args,
1976: vm_map_copy_t *copy_result) /* OUT */
1.1 root 1977: {
1978: vm_map_copy_discard((vm_map_copy_t) cont_args);
1979: if (copy_result != (vm_map_copy_t *)0)
1980: *copy_result = VM_MAP_COPY_NULL;
1981: return(KERN_SUCCESS);
1982: }
1983:
1984: /*
1985: * Routine: vm_map_copy_overwrite
1986: *
1987: * Description:
1988: * Copy the memory described by the map copy
1989: * object (copy; returned by vm_map_copyin) onto
1990: * the specified destination region (dst_map, dst_addr).
1991: * The destination must be writeable.
1992: *
1993: * Unlike vm_map_copyout, this routine actually
1994: * writes over previously-mapped memory. If the
1995: * previous mapping was to a permanent (user-supplied)
1996: * memory object, it is preserved.
1997: *
1998: * The attributes (protection and inheritance) of the
1999: * destination region are preserved.
2000: *
2001: * If successful, consumes the copy object.
2002: * Otherwise, the caller is responsible for it.
2003: *
2004: * Implementation notes:
2005: * To overwrite temporary virtual memory, it is
2006: * sufficient to remove the previous mapping and insert
2007: * the new copy. This replacement is done either on
2008: * the whole region (if no permanent virtual memory
2009: * objects are embedded in the destination region) or
2010: * in individual map entries.
2011: *
2012: * To overwrite permanent virtual memory, it is
2013: * necessary to copy each page, as the external
2014: * memory management interface currently does not
2015: * provide any optimizations.
2016: *
2017: * Once a page of permanent memory has been overwritten,
2018: * it is impossible to interrupt this function; otherwise,
2019: * the call would be neither atomic nor location-independent.
2020: * The kernel-state portion of a user thread must be
2021: * interruptible.
2022: *
2023: * It may be expensive to forward all requests that might
2024: * overwrite permanent memory (vm_write, vm_copy) to
2025: * uninterruptible kernel threads. This routine may be
2026: * called by interruptible threads; however, success is
2027: * not guaranteed -- if the request cannot be performed
2028: * atomically and interruptibly, an error indication is
2029: * returned.
2030: */
1.1.1.4 root 2031: kern_return_t vm_map_copy_overwrite(
2032: vm_map_t dst_map,
2033: vm_offset_t dst_addr,
2034: vm_map_copy_t copy,
2035: boolean_t interruptible)
1.1 root 2036: {
2037: vm_size_t size;
2038: vm_offset_t start;
2039: vm_map_entry_t tmp_entry;
2040: vm_map_entry_t entry;
2041:
2042: boolean_t contains_permanent_objects = FALSE;
2043:
2044: interruptible = FALSE; /* XXX */
2045:
2046: /*
2047: * Check for null copy object.
2048: */
2049:
2050: if (copy == VM_MAP_COPY_NULL)
2051: return(KERN_SUCCESS);
2052:
2053: /*
2054: * Only works for entry lists at the moment. Will
2055: * support page lists LATER.
2056: */
2057:
2058: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2059:
2060: /*
2061: * Currently this routine only handles page-aligned
2062: * regions. Eventually, it should handle misalignments
2063: * by actually copying pages.
2064: */
2065:
2066: if (!page_aligned(copy->offset) ||
2067: !page_aligned(copy->size) ||
2068: !page_aligned(dst_addr))
2069: return(KERN_INVALID_ARGUMENT);
2070:
2071: size = copy->size;
2072:
2073: if (size == 0) {
2074: vm_map_copy_discard(copy);
2075: return(KERN_SUCCESS);
2076: }
2077:
2078: /*
2079: * Verify that the destination is all writeable
2080: * initially.
2081: */
2082: start_pass_1:
2083: vm_map_lock(dst_map);
2084: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2085: vm_map_unlock(dst_map);
2086: return(KERN_INVALID_ADDRESS);
2087: }
2088: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2089: for (entry = tmp_entry;;) {
2090: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2091: vm_map_entry_t next = entry->vme_next;
2092:
2093: if ( ! (entry->protection & VM_PROT_WRITE)) {
2094: vm_map_unlock(dst_map);
2095: return(KERN_PROTECTION_FAILURE);
2096: }
2097:
2098: /*
2099: * If the entry is in transition, we must wait
2100: * for it to exit that state. Anything could happen
2101: * when we unlock the map, so start over.
2102: */
2103: if (entry->in_transition) {
2104:
2105: /*
2106: * Say that we are waiting, and wait for entry.
2107: */
2108: entry->needs_wakeup = TRUE;
2109: vm_map_entry_wait(dst_map, FALSE);
2110:
2111: goto start_pass_1;
2112: }
2113:
2114: if (size <= sub_size)
2115: break;
2116:
2117: if ((next == vm_map_to_entry(dst_map)) ||
2118: (next->vme_start != entry->vme_end)) {
2119: vm_map_unlock(dst_map);
2120: return(KERN_INVALID_ADDRESS);
2121: }
2122:
2123:
2124: /*
2125: * Check for permanent objects in the destination.
2126: */
2127:
2128: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2129: !entry->object.vm_object->temporary)
2130: contains_permanent_objects = TRUE;
2131:
2132: size -= sub_size;
2133: entry = next;
2134: }
2135:
2136: /*
2137: * If there are permanent objects in the destination, then
2138: * the copy cannot be interrupted.
2139: */
2140:
1.1.1.3 root 2141: if (interruptible && contains_permanent_objects) {
2142: vm_map_unlock(dst_map);
1.1 root 2143: return(KERN_FAILURE); /* XXX */
1.1.1.3 root 2144: }
1.1 root 2145:
2146: /*
2147: * XXXO If there are no permanent objects in the destination,
1.1.1.3 root 2148: * XXXO and the source and destination map entry caches match,
1.1.1.2 root 2149: * XXXO and the destination map entry is not shared,
1.1 root 2150: * XXXO then the map entries can be deleted and replaced
2151: * XXXO with those from the copy. The following code is the
2152: * XXXO basic idea of what to do, but there are lots of annoying
2153: * XXXO little details about getting protection and inheritance
2154: * XXXO right. Should add protection, inheritance, and sharing checks
2155: * XXXO to the above pass and make sure that no wiring is involved.
2156: */
2157: /*
2158: * if (!contains_permanent_objects &&
2159: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2160: *
2161: * *
2162: * * Run over copy and adjust entries. Steal code
2163: * * from vm_map_copyout() to do this.
2164: * *
2165: *
2166: * tmp_entry = tmp_entry->vme_prev;
2167: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2168: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2169: *
2170: * vm_map_unlock(dst_map);
2171: * vm_map_copy_discard(copy);
2172: * }
2173: */
2174: /*
2175: *
2176: * Make a second pass, overwriting the data
2177: * At the beginning of each loop iteration,
2178: * the next entry to be overwritten is "tmp_entry"
2179: * (initially, the value returned from the lookup above),
2180: * and the starting address expected in that entry
2181: * is "start".
2182: */
2183:
2184: start = dst_addr;
2185:
2186: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2187: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2188: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2189: vm_object_t object;
1.1.1.2 root 2190:
1.1 root 2191: entry = tmp_entry;
2192: size = (entry->vme_end - entry->vme_start);
2193: /*
2194: * Make sure that no holes popped up in the
2195: * address map, and that the protection is
2196: * still valid, in case the map was unlocked
2197: * earlier.
2198: */
2199:
2200: if (entry->vme_start != start) {
2201: vm_map_unlock(dst_map);
2202: return(KERN_INVALID_ADDRESS);
2203: }
2204: assert(entry != vm_map_to_entry(dst_map));
2205:
2206: /*
2207: * Check protection again
2208: */
2209:
2210: if ( ! (entry->protection & VM_PROT_WRITE)) {
2211: vm_map_unlock(dst_map);
2212: return(KERN_PROTECTION_FAILURE);
2213: }
2214:
2215: /*
2216: * Adjust to source size first
2217: */
2218:
2219: if (copy_size < size) {
2220: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2221: size = copy_size;
2222: }
2223:
2224: /*
2225: * Adjust to destination size
2226: */
2227:
2228: if (size < copy_size) {
2229: vm_map_copy_clip_end(copy, copy_entry,
2230: copy_entry->vme_start + size);
2231: copy_size = size;
2232: }
2233:
2234: assert((entry->vme_end - entry->vme_start) == size);
2235: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2236: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2237:
2238: /*
2239: * If the destination contains temporary unshared memory,
2240: * we can perform the copy by throwing it away and
2241: * installing the source data.
2242: */
2243:
2244: object = entry->object.vm_object;
2245: if (!entry->is_shared &&
2246: ((object == VM_OBJECT_NULL) || object->temporary)) {
2247: vm_object_t old_object = entry->object.vm_object;
2248: vm_offset_t old_offset = entry->offset;
2249:
2250: entry->object = copy_entry->object;
2251: entry->offset = copy_entry->offset;
2252: entry->needs_copy = copy_entry->needs_copy;
2253: entry->wired_count = 0;
1.1.1.5 root 2254: if (entry->user_wired_count)
2255: dst_map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 2256: entry->user_wired_count = 0;
2257:
2258: vm_map_copy_entry_unlink(copy, copy_entry);
2259: vm_map_copy_entry_dispose(copy, copy_entry);
2260:
2261: vm_object_pmap_protect(
2262: old_object,
2263: old_offset,
2264: size,
2265: dst_map->pmap,
2266: tmp_entry->vme_start,
2267: VM_PROT_NONE);
2268:
2269: vm_object_deallocate(old_object);
2270:
2271: /*
2272: * Set up for the next iteration. The map
2273: * has not been unlocked, so the next
2274: * address should be at the end of this
2275: * entry, and the next map entry should be
2276: * the one following it.
2277: */
2278:
2279: start = tmp_entry->vme_end;
2280: tmp_entry = tmp_entry->vme_next;
2281: } else {
2282: vm_map_version_t version;
2283: vm_object_t dst_object = entry->object.vm_object;
2284: vm_offset_t dst_offset = entry->offset;
2285: kern_return_t r;
2286:
2287: /*
2288: * Take an object reference, and record
2289: * the map version information so that the
2290: * map can be safely unlocked.
2291: */
2292:
2293: vm_object_reference(dst_object);
2294:
2295: version.main_timestamp = dst_map->timestamp;
2296:
2297: vm_map_unlock(dst_map);
2298:
2299: /*
2300: * Copy as much as possible in one pass
2301: */
2302:
2303: copy_size = size;
2304: r = vm_fault_copy(
2305: copy_entry->object.vm_object,
2306: copy_entry->offset,
2307: ©_size,
2308: dst_object,
2309: dst_offset,
2310: dst_map,
2311: &version,
2312: FALSE /* XXX interruptible */ );
2313:
2314: /*
2315: * Release the object reference
2316: */
2317:
2318: vm_object_deallocate(dst_object);
2319:
2320: /*
2321: * If a hard error occurred, return it now
2322: */
2323:
2324: if (r != KERN_SUCCESS)
2325: return(r);
2326:
2327: if (copy_size != 0) {
2328: /*
2329: * Dispose of the copied region
2330: */
2331:
2332: vm_map_copy_clip_end(copy, copy_entry,
2333: copy_entry->vme_start + copy_size);
2334: vm_map_copy_entry_unlink(copy, copy_entry);
2335: vm_object_deallocate(copy_entry->object.vm_object);
2336: vm_map_copy_entry_dispose(copy, copy_entry);
2337: }
2338:
2339: /*
2340: * Pick up in the destination map where we left off.
2341: *
2342: * Use the version information to avoid a lookup
2343: * in the normal case.
2344: */
2345:
2346: start += copy_size;
2347: vm_map_lock(dst_map);
2348: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2349: /* We can safely use saved tmp_entry value */
2350:
2351: vm_map_clip_end(dst_map, tmp_entry, start);
2352: tmp_entry = tmp_entry->vme_next;
2353: } else {
2354: /* Must do lookup of tmp_entry */
2355:
2356: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2357: vm_map_unlock(dst_map);
2358: return(KERN_INVALID_ADDRESS);
2359: }
2360: vm_map_clip_start(dst_map, tmp_entry, start);
2361: }
2362: }
2363:
2364: }
2365: vm_map_unlock(dst_map);
2366:
2367: /*
2368: * Throw away the vm_map_copy object
2369: */
2370: vm_map_copy_discard(copy);
2371:
2372: return(KERN_SUCCESS);
2373: }
2374:
2375: /*
2376: * Macro: vm_map_copy_insert
1.1.1.2 root 2377: *
1.1 root 2378: * Description:
2379: * Link a copy chain ("copy") into a map at the
2380: * specified location (after "where").
2381: * Side effects:
2382: * The copy chain is destroyed.
2383: * Warning:
2384: * The arguments are evaluated multiple times.
2385: */
2386: #define vm_map_copy_insert(map, where, copy) \
2387: MACRO_BEGIN \
1.1.1.3 root 2388: struct rbtree_node *node, *tmp; \
2389: rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \
2390: rbtree_insert(&(map)->hdr.tree, node, \
2391: vm_map_entry_cmp_insert); \
1.1 root 2392: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \
2393: ->vme_next = ((where)->vme_next); \
2394: ((where)->vme_next = vm_map_copy_first_entry(copy)) \
2395: ->vme_prev = (where); \
2396: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \
1.1.1.3 root 2397: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
1.1 root 2398: MACRO_END
2399:
2400: /*
2401: * Routine: vm_map_copyout
2402: *
2403: * Description:
2404: * Copy out a copy chain ("copy") into newly-allocated
2405: * space in the destination map.
2406: *
2407: * If successful, consumes the copy object.
2408: * Otherwise, the caller is responsible for it.
2409: */
1.1.1.4 root 2410: kern_return_t vm_map_copyout(
2411: vm_map_t dst_map,
2412: vm_offset_t *dst_addr, /* OUT */
2413: vm_map_copy_t copy)
1.1 root 2414: {
2415: vm_size_t size;
2416: vm_size_t adjustment;
2417: vm_offset_t start;
2418: vm_offset_t vm_copy_start;
2419: vm_map_entry_t last;
2420: vm_map_entry_t entry;
2421:
2422: /*
2423: * Check for null copy object.
2424: */
2425:
2426: if (copy == VM_MAP_COPY_NULL) {
2427: *dst_addr = 0;
2428: return(KERN_SUCCESS);
2429: }
2430:
2431: /*
2432: * Check for special copy object, created
2433: * by vm_map_copyin_object.
2434: */
2435:
2436: if (copy->type == VM_MAP_COPY_OBJECT) {
2437: vm_object_t object = copy->cpy_object;
2438: vm_size_t offset = copy->offset;
2439: vm_size_t tmp_size = copy->size;
2440: kern_return_t kr;
2441:
2442: *dst_addr = 0;
2443: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2444: (vm_offset_t) 0, TRUE,
2445: object, offset, FALSE,
2446: VM_PROT_DEFAULT, VM_PROT_ALL,
2447: VM_INHERIT_DEFAULT);
2448: if (kr != KERN_SUCCESS)
2449: return(kr);
1.1.1.3 root 2450: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2451: return(KERN_SUCCESS);
2452: }
2453:
2454: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2455: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2456:
2457: /*
2458: * Find space for the data
2459: */
2460:
2461: vm_copy_start = trunc_page(copy->offset);
2462: size = round_page(copy->offset + copy->size) - vm_copy_start;
2463:
2464: StartAgain: ;
2465:
2466: vm_map_lock(dst_map);
2467: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
2468: vm_map_min(dst_map) : last->vme_end;
2469:
2470: while (TRUE) {
2471: vm_map_entry_t next = last->vme_next;
2472: vm_offset_t end = start + size;
2473:
2474: if ((end > dst_map->max_offset) || (end < start)) {
2475: if (dst_map->wait_for_space) {
2476: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
2477: assert_wait((event_t) dst_map, TRUE);
2478: vm_map_unlock(dst_map);
2479: thread_block((void (*)()) 0);
2480: goto StartAgain;
2481: }
2482: }
2483: vm_map_unlock(dst_map);
1.1.1.3 root 2484: printf_once("no more room for vm_map_copyout in %p\n", dst_map);
1.1 root 2485: return(KERN_NO_SPACE);
2486: }
2487:
2488: if ((next == vm_map_to_entry(dst_map)) ||
2489: (next->vme_start >= end))
2490: break;
2491:
2492: last = next;
2493: start = last->vme_end;
2494: }
2495:
2496: /*
2497: * Since we're going to just drop the map
2498: * entries from the copy into the destination
2499: * map, they must come from the same pool.
2500: */
2501:
2502: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2503: /*
2504: * Mismatches occur when dealing with the default
2505: * pager.
2506: */
2507: vm_map_entry_t next, new;
2508:
2509: entry = vm_map_copy_first_entry(copy);
2510:
2511: /*
2512: * Reinitialize the copy so that vm_map_copy_entry_link
2513: * will work.
2514: */
2515: copy->cpy_hdr.nentries = 0;
2516: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
1.1.1.6 ! root 2517: rbtree_init(©->cpy_hdr.tree);
1.1 root 2518: vm_map_copy_first_entry(copy) =
2519: vm_map_copy_last_entry(copy) =
2520: vm_map_copy_to_entry(copy);
2521:
2522: /*
2523: * Copy each entry.
2524: */
2525: while (entry != vm_map_copy_to_entry(copy)) {
2526: new = vm_map_copy_entry_create(copy);
2527: vm_map_entry_copy_full(new, entry);
2528: vm_map_copy_entry_link(copy,
2529: vm_map_copy_last_entry(copy),
2530: new);
2531: next = entry->vme_next;
1.1.1.6 ! root 2532: kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry);
1.1 root 2533: entry = next;
2534: }
2535: }
2536:
2537: /*
2538: * Adjust the addresses in the copy chain, and
2539: * reset the region attributes.
2540: */
2541:
2542: adjustment = start - vm_copy_start;
2543: for (entry = vm_map_copy_first_entry(copy);
2544: entry != vm_map_copy_to_entry(copy);
2545: entry = entry->vme_next) {
2546: entry->vme_start += adjustment;
2547: entry->vme_end += adjustment;
2548:
2549: entry->inheritance = VM_INHERIT_DEFAULT;
2550: entry->protection = VM_PROT_DEFAULT;
2551: entry->max_protection = VM_PROT_ALL;
2552: entry->projected_on = 0;
2553:
2554: /*
2555: * If the entry is now wired,
2556: * map the pages into the destination map.
2557: */
2558: if (entry->wired_count != 0) {
1.1.1.4 root 2559: vm_offset_t va;
2560: vm_offset_t offset;
2561: vm_object_t object;
1.1 root 2562:
2563: object = entry->object.vm_object;
2564: offset = entry->offset;
2565: va = entry->vme_start;
2566:
2567: pmap_pageable(dst_map->pmap,
2568: entry->vme_start,
2569: entry->vme_end,
2570: TRUE);
2571:
2572: while (va < entry->vme_end) {
1.1.1.4 root 2573: vm_page_t m;
1.1 root 2574:
2575: /*
2576: * Look up the page in the object.
2577: * Assert that the page will be found in the
2578: * top object:
2579: * either
2580: * the object was newly created by
2581: * vm_object_copy_slowly, and has
2582: * copies of all of the pages from
2583: * the source object
2584: * or
2585: * the object was moved from the old
2586: * map entry; because the old map
2587: * entry was wired, all of the pages
2588: * were in the top-level object.
2589: * (XXX not true if we wire pages for
2590: * reading)
2591: */
2592: vm_object_lock(object);
2593: vm_object_paging_begin(object);
2594:
2595: m = vm_page_lookup(object, offset);
2596: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2597: m->absent)
2598: panic("vm_map_copyout: wiring 0x%x", m);
2599:
2600: m->busy = TRUE;
2601: vm_object_unlock(object);
2602:
2603: PMAP_ENTER(dst_map->pmap, va, m,
2604: entry->protection, TRUE);
2605:
2606: vm_object_lock(object);
2607: PAGE_WAKEUP_DONE(m);
2608: /* the page is wired, so we don't have to activate */
2609: vm_object_paging_end(object);
2610: vm_object_unlock(object);
2611:
2612: offset += PAGE_SIZE;
2613: va += PAGE_SIZE;
2614: }
2615: }
2616:
2617:
2618: }
2619:
2620: /*
2621: * Correct the page alignment for the result
2622: */
2623:
2624: *dst_addr = start + (copy->offset - vm_copy_start);
2625:
2626: /*
2627: * Update the hints and the map size
2628: */
2629:
2630: if (dst_map->first_free == last)
2631: dst_map->first_free = vm_map_copy_last_entry(copy);
2632: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2633:
2634: dst_map->size += size;
2635:
2636: /*
2637: * Link in the copy
2638: */
2639:
2640: vm_map_copy_insert(dst_map, last, copy);
2641:
2642: vm_map_unlock(dst_map);
2643:
2644: /*
2645: * XXX If wiring_required, call vm_map_pageable
2646: */
2647:
2648: return(KERN_SUCCESS);
2649: }
2650:
2651: /*
2652: *
2653: * vm_map_copyout_page_list:
2654: *
2655: * Version of vm_map_copyout() for page list vm map copies.
2656: *
2657: */
1.1.1.4 root 2658: kern_return_t vm_map_copyout_page_list(
2659: vm_map_t dst_map,
2660: vm_offset_t *dst_addr, /* OUT */
2661: vm_map_copy_t copy)
1.1 root 2662: {
2663: vm_size_t size;
2664: vm_offset_t start;
2665: vm_offset_t end;
2666: vm_offset_t offset;
2667: vm_map_entry_t last;
2668: vm_object_t object;
2669: vm_page_t *page_list, m;
2670: vm_map_entry_t entry;
2671: vm_offset_t old_last_offset;
2672: boolean_t cont_invoked, needs_wakeup = FALSE;
2673: kern_return_t result = KERN_SUCCESS;
2674: vm_map_copy_t orig_copy;
2675: vm_offset_t dst_offset;
2676: boolean_t must_wire;
2677:
2678: /*
2679: * Make sure the pages are stolen, because we are
2680: * going to put them in a new object. Assume that
2681: * all pages are identical to first in this regard.
2682: */
2683:
2684: page_list = ©->cpy_page_list[0];
2685: if ((*page_list)->tabled)
2686: vm_map_copy_steal_pages(copy);
2687:
2688: /*
2689: * Find space for the data
2690: */
2691:
2692: size = round_page(copy->offset + copy->size) -
2693: trunc_page(copy->offset);
2694: StartAgain:
2695: vm_map_lock(dst_map);
2696: must_wire = dst_map->wiring_required;
2697:
2698: last = dst_map->first_free;
2699: if (last == vm_map_to_entry(dst_map)) {
2700: start = vm_map_min(dst_map);
2701: } else {
2702: start = last->vme_end;
2703: }
2704:
2705: while (TRUE) {
2706: vm_map_entry_t next = last->vme_next;
2707: end = start + size;
2708:
2709: if ((end > dst_map->max_offset) || (end < start)) {
2710: if (dst_map->wait_for_space) {
2711: if (size <= (dst_map->max_offset -
2712: dst_map->min_offset)) {
2713: assert_wait((event_t) dst_map, TRUE);
2714: vm_map_unlock(dst_map);
2715: thread_block((void (*)()) 0);
2716: goto StartAgain;
2717: }
2718: }
2719: vm_map_unlock(dst_map);
1.1.1.3 root 2720: printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map);
1.1 root 2721: return(KERN_NO_SPACE);
2722: }
2723:
2724: if ((next == vm_map_to_entry(dst_map)) ||
2725: (next->vme_start >= end)) {
2726: break;
2727: }
2728:
2729: last = next;
2730: start = last->vme_end;
2731: }
2732:
2733: /*
2734: * See whether we can avoid creating a new entry (and object) by
2735: * extending one of our neighbors. [So far, we only attempt to
2736: * extend from below.]
2737: *
2738: * The code path below here is a bit twisted. If any of the
2739: * extension checks fails, we branch to create_object. If
2740: * it all works, we fall out the bottom and goto insert_pages.
2741: */
2742: if (last == vm_map_to_entry(dst_map) ||
2743: last->vme_end != start ||
2744: last->is_shared != FALSE ||
2745: last->is_sub_map != FALSE ||
2746: last->inheritance != VM_INHERIT_DEFAULT ||
2747: last->protection != VM_PROT_DEFAULT ||
2748: last->max_protection != VM_PROT_ALL ||
2749: (must_wire ? (last->wired_count != 1 ||
2750: last->user_wired_count != 1) :
2751: (last->wired_count != 0))) {
2752: goto create_object;
2753: }
1.1.1.2 root 2754:
1.1 root 2755: /*
2756: * If this entry needs an object, make one.
2757: */
2758: if (last->object.vm_object == VM_OBJECT_NULL) {
2759: object = vm_object_allocate(
2760: (vm_size_t)(last->vme_end - last->vme_start + size));
2761: last->object.vm_object = object;
2762: last->offset = 0;
2763: vm_object_lock(object);
2764: }
2765: else {
2766: vm_offset_t prev_offset = last->offset;
2767: vm_size_t prev_size = start - last->vme_start;
2768: vm_size_t new_size;
2769:
2770: /*
2771: * This is basically vm_object_coalesce.
2772: */
2773:
2774: object = last->object.vm_object;
2775: vm_object_lock(object);
2776:
2777: /*
2778: * Try to collapse the object first
2779: */
2780: vm_object_collapse(object);
2781:
2782: /*
2783: * Can't coalesce if pages not mapped to
2784: * last may be in use anyway:
2785: * . more than one reference
2786: * . paged out
2787: * . shadows another object
2788: * . has a copy elsewhere
2789: * . paging references (pages might be in page-list)
2790: */
2791:
2792: if ((object->ref_count > 1) ||
2793: object->pager_created ||
2794: (object->shadow != VM_OBJECT_NULL) ||
2795: (object->copy != VM_OBJECT_NULL) ||
2796: (object->paging_in_progress != 0)) {
2797: vm_object_unlock(object);
2798: goto create_object;
2799: }
2800:
2801: /*
2802: * Extend the object if necessary. Don't have to call
2803: * vm_object_page_remove because the pages aren't mapped,
2804: * and vm_page_replace will free up any old ones it encounters.
2805: */
2806: new_size = prev_offset + prev_size + size;
2807: if (new_size > object->size)
2808: object->size = new_size;
2809: }
2810:
2811: /*
2812: * Coalesced the two objects - can extend
2813: * the previous map entry to include the
2814: * new range.
2815: */
2816: dst_map->size += size;
2817: last->vme_end = end;
2818:
2819: SAVE_HINT(dst_map, last);
2820:
2821: goto insert_pages;
2822:
2823: create_object:
2824:
2825: /*
2826: * Create object
2827: */
2828: object = vm_object_allocate(size);
2829:
2830: /*
2831: * Create entry
2832: */
2833:
2834: entry = vm_map_entry_create(dst_map);
2835:
2836: entry->object.vm_object = object;
2837: entry->offset = 0;
2838:
2839: entry->is_shared = FALSE;
2840: entry->is_sub_map = FALSE;
2841: entry->needs_copy = FALSE;
2842:
2843: if (must_wire) {
2844: entry->wired_count = 1;
1.1.1.5 root 2845: dst_map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 2846: entry->user_wired_count = 1;
2847: } else {
2848: entry->wired_count = 0;
2849: entry->user_wired_count = 0;
2850: }
2851:
2852: entry->in_transition = TRUE;
2853: entry->needs_wakeup = FALSE;
2854:
2855: entry->vme_start = start;
2856: entry->vme_end = start + size;
1.1.1.2 root 2857:
1.1 root 2858: entry->inheritance = VM_INHERIT_DEFAULT;
2859: entry->protection = VM_PROT_DEFAULT;
2860: entry->max_protection = VM_PROT_ALL;
2861: entry->projected_on = 0;
2862:
2863: vm_object_lock(object);
2864:
2865: /*
2866: * Update the hints and the map size
2867: */
2868: if (dst_map->first_free == last) {
2869: dst_map->first_free = entry;
2870: }
2871: SAVE_HINT(dst_map, entry);
2872: dst_map->size += size;
2873:
2874: /*
2875: * Link in the entry
2876: */
2877: vm_map_entry_link(dst_map, last, entry);
2878: last = entry;
2879:
2880: /*
1.1.1.2 root 2881: * Transfer pages into new object.
1.1 root 2882: * Scan page list in vm_map_copy.
2883: */
2884: insert_pages:
2885: dst_offset = copy->offset & PAGE_MASK;
2886: cont_invoked = FALSE;
2887: orig_copy = copy;
2888: last->in_transition = TRUE;
2889: old_last_offset = last->offset
2890: + (start - last->vme_start);
2891:
2892: vm_page_lock_queues();
2893:
2894: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2895: m = *page_list;
2896: assert(m && !m->tabled);
2897:
2898: /*
2899: * Must clear busy bit in page before inserting it.
2900: * Ok to skip wakeup logic because nobody else
2901: * can possibly know about this page.
2902: * The page is dirty in its new object.
2903: */
2904:
2905: assert(!m->wanted);
2906:
2907: m->busy = FALSE;
2908: m->dirty = TRUE;
2909: vm_page_replace(m, object, old_last_offset + offset);
2910: if (must_wire) {
2911: vm_page_wire(m);
2912: PMAP_ENTER(dst_map->pmap,
2913: last->vme_start + m->offset - last->offset,
2914: m, last->protection, TRUE);
2915: } else {
2916: vm_page_activate(m);
2917: }
2918:
2919: *page_list++ = VM_PAGE_NULL;
2920: if (--(copy->cpy_npages) == 0 &&
2921: vm_map_copy_has_cont(copy)) {
2922: vm_map_copy_t new_copy;
2923:
2924: /*
2925: * Ok to unlock map because entry is
2926: * marked in_transition.
2927: */
2928: cont_invoked = TRUE;
2929: vm_page_unlock_queues();
2930: vm_object_unlock(object);
2931: vm_map_unlock(dst_map);
2932: vm_map_copy_invoke_cont(copy, &new_copy, &result);
2933:
2934: if (result == KERN_SUCCESS) {
2935:
2936: /*
2937: * If we got back a copy with real pages,
2938: * steal them now. Either all of the
2939: * pages in the list are tabled or none
2940: * of them are; mixtures are not possible.
2941: *
2942: * Save original copy for consume on
2943: * success logic at end of routine.
2944: */
2945: if (copy != orig_copy)
2946: vm_map_copy_discard(copy);
2947:
2948: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
2949: page_list = ©->cpy_page_list[0];
2950: if ((*page_list)->tabled)
2951: vm_map_copy_steal_pages(copy);
2952: }
2953: }
2954: else {
2955: /*
2956: * Continuation failed.
2957: */
2958: vm_map_lock(dst_map);
2959: goto error;
2960: }
2961:
2962: vm_map_lock(dst_map);
2963: vm_object_lock(object);
2964: vm_page_lock_queues();
2965: }
2966: }
2967:
2968: vm_page_unlock_queues();
2969: vm_object_unlock(object);
2970:
2971: *dst_addr = start + dst_offset;
1.1.1.2 root 2972:
1.1 root 2973: /*
2974: * Clear the in transition bits. This is easy if we
2975: * didn't have a continuation.
2976: */
2977: error:
2978: if (!cont_invoked) {
2979: /*
2980: * We didn't unlock the map, so nobody could
2981: * be waiting.
2982: */
2983: last->in_transition = FALSE;
2984: assert(!last->needs_wakeup);
2985: needs_wakeup = FALSE;
2986: }
2987: else {
2988: if (!vm_map_lookup_entry(dst_map, start, &entry))
2989: panic("vm_map_copyout_page_list: missing entry");
2990:
2991: /*
2992: * Clear transition bit for all constituent entries that
2993: * were in the original entry. Also check for waiters.
2994: */
2995: while((entry != vm_map_to_entry(dst_map)) &&
2996: (entry->vme_start < end)) {
2997: assert(entry->in_transition);
2998: entry->in_transition = FALSE;
2999: if(entry->needs_wakeup) {
3000: entry->needs_wakeup = FALSE;
3001: needs_wakeup = TRUE;
3002: }
3003: entry = entry->vme_next;
3004: }
3005: }
1.1.1.2 root 3006:
1.1 root 3007: if (result != KERN_SUCCESS)
3008: vm_map_delete(dst_map, start, end);
3009:
3010: vm_map_unlock(dst_map);
3011:
3012: if (needs_wakeup)
3013: vm_map_entry_wakeup(dst_map);
3014:
3015: /*
3016: * Consume on success logic.
3017: */
3018: if (copy != orig_copy) {
1.1.1.3 root 3019: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 3020: }
3021: if (result == KERN_SUCCESS) {
1.1.1.3 root 3022: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
1.1 root 3023: }
1.1.1.2 root 3024:
1.1 root 3025: return(result);
3026: }
3027:
3028: /*
3029: * Routine: vm_map_copyin
3030: *
3031: * Description:
3032: * Copy the specified region (src_addr, len) from the
3033: * source address space (src_map), possibly removing
3034: * the region from the source address space (src_destroy).
3035: *
3036: * Returns:
3037: * A vm_map_copy_t object (copy_result), suitable for
3038: * insertion into another address space (using vm_map_copyout),
3039: * copying over another address space region (using
3040: * vm_map_copy_overwrite). If the copy is unused, it
3041: * should be destroyed (using vm_map_copy_discard).
3042: *
3043: * In/out conditions:
3044: * The source map should not be locked on entry.
3045: */
1.1.1.4 root 3046: kern_return_t vm_map_copyin(
3047: vm_map_t src_map,
3048: vm_offset_t src_addr,
3049: vm_size_t len,
3050: boolean_t src_destroy,
3051: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3052: {
3053: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3054: * in multi-level lookup, this
3055: * entry contains the actual
3056: * vm_object/offset.
3057: */
3058:
3059: vm_offset_t src_start; /* Start of current entry --
3060: * where copy is taking place now
3061: */
3062: vm_offset_t src_end; /* End of entire region to be
3063: * copied */
3064:
3065: vm_map_copy_t copy; /* Resulting copy */
3066:
3067: /*
3068: * Check for copies of zero bytes.
3069: */
3070:
3071: if (len == 0) {
3072: *copy_result = VM_MAP_COPY_NULL;
3073: return(KERN_SUCCESS);
3074: }
3075:
3076: /*
3077: * Compute start and end of region
3078: */
3079:
3080: src_start = trunc_page(src_addr);
3081: src_end = round_page(src_addr + len);
3082:
3083: /*
3084: * Check that the end address doesn't overflow
3085: */
3086:
3087: if (src_end <= src_start)
3088: if ((src_end < src_start) || (src_start != 0))
3089: return(KERN_INVALID_ADDRESS);
3090:
3091: /*
3092: * Allocate a header element for the list.
3093: *
1.1.1.2 root 3094: * Use the start and end in the header to
1.1 root 3095: * remember the endpoints prior to rounding.
3096: */
3097:
1.1.1.3 root 3098: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3099: vm_map_copy_first_entry(copy) =
3100: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3101: copy->type = VM_MAP_COPY_ENTRY_LIST;
3102: copy->cpy_hdr.nentries = 0;
3103: copy->cpy_hdr.entries_pageable = TRUE;
1.1.1.3 root 3104: rbtree_init(©->cpy_hdr.tree);
1.1 root 3105:
3106: copy->offset = src_addr;
3107: copy->size = len;
1.1.1.2 root 3108:
1.1 root 3109: #define RETURN(x) \
3110: MACRO_BEGIN \
3111: vm_map_unlock(src_map); \
3112: vm_map_copy_discard(copy); \
3113: MACRO_RETURN(x); \
3114: MACRO_END
3115:
3116: /*
3117: * Find the beginning of the region.
3118: */
3119:
3120: vm_map_lock(src_map);
3121:
3122: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3123: RETURN(KERN_INVALID_ADDRESS);
3124: vm_map_clip_start(src_map, tmp_entry, src_start);
3125:
3126: /*
3127: * Go through entries until we get to the end.
3128: */
3129:
3130: while (TRUE) {
3131: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3132: vm_size_t src_size; /* Size of source
3133: * map entry (in both
3134: * maps)
3135: */
3136:
3137: vm_object_t src_object; /* Object to copy */
3138: vm_offset_t src_offset;
3139:
3140: boolean_t src_needs_copy; /* Should source map
3141: * be made read-only
3142: * for copy-on-write?
3143: */
3144:
3145: vm_map_entry_t new_entry; /* Map entry for copy */
3146: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3147:
3148: boolean_t was_wired; /* Was source wired? */
3149: vm_map_version_t version; /* Version before locks
3150: * dropped to make copy
3151: */
3152:
3153: /*
3154: * Verify that the region can be read.
3155: */
3156:
3157: if (! (src_entry->protection & VM_PROT_READ))
3158: RETURN(KERN_PROTECTION_FAILURE);
3159:
3160: /*
3161: * Clip against the endpoints of the entire region.
3162: */
3163:
3164: vm_map_clip_end(src_map, src_entry, src_end);
3165:
3166: src_size = src_entry->vme_end - src_start;
3167: src_object = src_entry->object.vm_object;
3168: src_offset = src_entry->offset;
3169: was_wired = (src_entry->wired_count != 0);
3170:
3171: /*
3172: * Create a new address map entry to
3173: * hold the result. Fill in the fields from
3174: * the appropriate source entries.
3175: */
3176:
3177: new_entry = vm_map_copy_entry_create(copy);
3178: vm_map_entry_copy(new_entry, src_entry);
3179:
3180: /*
3181: * Attempt non-blocking copy-on-write optimizations.
3182: */
3183:
3184: if (src_destroy &&
3185: (src_object == VM_OBJECT_NULL ||
3186: (src_object->temporary && !src_object->use_shared_copy)))
3187: {
3188: /*
3189: * If we are destroying the source, and the object
3190: * is temporary, and not shared writable,
3191: * we can move the object reference
3192: * from the source to the copy. The copy is
3193: * copy-on-write only if the source is.
3194: * We make another reference to the object, because
3195: * destroying the source entry will deallocate it.
3196: */
3197: vm_object_reference(src_object);
3198:
3199: /*
3200: * Copy is always unwired. vm_map_copy_entry
3201: * set its wired count to zero.
3202: */
3203:
3204: goto CopySuccessful;
3205: }
3206:
3207: if (!was_wired &&
3208: vm_object_copy_temporary(
3209: &new_entry->object.vm_object,
3210: &new_entry->offset,
3211: &src_needs_copy,
3212: &new_entry_needs_copy)) {
3213:
3214: new_entry->needs_copy = new_entry_needs_copy;
3215:
3216: /*
3217: * Handle copy-on-write obligations
3218: */
3219:
3220: if (src_needs_copy && !tmp_entry->needs_copy) {
3221: vm_object_pmap_protect(
3222: src_object,
3223: src_offset,
3224: src_size,
3225: (src_entry->is_shared ? PMAP_NULL
3226: : src_map->pmap),
3227: src_entry->vme_start,
3228: src_entry->protection &
3229: ~VM_PROT_WRITE);
3230:
3231: tmp_entry->needs_copy = TRUE;
3232: }
3233:
3234: /*
3235: * The map has never been unlocked, so it's safe to
3236: * move to the next entry rather than doing another
3237: * lookup.
3238: */
3239:
3240: goto CopySuccessful;
3241: }
3242:
3243: new_entry->needs_copy = FALSE;
3244:
3245: /*
3246: * Take an object reference, so that we may
3247: * release the map lock(s).
3248: */
3249:
3250: assert(src_object != VM_OBJECT_NULL);
3251: vm_object_reference(src_object);
3252:
3253: /*
3254: * Record the timestamp for later verification.
3255: * Unlock the map.
3256: */
3257:
3258: version.main_timestamp = src_map->timestamp;
3259: vm_map_unlock(src_map);
3260:
3261: /*
3262: * Perform the copy
3263: */
3264:
3265: if (was_wired) {
3266: vm_object_lock(src_object);
3267: (void) vm_object_copy_slowly(
3268: src_object,
3269: src_offset,
3270: src_size,
3271: FALSE,
3272: &new_entry->object.vm_object);
3273: new_entry->offset = 0;
3274: new_entry->needs_copy = FALSE;
3275: } else {
3276: kern_return_t result;
3277:
3278: result = vm_object_copy_strategically(src_object,
3279: src_offset,
3280: src_size,
3281: &new_entry->object.vm_object,
3282: &new_entry->offset,
3283: &new_entry_needs_copy);
3284:
3285: new_entry->needs_copy = new_entry_needs_copy;
1.1.1.2 root 3286:
1.1 root 3287:
3288: if (result != KERN_SUCCESS) {
3289: vm_map_copy_entry_dispose(copy, new_entry);
3290:
3291: vm_map_lock(src_map);
3292: RETURN(result);
3293: }
3294:
3295: }
3296:
3297: /*
3298: * Throw away the extra reference
3299: */
3300:
3301: vm_object_deallocate(src_object);
3302:
3303: /*
3304: * Verify that the map has not substantially
3305: * changed while the copy was being made.
3306: */
3307:
3308: vm_map_lock(src_map); /* Increments timestamp once! */
3309:
3310: if ((version.main_timestamp + 1) == src_map->timestamp)
3311: goto CopySuccessful;
3312:
3313: /*
3314: * Simple version comparison failed.
3315: *
3316: * Retry the lookup and verify that the
3317: * same object/offset are still present.
3318: *
3319: * [Note: a memory manager that colludes with
3320: * the calling task can detect that we have
3321: * cheated. While the map was unlocked, the
3322: * mapping could have been changed and restored.]
3323: */
3324:
3325: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3326: vm_map_copy_entry_dispose(copy, new_entry);
3327: RETURN(KERN_INVALID_ADDRESS);
3328: }
3329:
3330: src_entry = tmp_entry;
3331: vm_map_clip_start(src_map, src_entry, src_start);
3332:
3333: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3334: goto VerificationFailed;
3335:
3336: if (src_entry->vme_end < new_entry->vme_end)
3337: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3338:
3339: if ((src_entry->object.vm_object != src_object) ||
3340: (src_entry->offset != src_offset) ) {
3341:
3342: /*
3343: * Verification failed.
3344: *
3345: * Start over with this top-level entry.
3346: */
3347:
3348: VerificationFailed: ;
3349:
3350: vm_object_deallocate(new_entry->object.vm_object);
3351: vm_map_copy_entry_dispose(copy, new_entry);
3352: tmp_entry = src_entry;
3353: continue;
3354: }
3355:
3356: /*
3357: * Verification succeeded.
3358: */
3359:
3360: CopySuccessful: ;
3361:
3362: /*
3363: * Link in the new copy entry.
3364: */
3365:
3366: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3367: new_entry);
1.1.1.2 root 3368:
1.1 root 3369: /*
3370: * Determine whether the entire region
3371: * has been copied.
3372: */
3373: src_start = new_entry->vme_end;
3374: if ((src_start >= src_end) && (src_end != 0))
3375: break;
3376:
3377: /*
3378: * Verify that there are no gaps in the region
3379: */
3380:
3381: tmp_entry = src_entry->vme_next;
3382: if (tmp_entry->vme_start != src_start)
3383: RETURN(KERN_INVALID_ADDRESS);
3384: }
3385:
3386: /*
3387: * If the source should be destroyed, do it now, since the
1.1.1.2 root 3388: * copy was successful.
1.1 root 3389: */
3390: if (src_destroy)
3391: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3392:
3393: vm_map_unlock(src_map);
3394:
3395: *copy_result = copy;
3396: return(KERN_SUCCESS);
3397:
3398: #undef RETURN
3399: }
3400:
3401: /*
3402: * vm_map_copyin_object:
3403: *
3404: * Create a copy object from an object.
3405: * Our caller donates an object reference.
3406: */
3407:
1.1.1.4 root 3408: kern_return_t vm_map_copyin_object(
3409: vm_object_t object,
3410: vm_offset_t offset, /* offset of region in object */
3411: vm_size_t size, /* size of region in object */
3412: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3413: {
3414: vm_map_copy_t copy; /* Resulting copy */
3415:
3416: /*
3417: * We drop the object into a special copy object
3418: * that contains the object directly. These copy objects
3419: * are distinguished by entries_pageable == FALSE
3420: * and null links.
3421: */
3422:
1.1.1.3 root 3423: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3424: vm_map_copy_first_entry(copy) =
3425: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3426: copy->type = VM_MAP_COPY_OBJECT;
3427: copy->cpy_object = object;
3428: copy->offset = offset;
3429: copy->size = size;
3430:
3431: *copy_result = copy;
3432: return(KERN_SUCCESS);
3433: }
3434:
3435: /*
3436: * vm_map_copyin_page_list_cont:
3437: *
3438: * Continuation routine for vm_map_copyin_page_list.
1.1.1.2 root 3439: *
1.1 root 3440: * If vm_map_copyin_page_list can't fit the entire vm range
3441: * into a single page list object, it creates a continuation.
3442: * When the target of the operation has used the pages in the
3443: * initial page list, it invokes the continuation, which calls
3444: * this routine. If an error happens, the continuation is aborted
3445: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3446: * pages are discarded from the initial page list before invoking
3447: * the continuation.
3448: *
3449: * NOTE: This is not the same sort of continuation used by
3450: * the scheduler.
3451: */
3452:
1.1.1.4 root 3453: kern_return_t vm_map_copyin_page_list_cont(
3454: vm_map_copyin_args_t cont_args,
3455: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3456: {
3457: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
1.1.1.4 root 3458: boolean_t do_abort, src_destroy, src_destroy_only;
1.1 root 3459:
3460: /*
3461: * Check for cases that only require memory destruction.
3462: */
3463: do_abort = (copy_result == (vm_map_copy_t *) 0);
3464: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3465: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3466:
3467: if (do_abort || src_destroy_only) {
1.1.1.2 root 3468: if (src_destroy)
1.1 root 3469: result = vm_map_remove(cont_args->map,
3470: cont_args->destroy_addr,
3471: cont_args->destroy_addr + cont_args->destroy_len);
3472: if (!do_abort)
3473: *copy_result = VM_MAP_COPY_NULL;
3474: }
3475: else {
3476: result = vm_map_copyin_page_list(cont_args->map,
3477: cont_args->src_addr, cont_args->src_len, src_destroy,
3478: cont_args->steal_pages, copy_result, TRUE);
3479:
3480: if (src_destroy && !cont_args->steal_pages &&
3481: vm_map_copy_has_cont(*copy_result)) {
3482: vm_map_copyin_args_t new_args;
3483: /*
3484: * Transfer old destroy info.
3485: */
3486: new_args = (vm_map_copyin_args_t)
3487: (*copy_result)->cpy_cont_args;
3488: new_args->destroy_addr = cont_args->destroy_addr;
3489: new_args->destroy_len = cont_args->destroy_len;
3490: }
3491: }
1.1.1.2 root 3492:
1.1 root 3493: vm_map_deallocate(cont_args->map);
3494: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3495:
3496: return(result);
3497: }
3498:
3499: /*
3500: * vm_map_copyin_page_list:
3501: *
3502: * This is a variant of vm_map_copyin that copies in a list of pages.
3503: * If steal_pages is TRUE, the pages are only in the returned list.
3504: * If steal_pages is FALSE, the pages are busy and still in their
3505: * objects. A continuation may be returned if not all the pages fit:
3506: * the recipient of this copy_result must be prepared to deal with it.
3507: */
3508:
1.1.1.4 root 3509: kern_return_t vm_map_copyin_page_list(
3510: vm_map_t src_map,
3511: vm_offset_t src_addr,
3512: vm_size_t len,
3513: boolean_t src_destroy,
3514: boolean_t steal_pages,
3515: vm_map_copy_t *copy_result, /* OUT */
3516: boolean_t is_cont)
1.1 root 3517: {
3518: vm_map_entry_t src_entry;
3519: vm_page_t m;
3520: vm_offset_t src_start;
3521: vm_offset_t src_end;
3522: vm_size_t src_size;
3523: vm_object_t src_object;
3524: vm_offset_t src_offset;
3525: vm_offset_t src_last_offset;
3526: vm_map_copy_t copy; /* Resulting copy */
3527: kern_return_t result = KERN_SUCCESS;
3528: boolean_t need_map_lookup;
3529: vm_map_copyin_args_t cont_args;
3530:
3531: /*
3532: * If steal_pages is FALSE, this leaves busy pages in
3533: * the object. A continuation must be used if src_destroy
3534: * is true in this case (!steal_pages && src_destroy).
3535: *
3536: * XXX Still have a more general problem of what happens
3537: * XXX if the same page occurs twice in a list. Deadlock
3538: * XXX can happen if vm_fault_page was called. A
3539: * XXX possible solution is to use a continuation if vm_fault_page
3540: * XXX is called and we cross a map entry boundary.
3541: */
3542:
3543: /*
3544: * Check for copies of zero bytes.
3545: */
3546:
3547: if (len == 0) {
3548: *copy_result = VM_MAP_COPY_NULL;
3549: return(KERN_SUCCESS);
3550: }
3551:
3552: /*
3553: * Compute start and end of region
3554: */
3555:
3556: src_start = trunc_page(src_addr);
3557: src_end = round_page(src_addr + len);
3558:
3559: /*
3560: * Check that the end address doesn't overflow
3561: */
3562:
3563: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3564: return KERN_INVALID_ADDRESS;
3565: }
3566:
3567: /*
3568: * Allocate a header element for the page list.
3569: *
3570: * Record original offset and size, as caller may not
3571: * be page-aligned.
3572: */
3573:
1.1.1.3 root 3574: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3575: copy->type = VM_MAP_COPY_PAGE_LIST;
3576: copy->cpy_npages = 0;
3577: copy->offset = src_addr;
3578: copy->size = len;
3579: copy->cpy_cont = ((kern_return_t (*)()) 0);
3580: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
1.1.1.2 root 3581:
1.1 root 3582: /*
3583: * Find the beginning of the region.
3584: */
3585:
3586: do_map_lookup:
3587:
3588: vm_map_lock(src_map);
3589:
3590: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3591: result = KERN_INVALID_ADDRESS;
3592: goto error;
3593: }
3594: need_map_lookup = FALSE;
3595:
3596: /*
3597: * Go through entries until we get to the end.
3598: */
3599:
3600: while (TRUE) {
3601:
3602: if (! (src_entry->protection & VM_PROT_READ)) {
3603: result = KERN_PROTECTION_FAILURE;
3604: goto error;
3605: }
3606:
3607: if (src_end > src_entry->vme_end)
3608: src_size = src_entry->vme_end - src_start;
3609: else
3610: src_size = src_end - src_start;
3611:
3612: src_object = src_entry->object.vm_object;
3613: src_offset = src_entry->offset +
3614: (src_start - src_entry->vme_start);
3615:
3616: /*
3617: * If src_object is NULL, allocate it now;
3618: * we're going to fault on it shortly.
3619: */
3620: if (src_object == VM_OBJECT_NULL) {
3621: src_object = vm_object_allocate((vm_size_t)
3622: src_entry->vme_end -
3623: src_entry->vme_start);
3624: src_entry->object.vm_object = src_object;
3625: }
3626:
3627: /*
3628: * Iterate over pages. Fault in ones that aren't present.
3629: */
3630: src_last_offset = src_offset + src_size;
3631: for (; (src_offset < src_last_offset && !need_map_lookup);
3632: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3633:
3634: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3635: make_continuation:
3636: /*
3637: * At this point we have the max number of
3638: * pages busy for this thread that we're
3639: * willing to allow. Stop here and record
3640: * arguments for the remainder. Note:
3641: * this means that this routine isn't atomic,
3642: * but that's the breaks. Note that only
3643: * the first vm_map_copy_t that comes back
3644: * from this routine has the right offset
3645: * and size; those from continuations are
3646: * page rounded, and short by the amount
3647: * already done.
3648: *
3649: * Reset src_end so the src_destroy
3650: * code at the bottom doesn't do
3651: * something stupid.
3652: */
3653:
1.1.1.2 root 3654: cont_args = (vm_map_copyin_args_t)
1.1 root 3655: kalloc(sizeof(vm_map_copyin_args_data_t));
3656: cont_args->map = src_map;
3657: vm_map_reference(src_map);
3658: cont_args->src_addr = src_start;
3659: cont_args->src_len = len - (src_start - src_addr);
3660: if (src_destroy) {
3661: cont_args->destroy_addr = cont_args->src_addr;
3662: cont_args->destroy_len = cont_args->src_len;
3663: }
3664: else {
3665: cont_args->destroy_addr = (vm_offset_t) 0;
3666: cont_args->destroy_len = (vm_offset_t) 0;
3667: }
3668: cont_args->steal_pages = steal_pages;
3669:
3670: copy->cpy_cont_args = (char *) cont_args;
3671: copy->cpy_cont = vm_map_copyin_page_list_cont;
3672:
3673: src_end = src_start;
3674: vm_map_clip_end(src_map, src_entry, src_end);
3675: break;
3676: }
3677:
3678: /*
3679: * Try to find the page of data.
3680: */
3681: vm_object_lock(src_object);
3682: vm_object_paging_begin(src_object);
3683: if (((m = vm_page_lookup(src_object, src_offset)) !=
3684: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3685: !m->absent && !m->error) {
3686:
3687: /*
3688: * This is the page. Mark it busy
3689: * and keep the paging reference on
3690: * the object whilst we do our thing.
3691: */
3692: m->busy = TRUE;
3693:
3694: /*
3695: * Also write-protect the page, so
3696: * that the map`s owner cannot change
3697: * the data. The busy bit will prevent
3698: * faults on the page from succeeding
3699: * until the copy is released; after
3700: * that, the page can be re-entered
3701: * as writable, since we didn`t alter
3702: * the map entry. This scheme is a
3703: * cheap copy-on-write.
3704: *
3705: * Don`t forget the protection and
3706: * the page_lock value!
3707: *
3708: * If the source is being destroyed
3709: * AND not shared writable, we don`t
3710: * have to protect the page, since
3711: * we will destroy the (only)
3712: * writable mapping later.
3713: */
3714: if (!src_destroy ||
3715: src_object->use_shared_copy)
3716: {
3717: pmap_page_protect(m->phys_addr,
3718: src_entry->protection
3719: & ~m->page_lock
3720: & ~VM_PROT_WRITE);
3721: }
3722:
3723: }
3724: else {
3725: vm_prot_t result_prot;
3726: vm_page_t top_page;
3727: kern_return_t kr;
1.1.1.2 root 3728:
1.1 root 3729: /*
3730: * Have to fault the page in; must
3731: * unlock the map to do so. While
3732: * the map is unlocked, anything
3733: * can happen, we must lookup the
3734: * map entry before continuing.
3735: */
3736: vm_map_unlock(src_map);
3737: need_map_lookup = TRUE;
3738: retry:
3739: result_prot = VM_PROT_READ;
1.1.1.2 root 3740:
1.1 root 3741: kr = vm_fault_page(src_object, src_offset,
3742: VM_PROT_READ, FALSE, FALSE,
3743: &result_prot, &m, &top_page,
3744: FALSE, (void (*)()) 0);
3745: /*
3746: * Cope with what happened.
3747: */
3748: switch (kr) {
3749: case VM_FAULT_SUCCESS:
3750: break;
3751: case VM_FAULT_INTERRUPTED: /* ??? */
3752: case VM_FAULT_RETRY:
3753: vm_object_lock(src_object);
3754: vm_object_paging_begin(src_object);
3755: goto retry;
3756: case VM_FAULT_MEMORY_SHORTAGE:
3757: VM_PAGE_WAIT((void (*)()) 0);
3758: vm_object_lock(src_object);
3759: vm_object_paging_begin(src_object);
3760: goto retry;
3761: case VM_FAULT_FICTITIOUS_SHORTAGE:
3762: vm_page_more_fictitious();
3763: vm_object_lock(src_object);
3764: vm_object_paging_begin(src_object);
3765: goto retry;
3766: case VM_FAULT_MEMORY_ERROR:
3767: /*
3768: * Something broke. If this
3769: * is a continuation, return
3770: * a partial result if possible,
3771: * else fail the whole thing.
3772: * In the continuation case, the
3773: * next continuation call will
3774: * get this error if it persists.
3775: */
3776: vm_map_lock(src_map);
3777: if (is_cont &&
3778: copy->cpy_npages != 0)
3779: goto make_continuation;
3780:
3781: result = KERN_MEMORY_ERROR;
3782: goto error;
3783: }
1.1.1.2 root 3784:
1.1 root 3785: if (top_page != VM_PAGE_NULL) {
3786: vm_object_lock(src_object);
3787: VM_PAGE_FREE(top_page);
3788: vm_object_paging_end(src_object);
3789: vm_object_unlock(src_object);
3790: }
3791:
3792: /*
3793: * We do not need to write-protect
3794: * the page, since it cannot have
3795: * been in the pmap (and we did not
3796: * enter it above). The busy bit
3797: * will protect the page from being
3798: * entered as writable until it is
3799: * unlocked.
3800: */
3801:
3802: }
3803:
3804: /*
3805: * The page is busy, its object is locked, and
3806: * we have a paging reference on it. Either
3807: * the map is locked, or need_map_lookup is
3808: * TRUE.
1.1.1.2 root 3809: *
1.1 root 3810: * Put the page in the page list.
3811: */
3812: copy->cpy_page_list[copy->cpy_npages++] = m;
3813: vm_object_unlock(m->object);
3814: }
1.1.1.2 root 3815:
1.1 root 3816: /*
3817: * DETERMINE whether the entire region
3818: * has been copied.
3819: */
3820: if (src_start >= src_end && src_end != 0) {
3821: if (need_map_lookup)
3822: vm_map_lock(src_map);
3823: break;
3824: }
3825:
3826: /*
3827: * If need_map_lookup is TRUE, have to start over with
3828: * another map lookup. Note that we dropped the map
3829: * lock (to call vm_fault_page) above only in this case.
3830: */
3831: if (need_map_lookup)
3832: goto do_map_lookup;
3833:
3834: /*
3835: * Verify that there are no gaps in the region
3836: */
3837:
3838: src_start = src_entry->vme_end;
3839: src_entry = src_entry->vme_next;
3840: if (src_entry->vme_start != src_start) {
3841: result = KERN_INVALID_ADDRESS;
3842: goto error;
3843: }
3844: }
3845:
3846: /*
3847: * If steal_pages is true, make sure all
3848: * pages in the copy are not in any object
3849: * We try to remove them from the original
3850: * object, but we may have to copy them.
3851: *
3852: * At this point every page in the list is busy
3853: * and holds a paging reference to its object.
3854: * When we're done stealing, every page is busy,
3855: * and in no object (m->tabled == FALSE).
3856: */
3857: src_start = trunc_page(src_addr);
3858: if (steal_pages) {
1.1.1.4 root 3859: int i;
1.1 root 3860: vm_offset_t unwire_end;
3861:
3862: unwire_end = src_start;
3863: for (i = 0; i < copy->cpy_npages; i++) {
3864:
3865: /*
3866: * Remove the page from its object if it
3867: * can be stolen. It can be stolen if:
3868: *
1.1.1.2 root 3869: * (1) The source is being destroyed,
1.1 root 3870: * the object is temporary, and
3871: * not shared.
3872: * (2) The page is not precious.
3873: *
3874: * The not shared check consists of two
3875: * parts: (a) there are no objects that
3876: * shadow this object. (b) it is not the
3877: * object in any shared map entries (i.e.,
3878: * use_shared_copy is not set).
3879: *
3880: * The first check (a) means that we can't
3881: * steal pages from objects that are not
3882: * at the top of their shadow chains. This
3883: * should not be a frequent occurrence.
3884: *
3885: * Stealing wired pages requires telling the
3886: * pmap module to let go of them.
1.1.1.2 root 3887: *
1.1 root 3888: * NOTE: stealing clean pages from objects
3889: * whose mappings survive requires a call to
3890: * the pmap module. Maybe later.
3891: */
3892: m = copy->cpy_page_list[i];
3893: src_object = m->object;
3894: vm_object_lock(src_object);
3895:
3896: if (src_destroy &&
3897: src_object->temporary &&
3898: (!src_object->shadowed) &&
3899: (!src_object->use_shared_copy) &&
3900: !m->precious) {
3901: vm_offset_t page_vaddr;
1.1.1.2 root 3902:
1.1 root 3903: page_vaddr = src_start + (i * PAGE_SIZE);
3904: if (m->wire_count > 0) {
3905:
3906: assert(m->wire_count == 1);
3907: /*
3908: * In order to steal a wired
3909: * page, we have to unwire it
3910: * first. We do this inline
3911: * here because we have the page.
3912: *
3913: * Step 1: Unwire the map entry.
3914: * Also tell the pmap module
3915: * that this piece of the
3916: * pmap is pageable.
3917: */
3918: vm_object_unlock(src_object);
3919: if (page_vaddr >= unwire_end) {
3920: if (!vm_map_lookup_entry(src_map,
3921: page_vaddr, &src_entry))
3922: panic("vm_map_copyin_page_list: missing wired map entry");
3923:
3924: vm_map_clip_start(src_map, src_entry,
3925: page_vaddr);
3926: vm_map_clip_end(src_map, src_entry,
3927: src_start + src_size);
3928:
3929: assert(src_entry->wired_count > 0);
3930: src_entry->wired_count = 0;
1.1.1.5 root 3931: if (src_entry->user_wired_count)
3932: src_map->user_wired -= src_entry->vme_end - src_entry->vme_start;
1.1 root 3933: src_entry->user_wired_count = 0;
3934: unwire_end = src_entry->vme_end;
3935: pmap_pageable(vm_map_pmap(src_map),
3936: page_vaddr, unwire_end, TRUE);
3937: }
3938:
3939: /*
3940: * Step 2: Unwire the page.
3941: * pmap_remove handles this for us.
3942: */
3943: vm_object_lock(src_object);
3944: }
3945:
3946: /*
3947: * Don't need to remove the mapping;
3948: * vm_map_delete will handle it.
3949: *
3950: * Steal the page. Setting the wire count
3951: * to zero is vm_page_unwire without
3952: * activating the page.
3953: */
3954: vm_page_lock_queues();
3955: vm_page_remove(m);
3956: if (m->wire_count > 0) {
3957: m->wire_count = 0;
3958: vm_page_wire_count--;
3959: } else {
3960: VM_PAGE_QUEUES_REMOVE(m);
3961: }
3962: vm_page_unlock_queues();
3963: }
3964: else {
3965: /*
3966: * Have to copy this page. Have to
3967: * unlock the map while copying,
3968: * hence no further page stealing.
3969: * Hence just copy all the pages.
3970: * Unlock the map while copying;
3971: * This means no further page stealing.
3972: */
3973: vm_object_unlock(src_object);
3974: vm_map_unlock(src_map);
3975:
3976: vm_map_copy_steal_pages(copy);
3977:
3978: vm_map_lock(src_map);
3979: break;
3980: }
3981:
3982: vm_object_paging_end(src_object);
3983: vm_object_unlock(src_object);
3984: }
3985:
3986: /*
3987: * If the source should be destroyed, do it now, since the
3988: * copy was successful.
3989: */
3990:
3991: if (src_destroy) {
3992: (void) vm_map_delete(src_map, src_start, src_end);
3993: }
3994: }
3995: else {
3996: /*
3997: * !steal_pages leaves busy pages in the map.
3998: * This will cause src_destroy to hang. Use
3999: * a continuation to prevent this.
4000: */
4001: if (src_destroy && !vm_map_copy_has_cont(copy)) {
1.1.1.2 root 4002: cont_args = (vm_map_copyin_args_t)
1.1 root 4003: kalloc(sizeof(vm_map_copyin_args_data_t));
4004: vm_map_reference(src_map);
4005: cont_args->map = src_map;
4006: cont_args->src_addr = (vm_offset_t) 0;
4007: cont_args->src_len = (vm_size_t) 0;
4008: cont_args->destroy_addr = src_start;
4009: cont_args->destroy_len = src_end - src_start;
4010: cont_args->steal_pages = FALSE;
4011:
4012: copy->cpy_cont_args = (char *) cont_args;
4013: copy->cpy_cont = vm_map_copyin_page_list_cont;
4014: }
1.1.1.2 root 4015:
1.1 root 4016: }
4017:
4018: vm_map_unlock(src_map);
4019:
4020: *copy_result = copy;
4021: return(result);
4022:
4023: error:
4024: vm_map_unlock(src_map);
4025: vm_map_copy_discard(copy);
4026: return(result);
4027: }
4028:
4029: /*
4030: * vm_map_fork:
4031: *
4032: * Create and return a new map based on the old
4033: * map, according to the inheritance values on the
4034: * regions in that map.
4035: *
4036: * The source map must not be locked.
4037: */
1.1.1.4 root 4038: vm_map_t vm_map_fork(vm_map_t old_map)
1.1 root 4039: {
4040: vm_map_t new_map;
4041: vm_map_entry_t old_entry;
4042: vm_map_entry_t new_entry;
4043: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4044: vm_size_t new_size = 0;
4045: vm_size_t entry_size;
4046: vm_object_t object;
4047:
4048: vm_map_lock(old_map);
4049:
4050: new_map = vm_map_create(new_pmap,
4051: old_map->min_offset,
4052: old_map->max_offset,
4053: old_map->hdr.entries_pageable);
4054:
4055: for (
4056: old_entry = vm_map_first_entry(old_map);
4057: old_entry != vm_map_to_entry(old_map);
4058: ) {
4059: if (old_entry->is_sub_map)
4060: panic("vm_map_fork: encountered a submap");
4061:
4062: entry_size = (old_entry->vme_end - old_entry->vme_start);
4063:
4064: switch (old_entry->inheritance) {
4065: case VM_INHERIT_NONE:
4066: break;
4067:
4068: case VM_INHERIT_SHARE:
4069: /*
4070: * New sharing code. New map entry
4071: * references original object. Temporary
4072: * objects use asynchronous copy algorithm for
4073: * future copies. First make sure we have
4074: * the right object. If we need a shadow,
4075: * or someone else already has one, then
4076: * make a new shadow and share it.
4077: */
4078:
4079: object = old_entry->object.vm_object;
4080: if (object == VM_OBJECT_NULL) {
4081: object = vm_object_allocate(
4082: (vm_size_t)(old_entry->vme_end -
4083: old_entry->vme_start));
4084: old_entry->offset = 0;
4085: old_entry->object.vm_object = object;
4086: assert(!old_entry->needs_copy);
4087: }
4088: else if (old_entry->needs_copy || object->shadowed ||
4089: (object->temporary && !old_entry->is_shared &&
4090: object->size > (vm_size_t)(old_entry->vme_end -
4091: old_entry->vme_start))) {
4092:
4093: assert(object->temporary);
4094: assert(!(object->shadowed && old_entry->is_shared));
4095: vm_object_shadow(
4096: &old_entry->object.vm_object,
4097: &old_entry->offset,
4098: (vm_size_t) (old_entry->vme_end -
4099: old_entry->vme_start));
1.1.1.2 root 4100:
1.1 root 4101: /*
4102: * If we're making a shadow for other than
4103: * copy on write reasons, then we have
4104: * to remove write permission.
4105: */
4106:
4107: if (!old_entry->needs_copy &&
4108: (old_entry->protection & VM_PROT_WRITE)) {
4109: pmap_protect(vm_map_pmap(old_map),
4110: old_entry->vme_start,
4111: old_entry->vme_end,
4112: old_entry->protection &
4113: ~VM_PROT_WRITE);
4114: }
4115: old_entry->needs_copy = FALSE;
4116: object = old_entry->object.vm_object;
4117: }
4118:
4119: /*
4120: * Set use_shared_copy to indicate that
4121: * object must use shared (delayed) copy-on
4122: * write. This is ignored for permanent objects.
4123: * Bump the reference count for the new entry
4124: */
4125:
4126: vm_object_lock(object);
4127: object->use_shared_copy = TRUE;
4128: object->ref_count++;
4129: vm_object_unlock(object);
4130:
1.1.1.3 root 4131: new_entry = vm_map_entry_create(new_map);
4132:
1.1 root 4133: if (old_entry->projected_on != 0) {
4134: /*
4135: * If entry is projected buffer, clone the
4136: * entry exactly.
4137: */
4138:
4139: vm_map_entry_copy_full(new_entry, old_entry);
4140:
4141: } else {
4142: /*
4143: * Clone the entry, using object ref from above.
4144: * Mark both entries as shared.
4145: */
4146:
4147: vm_map_entry_copy(new_entry, old_entry);
4148: old_entry->is_shared = TRUE;
4149: new_entry->is_shared = TRUE;
4150: }
4151:
4152: /*
4153: * Insert the entry into the new map -- we
4154: * know we're inserting at the end of the new
4155: * map.
4156: */
4157:
4158: vm_map_entry_link(
4159: new_map,
4160: vm_map_last_entry(new_map),
4161: new_entry);
4162:
4163: /*
4164: * Update the physical map
4165: */
4166:
4167: pmap_copy(new_map->pmap, old_map->pmap,
4168: new_entry->vme_start,
4169: entry_size,
4170: old_entry->vme_start);
4171:
4172: new_size += entry_size;
4173: break;
4174:
4175: case VM_INHERIT_COPY:
4176: if (old_entry->wired_count == 0) {
4177: boolean_t src_needs_copy;
4178: boolean_t new_entry_needs_copy;
4179:
4180: new_entry = vm_map_entry_create(new_map);
4181: vm_map_entry_copy(new_entry, old_entry);
4182:
4183: if (vm_object_copy_temporary(
4184: &new_entry->object.vm_object,
4185: &new_entry->offset,
4186: &src_needs_copy,
4187: &new_entry_needs_copy)) {
4188:
4189: /*
4190: * Handle copy-on-write obligations
4191: */
4192:
4193: if (src_needs_copy && !old_entry->needs_copy) {
4194: vm_object_pmap_protect(
4195: old_entry->object.vm_object,
4196: old_entry->offset,
4197: entry_size,
4198: (old_entry->is_shared ?
4199: PMAP_NULL :
4200: old_map->pmap),
4201: old_entry->vme_start,
4202: old_entry->protection &
4203: ~VM_PROT_WRITE);
4204:
4205: old_entry->needs_copy = TRUE;
4206: }
4207:
4208: new_entry->needs_copy = new_entry_needs_copy;
4209:
4210: /*
4211: * Insert the entry at the end
4212: * of the map.
4213: */
4214:
4215: vm_map_entry_link(new_map,
4216: vm_map_last_entry(new_map),
4217: new_entry);
4218:
4219:
4220: new_size += entry_size;
4221: break;
4222: }
4223:
4224: vm_map_entry_dispose(new_map, new_entry);
4225: }
4226:
4227: /* INNER BLOCK (copy cannot be optimized) */ {
4228:
4229: vm_offset_t start = old_entry->vme_start;
4230: vm_map_copy_t copy;
4231: vm_map_entry_t last = vm_map_last_entry(new_map);
4232:
4233: vm_map_unlock(old_map);
4234: if (vm_map_copyin(old_map,
4235: start,
4236: entry_size,
4237: FALSE,
1.1.1.2 root 4238: ©)
1.1 root 4239: != KERN_SUCCESS) {
4240: vm_map_lock(old_map);
4241: if (!vm_map_lookup_entry(old_map, start, &last))
4242: last = last->vme_next;
4243: old_entry = last;
4244: /*
4245: * For some error returns, want to
4246: * skip to the next element.
4247: */
4248:
4249: continue;
4250: }
4251:
4252: /*
4253: * Insert the copy into the new map
4254: */
4255:
4256: vm_map_copy_insert(new_map, last, copy);
4257: new_size += entry_size;
4258:
4259: /*
4260: * Pick up the traversal at the end of
4261: * the copied region.
4262: */
4263:
4264: vm_map_lock(old_map);
4265: start += entry_size;
4266: if (!vm_map_lookup_entry(old_map, start, &last))
4267: last = last->vme_next;
4268: else
4269: vm_map_clip_start(old_map, last, start);
4270: old_entry = last;
4271:
4272: continue;
4273: /* INNER BLOCK (copy cannot be optimized) */ }
4274: }
4275: old_entry = old_entry->vme_next;
4276: }
4277:
4278: new_map->size = new_size;
4279: vm_map_unlock(old_map);
4280:
4281: return(new_map);
4282: }
4283:
4284: /*
4285: * vm_map_lookup:
4286: *
4287: * Finds the VM object, offset, and
4288: * protection for a given virtual address in the
4289: * specified map, assuming a page fault of the
4290: * type specified.
4291: *
4292: * Returns the (object, offset, protection) for
4293: * this address, whether it is wired down, and whether
4294: * this map has the only reference to the data in question.
4295: * In order to later verify this lookup, a "version"
4296: * is returned.
4297: *
4298: * The map should not be locked; it will not be
4299: * locked on exit. In order to guarantee the
4300: * existence of the returned object, it is returned
4301: * locked.
4302: *
4303: * If a lookup is requested with "write protection"
4304: * specified, the map may be changed to perform virtual
4305: * copying operations, although the data referenced will
4306: * remain the same.
4307: */
1.1.1.4 root 4308: kern_return_t vm_map_lookup(
4309: vm_map_t *var_map, /* IN/OUT */
4310: vm_offset_t vaddr,
4311: vm_prot_t fault_type,
4312:
4313: vm_map_version_t *out_version, /* OUT */
4314: vm_object_t *object, /* OUT */
4315: vm_offset_t *offset, /* OUT */
4316: vm_prot_t *out_prot, /* OUT */
4317: boolean_t *wired) /* OUT */
4318: {
4319: vm_map_entry_t entry;
4320: vm_map_t map = *var_map;
4321: vm_prot_t prot;
1.1 root 4322:
4323: RetryLookup: ;
4324:
4325: /*
4326: * Lookup the faulting address.
4327: */
4328:
4329: vm_map_lock_read(map);
4330:
4331: #define RETURN(why) \
4332: { \
4333: vm_map_unlock_read(map); \
4334: return(why); \
4335: }
4336:
4337: /*
4338: * If the map has an interesting hint, try it before calling
4339: * full blown lookup routine.
4340: */
4341:
4342: simple_lock(&map->hint_lock);
4343: entry = map->hint;
4344: simple_unlock(&map->hint_lock);
4345:
4346: if ((entry == vm_map_to_entry(map)) ||
4347: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4348: vm_map_entry_t tmp_entry;
4349:
4350: /*
4351: * Entry was either not a valid hint, or the vaddr
4352: * was not contained in the entry, so do a full lookup.
4353: */
4354: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4355: RETURN(KERN_INVALID_ADDRESS);
4356:
4357: entry = tmp_entry;
4358: }
4359:
4360: /*
4361: * Handle submaps.
4362: */
4363:
4364: if (entry->is_sub_map) {
4365: vm_map_t old_map = map;
4366:
4367: *var_map = map = entry->object.sub_map;
4368: vm_map_unlock_read(old_map);
4369: goto RetryLookup;
4370: }
1.1.1.2 root 4371:
1.1 root 4372: /*
4373: * Check whether this task is allowed to have
4374: * this page.
4375: */
4376:
4377: prot = entry->protection;
4378:
1.1.1.3 root 4379: if ((fault_type & (prot)) != fault_type) {
1.1 root 4380: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4381: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4382: } else {
4383: RETURN(KERN_PROTECTION_FAILURE);
4384: }
1.1.1.3 root 4385: }
1.1 root 4386:
4387: /*
4388: * If this page is not pageable, we have to get
4389: * it for all possible accesses.
4390: */
4391:
1.1.1.3 root 4392: if ((*wired = (entry->wired_count != 0)))
1.1 root 4393: prot = fault_type = entry->protection;
4394:
4395: /*
4396: * If the entry was copy-on-write, we either ...
4397: */
4398:
4399: if (entry->needs_copy) {
4400: /*
4401: * If we want to write the page, we may as well
4402: * handle that now since we've got the map locked.
4403: *
4404: * If we don't need to write the page, we just
4405: * demote the permissions allowed.
4406: */
4407:
4408: if (fault_type & VM_PROT_WRITE) {
4409: /*
4410: * Make a new object, and place it in the
4411: * object chain. Note that no new references
4412: * have appeared -- one just moved from the
4413: * map to the new object.
4414: */
4415:
4416: if (vm_map_lock_read_to_write(map)) {
4417: goto RetryLookup;
4418: }
4419: map->timestamp++;
4420:
4421: vm_object_shadow(
4422: &entry->object.vm_object,
4423: &entry->offset,
4424: (vm_size_t) (entry->vme_end - entry->vme_start));
1.1.1.2 root 4425:
1.1 root 4426: entry->needs_copy = FALSE;
1.1.1.2 root 4427:
1.1 root 4428: vm_map_lock_write_to_read(map);
4429: }
4430: else {
4431: /*
4432: * We're attempting to read a copy-on-write
4433: * page -- don't allow writes.
4434: */
4435:
4436: prot &= (~VM_PROT_WRITE);
4437: }
4438: }
4439:
4440: /*
4441: * Create an object if necessary.
4442: */
4443: if (entry->object.vm_object == VM_OBJECT_NULL) {
4444:
4445: if (vm_map_lock_read_to_write(map)) {
4446: goto RetryLookup;
4447: }
4448:
4449: entry->object.vm_object = vm_object_allocate(
4450: (vm_size_t)(entry->vme_end - entry->vme_start));
4451: entry->offset = 0;
4452: vm_map_lock_write_to_read(map);
4453: }
4454:
4455: /*
4456: * Return the object/offset from this entry. If the entry
4457: * was copy-on-write or empty, it has been fixed up. Also
4458: * return the protection.
4459: */
4460:
4461: *offset = (vaddr - entry->vme_start) + entry->offset;
4462: *object = entry->object.vm_object;
4463: *out_prot = prot;
4464:
4465: /*
4466: * Lock the object to prevent it from disappearing
4467: */
4468:
4469: vm_object_lock(*object);
4470:
4471: /*
4472: * Save the version number and unlock the map.
4473: */
4474:
4475: out_version->main_timestamp = map->timestamp;
4476:
4477: RETURN(KERN_SUCCESS);
1.1.1.2 root 4478:
1.1 root 4479: #undef RETURN
4480: }
4481:
4482: /*
4483: * vm_map_verify:
4484: *
4485: * Verifies that the map in question has not changed
4486: * since the given version. If successful, the map
4487: * will not change until vm_map_verify_done() is called.
4488: */
1.1.1.4 root 4489: boolean_t vm_map_verify(
4490: vm_map_t map,
4491: vm_map_version_t *version) /* REF */
1.1 root 4492: {
4493: boolean_t result;
4494:
4495: vm_map_lock_read(map);
4496: result = (map->timestamp == version->main_timestamp);
4497:
4498: if (!result)
4499: vm_map_unlock_read(map);
4500:
4501: return(result);
4502: }
4503:
4504: /*
4505: * vm_map_verify_done:
4506: *
4507: * Releases locks acquired by a vm_map_verify.
4508: *
4509: * This is now a macro in vm/vm_map.h. It does a
4510: * vm_map_unlock_read on the map.
4511: */
4512:
4513: /*
4514: * vm_region:
4515: *
4516: * User call to obtain information about a region in
4517: * a task's address map.
4518: */
4519:
1.1.1.4 root 4520: kern_return_t vm_region(
4521: vm_map_t map,
4522: vm_offset_t *address, /* IN/OUT */
4523: vm_size_t *size, /* OUT */
4524: vm_prot_t *protection, /* OUT */
4525: vm_prot_t *max_protection, /* OUT */
4526: vm_inherit_t *inheritance, /* OUT */
4527: boolean_t *is_shared, /* OUT */
4528: ipc_port_t *object_name, /* OUT */
4529: vm_offset_t *offset_in_object) /* OUT */
1.1 root 4530: {
4531: vm_map_entry_t tmp_entry;
4532: vm_map_entry_t entry;
4533: vm_offset_t tmp_offset;
4534: vm_offset_t start;
4535:
4536: if (map == VM_MAP_NULL)
4537: return(KERN_INVALID_ARGUMENT);
4538:
4539: start = *address;
4540:
4541: vm_map_lock_read(map);
4542: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4543: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4544: vm_map_unlock_read(map);
4545: return(KERN_NO_SPACE);
4546: }
4547: } else {
4548: entry = tmp_entry;
4549: }
4550:
4551: start = entry->vme_start;
4552: *protection = entry->protection;
4553: *max_protection = entry->max_protection;
4554: *inheritance = entry->inheritance;
4555: *address = start;
4556: *size = (entry->vme_end - start);
4557:
4558: tmp_offset = entry->offset;
4559:
4560:
4561: if (entry->is_sub_map) {
4562: *is_shared = FALSE;
4563: *object_name = IP_NULL;
4564: *offset_in_object = tmp_offset;
4565: } else {
4566: *is_shared = entry->is_shared;
4567: *object_name = vm_object_name(entry->object.vm_object);
4568: *offset_in_object = tmp_offset;
4569: }
4570:
4571: vm_map_unlock_read(map);
4572:
4573: return(KERN_SUCCESS);
4574: }
4575:
4576: /*
4577: * Routine: vm_map_simplify
4578: *
4579: * Description:
4580: * Attempt to simplify the map representation in
4581: * the vicinity of the given starting address.
4582: * Note:
4583: * This routine is intended primarily to keep the
4584: * kernel maps more compact -- they generally don't
4585: * benefit from the "expand a map entry" technology
4586: * at allocation time because the adjacent entry
4587: * is often wired down.
4588: */
1.1.1.4 root 4589: void vm_map_simplify(
4590: vm_map_t map,
4591: vm_offset_t start)
1.1 root 4592: {
4593: vm_map_entry_t this_entry;
4594: vm_map_entry_t prev_entry;
4595:
4596: vm_map_lock(map);
4597: if (
4598: (vm_map_lookup_entry(map, start, &this_entry)) &&
4599: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4600:
4601: (prev_entry->vme_end == start) &&
4602:
4603: (prev_entry->is_shared == FALSE) &&
4604: (prev_entry->is_sub_map == FALSE) &&
4605:
4606: (this_entry->is_shared == FALSE) &&
4607: (this_entry->is_sub_map == FALSE) &&
4608:
4609: (prev_entry->inheritance == this_entry->inheritance) &&
4610: (prev_entry->protection == this_entry->protection) &&
4611: (prev_entry->max_protection == this_entry->max_protection) &&
4612: (prev_entry->wired_count == this_entry->wired_count) &&
4613: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4614:
4615: (prev_entry->needs_copy == this_entry->needs_copy) &&
4616:
4617: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4618: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4619: == this_entry->offset) &&
4620: (prev_entry->projected_on == 0) &&
1.1.1.2 root 4621: (this_entry->projected_on == 0)
1.1 root 4622: ) {
4623: if (map->first_free == this_entry)
4624: map->first_free = prev_entry;
4625:
4626: SAVE_HINT(map, prev_entry);
4627: vm_map_entry_unlink(map, this_entry);
4628: prev_entry->vme_end = this_entry->vme_end;
4629: vm_object_deallocate(this_entry->object.vm_object);
4630: vm_map_entry_dispose(map, this_entry);
4631: }
4632: vm_map_unlock(map);
4633: }
4634:
4635:
4636: /*
4637: * Routine: vm_map_machine_attribute
4638: * Purpose:
4639: * Provide machine-specific attributes to mappings,
4640: * such as cachability etc. for machines that provide
4641: * them. NUMA architectures and machines with big/strange
4642: * caches will use this.
4643: * Note:
4644: * Responsibilities for locking and checking are handled here,
4645: * everything else in the pmap module. If any non-volatile
4646: * information must be kept, the pmap module should handle
4647: * it itself. [This assumes that attributes do not
4648: * need to be inherited, which seems ok to me]
4649: */
1.1.1.4 root 4650: kern_return_t vm_map_machine_attribute(
4651: vm_map_t map,
4652: vm_offset_t address,
4653: vm_size_t size,
4654: vm_machine_attribute_t attribute,
4655: vm_machine_attribute_val_t* value) /* IN/OUT */
1.1 root 4656: {
4657: kern_return_t ret;
4658:
4659: if (address < vm_map_min(map) ||
4660: (address + size) > vm_map_max(map))
4661: return KERN_INVALID_ARGUMENT;
4662:
4663: vm_map_lock(map);
4664:
4665: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4666:
4667: vm_map_unlock(map);
4668:
4669: return ret;
4670: }
4671:
4672:
4673: #if MACH_KDB
4674:
4675: #define printf kdbprintf
4676:
4677: /*
4678: * vm_map_print: [ debug ]
4679: */
1.1.1.6 ! root 4680: void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const char *modif)
1.1 root 4681: {
1.1.1.6 ! root 4682: vm_map_t map;
1.1.1.4 root 4683: vm_map_entry_t entry;
1.1 root 4684:
1.1.1.6 ! root 4685: if (!have_addr)
! 4686: map = current_thread()->task->map;
! 4687: else
! 4688: map = (vm_map_t)addr;
! 4689:
! 4690: iprintf("Map 0x%X: pmap=0x%X,",
1.1 root 4691: (vm_offset_t) map, (vm_offset_t) (map->pmap));
4692: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4693: printf("version=%d\n", map->timestamp);
1.1.1.6 ! root 4694: indent += 1;
1.1 root 4695: for (entry = vm_map_first_entry(map);
4696: entry != vm_map_to_entry(map);
4697: entry = entry->vme_next) {
4698: static char *inheritance_name[3] = { "share", "copy", "none"};
4699:
4700: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
1.1.1.6 ! root 4701: printf("start=0x%X, end=0x%X\n",
1.1 root 4702: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
1.1.1.6 ! root 4703: iprintf("prot=%X/%X/%s, ",
1.1 root 4704: entry->protection,
4705: entry->max_protection,
4706: inheritance_name[entry->inheritance]);
4707: if (entry->wired_count != 0) {
4708: printf("wired(");
4709: if (entry->user_wired_count != 0)
4710: printf("u");
4711: if (entry->wired_count >
4712: ((entry->user_wired_count == 0) ? 0 : 1))
4713: printf("k");
4714: printf(") ");
4715: }
4716: if (entry->in_transition) {
4717: printf("in transition");
4718: if (entry->needs_wakeup)
4719: printf("(wake request)");
4720: printf(", ");
4721: }
4722: if (entry->is_sub_map) {
4723: printf("submap=0x%X, offset=0x%X\n",
4724: (vm_offset_t) entry->object.sub_map,
4725: (vm_offset_t) entry->offset);
4726: } else {
4727: printf("object=0x%X, offset=0x%X",
4728: (vm_offset_t) entry->object.vm_object,
4729: (vm_offset_t) entry->offset);
4730: if (entry->is_shared)
4731: printf(", shared");
4732: if (entry->needs_copy)
4733: printf(", copy needed");
4734: printf("\n");
4735:
4736: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4737: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
1.1.1.6 ! root 4738: indent += 1;
1.1 root 4739: vm_object_print(entry->object.vm_object);
1.1.1.6 ! root 4740: indent -= 1;
1.1 root 4741: }
4742: }
4743: }
1.1.1.6 ! root 4744: indent -= 1;
1.1 root 4745: }
4746:
4747: /*
4748: * Routine: vm_map_copy_print
4749: * Purpose:
4750: * Pretty-print a copy object for ddb.
4751: */
4752:
4753: void vm_map_copy_print(copy)
1.1.1.4 root 4754: const vm_map_copy_t copy;
1.1 root 4755: {
4756: int i, npages;
4757:
4758: printf("copy object 0x%x\n", copy);
4759:
1.1.1.6 ! root 4760: indent += 1;
1.1 root 4761:
4762: iprintf("type=%d", copy->type);
4763: switch (copy->type) {
4764: case VM_MAP_COPY_ENTRY_LIST:
4765: printf("[entry_list]");
4766: break;
1.1.1.2 root 4767:
1.1 root 4768: case VM_MAP_COPY_OBJECT:
4769: printf("[object]");
4770: break;
1.1.1.2 root 4771:
1.1 root 4772: case VM_MAP_COPY_PAGE_LIST:
4773: printf("[page_list]");
4774: break;
4775:
4776: default:
4777: printf("[bad type]");
4778: break;
4779: }
4780: printf(", offset=0x%x", copy->offset);
4781: printf(", size=0x%x\n", copy->size);
4782:
4783: switch (copy->type) {
4784: case VM_MAP_COPY_ENTRY_LIST:
4785: /* XXX add stuff here */
4786: break;
4787:
4788: case VM_MAP_COPY_OBJECT:
4789: iprintf("object=0x%x\n", copy->cpy_object);
4790: break;
4791:
4792: case VM_MAP_COPY_PAGE_LIST:
4793: iprintf("npages=%d", copy->cpy_npages);
4794: printf(", cont=%x", copy->cpy_cont);
4795: printf(", cont_args=%x\n", copy->cpy_cont_args);
4796: if (copy->cpy_npages < 0) {
4797: npages = 0;
4798: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4799: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4800: } else {
4801: npages = copy->cpy_npages;
4802: }
4803: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4804: for (i = 0; i < npages - 1; i++) {
4805: printf("0x%x, ", copy->cpy_page_list[i]);
4806: }
4807: if (npages > 0) {
4808: printf("0x%x", copy->cpy_page_list[npages - 1]);
4809: }
4810: printf("}\n");
4811: break;
4812: }
4813:
1.1.1.6 ! root 4814: indent -= 1;
1.1 root 4815: }
1.1.1.2 root 4816: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.