![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
1.1.1.3 root 37: #include <kern/printf.h>
1.1 root 38: #include <mach/kern_return.h>
39: #include <mach/port.h>
40: #include <mach/vm_attributes.h>
41: #include <mach/vm_param.h>
42: #include <kern/assert.h>
1.1.1.3 root 43: #include <kern/debug.h>
44: #include <kern/kalloc.h>
45: #include <kern/rbtree.h>
46: #include <kern/slab.h>
47: #include <vm/pmap.h>
1.1 root 48: #include <vm/vm_fault.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
1.1.1.3 root 52: #include <vm/vm_resident.h>
1.1 root 53: #include <vm/vm_kern.h>
54: #include <ipc/ipc_port.h>
55:
1.1.1.3 root 56: #if MACH_KDB
57: #include <ddb/db_output.h>
58: #include <vm/vm_print.h>
59: #endif /* MACH_KDB */
60:
1.1 root 61: /*
62: * Macros to copy a vm_map_entry. We must be careful to correctly
63: * manage the wired page count. vm_map_entry_copy() creates a new
64: * map entry to the same memory - the wired count in the new entry
65: * must be set to zero. vm_map_entry_copy_full() creates a new
66: * entry that is identical to the old entry. This preserves the
1.1.1.3 root 67: * wire count; it's used for map splitting and cache changing in
1.1 root 68: * vm_map_copyout.
69: */
70: #define vm_map_entry_copy(NEW,OLD) \
71: MACRO_BEGIN \
72: *(NEW) = *(OLD); \
73: (NEW)->is_shared = FALSE; \
74: (NEW)->needs_wakeup = FALSE; \
75: (NEW)->in_transition = FALSE; \
76: (NEW)->wired_count = 0; \
77: (NEW)->user_wired_count = 0; \
78: MACRO_END
79:
80: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
81:
82: /*
83: * Virtual memory maps provide for the mapping, protection,
84: * and sharing of virtual memory objects. In addition,
85: * this module provides for an efficient virtual copy of
86: * memory from one map to another.
87: *
88: * Synchronization is required prior to most operations.
89: *
90: * Maps consist of an ordered doubly-linked list of simple
1.1.1.3 root 91: * entries; a hint and a red-black tree are used to speed up lookups.
1.1 root 92: *
93: * Sharing maps have been deleted from this version of Mach.
94: * All shared objects are now mapped directly into the respective
95: * maps. This requires a change in the copy on write strategy;
96: * the asymmetric (delayed) strategy is used for shared temporary
97: * objects instead of the symmetric (shadow) strategy. This is
98: * selected by the (new) use_shared_copy bit in the object. See
99: * vm_object_copy_temporary in vm_object.c for details. All maps
100: * are now "top level" maps (either task map, kernel map or submap
1.1.1.2 root 101: * of the kernel map).
1.1 root 102: *
103: * Since portions of maps are specified by start/end addreses,
104: * which may not align with existing map entries, all
105: * routines merely "clip" entries to these start/end values.
106: * [That is, an entry is split into two, bordering at a
107: * start or end value.] Note that these clippings may not
108: * always be necessary (as the two resulting entries are then
109: * not changed); however, the clipping is done for convenience.
110: * No attempt is currently made to "glue back together" two
111: * abutting entries.
112: *
113: * The symmetric (shadow) copy strategy implements virtual copy
114: * by copying VM object references from one map to
115: * another, and then marking both regions as copy-on-write.
116: * It is important to note that only one writeable reference
117: * to a VM object region exists in any map when this strategy
118: * is used -- this means that shadow object creation can be
119: * delayed until a write operation occurs. The asymmetric (delayed)
120: * strategy allows multiple maps to have writeable references to
121: * the same region of a vm object, and hence cannot delay creating
122: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
123: * Copying of permanent objects is completely different; see
124: * vm_object_copy_strategically() in vm_object.c.
125: */
126:
1.1.1.3 root 127: struct kmem_cache vm_map_cache; /* cache for vm_map structures */
128: struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
129: struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
130: struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
1.1 root 131:
132: /*
133: * Placeholder object for submap operations. This object is dropped
134: * into the range by a call to vm_map_find, and removed when
135: * vm_map_submap creates the submap.
136: */
137:
1.1.1.3 root 138: static struct vm_object vm_submap_object_store;
139: vm_object_t vm_submap_object = &vm_submap_object_store;
1.1 root 140:
141: /*
142: * vm_map_init:
143: *
144: * Initialize the vm_map module. Must be called before
145: * any other vm_map routines.
146: *
1.1.1.3 root 147: * Map and entry structures are allocated from caches -- we must
148: * initialize those caches.
1.1 root 149: *
1.1.1.3 root 150: * There are three caches of interest:
1.1 root 151: *
1.1.1.3 root 152: * vm_map_cache: used to allocate maps.
153: * vm_map_entry_cache: used to allocate map entries.
154: * vm_map_kentry_cache: used to allocate map entries for the kernel.
155: *
156: * Kernel map entries are allocated from a special cache, using a custom
157: * page allocation function to avoid recursion. It would be difficult
158: * (perhaps impossible) for the kernel to allocate more memory to an entry
159: * cache when it became empty since the very act of allocating memory
160: * implies the creation of a new entry.
1.1 root 161: */
162:
163: vm_offset_t kentry_data;
1.1.1.3 root 164: vm_size_t kentry_data_size = KENTRY_DATA_SIZE;
1.1 root 165:
1.1.1.3 root 166: static vm_offset_t kentry_pagealloc(vm_size_t size)
1.1 root 167: {
1.1.1.3 root 168: vm_offset_t result;
1.1 root 169:
1.1.1.3 root 170: if (size > kentry_data_size)
171: panic("vm_map: kentry memory exhausted");
1.1 root 172:
1.1.1.3 root 173: result = kentry_data;
174: kentry_data += size;
175: kentry_data_size -= size;
176: return result;
177: }
178:
179: void vm_map_init(void)
180: {
181: kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
182: NULL, NULL, NULL, 0);
183: kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
184: sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0);
185: kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
186: sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc,
187: NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB
188: | KMEM_CACHE_NORECLAIM);
189: kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
190: sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0);
1.1 root 191:
192: /*
193: * Submap object is initialized by vm_object_init.
194: */
195: }
196:
1.1.1.4 root 197: void vm_map_setup(
198: vm_map_t map,
199: pmap_t pmap,
200: vm_offset_t min,
201: vm_offset_t max,
202: boolean_t pageable)
1.1.1.3 root 203: {
204: vm_map_first_entry(map) = vm_map_to_entry(map);
205: vm_map_last_entry(map) = vm_map_to_entry(map);
206: map->hdr.nentries = 0;
207: map->hdr.entries_pageable = pageable;
208: rbtree_init(&map->hdr.tree);
209:
210: map->size = 0;
1.1.1.5 ! root 211: map->user_wired = 0;
1.1.1.3 root 212: map->ref_count = 1;
213: map->pmap = pmap;
214: map->min_offset = min;
215: map->max_offset = max;
216: map->wiring_required = FALSE;
217: map->wait_for_space = FALSE;
218: map->first_free = vm_map_to_entry(map);
219: map->hint = vm_map_to_entry(map);
220: vm_map_lock_init(map);
221: simple_lock_init(&map->ref_lock);
222: simple_lock_init(&map->hint_lock);
223: }
224:
1.1 root 225: /*
226: * vm_map_create:
227: *
228: * Creates and returns a new empty VM map with
229: * the given physical map structure, and having
230: * the given lower and upper address bounds.
231: */
1.1.1.4 root 232: vm_map_t vm_map_create(
233: pmap_t pmap,
234: vm_offset_t min,
235: vm_offset_t max,
236: boolean_t pageable)
1.1 root 237: {
1.1.1.4 root 238: vm_map_t result;
1.1 root 239:
1.1.1.3 root 240: result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
1.1 root 241: if (result == VM_MAP_NULL)
242: panic("vm_map_create");
243:
1.1.1.3 root 244: vm_map_setup(result, pmap, min, max, pageable);
1.1 root 245:
246: return(result);
247: }
248:
249: /*
250: * vm_map_entry_create: [ internal use only ]
251: *
252: * Allocates a VM map entry for insertion in the
253: * given map (or map copy). No fields are filled.
254: */
255: #define vm_map_entry_create(map) \
256: _vm_map_entry_create(&(map)->hdr)
257:
258: #define vm_map_copy_entry_create(copy) \
259: _vm_map_entry_create(&(copy)->cpy_hdr)
260:
261: vm_map_entry_t _vm_map_entry_create(map_header)
1.1.1.4 root 262: const struct vm_map_header *map_header;
1.1 root 263: {
1.1.1.4 root 264: kmem_cache_t cache;
265: vm_map_entry_t entry;
1.1 root 266:
267: if (map_header->entries_pageable)
1.1.1.3 root 268: cache = &vm_map_entry_cache;
1.1 root 269: else
1.1.1.3 root 270: cache = &vm_map_kentry_cache;
1.1 root 271:
1.1.1.3 root 272: entry = (vm_map_entry_t) kmem_cache_alloc(cache);
1.1 root 273: if (entry == VM_MAP_ENTRY_NULL)
274: panic("vm_map_entry_create");
275:
276: return(entry);
277: }
278:
279: /*
280: * vm_map_entry_dispose: [ internal use only ]
281: *
282: * Inverse of vm_map_entry_create.
283: */
284: #define vm_map_entry_dispose(map, entry) \
285: _vm_map_entry_dispose(&(map)->hdr, (entry))
286:
287: #define vm_map_copy_entry_dispose(map, entry) \
288: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
289:
290: void _vm_map_entry_dispose(map_header, entry)
1.1.1.4 root 291: const struct vm_map_header *map_header;
292: vm_map_entry_t entry;
1.1 root 293: {
1.1.1.4 root 294: kmem_cache_t cache;
1.1 root 295:
296: if (map_header->entries_pageable)
1.1.1.3 root 297: cache = &vm_map_entry_cache;
298: else
299: cache = &vm_map_kentry_cache;
300:
301: kmem_cache_free(cache, (vm_offset_t) entry);
302: }
303:
304: /*
305: * Red-black tree lookup/insert comparison functions
306: */
307: static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
308: const struct rbtree_node *node)
309: {
310: struct vm_map_entry *entry;
311:
312: entry = rbtree_entry(node, struct vm_map_entry, tree_node);
313:
314: if (addr < entry->vme_start)
315: return -1;
316: else if (addr < entry->vme_end)
317: return 0;
1.1 root 318: else
1.1.1.3 root 319: return 1;
320: }
321:
322: static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
323: const struct rbtree_node *b)
324: {
325: struct vm_map_entry *entry;
1.1 root 326:
1.1.1.3 root 327: entry = rbtree_entry(a, struct vm_map_entry, tree_node);
328: return vm_map_entry_cmp_lookup(entry->vme_start, b);
1.1 root 329: }
330:
331: /*
332: * vm_map_entry_{un,}link:
333: *
334: * Insert/remove entries from maps (or map copies).
1.1.1.3 root 335: *
336: * The start and end addresses of the entries must be properly set
337: * before using these macros.
1.1 root 338: */
339: #define vm_map_entry_link(map, after_where, entry) \
340: _vm_map_entry_link(&(map)->hdr, after_where, entry)
341:
342: #define vm_map_copy_entry_link(copy, after_where, entry) \
343: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
344:
345: #define _vm_map_entry_link(hdr, after_where, entry) \
346: MACRO_BEGIN \
347: (hdr)->nentries++; \
348: (entry)->vme_prev = (after_where); \
349: (entry)->vme_next = (after_where)->vme_next; \
350: (entry)->vme_prev->vme_next = \
351: (entry)->vme_next->vme_prev = (entry); \
1.1.1.3 root 352: rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \
353: vm_map_entry_cmp_insert); \
1.1 root 354: MACRO_END
355:
356: #define vm_map_entry_unlink(map, entry) \
357: _vm_map_entry_unlink(&(map)->hdr, entry)
358:
359: #define vm_map_copy_entry_unlink(copy, entry) \
360: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
361:
362: #define _vm_map_entry_unlink(hdr, entry) \
363: MACRO_BEGIN \
364: (hdr)->nentries--; \
365: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
366: (entry)->vme_prev->vme_next = (entry)->vme_next; \
1.1.1.3 root 367: rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
1.1 root 368: MACRO_END
369:
370: /*
371: * vm_map_reference:
372: *
373: * Creates another valid reference to the given map.
374: *
375: */
1.1.1.4 root 376: void vm_map_reference(vm_map_t map)
1.1 root 377: {
378: if (map == VM_MAP_NULL)
379: return;
380:
381: simple_lock(&map->ref_lock);
382: map->ref_count++;
383: simple_unlock(&map->ref_lock);
384: }
385:
386: /*
387: * vm_map_deallocate:
388: *
389: * Removes a reference from the specified map,
390: * destroying it if no references remain.
391: * The map should not be locked.
392: */
1.1.1.4 root 393: void vm_map_deallocate(vm_map_t map)
1.1 root 394: {
1.1.1.4 root 395: int c;
1.1 root 396:
397: if (map == VM_MAP_NULL)
398: return;
399:
400: simple_lock(&map->ref_lock);
401: c = --map->ref_count;
402: simple_unlock(&map->ref_lock);
403:
404: if (c > 0) {
405: return;
406: }
407:
408: projected_buffer_collect(map);
409: (void) vm_map_delete(map, map->min_offset, map->max_offset);
410:
411: pmap_destroy(map->pmap);
412:
1.1.1.3 root 413: kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
1.1 root 414: }
415:
416: /*
417: * SAVE_HINT:
418: *
419: * Saves the specified entry as the hint for
420: * future lookups. Performs necessary interlocks.
421: */
422: #define SAVE_HINT(map,value) \
423: simple_lock(&(map)->hint_lock); \
424: (map)->hint = (value); \
425: simple_unlock(&(map)->hint_lock);
426:
427: /*
428: * vm_map_lookup_entry: [ internal use only ]
429: *
430: * Finds the map entry containing (or
431: * immediately preceding) the specified address
432: * in the given map; the entry is returned
433: * in the "entry" parameter. The boolean
434: * result indicates whether the address is
435: * actually contained in the map.
436: */
1.1.1.4 root 437: boolean_t vm_map_lookup_entry(
438: vm_map_t map,
439: vm_offset_t address,
440: vm_map_entry_t *entry) /* OUT */
1.1 root 441: {
1.1.1.4 root 442: struct rbtree_node *node;
443: vm_map_entry_t hint;
1.1 root 444:
445: /*
1.1.1.3 root 446: * First, make a quick check to see if we are already
447: * looking at the entry we want (which is often the case).
1.1 root 448: */
449:
450: simple_lock(&map->hint_lock);
1.1.1.3 root 451: hint = map->hint;
1.1 root 452: simple_unlock(&map->hint_lock);
453:
1.1.1.3 root 454: if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) {
455: if (address < hint->vme_end) {
456: *entry = hint;
1.1 root 457: return(TRUE);
1.1.1.3 root 458: } else {
459: vm_map_entry_t next = hint->vme_next;
460:
461: if ((next == vm_map_to_entry(map))
462: || (address < next->vme_start)) {
463: *entry = hint;
464: return(FALSE);
465: }
1.1 root 466: }
467: }
468:
469: /*
1.1.1.3 root 470: * If the hint didn't help, use the red-black tree.
1.1 root 471: */
472:
1.1.1.3 root 473: node = rbtree_lookup_nearest(&map->hdr.tree, address,
474: vm_map_entry_cmp_lookup, RBTREE_LEFT);
1.1 root 475:
1.1.1.3 root 476: if (node == NULL) {
477: *entry = vm_map_to_entry(map);
478: SAVE_HINT(map, *entry);
479: return(FALSE);
480: } else {
481: *entry = rbtree_entry(node, struct vm_map_entry, tree_node);
482: SAVE_HINT(map, *entry);
483: return((address < (*entry)->vme_end) ? TRUE : FALSE);
1.1 root 484: }
485: }
486:
487: /*
488: * Routine: invalid_user_access
489: *
490: * Verifies whether user access is valid.
491: */
492:
493: boolean_t
1.1.1.4 root 494: invalid_user_access(
495: vm_map_t map,
496: vm_offset_t start,
497: vm_offset_t end,
498: vm_prot_t prot)
1.1 root 499: {
500: vm_map_entry_t entry;
501:
502: return (map == VM_MAP_NULL || map == kernel_map ||
503: !vm_map_lookup_entry(map, start, &entry) ||
504: entry->vme_end < end ||
505: (prot & ~(entry->protection)));
506: }
507:
508:
509: /*
510: * Routine: vm_map_find_entry
511: * Purpose:
512: * Allocate a range in the specified virtual address map,
513: * returning the entry allocated for that range.
514: * Used by kmem_alloc, etc. Returns wired entries.
515: *
516: * The map must be locked.
517: *
518: * If an entry is allocated, the object/offset fields
519: * are initialized to zero. If an object is supplied,
520: * then an existing entry may be extended.
521: */
1.1.1.4 root 522: kern_return_t vm_map_find_entry(
523: vm_map_t map,
524: vm_offset_t *address, /* OUT */
525: vm_size_t size,
526: vm_offset_t mask,
527: vm_object_t object,
528: vm_map_entry_t *o_entry) /* OUT */
1.1 root 529: {
1.1.1.4 root 530: vm_map_entry_t entry, new_entry;
531: vm_offset_t start;
532: vm_offset_t end;
1.1 root 533:
534: /*
535: * Look for the first possible address;
536: * if there's already something at this
537: * address, we have to start after it.
538: */
539:
540: if ((entry = map->first_free) == vm_map_to_entry(map))
541: start = map->min_offset;
542: else
543: start = entry->vme_end;
544:
545: /*
546: * In any case, the "entry" always precedes
547: * the proposed new region throughout the loop:
548: */
549:
550: while (TRUE) {
1.1.1.4 root 551: vm_map_entry_t next;
1.1 root 552:
553: /*
554: * Find the end of the proposed new region.
555: * Be sure we didn't go beyond the end, or
556: * wrap around the address.
557: */
558:
1.1.1.3 root 559: if (((start + mask) & ~mask) < start) {
560: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 561: return(KERN_NO_SPACE);
1.1.1.3 root 562: }
1.1 root 563: start = ((start + mask) & ~mask);
564: end = start + size;
565:
1.1.1.3 root 566: if ((end > map->max_offset) || (end < start)) {
567: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 568: return(KERN_NO_SPACE);
1.1.1.3 root 569: }
1.1 root 570:
571: /*
572: * If there are no more entries, we must win.
573: */
574:
575: next = entry->vme_next;
576: if (next == vm_map_to_entry(map))
577: break;
578:
579: /*
580: * If there is another entry, it must be
581: * after the end of the potential new region.
582: */
583:
584: if (next->vme_start >= end)
585: break;
586:
587: /*
588: * Didn't fit -- move to the next entry.
589: */
590:
591: entry = next;
592: start = entry->vme_end;
593: }
594:
595: /*
596: * At this point,
597: * "start" and "end" should define the endpoints of the
598: * available new range, and
599: * "entry" should refer to the region before the new
600: * range, and
601: *
602: * the map should be locked.
603: */
604:
605: *address = start;
606:
607: /*
608: * See whether we can avoid creating a new entry by
609: * extending one of our neighbors. [So far, we only attempt to
610: * extend from below.]
611: */
612:
613: if ((object != VM_OBJECT_NULL) &&
614: (entry != vm_map_to_entry(map)) &&
615: (entry->vme_end == start) &&
616: (!entry->is_shared) &&
617: (!entry->is_sub_map) &&
618: (entry->object.vm_object == object) &&
619: (entry->needs_copy == FALSE) &&
620: (entry->inheritance == VM_INHERIT_DEFAULT) &&
621: (entry->protection == VM_PROT_DEFAULT) &&
622: (entry->max_protection == VM_PROT_ALL) &&
623: (entry->wired_count == 1) &&
624: (entry->user_wired_count == 0) &&
625: (entry->projected_on == 0)) {
626: /*
627: * Because this is a special case,
628: * we don't need to use vm_object_coalesce.
629: */
630:
631: entry->vme_end = end;
632: new_entry = entry;
633: } else {
634: new_entry = vm_map_entry_create(map);
635:
636: new_entry->vme_start = start;
637: new_entry->vme_end = end;
638:
639: new_entry->is_shared = FALSE;
640: new_entry->is_sub_map = FALSE;
641: new_entry->object.vm_object = VM_OBJECT_NULL;
642: new_entry->offset = (vm_offset_t) 0;
643:
644: new_entry->needs_copy = FALSE;
645:
646: new_entry->inheritance = VM_INHERIT_DEFAULT;
647: new_entry->protection = VM_PROT_DEFAULT;
648: new_entry->max_protection = VM_PROT_ALL;
649: new_entry->wired_count = 1;
650: new_entry->user_wired_count = 0;
651:
652: new_entry->in_transition = FALSE;
653: new_entry->needs_wakeup = FALSE;
654: new_entry->projected_on = 0;
655:
656: /*
657: * Insert the new entry into the list
658: */
659:
660: vm_map_entry_link(map, entry, new_entry);
661: }
662:
663: map->size += size;
664:
665: /*
666: * Update the free space hint and the lookup hint
667: */
668:
669: map->first_free = new_entry;
670: SAVE_HINT(map, new_entry);
671:
672: *o_entry = new_entry;
673: return(KERN_SUCCESS);
674: }
675:
1.1.1.4 root 676: boolean_t vm_map_pmap_enter_print = FALSE;
677: boolean_t vm_map_pmap_enter_enable = FALSE;
1.1 root 678:
679: /*
680: * Routine: vm_map_pmap_enter
681: *
682: * Description:
683: * Force pages from the specified object to be entered into
684: * the pmap at the specified address if they are present.
685: * As soon as a page not found in the object the scan ends.
686: *
687: * Returns:
1.1.1.2 root 688: * Nothing.
1.1 root 689: *
690: * In/out conditions:
691: * The source map should not be locked on entry.
692: */
693: void
1.1.1.4 root 694: vm_map_pmap_enter(
695: vm_map_t map,
696: vm_offset_t addr,
697: vm_offset_t end_addr,
698: vm_object_t object,
699: vm_offset_t offset,
700: vm_prot_t protection)
1.1 root 701: {
702: while (addr < end_addr) {
1.1.1.4 root 703: vm_page_t m;
1.1 root 704:
705: vm_object_lock(object);
706: vm_object_paging_begin(object);
707:
708: m = vm_page_lookup(object, offset);
709: if (m == VM_PAGE_NULL || m->absent) {
710: vm_object_paging_end(object);
711: vm_object_unlock(object);
712: return;
713: }
714:
715: if (vm_map_pmap_enter_print) {
716: printf("vm_map_pmap_enter:");
1.1.1.3 root 717: printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
1.1 root 718: map, addr, object, offset);
719: }
720:
721: m->busy = TRUE;
722: vm_object_unlock(object);
723:
724: PMAP_ENTER(map->pmap, addr, m,
725: protection, FALSE);
726:
727: vm_object_lock(object);
728: PAGE_WAKEUP_DONE(m);
729: vm_page_lock_queues();
730: if (!m->active && !m->inactive)
731: vm_page_activate(m);
732: vm_page_unlock_queues();
733: vm_object_paging_end(object);
734: vm_object_unlock(object);
735:
736: offset += PAGE_SIZE;
737: addr += PAGE_SIZE;
738: }
739: }
740:
741: /*
742: * Routine: vm_map_enter
743: *
744: * Description:
745: * Allocate a range in the specified virtual address map.
746: * The resulting range will refer to memory defined by
747: * the given memory object and offset into that object.
748: *
749: * Arguments are as defined in the vm_map call.
750: */
751: kern_return_t vm_map_enter(
1.1.1.4 root 752: vm_map_t map,
753: vm_offset_t *address, /* IN/OUT */
754: vm_size_t size,
755: vm_offset_t mask,
756: boolean_t anywhere,
757: vm_object_t object,
758: vm_offset_t offset,
759: boolean_t needs_copy,
760: vm_prot_t cur_protection,
761: vm_prot_t max_protection,
762: vm_inherit_t inheritance)
763: {
764: vm_map_entry_t entry;
765: vm_offset_t start;
766: vm_offset_t end;
767: kern_return_t result = KERN_SUCCESS;
1.1 root 768:
769: #define RETURN(value) { result = value; goto BailOut; }
770:
1.1.1.3 root 771: if (size == 0)
772: return KERN_INVALID_ARGUMENT;
773:
1.1 root 774: StartAgain: ;
775:
776: start = *address;
777:
778: if (anywhere) {
779: vm_map_lock(map);
780:
781: /*
782: * Calculate the first possible address.
783: */
784:
785: if (start < map->min_offset)
786: start = map->min_offset;
787: if (start > map->max_offset)
788: RETURN(KERN_NO_SPACE);
789:
790: /*
791: * Look for the first possible address;
792: * if there's already something at this
793: * address, we have to start after it.
794: */
795:
796: if (start == map->min_offset) {
797: if ((entry = map->first_free) != vm_map_to_entry(map))
798: start = entry->vme_end;
799: } else {
800: vm_map_entry_t tmp_entry;
801: if (vm_map_lookup_entry(map, start, &tmp_entry))
802: start = tmp_entry->vme_end;
803: entry = tmp_entry;
804: }
805:
806: /*
807: * In any case, the "entry" always precedes
808: * the proposed new region throughout the
809: * loop:
810: */
811:
812: while (TRUE) {
1.1.1.4 root 813: vm_map_entry_t next;
1.1 root 814:
815: /*
816: * Find the end of the proposed new region.
817: * Be sure we didn't go beyond the end, or
818: * wrap around the address.
819: */
820:
1.1.1.3 root 821: if (((start + mask) & ~mask) < start) {
822: printf_once("no more room for vm_map_enter in %p\n", map);
823: RETURN(KERN_NO_SPACE);
824: }
1.1 root 825: start = ((start + mask) & ~mask);
826: end = start + size;
827:
828: if ((end > map->max_offset) || (end < start)) {
829: if (map->wait_for_space) {
830: if (size <= (map->max_offset -
831: map->min_offset)) {
832: assert_wait((event_t) map, TRUE);
833: vm_map_unlock(map);
834: thread_block((void (*)()) 0);
835: goto StartAgain;
836: }
837: }
1.1.1.2 root 838:
1.1.1.3 root 839: printf_once("no more room for vm_map_enter in %p\n", map);
1.1 root 840: RETURN(KERN_NO_SPACE);
841: }
842:
843: /*
844: * If there are no more entries, we must win.
845: */
846:
847: next = entry->vme_next;
848: if (next == vm_map_to_entry(map))
849: break;
850:
851: /*
852: * If there is another entry, it must be
853: * after the end of the potential new region.
854: */
855:
856: if (next->vme_start >= end)
857: break;
858:
859: /*
860: * Didn't fit -- move to the next entry.
861: */
862:
863: entry = next;
864: start = entry->vme_end;
865: }
866: *address = start;
867: } else {
868: vm_map_entry_t temp_entry;
869:
870: /*
871: * Verify that:
872: * the address doesn't itself violate
873: * the mask requirement.
874: */
875:
876: if ((start & mask) != 0)
877: return(KERN_NO_SPACE);
878:
879: vm_map_lock(map);
880:
881: /*
882: * ... the address is within bounds
883: */
884:
885: end = start + size;
886:
887: if ((start < map->min_offset) ||
888: (end > map->max_offset) ||
889: (start >= end)) {
890: RETURN(KERN_INVALID_ADDRESS);
891: }
892:
893: /*
894: * ... the starting address isn't allocated
895: */
896:
897: if (vm_map_lookup_entry(map, start, &temp_entry))
898: RETURN(KERN_NO_SPACE);
899:
900: entry = temp_entry;
901:
902: /*
903: * ... the next region doesn't overlap the
904: * end point.
905: */
906:
907: if ((entry->vme_next != vm_map_to_entry(map)) &&
908: (entry->vme_next->vme_start < end))
909: RETURN(KERN_NO_SPACE);
910: }
911:
912: /*
913: * At this point,
914: * "start" and "end" should define the endpoints of the
915: * available new range, and
916: * "entry" should refer to the region before the new
917: * range, and
918: *
919: * the map should be locked.
920: */
921:
922: /*
923: * See whether we can avoid creating a new entry (and object) by
924: * extending one of our neighbors. [So far, we only attempt to
925: * extend from below.]
926: */
927:
928: if ((object == VM_OBJECT_NULL) &&
929: (entry != vm_map_to_entry(map)) &&
930: (entry->vme_end == start) &&
931: (!entry->is_shared) &&
932: (!entry->is_sub_map) &&
933: (entry->inheritance == inheritance) &&
934: (entry->protection == cur_protection) &&
935: (entry->max_protection == max_protection) &&
936: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
1.1.1.2 root 937: (entry->projected_on == 0)) {
1.1 root 938: if (vm_object_coalesce(entry->object.vm_object,
939: VM_OBJECT_NULL,
940: entry->offset,
941: (vm_offset_t) 0,
942: (vm_size_t)(entry->vme_end - entry->vme_start),
943: (vm_size_t)(end - entry->vme_end))) {
944:
945: /*
946: * Coalesced the two objects - can extend
947: * the previous map entry to include the
948: * new range.
949: */
950: map->size += (end - entry->vme_end);
951: entry->vme_end = end;
952: RETURN(KERN_SUCCESS);
953: }
954: }
955:
956: /*
957: * Create a new entry
958: */
959:
960: /**/ {
1.1.1.4 root 961: vm_map_entry_t new_entry;
1.1 root 962:
963: new_entry = vm_map_entry_create(map);
964:
965: new_entry->vme_start = start;
966: new_entry->vme_end = end;
967:
968: new_entry->is_shared = FALSE;
969: new_entry->is_sub_map = FALSE;
970: new_entry->object.vm_object = object;
971: new_entry->offset = offset;
972:
973: new_entry->needs_copy = needs_copy;
974:
975: new_entry->inheritance = inheritance;
976: new_entry->protection = cur_protection;
977: new_entry->max_protection = max_protection;
978: new_entry->wired_count = 0;
979: new_entry->user_wired_count = 0;
980:
981: new_entry->in_transition = FALSE;
982: new_entry->needs_wakeup = FALSE;
983: new_entry->projected_on = 0;
984:
985: /*
986: * Insert the new entry into the list
987: */
988:
989: vm_map_entry_link(map, entry, new_entry);
990: map->size += size;
991:
992: /*
993: * Update the free space hint and the lookup hint
994: */
995:
996: if ((map->first_free == entry) &&
997: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
998: >= new_entry->vme_start))
999: map->first_free = new_entry;
1000:
1001: SAVE_HINT(map, new_entry);
1002:
1003: vm_map_unlock(map);
1004:
1005: if ((object != VM_OBJECT_NULL) &&
1006: (vm_map_pmap_enter_enable) &&
1007: (!anywhere) &&
1.1.1.2 root 1008: (!needs_copy) &&
1.1 root 1009: (size < (128*1024))) {
1.1.1.2 root 1010: vm_map_pmap_enter(map, start, end,
1.1 root 1011: object, offset, cur_protection);
1012: }
1013:
1014: return(result);
1015: /**/ }
1016:
1017: BailOut: ;
1018:
1019: vm_map_unlock(map);
1020: return(result);
1021:
1022: #undef RETURN
1023: }
1024:
1025: /*
1026: * vm_map_clip_start: [ internal use only ]
1027: *
1028: * Asserts that the given entry begins at or after
1029: * the specified address; if necessary,
1030: * it splits the entry into two.
1031: */
1032: #define vm_map_clip_start(map, entry, startaddr) \
1033: MACRO_BEGIN \
1034: if ((startaddr) > (entry)->vme_start) \
1035: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1036: MACRO_END
1037:
1038: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1039: MACRO_BEGIN \
1040: if ((startaddr) > (entry)->vme_start) \
1041: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1042: MACRO_END
1043:
1044: /*
1045: * This routine is called only when it is known that
1046: * the entry must be split.
1047: */
1.1.1.4 root 1048: void _vm_map_clip_start(
1049: struct vm_map_header *map_header,
1050: vm_map_entry_t entry,
1051: vm_offset_t start)
1.1 root 1052: {
1.1.1.4 root 1053: vm_map_entry_t new_entry;
1.1 root 1054:
1055: /*
1056: * Split off the front portion --
1057: * note that we must insert the new
1058: * entry BEFORE this one, so that
1059: * this entry has the specified starting
1060: * address.
1061: */
1062:
1063: new_entry = _vm_map_entry_create(map_header);
1064: vm_map_entry_copy_full(new_entry, entry);
1065:
1066: new_entry->vme_end = start;
1067: entry->offset += (start - entry->vme_start);
1068: entry->vme_start = start;
1069:
1070: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1071:
1072: if (entry->is_sub_map)
1073: vm_map_reference(new_entry->object.sub_map);
1074: else
1075: vm_object_reference(new_entry->object.vm_object);
1076: }
1077:
1078: /*
1079: * vm_map_clip_end: [ internal use only ]
1080: *
1081: * Asserts that the given entry ends at or before
1082: * the specified address; if necessary,
1083: * it splits the entry into two.
1084: */
1085: #define vm_map_clip_end(map, entry, endaddr) \
1086: MACRO_BEGIN \
1087: if ((endaddr) < (entry)->vme_end) \
1088: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1089: MACRO_END
1090:
1091: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1092: MACRO_BEGIN \
1093: if ((endaddr) < (entry)->vme_end) \
1094: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1095: MACRO_END
1096:
1097: /*
1098: * This routine is called only when it is known that
1099: * the entry must be split.
1100: */
1.1.1.4 root 1101: void _vm_map_clip_end(
1102: struct vm_map_header *map_header,
1103: vm_map_entry_t entry,
1104: vm_offset_t end)
1.1 root 1105: {
1.1.1.4 root 1106: vm_map_entry_t new_entry;
1.1 root 1107:
1108: /*
1109: * Create a new entry and insert it
1110: * AFTER the specified entry
1111: */
1112:
1113: new_entry = _vm_map_entry_create(map_header);
1114: vm_map_entry_copy_full(new_entry, entry);
1115:
1116: new_entry->vme_start = entry->vme_end = end;
1117: new_entry->offset += (end - entry->vme_start);
1118:
1119: _vm_map_entry_link(map_header, entry, new_entry);
1120:
1121: if (entry->is_sub_map)
1122: vm_map_reference(new_entry->object.sub_map);
1123: else
1124: vm_object_reference(new_entry->object.vm_object);
1125: }
1126:
1127: /*
1128: * VM_MAP_RANGE_CHECK: [ internal use only ]
1129: *
1130: * Asserts that the starting and ending region
1131: * addresses fall within the valid range of the map.
1132: */
1133: #define VM_MAP_RANGE_CHECK(map, start, end) \
1134: { \
1135: if (start < vm_map_min(map)) \
1136: start = vm_map_min(map); \
1137: if (end > vm_map_max(map)) \
1138: end = vm_map_max(map); \
1139: if (start > end) \
1140: start = end; \
1141: }
1142:
1143: /*
1144: * vm_map_submap: [ kernel use only ]
1145: *
1146: * Mark the given range as handled by a subordinate map.
1147: *
1148: * This range must have been created with vm_map_find using
1149: * the vm_submap_object, and no other operations may have been
1150: * performed on this range prior to calling vm_map_submap.
1151: *
1152: * Only a limited number of operations can be performed
1153: * within this rage after calling vm_map_submap:
1154: * vm_fault
1155: * [Don't try vm_map_copyin!]
1156: *
1157: * To remove a submapping, one must first remove the
1158: * range from the superior map, and then destroy the
1159: * submap (if desired). [Better yet, don't try it.]
1160: */
1.1.1.4 root 1161: kern_return_t vm_map_submap(
1162: vm_map_t map,
1163: vm_offset_t start,
1164: vm_offset_t end,
1165: vm_map_t submap)
1.1 root 1166: {
1167: vm_map_entry_t entry;
1.1.1.4 root 1168: kern_return_t result = KERN_INVALID_ARGUMENT;
1169: vm_object_t object;
1.1 root 1170:
1171: vm_map_lock(map);
1172:
1173: VM_MAP_RANGE_CHECK(map, start, end);
1174:
1175: if (vm_map_lookup_entry(map, start, &entry)) {
1176: vm_map_clip_start(map, entry, start);
1177: }
1178: else
1179: entry = entry->vme_next;
1180:
1181: vm_map_clip_end(map, entry, end);
1182:
1183: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1184: (!entry->is_sub_map) &&
1185: ((object = entry->object.vm_object) == vm_submap_object) &&
1186: (object->resident_page_count == 0) &&
1187: (object->copy == VM_OBJECT_NULL) &&
1188: (object->shadow == VM_OBJECT_NULL) &&
1189: (!object->pager_created)) {
1190: entry->object.vm_object = VM_OBJECT_NULL;
1191: vm_object_deallocate(object);
1192: entry->is_sub_map = TRUE;
1193: vm_map_reference(entry->object.sub_map = submap);
1194: result = KERN_SUCCESS;
1195: }
1196: vm_map_unlock(map);
1197:
1198: return(result);
1199: }
1200:
1201: /*
1202: * vm_map_protect:
1203: *
1204: * Sets the protection of the specified address
1205: * region in the target map. If "set_max" is
1206: * specified, the maximum protection is to be set;
1207: * otherwise, only the current protection is affected.
1208: */
1.1.1.4 root 1209: kern_return_t vm_map_protect(
1210: vm_map_t map,
1211: vm_offset_t start,
1212: vm_offset_t end,
1213: vm_prot_t new_prot,
1214: boolean_t set_max)
1.1 root 1215: {
1.1.1.4 root 1216: vm_map_entry_t current;
1217: vm_map_entry_t entry;
1.1 root 1218:
1219: vm_map_lock(map);
1220:
1221: VM_MAP_RANGE_CHECK(map, start, end);
1222:
1223: if (vm_map_lookup_entry(map, start, &entry)) {
1224: vm_map_clip_start(map, entry, start);
1225: }
1226: else
1227: entry = entry->vme_next;
1228:
1229: /*
1230: * Make a first pass to check for protection
1231: * violations.
1232: */
1233:
1234: current = entry;
1235: while ((current != vm_map_to_entry(map)) &&
1236: (current->vme_start < end)) {
1237:
1238: if (current->is_sub_map) {
1239: vm_map_unlock(map);
1240: return(KERN_INVALID_ARGUMENT);
1241: }
1242: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1243: != new_prot) {
1244: vm_map_unlock(map);
1245: return(KERN_PROTECTION_FAILURE);
1246: }
1247:
1248: current = current->vme_next;
1249: }
1250:
1251: /*
1252: * Go back and fix up protections.
1253: * [Note that clipping is not necessary the second time.]
1254: */
1255:
1256: current = entry;
1257:
1258: while ((current != vm_map_to_entry(map)) &&
1259: (current->vme_start < end)) {
1260:
1261: vm_prot_t old_prot;
1262:
1263: vm_map_clip_end(map, current, end);
1264:
1265: old_prot = current->protection;
1266: if (set_max)
1267: current->protection =
1268: (current->max_protection = new_prot) &
1269: old_prot;
1270: else
1271: current->protection = new_prot;
1272:
1273: /*
1274: * Update physical map if necessary.
1275: */
1276:
1277: if (current->protection != old_prot) {
1278: pmap_protect(map->pmap, current->vme_start,
1279: current->vme_end,
1280: current->protection);
1281: }
1282: current = current->vme_next;
1283: }
1284:
1285: vm_map_unlock(map);
1286: return(KERN_SUCCESS);
1287: }
1288:
1289: /*
1290: * vm_map_inherit:
1291: *
1292: * Sets the inheritance of the specified address
1293: * range in the target map. Inheritance
1294: * affects how the map will be shared with
1295: * child maps at the time of vm_map_fork.
1296: */
1.1.1.4 root 1297: kern_return_t vm_map_inherit(
1298: vm_map_t map,
1299: vm_offset_t start,
1300: vm_offset_t end,
1301: vm_inherit_t new_inheritance)
1.1 root 1302: {
1.1.1.4 root 1303: vm_map_entry_t entry;
1.1 root 1304: vm_map_entry_t temp_entry;
1305:
1306: vm_map_lock(map);
1307:
1308: VM_MAP_RANGE_CHECK(map, start, end);
1309:
1310: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1311: entry = temp_entry;
1312: vm_map_clip_start(map, entry, start);
1313: }
1314: else
1315: entry = temp_entry->vme_next;
1316:
1317: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1318: vm_map_clip_end(map, entry, end);
1319:
1320: entry->inheritance = new_inheritance;
1321:
1322: entry = entry->vme_next;
1323: }
1324:
1325: vm_map_unlock(map);
1326: return(KERN_SUCCESS);
1327: }
1328:
1329: /*
1330: * vm_map_pageable_common:
1331: *
1332: * Sets the pageability of the specified address
1333: * range in the target map. Regions specified
1334: * as not pageable require locked-down physical
1335: * memory and physical page maps. access_type indicates
1336: * types of accesses that must not generate page faults.
1337: * This is checked against protection of memory being locked-down.
1338: * access_type of VM_PROT_NONE makes memory pageable.
1339: *
1340: * The map must not be locked, but a reference
1341: * must remain to the map throughout the call.
1342: *
1343: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1344: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1345: */
1.1.1.4 root 1346: kern_return_t vm_map_pageable_common(
1347: vm_map_t map,
1348: vm_offset_t start,
1349: vm_offset_t end,
1350: vm_prot_t access_type,
1351: boolean_t user_wire)
1.1 root 1352: {
1.1.1.4 root 1353: vm_map_entry_t entry;
1.1 root 1354: vm_map_entry_t start_entry;
1355:
1356: vm_map_lock(map);
1357:
1358: VM_MAP_RANGE_CHECK(map, start, end);
1359:
1360: if (vm_map_lookup_entry(map, start, &start_entry)) {
1361: entry = start_entry;
1362: /*
1363: * vm_map_clip_start will be done later.
1364: */
1365: }
1366: else {
1367: /*
1368: * Start address is not in map; this is fatal.
1369: */
1370: vm_map_unlock(map);
1371: return(KERN_FAILURE);
1372: }
1373:
1374: /*
1375: * Actions are rather different for wiring and unwiring,
1376: * so we have two separate cases.
1377: */
1378:
1379: if (access_type == VM_PROT_NONE) {
1380:
1381: vm_map_clip_start(map, entry, start);
1382:
1383: /*
1384: * Unwiring. First ensure that the range to be
1385: * unwired is really wired down.
1386: */
1387: while ((entry != vm_map_to_entry(map)) &&
1388: (entry->vme_start < end)) {
1389:
1390: if ((entry->wired_count == 0) ||
1.1.1.2 root 1391: ((entry->vme_end < end) &&
1.1 root 1392: ((entry->vme_next == vm_map_to_entry(map)) ||
1393: (entry->vme_next->vme_start > entry->vme_end))) ||
1394: (user_wire && (entry->user_wired_count == 0))) {
1395: vm_map_unlock(map);
1396: return(KERN_INVALID_ARGUMENT);
1397: }
1398: entry = entry->vme_next;
1399: }
1400:
1401: /*
1402: * Now decrement the wiring count for each region.
1403: * If a region becomes completely unwired,
1404: * unwire its physical pages and mappings.
1405: */
1406: entry = start_entry;
1407: while ((entry != vm_map_to_entry(map)) &&
1408: (entry->vme_start < end)) {
1409: vm_map_clip_end(map, entry, end);
1410:
1411: if (user_wire) {
1412: if (--(entry->user_wired_count) == 0)
1.1.1.5 ! root 1413: {
! 1414: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1415: entry->wired_count--;
1.1.1.5 ! root 1416: }
1.1 root 1417: }
1418: else {
1419: entry->wired_count--;
1420: }
1.1.1.2 root 1421:
1.1 root 1422: if (entry->wired_count == 0)
1423: vm_fault_unwire(map, entry);
1424:
1425: entry = entry->vme_next;
1426: }
1427: }
1428:
1429: else {
1430: /*
1431: * Wiring. We must do this in two passes:
1432: *
1433: * 1. Holding the write lock, we create any shadow
1434: * or zero-fill objects that need to be created.
1435: * Then we clip each map entry to the region to be
1436: * wired and increment its wiring count. We
1437: * create objects before clipping the map entries
1438: * to avoid object proliferation.
1439: *
1440: * 2. We downgrade to a read lock, and call
1441: * vm_fault_wire to fault in the pages for any
1442: * newly wired area (wired_count is 1).
1443: *
1444: * Downgrading to a read lock for vm_fault_wire avoids
1445: * a possible deadlock with another thread that may have
1446: * faulted on one of the pages to be wired (it would mark
1447: * the page busy, blocking us, then in turn block on the
1448: * map lock that we hold). Because of problems in the
1449: * recursive lock package, we cannot upgrade to a write
1450: * lock in vm_map_lookup. Thus, any actions that require
1451: * the write lock must be done beforehand. Because we
1452: * keep the read lock on the map, the copy-on-write
1453: * status of the entries we modify here cannot change.
1454: */
1455:
1456: /*
1457: * Pass 1.
1458: */
1459: while ((entry != vm_map_to_entry(map)) &&
1460: (entry->vme_start < end)) {
1461: vm_map_clip_end(map, entry, end);
1462:
1463: if (entry->wired_count == 0) {
1464:
1465: /*
1466: * Perform actions of vm_map_lookup that need
1467: * the write lock on the map: create a shadow
1468: * object for a copy-on-write region, or an
1469: * object for a zero-fill region.
1470: */
1471: if (entry->needs_copy &&
1472: ((entry->protection & VM_PROT_WRITE) != 0)) {
1473:
1474: vm_object_shadow(&entry->object.vm_object,
1475: &entry->offset,
1476: (vm_size_t)(entry->vme_end
1477: - entry->vme_start));
1478: entry->needs_copy = FALSE;
1479: }
1480: if (entry->object.vm_object == VM_OBJECT_NULL) {
1481: entry->object.vm_object =
1482: vm_object_allocate(
1.1.1.2 root 1483: (vm_size_t)(entry->vme_end
1.1 root 1484: - entry->vme_start));
1485: entry->offset = (vm_offset_t)0;
1486: }
1487: }
1488: vm_map_clip_start(map, entry, start);
1489: vm_map_clip_end(map, entry, end);
1490:
1491: if (user_wire) {
1492: if ((entry->user_wired_count)++ == 0)
1.1.1.5 ! root 1493: {
! 1494: map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 1495: entry->wired_count++;
1.1.1.5 ! root 1496: }
1.1 root 1497: }
1498: else {
1499: entry->wired_count++;
1500: }
1501:
1502: /*
1503: * Check for holes and protection mismatch.
1504: * Holes: Next entry should be contiguous unless
1505: * this is the end of the region.
1506: * Protection: Access requested must be allowed.
1507: */
1.1.1.2 root 1508: if (((entry->vme_end < end) &&
1.1 root 1509: ((entry->vme_next == vm_map_to_entry(map)) ||
1510: (entry->vme_next->vme_start > entry->vme_end))) ||
1511: ((entry->protection & access_type) != access_type)) {
1512: /*
1513: * Found a hole or protection problem.
1514: * Object creation actions
1515: * do not need to be undone, but the
1516: * wired counts need to be restored.
1517: */
1518: while ((entry != vm_map_to_entry(map)) &&
1519: (entry->vme_end > start)) {
1520: if (user_wire) {
1521: if (--(entry->user_wired_count) == 0)
1.1.1.5 ! root 1522: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1523: entry->wired_count--;
1524: }
1525: else {
1526: entry->wired_count--;
1527: }
1528:
1529: entry = entry->vme_prev;
1530: }
1531:
1532: vm_map_unlock(map);
1533: return(KERN_FAILURE);
1534: }
1535: entry = entry->vme_next;
1536: }
1537:
1538: /*
1539: * Pass 2.
1540: */
1541:
1542: /*
1543: * HACK HACK HACK HACK
1544: *
1545: * If we are wiring in the kernel map or a submap of it,
1546: * unlock the map to avoid deadlocks. We trust that the
1547: * kernel threads are well-behaved, and therefore will
1548: * not do anything destructive to this region of the map
1549: * while we have it unlocked. We cannot trust user threads
1550: * to do the same.
1551: *
1552: * HACK HACK HACK HACK
1553: */
1554: if (vm_map_pmap(map) == kernel_pmap) {
1555: vm_map_unlock(map); /* trust me ... */
1556: }
1557: else {
1558: vm_map_lock_set_recursive(map);
1559: vm_map_lock_write_to_read(map);
1560: }
1561:
1562: entry = start_entry;
1563: while (entry != vm_map_to_entry(map) &&
1564: entry->vme_start < end) {
1565: /*
1566: * Wiring cases:
1567: * Kernel: wired == 1 && user_wired == 0
1568: * User: wired == 1 && user_wired == 1
1569: *
1570: * Don't need to wire if either is > 1. wired = 0 &&
1571: * user_wired == 1 can't happen.
1572: */
1573:
1574: /*
1575: * XXX This assumes that the faults always succeed.
1576: */
1577: if ((entry->wired_count == 1) &&
1578: (entry->user_wired_count <= 1)) {
1579: vm_fault_wire(map, entry);
1580: }
1581: entry = entry->vme_next;
1582: }
1583:
1584: if (vm_map_pmap(map) == kernel_pmap) {
1585: vm_map_lock(map);
1586: }
1587: else {
1588: vm_map_lock_clear_recursive(map);
1589: }
1590: }
1591:
1592: vm_map_unlock(map);
1593:
1594: return(KERN_SUCCESS);
1595: }
1596:
1597: /*
1598: * vm_map_entry_delete: [ internal use only ]
1599: *
1600: * Deallocate the given entry from the target map.
1.1.1.2 root 1601: */
1.1.1.4 root 1602: void vm_map_entry_delete(
1603: vm_map_t map,
1604: vm_map_entry_t entry)
1.1 root 1605: {
1.1.1.4 root 1606: vm_offset_t s, e;
1607: vm_object_t object;
1.1 root 1608: extern vm_object_t kernel_object;
1609:
1610: s = entry->vme_start;
1611: e = entry->vme_end;
1612:
1613: /*Check if projected buffer*/
1614: if (map != kernel_map && entry->projected_on != 0) {
1615: /*Check if projected kernel entry is persistent;
1616: may only manipulate directly if it is*/
1617: if (entry->projected_on->projected_on == 0)
1618: entry->wired_count = 0; /*Avoid unwire fault*/
1619: else
1620: return;
1621: }
1622:
1623: /*
1624: * Get the object. Null objects cannot have pmap entries.
1625: */
1626:
1627: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1628:
1629: /*
1630: * Unwire before removing addresses from the pmap;
1631: * otherwise, unwiring will put the entries back in
1632: * the pmap.
1633: */
1634:
1635: if (entry->wired_count != 0) {
1636: vm_fault_unwire(map, entry);
1637: entry->wired_count = 0;
1.1.1.5 ! root 1638: if (entry->user_wired_count)
! 1639: map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 1640: entry->user_wired_count = 0;
1641: }
1642:
1643: /*
1644: * If the object is shared, we must remove
1645: * *all* references to this data, since we can't
1646: * find all of the physical maps which are sharing
1647: * it.
1648: */
1649:
1650: if (object == kernel_object) {
1651: vm_object_lock(object);
1652: vm_object_page_remove(object, entry->offset,
1653: entry->offset + (e - s));
1654: vm_object_unlock(object);
1655: } else if (entry->is_shared) {
1656: vm_object_pmap_remove(object,
1657: entry->offset,
1658: entry->offset + (e - s));
1659: }
1660: else {
1661: pmap_remove(map->pmap, s, e);
1662: }
1663: }
1664:
1665: /*
1666: * Deallocate the object only after removing all
1667: * pmap entries pointing to its pages.
1668: */
1669:
1670: if (entry->is_sub_map)
1671: vm_map_deallocate(entry->object.sub_map);
1672: else
1673: vm_object_deallocate(entry->object.vm_object);
1674:
1675: vm_map_entry_unlink(map, entry);
1676: map->size -= e - s;
1677:
1678: vm_map_entry_dispose(map, entry);
1679: }
1680:
1681: /*
1682: * vm_map_delete: [ internal use only ]
1683: *
1684: * Deallocates the given address range from the target
1685: * map.
1686: */
1687:
1.1.1.4 root 1688: kern_return_t vm_map_delete(
1689: vm_map_t map,
1690: vm_offset_t start,
1691: vm_offset_t end)
1.1 root 1692: {
1693: vm_map_entry_t entry;
1694: vm_map_entry_t first_entry;
1695:
1696: /*
1697: * Find the start of the region, and clip it
1698: */
1699:
1700: if (!vm_map_lookup_entry(map, start, &first_entry))
1701: entry = first_entry->vme_next;
1702: else {
1703: entry = first_entry;
1704: vm_map_clip_start(map, entry, start);
1705:
1706: /*
1707: * Fix the lookup hint now, rather than each
1708: * time though the loop.
1709: */
1710:
1711: SAVE_HINT(map, entry->vme_prev);
1712: }
1713:
1714: /*
1715: * Save the free space hint
1716: */
1717:
1718: if (map->first_free->vme_start >= start)
1719: map->first_free = entry->vme_prev;
1720:
1721: /*
1722: * Step through all entries in this region
1723: */
1724:
1725: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1726: vm_map_entry_t next;
1727:
1728: vm_map_clip_end(map, entry, end);
1729:
1730: /*
1731: * If the entry is in transition, we must wait
1732: * for it to exit that state. It could be clipped
1733: * while we leave the map unlocked.
1734: */
1735: if(entry->in_transition) {
1736: /*
1737: * Say that we are waiting, and wait for entry.
1738: */
1739: entry->needs_wakeup = TRUE;
1740: vm_map_entry_wait(map, FALSE);
1741: vm_map_lock(map);
1742:
1743: /*
1744: * The entry could have been clipped or it
1745: * may not exist anymore. look it up again.
1746: */
1747: if(!vm_map_lookup_entry(map, start, &entry)) {
1748: entry = entry->vme_next;
1749: }
1750: continue;
1751: }
1752:
1753: next = entry->vme_next;
1754:
1755: vm_map_entry_delete(map, entry);
1756: entry = next;
1757: }
1758:
1759: if (map->wait_for_space)
1760: thread_wakeup((event_t) map);
1761:
1762: return(KERN_SUCCESS);
1763: }
1764:
1765: /*
1766: * vm_map_remove:
1767: *
1768: * Remove the given address range from the target map.
1769: * This is the exported form of vm_map_delete.
1770: */
1.1.1.4 root 1771: kern_return_t vm_map_remove(
1772: vm_map_t map,
1773: vm_offset_t start,
1774: vm_offset_t end)
1.1 root 1775: {
1.1.1.4 root 1776: kern_return_t result;
1.1 root 1777:
1778: vm_map_lock(map);
1779: VM_MAP_RANGE_CHECK(map, start, end);
1780: result = vm_map_delete(map, start, end);
1781: vm_map_unlock(map);
1782:
1783: return(result);
1784: }
1785:
1786:
1787: /*
1788: * vm_map_copy_steal_pages:
1789: *
1790: * Steal all the pages from a vm_map_copy page_list by copying ones
1791: * that have not already been stolen.
1792: */
1793: void
1.1.1.4 root 1794: vm_map_copy_steal_pages(vm_map_copy_t copy)
1.1 root 1795: {
1.1.1.4 root 1796: vm_page_t m, new_m;
1797: int i;
1798: vm_object_t object;
1.1 root 1799:
1800: for (i = 0; i < copy->cpy_npages; i++) {
1801:
1802: /*
1803: * If the page is not tabled, then it's already stolen.
1804: */
1805: m = copy->cpy_page_list[i];
1806: if (!m->tabled)
1807: continue;
1808:
1809: /*
1810: * Page was not stolen, get a new
1811: * one and do the copy now.
1812: */
1.1.1.2 root 1813: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) {
1.1 root 1814: VM_PAGE_WAIT((void(*)()) 0);
1815: }
1816:
1817: vm_page_copy(m, new_m);
1818:
1819: object = m->object;
1820: vm_object_lock(object);
1821: vm_page_lock_queues();
1822: if (!m->active && !m->inactive)
1823: vm_page_activate(m);
1824: vm_page_unlock_queues();
1825: PAGE_WAKEUP_DONE(m);
1826: vm_object_paging_end(object);
1827: vm_object_unlock(object);
1828:
1829: copy->cpy_page_list[i] = new_m;
1830: }
1831: }
1832:
1833: /*
1834: * vm_map_copy_page_discard:
1835: *
1836: * Get rid of the pages in a page_list copy. If the pages are
1837: * stolen, they are freed. If the pages are not stolen, they
1838: * are unbusied, and associated state is cleaned up.
1839: */
1.1.1.4 root 1840: void vm_map_copy_page_discard(vm_map_copy_t copy)
1.1 root 1841: {
1842: while (copy->cpy_npages > 0) {
1843: vm_page_t m;
1844:
1845: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1846: VM_PAGE_NULL) {
1847:
1848: /*
1849: * If it's not in the table, then it's
1850: * a stolen page that goes back
1851: * to the free list. Else it belongs
1852: * to some object, and we hold a
1853: * paging reference on that object.
1854: */
1855: if (!m->tabled) {
1856: VM_PAGE_FREE(m);
1857: }
1858: else {
1859: vm_object_t object;
1860:
1861: object = m->object;
1862:
1863: vm_object_lock(object);
1864: vm_page_lock_queues();
1865: if (!m->active && !m->inactive)
1866: vm_page_activate(m);
1867: vm_page_unlock_queues();
1868:
1869: PAGE_WAKEUP_DONE(m);
1870: vm_object_paging_end(object);
1871: vm_object_unlock(object);
1872: }
1873: }
1874: }
1875: }
1876:
1877: /*
1878: * Routine: vm_map_copy_discard
1879: *
1880: * Description:
1881: * Dispose of a map copy object (returned by
1882: * vm_map_copyin).
1883: */
1884: void
1.1.1.4 root 1885: vm_map_copy_discard(vm_map_copy_t copy)
1.1 root 1886: {
1887: free_next_copy:
1888: if (copy == VM_MAP_COPY_NULL)
1889: return;
1890:
1891: switch (copy->type) {
1892: case VM_MAP_COPY_ENTRY_LIST:
1893: while (vm_map_copy_first_entry(copy) !=
1894: vm_map_copy_to_entry(copy)) {
1895: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
1896:
1897: vm_map_copy_entry_unlink(copy, entry);
1898: vm_object_deallocate(entry->object.vm_object);
1899: vm_map_copy_entry_dispose(copy, entry);
1900: }
1901: break;
1902: case VM_MAP_COPY_OBJECT:
1903: vm_object_deallocate(copy->cpy_object);
1904: break;
1905: case VM_MAP_COPY_PAGE_LIST:
1906:
1907: /*
1908: * To clean this up, we have to unbusy all the pages
1909: * and release the paging references in their objects.
1910: */
1911: if (copy->cpy_npages > 0)
1912: vm_map_copy_page_discard(copy);
1913:
1914: /*
1915: * If there's a continuation, abort it. The
1916: * abort routine releases any storage.
1917: */
1918: if (vm_map_copy_has_cont(copy)) {
1919:
1920: /*
1921: * Special case: recognize
1922: * vm_map_copy_discard_cont and optimize
1923: * here to avoid tail recursion.
1924: */
1925: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1.1.1.4 root 1926: vm_map_copy_t new_copy;
1.1 root 1927:
1928: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1.1.1.3 root 1929: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1930: copy = new_copy;
1931: goto free_next_copy;
1932: }
1933: else {
1934: vm_map_copy_abort_cont(copy);
1935: }
1936: }
1937:
1938: break;
1939: }
1.1.1.3 root 1940: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1941: }
1942:
1943: /*
1944: * Routine: vm_map_copy_copy
1945: *
1946: * Description:
1947: * Move the information in a map copy object to
1948: * a new map copy object, leaving the old one
1949: * empty.
1950: *
1951: * This is used by kernel routines that need
1952: * to look at out-of-line data (in copyin form)
1953: * before deciding whether to return SUCCESS.
1954: * If the routine returns FAILURE, the original
1955: * copy object will be deallocated; therefore,
1956: * these routines must make a copy of the copy
1957: * object and leave the original empty so that
1958: * deallocation will not fail.
1959: */
1960: vm_map_copy_t
1.1.1.4 root 1961: vm_map_copy_copy(vm_map_copy_t copy)
1.1 root 1962: {
1963: vm_map_copy_t new_copy;
1964:
1965: if (copy == VM_MAP_COPY_NULL)
1966: return VM_MAP_COPY_NULL;
1967:
1968: /*
1969: * Allocate a new copy object, and copy the information
1970: * from the old one into it.
1971: */
1972:
1.1.1.3 root 1973: new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 1974: *new_copy = *copy;
1975:
1976: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
1977: /*
1978: * The links in the entry chain must be
1979: * changed to point to the new copy object.
1980: */
1981: vm_map_copy_first_entry(copy)->vme_prev
1982: = vm_map_copy_to_entry(new_copy);
1983: vm_map_copy_last_entry(copy)->vme_next
1984: = vm_map_copy_to_entry(new_copy);
1985: }
1986:
1987: /*
1988: * Change the old copy object into one that contains
1989: * nothing to be deallocated.
1990: */
1991: copy->type = VM_MAP_COPY_OBJECT;
1992: copy->cpy_object = VM_OBJECT_NULL;
1993:
1994: /*
1995: * Return the new object.
1996: */
1997: return new_copy;
1998: }
1999:
2000: /*
2001: * Routine: vm_map_copy_discard_cont
2002: *
2003: * Description:
2004: * A version of vm_map_copy_discard that can be called
2005: * as a continuation from a vm_map_copy page list.
2006: */
1.1.1.4 root 2007: kern_return_t vm_map_copy_discard_cont(
2008: vm_map_copyin_args_t cont_args,
2009: vm_map_copy_t *copy_result) /* OUT */
1.1 root 2010: {
2011: vm_map_copy_discard((vm_map_copy_t) cont_args);
2012: if (copy_result != (vm_map_copy_t *)0)
2013: *copy_result = VM_MAP_COPY_NULL;
2014: return(KERN_SUCCESS);
2015: }
2016:
2017: /*
2018: * Routine: vm_map_copy_overwrite
2019: *
2020: * Description:
2021: * Copy the memory described by the map copy
2022: * object (copy; returned by vm_map_copyin) onto
2023: * the specified destination region (dst_map, dst_addr).
2024: * The destination must be writeable.
2025: *
2026: * Unlike vm_map_copyout, this routine actually
2027: * writes over previously-mapped memory. If the
2028: * previous mapping was to a permanent (user-supplied)
2029: * memory object, it is preserved.
2030: *
2031: * The attributes (protection and inheritance) of the
2032: * destination region are preserved.
2033: *
2034: * If successful, consumes the copy object.
2035: * Otherwise, the caller is responsible for it.
2036: *
2037: * Implementation notes:
2038: * To overwrite temporary virtual memory, it is
2039: * sufficient to remove the previous mapping and insert
2040: * the new copy. This replacement is done either on
2041: * the whole region (if no permanent virtual memory
2042: * objects are embedded in the destination region) or
2043: * in individual map entries.
2044: *
2045: * To overwrite permanent virtual memory, it is
2046: * necessary to copy each page, as the external
2047: * memory management interface currently does not
2048: * provide any optimizations.
2049: *
2050: * Once a page of permanent memory has been overwritten,
2051: * it is impossible to interrupt this function; otherwise,
2052: * the call would be neither atomic nor location-independent.
2053: * The kernel-state portion of a user thread must be
2054: * interruptible.
2055: *
2056: * It may be expensive to forward all requests that might
2057: * overwrite permanent memory (vm_write, vm_copy) to
2058: * uninterruptible kernel threads. This routine may be
2059: * called by interruptible threads; however, success is
2060: * not guaranteed -- if the request cannot be performed
2061: * atomically and interruptibly, an error indication is
2062: * returned.
2063: */
1.1.1.4 root 2064: kern_return_t vm_map_copy_overwrite(
2065: vm_map_t dst_map,
2066: vm_offset_t dst_addr,
2067: vm_map_copy_t copy,
2068: boolean_t interruptible)
1.1 root 2069: {
2070: vm_size_t size;
2071: vm_offset_t start;
2072: vm_map_entry_t tmp_entry;
2073: vm_map_entry_t entry;
2074:
2075: boolean_t contains_permanent_objects = FALSE;
2076:
2077: interruptible = FALSE; /* XXX */
2078:
2079: /*
2080: * Check for null copy object.
2081: */
2082:
2083: if (copy == VM_MAP_COPY_NULL)
2084: return(KERN_SUCCESS);
2085:
2086: /*
2087: * Only works for entry lists at the moment. Will
2088: * support page lists LATER.
2089: */
2090:
2091: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2092:
2093: /*
2094: * Currently this routine only handles page-aligned
2095: * regions. Eventually, it should handle misalignments
2096: * by actually copying pages.
2097: */
2098:
2099: if (!page_aligned(copy->offset) ||
2100: !page_aligned(copy->size) ||
2101: !page_aligned(dst_addr))
2102: return(KERN_INVALID_ARGUMENT);
2103:
2104: size = copy->size;
2105:
2106: if (size == 0) {
2107: vm_map_copy_discard(copy);
2108: return(KERN_SUCCESS);
2109: }
2110:
2111: /*
2112: * Verify that the destination is all writeable
2113: * initially.
2114: */
2115: start_pass_1:
2116: vm_map_lock(dst_map);
2117: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2118: vm_map_unlock(dst_map);
2119: return(KERN_INVALID_ADDRESS);
2120: }
2121: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2122: for (entry = tmp_entry;;) {
2123: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2124: vm_map_entry_t next = entry->vme_next;
2125:
2126: if ( ! (entry->protection & VM_PROT_WRITE)) {
2127: vm_map_unlock(dst_map);
2128: return(KERN_PROTECTION_FAILURE);
2129: }
2130:
2131: /*
2132: * If the entry is in transition, we must wait
2133: * for it to exit that state. Anything could happen
2134: * when we unlock the map, so start over.
2135: */
2136: if (entry->in_transition) {
2137:
2138: /*
2139: * Say that we are waiting, and wait for entry.
2140: */
2141: entry->needs_wakeup = TRUE;
2142: vm_map_entry_wait(dst_map, FALSE);
2143:
2144: goto start_pass_1;
2145: }
2146:
2147: if (size <= sub_size)
2148: break;
2149:
2150: if ((next == vm_map_to_entry(dst_map)) ||
2151: (next->vme_start != entry->vme_end)) {
2152: vm_map_unlock(dst_map);
2153: return(KERN_INVALID_ADDRESS);
2154: }
2155:
2156:
2157: /*
2158: * Check for permanent objects in the destination.
2159: */
2160:
2161: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2162: !entry->object.vm_object->temporary)
2163: contains_permanent_objects = TRUE;
2164:
2165: size -= sub_size;
2166: entry = next;
2167: }
2168:
2169: /*
2170: * If there are permanent objects in the destination, then
2171: * the copy cannot be interrupted.
2172: */
2173:
1.1.1.3 root 2174: if (interruptible && contains_permanent_objects) {
2175: vm_map_unlock(dst_map);
1.1 root 2176: return(KERN_FAILURE); /* XXX */
1.1.1.3 root 2177: }
1.1 root 2178:
2179: /*
2180: * XXXO If there are no permanent objects in the destination,
1.1.1.3 root 2181: * XXXO and the source and destination map entry caches match,
1.1.1.2 root 2182: * XXXO and the destination map entry is not shared,
1.1 root 2183: * XXXO then the map entries can be deleted and replaced
2184: * XXXO with those from the copy. The following code is the
2185: * XXXO basic idea of what to do, but there are lots of annoying
2186: * XXXO little details about getting protection and inheritance
2187: * XXXO right. Should add protection, inheritance, and sharing checks
2188: * XXXO to the above pass and make sure that no wiring is involved.
2189: */
2190: /*
2191: * if (!contains_permanent_objects &&
2192: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2193: *
2194: * *
2195: * * Run over copy and adjust entries. Steal code
2196: * * from vm_map_copyout() to do this.
2197: * *
2198: *
2199: * tmp_entry = tmp_entry->vme_prev;
2200: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2201: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2202: *
2203: * vm_map_unlock(dst_map);
2204: * vm_map_copy_discard(copy);
2205: * }
2206: */
2207: /*
2208: *
2209: * Make a second pass, overwriting the data
2210: * At the beginning of each loop iteration,
2211: * the next entry to be overwritten is "tmp_entry"
2212: * (initially, the value returned from the lookup above),
2213: * and the starting address expected in that entry
2214: * is "start".
2215: */
2216:
2217: start = dst_addr;
2218:
2219: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2220: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2221: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2222: vm_object_t object;
1.1.1.2 root 2223:
1.1 root 2224: entry = tmp_entry;
2225: size = (entry->vme_end - entry->vme_start);
2226: /*
2227: * Make sure that no holes popped up in the
2228: * address map, and that the protection is
2229: * still valid, in case the map was unlocked
2230: * earlier.
2231: */
2232:
2233: if (entry->vme_start != start) {
2234: vm_map_unlock(dst_map);
2235: return(KERN_INVALID_ADDRESS);
2236: }
2237: assert(entry != vm_map_to_entry(dst_map));
2238:
2239: /*
2240: * Check protection again
2241: */
2242:
2243: if ( ! (entry->protection & VM_PROT_WRITE)) {
2244: vm_map_unlock(dst_map);
2245: return(KERN_PROTECTION_FAILURE);
2246: }
2247:
2248: /*
2249: * Adjust to source size first
2250: */
2251:
2252: if (copy_size < size) {
2253: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2254: size = copy_size;
2255: }
2256:
2257: /*
2258: * Adjust to destination size
2259: */
2260:
2261: if (size < copy_size) {
2262: vm_map_copy_clip_end(copy, copy_entry,
2263: copy_entry->vme_start + size);
2264: copy_size = size;
2265: }
2266:
2267: assert((entry->vme_end - entry->vme_start) == size);
2268: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2269: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2270:
2271: /*
2272: * If the destination contains temporary unshared memory,
2273: * we can perform the copy by throwing it away and
2274: * installing the source data.
2275: */
2276:
2277: object = entry->object.vm_object;
2278: if (!entry->is_shared &&
2279: ((object == VM_OBJECT_NULL) || object->temporary)) {
2280: vm_object_t old_object = entry->object.vm_object;
2281: vm_offset_t old_offset = entry->offset;
2282:
2283: entry->object = copy_entry->object;
2284: entry->offset = copy_entry->offset;
2285: entry->needs_copy = copy_entry->needs_copy;
2286: entry->wired_count = 0;
1.1.1.5 ! root 2287: if (entry->user_wired_count)
! 2288: dst_map->user_wired -= entry->vme_end - entry->vme_start;
1.1 root 2289: entry->user_wired_count = 0;
2290:
2291: vm_map_copy_entry_unlink(copy, copy_entry);
2292: vm_map_copy_entry_dispose(copy, copy_entry);
2293:
2294: vm_object_pmap_protect(
2295: old_object,
2296: old_offset,
2297: size,
2298: dst_map->pmap,
2299: tmp_entry->vme_start,
2300: VM_PROT_NONE);
2301:
2302: vm_object_deallocate(old_object);
2303:
2304: /*
2305: * Set up for the next iteration. The map
2306: * has not been unlocked, so the next
2307: * address should be at the end of this
2308: * entry, and the next map entry should be
2309: * the one following it.
2310: */
2311:
2312: start = tmp_entry->vme_end;
2313: tmp_entry = tmp_entry->vme_next;
2314: } else {
2315: vm_map_version_t version;
2316: vm_object_t dst_object = entry->object.vm_object;
2317: vm_offset_t dst_offset = entry->offset;
2318: kern_return_t r;
2319:
2320: /*
2321: * Take an object reference, and record
2322: * the map version information so that the
2323: * map can be safely unlocked.
2324: */
2325:
2326: vm_object_reference(dst_object);
2327:
2328: version.main_timestamp = dst_map->timestamp;
2329:
2330: vm_map_unlock(dst_map);
2331:
2332: /*
2333: * Copy as much as possible in one pass
2334: */
2335:
2336: copy_size = size;
2337: r = vm_fault_copy(
2338: copy_entry->object.vm_object,
2339: copy_entry->offset,
2340: ©_size,
2341: dst_object,
2342: dst_offset,
2343: dst_map,
2344: &version,
2345: FALSE /* XXX interruptible */ );
2346:
2347: /*
2348: * Release the object reference
2349: */
2350:
2351: vm_object_deallocate(dst_object);
2352:
2353: /*
2354: * If a hard error occurred, return it now
2355: */
2356:
2357: if (r != KERN_SUCCESS)
2358: return(r);
2359:
2360: if (copy_size != 0) {
2361: /*
2362: * Dispose of the copied region
2363: */
2364:
2365: vm_map_copy_clip_end(copy, copy_entry,
2366: copy_entry->vme_start + copy_size);
2367: vm_map_copy_entry_unlink(copy, copy_entry);
2368: vm_object_deallocate(copy_entry->object.vm_object);
2369: vm_map_copy_entry_dispose(copy, copy_entry);
2370: }
2371:
2372: /*
2373: * Pick up in the destination map where we left off.
2374: *
2375: * Use the version information to avoid a lookup
2376: * in the normal case.
2377: */
2378:
2379: start += copy_size;
2380: vm_map_lock(dst_map);
2381: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2382: /* We can safely use saved tmp_entry value */
2383:
2384: vm_map_clip_end(dst_map, tmp_entry, start);
2385: tmp_entry = tmp_entry->vme_next;
2386: } else {
2387: /* Must do lookup of tmp_entry */
2388:
2389: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2390: vm_map_unlock(dst_map);
2391: return(KERN_INVALID_ADDRESS);
2392: }
2393: vm_map_clip_start(dst_map, tmp_entry, start);
2394: }
2395: }
2396:
2397: }
2398: vm_map_unlock(dst_map);
2399:
2400: /*
2401: * Throw away the vm_map_copy object
2402: */
2403: vm_map_copy_discard(copy);
2404:
2405: return(KERN_SUCCESS);
2406: }
2407:
2408: /*
2409: * Macro: vm_map_copy_insert
1.1.1.2 root 2410: *
1.1 root 2411: * Description:
2412: * Link a copy chain ("copy") into a map at the
2413: * specified location (after "where").
2414: * Side effects:
2415: * The copy chain is destroyed.
2416: * Warning:
2417: * The arguments are evaluated multiple times.
2418: */
2419: #define vm_map_copy_insert(map, where, copy) \
2420: MACRO_BEGIN \
1.1.1.3 root 2421: struct rbtree_node *node, *tmp; \
2422: rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \
2423: rbtree_insert(&(map)->hdr.tree, node, \
2424: vm_map_entry_cmp_insert); \
1.1 root 2425: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \
2426: ->vme_next = ((where)->vme_next); \
2427: ((where)->vme_next = vm_map_copy_first_entry(copy)) \
2428: ->vme_prev = (where); \
2429: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \
1.1.1.3 root 2430: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
1.1 root 2431: MACRO_END
2432:
2433: /*
2434: * Routine: vm_map_copyout
2435: *
2436: * Description:
2437: * Copy out a copy chain ("copy") into newly-allocated
2438: * space in the destination map.
2439: *
2440: * If successful, consumes the copy object.
2441: * Otherwise, the caller is responsible for it.
2442: */
1.1.1.4 root 2443: kern_return_t vm_map_copyout(
2444: vm_map_t dst_map,
2445: vm_offset_t *dst_addr, /* OUT */
2446: vm_map_copy_t copy)
1.1 root 2447: {
2448: vm_size_t size;
2449: vm_size_t adjustment;
2450: vm_offset_t start;
2451: vm_offset_t vm_copy_start;
2452: vm_map_entry_t last;
2453: vm_map_entry_t entry;
2454:
2455: /*
2456: * Check for null copy object.
2457: */
2458:
2459: if (copy == VM_MAP_COPY_NULL) {
2460: *dst_addr = 0;
2461: return(KERN_SUCCESS);
2462: }
2463:
2464: /*
2465: * Check for special copy object, created
2466: * by vm_map_copyin_object.
2467: */
2468:
2469: if (copy->type == VM_MAP_COPY_OBJECT) {
2470: vm_object_t object = copy->cpy_object;
2471: vm_size_t offset = copy->offset;
2472: vm_size_t tmp_size = copy->size;
2473: kern_return_t kr;
2474:
2475: *dst_addr = 0;
2476: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2477: (vm_offset_t) 0, TRUE,
2478: object, offset, FALSE,
2479: VM_PROT_DEFAULT, VM_PROT_ALL,
2480: VM_INHERIT_DEFAULT);
2481: if (kr != KERN_SUCCESS)
2482: return(kr);
1.1.1.3 root 2483: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2484: return(KERN_SUCCESS);
2485: }
2486:
2487: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2488: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2489:
2490: /*
2491: * Find space for the data
2492: */
2493:
2494: vm_copy_start = trunc_page(copy->offset);
2495: size = round_page(copy->offset + copy->size) - vm_copy_start;
2496:
2497: StartAgain: ;
2498:
2499: vm_map_lock(dst_map);
2500: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
2501: vm_map_min(dst_map) : last->vme_end;
2502:
2503: while (TRUE) {
2504: vm_map_entry_t next = last->vme_next;
2505: vm_offset_t end = start + size;
2506:
2507: if ((end > dst_map->max_offset) || (end < start)) {
2508: if (dst_map->wait_for_space) {
2509: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
2510: assert_wait((event_t) dst_map, TRUE);
2511: vm_map_unlock(dst_map);
2512: thread_block((void (*)()) 0);
2513: goto StartAgain;
2514: }
2515: }
2516: vm_map_unlock(dst_map);
1.1.1.3 root 2517: printf_once("no more room for vm_map_copyout in %p\n", dst_map);
1.1 root 2518: return(KERN_NO_SPACE);
2519: }
2520:
2521: if ((next == vm_map_to_entry(dst_map)) ||
2522: (next->vme_start >= end))
2523: break;
2524:
2525: last = next;
2526: start = last->vme_end;
2527: }
2528:
2529: /*
2530: * Since we're going to just drop the map
2531: * entries from the copy into the destination
2532: * map, they must come from the same pool.
2533: */
2534:
2535: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2536: /*
2537: * Mismatches occur when dealing with the default
2538: * pager.
2539: */
1.1.1.3 root 2540: kmem_cache_t old_cache;
1.1 root 2541: vm_map_entry_t next, new;
2542:
2543: /*
1.1.1.3 root 2544: * Find the cache that the copies were allocated from
1.1 root 2545: */
1.1.1.3 root 2546: old_cache = (copy->cpy_hdr.entries_pageable)
2547: ? &vm_map_entry_cache
2548: : &vm_map_kentry_cache;
1.1 root 2549: entry = vm_map_copy_first_entry(copy);
2550:
2551: /*
2552: * Reinitialize the copy so that vm_map_copy_entry_link
2553: * will work.
2554: */
2555: copy->cpy_hdr.nentries = 0;
2556: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
2557: vm_map_copy_first_entry(copy) =
2558: vm_map_copy_last_entry(copy) =
2559: vm_map_copy_to_entry(copy);
2560:
2561: /*
2562: * Copy each entry.
2563: */
2564: while (entry != vm_map_copy_to_entry(copy)) {
2565: new = vm_map_copy_entry_create(copy);
2566: vm_map_entry_copy_full(new, entry);
2567: vm_map_copy_entry_link(copy,
2568: vm_map_copy_last_entry(copy),
2569: new);
2570: next = entry->vme_next;
1.1.1.3 root 2571: kmem_cache_free(old_cache, (vm_offset_t) entry);
1.1 root 2572: entry = next;
2573: }
2574: }
2575:
2576: /*
2577: * Adjust the addresses in the copy chain, and
2578: * reset the region attributes.
2579: */
2580:
2581: adjustment = start - vm_copy_start;
2582: for (entry = vm_map_copy_first_entry(copy);
2583: entry != vm_map_copy_to_entry(copy);
2584: entry = entry->vme_next) {
2585: entry->vme_start += adjustment;
2586: entry->vme_end += adjustment;
2587:
2588: entry->inheritance = VM_INHERIT_DEFAULT;
2589: entry->protection = VM_PROT_DEFAULT;
2590: entry->max_protection = VM_PROT_ALL;
2591: entry->projected_on = 0;
2592:
2593: /*
2594: * If the entry is now wired,
2595: * map the pages into the destination map.
2596: */
2597: if (entry->wired_count != 0) {
1.1.1.4 root 2598: vm_offset_t va;
2599: vm_offset_t offset;
2600: vm_object_t object;
1.1 root 2601:
2602: object = entry->object.vm_object;
2603: offset = entry->offset;
2604: va = entry->vme_start;
2605:
2606: pmap_pageable(dst_map->pmap,
2607: entry->vme_start,
2608: entry->vme_end,
2609: TRUE);
2610:
2611: while (va < entry->vme_end) {
1.1.1.4 root 2612: vm_page_t m;
1.1 root 2613:
2614: /*
2615: * Look up the page in the object.
2616: * Assert that the page will be found in the
2617: * top object:
2618: * either
2619: * the object was newly created by
2620: * vm_object_copy_slowly, and has
2621: * copies of all of the pages from
2622: * the source object
2623: * or
2624: * the object was moved from the old
2625: * map entry; because the old map
2626: * entry was wired, all of the pages
2627: * were in the top-level object.
2628: * (XXX not true if we wire pages for
2629: * reading)
2630: */
2631: vm_object_lock(object);
2632: vm_object_paging_begin(object);
2633:
2634: m = vm_page_lookup(object, offset);
2635: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2636: m->absent)
2637: panic("vm_map_copyout: wiring 0x%x", m);
2638:
2639: m->busy = TRUE;
2640: vm_object_unlock(object);
2641:
2642: PMAP_ENTER(dst_map->pmap, va, m,
2643: entry->protection, TRUE);
2644:
2645: vm_object_lock(object);
2646: PAGE_WAKEUP_DONE(m);
2647: /* the page is wired, so we don't have to activate */
2648: vm_object_paging_end(object);
2649: vm_object_unlock(object);
2650:
2651: offset += PAGE_SIZE;
2652: va += PAGE_SIZE;
2653: }
2654: }
2655:
2656:
2657: }
2658:
2659: /*
2660: * Correct the page alignment for the result
2661: */
2662:
2663: *dst_addr = start + (copy->offset - vm_copy_start);
2664:
2665: /*
2666: * Update the hints and the map size
2667: */
2668:
2669: if (dst_map->first_free == last)
2670: dst_map->first_free = vm_map_copy_last_entry(copy);
2671: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2672:
2673: dst_map->size += size;
2674:
2675: /*
2676: * Link in the copy
2677: */
2678:
2679: vm_map_copy_insert(dst_map, last, copy);
2680:
2681: vm_map_unlock(dst_map);
2682:
2683: /*
2684: * XXX If wiring_required, call vm_map_pageable
2685: */
2686:
2687: return(KERN_SUCCESS);
2688: }
2689:
2690: /*
2691: *
2692: * vm_map_copyout_page_list:
2693: *
2694: * Version of vm_map_copyout() for page list vm map copies.
2695: *
2696: */
1.1.1.4 root 2697: kern_return_t vm_map_copyout_page_list(
2698: vm_map_t dst_map,
2699: vm_offset_t *dst_addr, /* OUT */
2700: vm_map_copy_t copy)
1.1 root 2701: {
2702: vm_size_t size;
2703: vm_offset_t start;
2704: vm_offset_t end;
2705: vm_offset_t offset;
2706: vm_map_entry_t last;
2707: vm_object_t object;
2708: vm_page_t *page_list, m;
2709: vm_map_entry_t entry;
2710: vm_offset_t old_last_offset;
2711: boolean_t cont_invoked, needs_wakeup = FALSE;
2712: kern_return_t result = KERN_SUCCESS;
2713: vm_map_copy_t orig_copy;
2714: vm_offset_t dst_offset;
2715: boolean_t must_wire;
2716:
2717: /*
2718: * Make sure the pages are stolen, because we are
2719: * going to put them in a new object. Assume that
2720: * all pages are identical to first in this regard.
2721: */
2722:
2723: page_list = ©->cpy_page_list[0];
2724: if ((*page_list)->tabled)
2725: vm_map_copy_steal_pages(copy);
2726:
2727: /*
2728: * Find space for the data
2729: */
2730:
2731: size = round_page(copy->offset + copy->size) -
2732: trunc_page(copy->offset);
2733: StartAgain:
2734: vm_map_lock(dst_map);
2735: must_wire = dst_map->wiring_required;
2736:
2737: last = dst_map->first_free;
2738: if (last == vm_map_to_entry(dst_map)) {
2739: start = vm_map_min(dst_map);
2740: } else {
2741: start = last->vme_end;
2742: }
2743:
2744: while (TRUE) {
2745: vm_map_entry_t next = last->vme_next;
2746: end = start + size;
2747:
2748: if ((end > dst_map->max_offset) || (end < start)) {
2749: if (dst_map->wait_for_space) {
2750: if (size <= (dst_map->max_offset -
2751: dst_map->min_offset)) {
2752: assert_wait((event_t) dst_map, TRUE);
2753: vm_map_unlock(dst_map);
2754: thread_block((void (*)()) 0);
2755: goto StartAgain;
2756: }
2757: }
2758: vm_map_unlock(dst_map);
1.1.1.3 root 2759: printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map);
1.1 root 2760: return(KERN_NO_SPACE);
2761: }
2762:
2763: if ((next == vm_map_to_entry(dst_map)) ||
2764: (next->vme_start >= end)) {
2765: break;
2766: }
2767:
2768: last = next;
2769: start = last->vme_end;
2770: }
2771:
2772: /*
2773: * See whether we can avoid creating a new entry (and object) by
2774: * extending one of our neighbors. [So far, we only attempt to
2775: * extend from below.]
2776: *
2777: * The code path below here is a bit twisted. If any of the
2778: * extension checks fails, we branch to create_object. If
2779: * it all works, we fall out the bottom and goto insert_pages.
2780: */
2781: if (last == vm_map_to_entry(dst_map) ||
2782: last->vme_end != start ||
2783: last->is_shared != FALSE ||
2784: last->is_sub_map != FALSE ||
2785: last->inheritance != VM_INHERIT_DEFAULT ||
2786: last->protection != VM_PROT_DEFAULT ||
2787: last->max_protection != VM_PROT_ALL ||
2788: (must_wire ? (last->wired_count != 1 ||
2789: last->user_wired_count != 1) :
2790: (last->wired_count != 0))) {
2791: goto create_object;
2792: }
1.1.1.2 root 2793:
1.1 root 2794: /*
2795: * If this entry needs an object, make one.
2796: */
2797: if (last->object.vm_object == VM_OBJECT_NULL) {
2798: object = vm_object_allocate(
2799: (vm_size_t)(last->vme_end - last->vme_start + size));
2800: last->object.vm_object = object;
2801: last->offset = 0;
2802: vm_object_lock(object);
2803: }
2804: else {
2805: vm_offset_t prev_offset = last->offset;
2806: vm_size_t prev_size = start - last->vme_start;
2807: vm_size_t new_size;
2808:
2809: /*
2810: * This is basically vm_object_coalesce.
2811: */
2812:
2813: object = last->object.vm_object;
2814: vm_object_lock(object);
2815:
2816: /*
2817: * Try to collapse the object first
2818: */
2819: vm_object_collapse(object);
2820:
2821: /*
2822: * Can't coalesce if pages not mapped to
2823: * last may be in use anyway:
2824: * . more than one reference
2825: * . paged out
2826: * . shadows another object
2827: * . has a copy elsewhere
2828: * . paging references (pages might be in page-list)
2829: */
2830:
2831: if ((object->ref_count > 1) ||
2832: object->pager_created ||
2833: (object->shadow != VM_OBJECT_NULL) ||
2834: (object->copy != VM_OBJECT_NULL) ||
2835: (object->paging_in_progress != 0)) {
2836: vm_object_unlock(object);
2837: goto create_object;
2838: }
2839:
2840: /*
2841: * Extend the object if necessary. Don't have to call
2842: * vm_object_page_remove because the pages aren't mapped,
2843: * and vm_page_replace will free up any old ones it encounters.
2844: */
2845: new_size = prev_offset + prev_size + size;
2846: if (new_size > object->size)
2847: object->size = new_size;
2848: }
2849:
2850: /*
2851: * Coalesced the two objects - can extend
2852: * the previous map entry to include the
2853: * new range.
2854: */
2855: dst_map->size += size;
2856: last->vme_end = end;
2857:
2858: SAVE_HINT(dst_map, last);
2859:
2860: goto insert_pages;
2861:
2862: create_object:
2863:
2864: /*
2865: * Create object
2866: */
2867: object = vm_object_allocate(size);
2868:
2869: /*
2870: * Create entry
2871: */
2872:
2873: entry = vm_map_entry_create(dst_map);
2874:
2875: entry->object.vm_object = object;
2876: entry->offset = 0;
2877:
2878: entry->is_shared = FALSE;
2879: entry->is_sub_map = FALSE;
2880: entry->needs_copy = FALSE;
2881:
2882: if (must_wire) {
2883: entry->wired_count = 1;
1.1.1.5 ! root 2884: dst_map->user_wired += entry->vme_end - entry->vme_start;
1.1 root 2885: entry->user_wired_count = 1;
2886: } else {
2887: entry->wired_count = 0;
2888: entry->user_wired_count = 0;
2889: }
2890:
2891: entry->in_transition = TRUE;
2892: entry->needs_wakeup = FALSE;
2893:
2894: entry->vme_start = start;
2895: entry->vme_end = start + size;
1.1.1.2 root 2896:
1.1 root 2897: entry->inheritance = VM_INHERIT_DEFAULT;
2898: entry->protection = VM_PROT_DEFAULT;
2899: entry->max_protection = VM_PROT_ALL;
2900: entry->projected_on = 0;
2901:
2902: vm_object_lock(object);
2903:
2904: /*
2905: * Update the hints and the map size
2906: */
2907: if (dst_map->first_free == last) {
2908: dst_map->first_free = entry;
2909: }
2910: SAVE_HINT(dst_map, entry);
2911: dst_map->size += size;
2912:
2913: /*
2914: * Link in the entry
2915: */
2916: vm_map_entry_link(dst_map, last, entry);
2917: last = entry;
2918:
2919: /*
1.1.1.2 root 2920: * Transfer pages into new object.
1.1 root 2921: * Scan page list in vm_map_copy.
2922: */
2923: insert_pages:
2924: dst_offset = copy->offset & PAGE_MASK;
2925: cont_invoked = FALSE;
2926: orig_copy = copy;
2927: last->in_transition = TRUE;
2928: old_last_offset = last->offset
2929: + (start - last->vme_start);
2930:
2931: vm_page_lock_queues();
2932:
2933: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2934: m = *page_list;
2935: assert(m && !m->tabled);
2936:
2937: /*
2938: * Must clear busy bit in page before inserting it.
2939: * Ok to skip wakeup logic because nobody else
2940: * can possibly know about this page.
2941: * The page is dirty in its new object.
2942: */
2943:
2944: assert(!m->wanted);
2945:
2946: m->busy = FALSE;
2947: m->dirty = TRUE;
2948: vm_page_replace(m, object, old_last_offset + offset);
2949: if (must_wire) {
2950: vm_page_wire(m);
2951: PMAP_ENTER(dst_map->pmap,
2952: last->vme_start + m->offset - last->offset,
2953: m, last->protection, TRUE);
2954: } else {
2955: vm_page_activate(m);
2956: }
2957:
2958: *page_list++ = VM_PAGE_NULL;
2959: if (--(copy->cpy_npages) == 0 &&
2960: vm_map_copy_has_cont(copy)) {
2961: vm_map_copy_t new_copy;
2962:
2963: /*
2964: * Ok to unlock map because entry is
2965: * marked in_transition.
2966: */
2967: cont_invoked = TRUE;
2968: vm_page_unlock_queues();
2969: vm_object_unlock(object);
2970: vm_map_unlock(dst_map);
2971: vm_map_copy_invoke_cont(copy, &new_copy, &result);
2972:
2973: if (result == KERN_SUCCESS) {
2974:
2975: /*
2976: * If we got back a copy with real pages,
2977: * steal them now. Either all of the
2978: * pages in the list are tabled or none
2979: * of them are; mixtures are not possible.
2980: *
2981: * Save original copy for consume on
2982: * success logic at end of routine.
2983: */
2984: if (copy != orig_copy)
2985: vm_map_copy_discard(copy);
2986:
2987: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
2988: page_list = ©->cpy_page_list[0];
2989: if ((*page_list)->tabled)
2990: vm_map_copy_steal_pages(copy);
2991: }
2992: }
2993: else {
2994: /*
2995: * Continuation failed.
2996: */
2997: vm_map_lock(dst_map);
2998: goto error;
2999: }
3000:
3001: vm_map_lock(dst_map);
3002: vm_object_lock(object);
3003: vm_page_lock_queues();
3004: }
3005: }
3006:
3007: vm_page_unlock_queues();
3008: vm_object_unlock(object);
3009:
3010: *dst_addr = start + dst_offset;
1.1.1.2 root 3011:
1.1 root 3012: /*
3013: * Clear the in transition bits. This is easy if we
3014: * didn't have a continuation.
3015: */
3016: error:
3017: if (!cont_invoked) {
3018: /*
3019: * We didn't unlock the map, so nobody could
3020: * be waiting.
3021: */
3022: last->in_transition = FALSE;
3023: assert(!last->needs_wakeup);
3024: needs_wakeup = FALSE;
3025: }
3026: else {
3027: if (!vm_map_lookup_entry(dst_map, start, &entry))
3028: panic("vm_map_copyout_page_list: missing entry");
3029:
3030: /*
3031: * Clear transition bit for all constituent entries that
3032: * were in the original entry. Also check for waiters.
3033: */
3034: while((entry != vm_map_to_entry(dst_map)) &&
3035: (entry->vme_start < end)) {
3036: assert(entry->in_transition);
3037: entry->in_transition = FALSE;
3038: if(entry->needs_wakeup) {
3039: entry->needs_wakeup = FALSE;
3040: needs_wakeup = TRUE;
3041: }
3042: entry = entry->vme_next;
3043: }
3044: }
1.1.1.2 root 3045:
1.1 root 3046: if (result != KERN_SUCCESS)
3047: vm_map_delete(dst_map, start, end);
3048:
3049: vm_map_unlock(dst_map);
3050:
3051: if (needs_wakeup)
3052: vm_map_entry_wakeup(dst_map);
3053:
3054: /*
3055: * Consume on success logic.
3056: */
3057: if (copy != orig_copy) {
1.1.1.3 root 3058: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 3059: }
3060: if (result == KERN_SUCCESS) {
1.1.1.3 root 3061: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
1.1 root 3062: }
1.1.1.2 root 3063:
1.1 root 3064: return(result);
3065: }
3066:
3067: /*
3068: * Routine: vm_map_copyin
3069: *
3070: * Description:
3071: * Copy the specified region (src_addr, len) from the
3072: * source address space (src_map), possibly removing
3073: * the region from the source address space (src_destroy).
3074: *
3075: * Returns:
3076: * A vm_map_copy_t object (copy_result), suitable for
3077: * insertion into another address space (using vm_map_copyout),
3078: * copying over another address space region (using
3079: * vm_map_copy_overwrite). If the copy is unused, it
3080: * should be destroyed (using vm_map_copy_discard).
3081: *
3082: * In/out conditions:
3083: * The source map should not be locked on entry.
3084: */
1.1.1.4 root 3085: kern_return_t vm_map_copyin(
3086: vm_map_t src_map,
3087: vm_offset_t src_addr,
3088: vm_size_t len,
3089: boolean_t src_destroy,
3090: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3091: {
3092: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3093: * in multi-level lookup, this
3094: * entry contains the actual
3095: * vm_object/offset.
3096: */
3097:
3098: vm_offset_t src_start; /* Start of current entry --
3099: * where copy is taking place now
3100: */
3101: vm_offset_t src_end; /* End of entire region to be
3102: * copied */
3103:
3104: vm_map_copy_t copy; /* Resulting copy */
3105:
3106: /*
3107: * Check for copies of zero bytes.
3108: */
3109:
3110: if (len == 0) {
3111: *copy_result = VM_MAP_COPY_NULL;
3112: return(KERN_SUCCESS);
3113: }
3114:
3115: /*
3116: * Compute start and end of region
3117: */
3118:
3119: src_start = trunc_page(src_addr);
3120: src_end = round_page(src_addr + len);
3121:
3122: /*
3123: * Check that the end address doesn't overflow
3124: */
3125:
3126: if (src_end <= src_start)
3127: if ((src_end < src_start) || (src_start != 0))
3128: return(KERN_INVALID_ADDRESS);
3129:
3130: /*
3131: * Allocate a header element for the list.
3132: *
1.1.1.2 root 3133: * Use the start and end in the header to
1.1 root 3134: * remember the endpoints prior to rounding.
3135: */
3136:
1.1.1.3 root 3137: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3138: vm_map_copy_first_entry(copy) =
3139: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3140: copy->type = VM_MAP_COPY_ENTRY_LIST;
3141: copy->cpy_hdr.nentries = 0;
3142: copy->cpy_hdr.entries_pageable = TRUE;
1.1.1.3 root 3143: rbtree_init(©->cpy_hdr.tree);
1.1 root 3144:
3145: copy->offset = src_addr;
3146: copy->size = len;
1.1.1.2 root 3147:
1.1 root 3148: #define RETURN(x) \
3149: MACRO_BEGIN \
3150: vm_map_unlock(src_map); \
3151: vm_map_copy_discard(copy); \
3152: MACRO_RETURN(x); \
3153: MACRO_END
3154:
3155: /*
3156: * Find the beginning of the region.
3157: */
3158:
3159: vm_map_lock(src_map);
3160:
3161: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3162: RETURN(KERN_INVALID_ADDRESS);
3163: vm_map_clip_start(src_map, tmp_entry, src_start);
3164:
3165: /*
3166: * Go through entries until we get to the end.
3167: */
3168:
3169: while (TRUE) {
3170: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3171: vm_size_t src_size; /* Size of source
3172: * map entry (in both
3173: * maps)
3174: */
3175:
3176: vm_object_t src_object; /* Object to copy */
3177: vm_offset_t src_offset;
3178:
3179: boolean_t src_needs_copy; /* Should source map
3180: * be made read-only
3181: * for copy-on-write?
3182: */
3183:
3184: vm_map_entry_t new_entry; /* Map entry for copy */
3185: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3186:
3187: boolean_t was_wired; /* Was source wired? */
3188: vm_map_version_t version; /* Version before locks
3189: * dropped to make copy
3190: */
3191:
3192: /*
3193: * Verify that the region can be read.
3194: */
3195:
3196: if (! (src_entry->protection & VM_PROT_READ))
3197: RETURN(KERN_PROTECTION_FAILURE);
3198:
3199: /*
3200: * Clip against the endpoints of the entire region.
3201: */
3202:
3203: vm_map_clip_end(src_map, src_entry, src_end);
3204:
3205: src_size = src_entry->vme_end - src_start;
3206: src_object = src_entry->object.vm_object;
3207: src_offset = src_entry->offset;
3208: was_wired = (src_entry->wired_count != 0);
3209:
3210: /*
3211: * Create a new address map entry to
3212: * hold the result. Fill in the fields from
3213: * the appropriate source entries.
3214: */
3215:
3216: new_entry = vm_map_copy_entry_create(copy);
3217: vm_map_entry_copy(new_entry, src_entry);
3218:
3219: /*
3220: * Attempt non-blocking copy-on-write optimizations.
3221: */
3222:
3223: if (src_destroy &&
3224: (src_object == VM_OBJECT_NULL ||
3225: (src_object->temporary && !src_object->use_shared_copy)))
3226: {
3227: /*
3228: * If we are destroying the source, and the object
3229: * is temporary, and not shared writable,
3230: * we can move the object reference
3231: * from the source to the copy. The copy is
3232: * copy-on-write only if the source is.
3233: * We make another reference to the object, because
3234: * destroying the source entry will deallocate it.
3235: */
3236: vm_object_reference(src_object);
3237:
3238: /*
3239: * Copy is always unwired. vm_map_copy_entry
3240: * set its wired count to zero.
3241: */
3242:
3243: goto CopySuccessful;
3244: }
3245:
3246: if (!was_wired &&
3247: vm_object_copy_temporary(
3248: &new_entry->object.vm_object,
3249: &new_entry->offset,
3250: &src_needs_copy,
3251: &new_entry_needs_copy)) {
3252:
3253: new_entry->needs_copy = new_entry_needs_copy;
3254:
3255: /*
3256: * Handle copy-on-write obligations
3257: */
3258:
3259: if (src_needs_copy && !tmp_entry->needs_copy) {
3260: vm_object_pmap_protect(
3261: src_object,
3262: src_offset,
3263: src_size,
3264: (src_entry->is_shared ? PMAP_NULL
3265: : src_map->pmap),
3266: src_entry->vme_start,
3267: src_entry->protection &
3268: ~VM_PROT_WRITE);
3269:
3270: tmp_entry->needs_copy = TRUE;
3271: }
3272:
3273: /*
3274: * The map has never been unlocked, so it's safe to
3275: * move to the next entry rather than doing another
3276: * lookup.
3277: */
3278:
3279: goto CopySuccessful;
3280: }
3281:
3282: new_entry->needs_copy = FALSE;
3283:
3284: /*
3285: * Take an object reference, so that we may
3286: * release the map lock(s).
3287: */
3288:
3289: assert(src_object != VM_OBJECT_NULL);
3290: vm_object_reference(src_object);
3291:
3292: /*
3293: * Record the timestamp for later verification.
3294: * Unlock the map.
3295: */
3296:
3297: version.main_timestamp = src_map->timestamp;
3298: vm_map_unlock(src_map);
3299:
3300: /*
3301: * Perform the copy
3302: */
3303:
3304: if (was_wired) {
3305: vm_object_lock(src_object);
3306: (void) vm_object_copy_slowly(
3307: src_object,
3308: src_offset,
3309: src_size,
3310: FALSE,
3311: &new_entry->object.vm_object);
3312: new_entry->offset = 0;
3313: new_entry->needs_copy = FALSE;
3314: } else {
3315: kern_return_t result;
3316:
3317: result = vm_object_copy_strategically(src_object,
3318: src_offset,
3319: src_size,
3320: &new_entry->object.vm_object,
3321: &new_entry->offset,
3322: &new_entry_needs_copy);
3323:
3324: new_entry->needs_copy = new_entry_needs_copy;
1.1.1.2 root 3325:
1.1 root 3326:
3327: if (result != KERN_SUCCESS) {
3328: vm_map_copy_entry_dispose(copy, new_entry);
3329:
3330: vm_map_lock(src_map);
3331: RETURN(result);
3332: }
3333:
3334: }
3335:
3336: /*
3337: * Throw away the extra reference
3338: */
3339:
3340: vm_object_deallocate(src_object);
3341:
3342: /*
3343: * Verify that the map has not substantially
3344: * changed while the copy was being made.
3345: */
3346:
3347: vm_map_lock(src_map); /* Increments timestamp once! */
3348:
3349: if ((version.main_timestamp + 1) == src_map->timestamp)
3350: goto CopySuccessful;
3351:
3352: /*
3353: * Simple version comparison failed.
3354: *
3355: * Retry the lookup and verify that the
3356: * same object/offset are still present.
3357: *
3358: * [Note: a memory manager that colludes with
3359: * the calling task can detect that we have
3360: * cheated. While the map was unlocked, the
3361: * mapping could have been changed and restored.]
3362: */
3363:
3364: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3365: vm_map_copy_entry_dispose(copy, new_entry);
3366: RETURN(KERN_INVALID_ADDRESS);
3367: }
3368:
3369: src_entry = tmp_entry;
3370: vm_map_clip_start(src_map, src_entry, src_start);
3371:
3372: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3373: goto VerificationFailed;
3374:
3375: if (src_entry->vme_end < new_entry->vme_end)
3376: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3377:
3378: if ((src_entry->object.vm_object != src_object) ||
3379: (src_entry->offset != src_offset) ) {
3380:
3381: /*
3382: * Verification failed.
3383: *
3384: * Start over with this top-level entry.
3385: */
3386:
3387: VerificationFailed: ;
3388:
3389: vm_object_deallocate(new_entry->object.vm_object);
3390: vm_map_copy_entry_dispose(copy, new_entry);
3391: tmp_entry = src_entry;
3392: continue;
3393: }
3394:
3395: /*
3396: * Verification succeeded.
3397: */
3398:
3399: CopySuccessful: ;
3400:
3401: /*
3402: * Link in the new copy entry.
3403: */
3404:
3405: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3406: new_entry);
1.1.1.2 root 3407:
1.1 root 3408: /*
3409: * Determine whether the entire region
3410: * has been copied.
3411: */
3412: src_start = new_entry->vme_end;
3413: if ((src_start >= src_end) && (src_end != 0))
3414: break;
3415:
3416: /*
3417: * Verify that there are no gaps in the region
3418: */
3419:
3420: tmp_entry = src_entry->vme_next;
3421: if (tmp_entry->vme_start != src_start)
3422: RETURN(KERN_INVALID_ADDRESS);
3423: }
3424:
3425: /*
3426: * If the source should be destroyed, do it now, since the
1.1.1.2 root 3427: * copy was successful.
1.1 root 3428: */
3429: if (src_destroy)
3430: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3431:
3432: vm_map_unlock(src_map);
3433:
3434: *copy_result = copy;
3435: return(KERN_SUCCESS);
3436:
3437: #undef RETURN
3438: }
3439:
3440: /*
3441: * vm_map_copyin_object:
3442: *
3443: * Create a copy object from an object.
3444: * Our caller donates an object reference.
3445: */
3446:
1.1.1.4 root 3447: kern_return_t vm_map_copyin_object(
3448: vm_object_t object,
3449: vm_offset_t offset, /* offset of region in object */
3450: vm_size_t size, /* size of region in object */
3451: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3452: {
3453: vm_map_copy_t copy; /* Resulting copy */
3454:
3455: /*
3456: * We drop the object into a special copy object
3457: * that contains the object directly. These copy objects
3458: * are distinguished by entries_pageable == FALSE
3459: * and null links.
3460: */
3461:
1.1.1.3 root 3462: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3463: vm_map_copy_first_entry(copy) =
3464: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3465: copy->type = VM_MAP_COPY_OBJECT;
3466: copy->cpy_object = object;
3467: copy->offset = offset;
3468: copy->size = size;
3469:
3470: *copy_result = copy;
3471: return(KERN_SUCCESS);
3472: }
3473:
3474: /*
3475: * vm_map_copyin_page_list_cont:
3476: *
3477: * Continuation routine for vm_map_copyin_page_list.
1.1.1.2 root 3478: *
1.1 root 3479: * If vm_map_copyin_page_list can't fit the entire vm range
3480: * into a single page list object, it creates a continuation.
3481: * When the target of the operation has used the pages in the
3482: * initial page list, it invokes the continuation, which calls
3483: * this routine. If an error happens, the continuation is aborted
3484: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3485: * pages are discarded from the initial page list before invoking
3486: * the continuation.
3487: *
3488: * NOTE: This is not the same sort of continuation used by
3489: * the scheduler.
3490: */
3491:
1.1.1.4 root 3492: kern_return_t vm_map_copyin_page_list_cont(
3493: vm_map_copyin_args_t cont_args,
3494: vm_map_copy_t *copy_result) /* OUT */
1.1 root 3495: {
3496: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
1.1.1.4 root 3497: boolean_t do_abort, src_destroy, src_destroy_only;
1.1 root 3498:
3499: /*
3500: * Check for cases that only require memory destruction.
3501: */
3502: do_abort = (copy_result == (vm_map_copy_t *) 0);
3503: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3504: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3505:
3506: if (do_abort || src_destroy_only) {
1.1.1.2 root 3507: if (src_destroy)
1.1 root 3508: result = vm_map_remove(cont_args->map,
3509: cont_args->destroy_addr,
3510: cont_args->destroy_addr + cont_args->destroy_len);
3511: if (!do_abort)
3512: *copy_result = VM_MAP_COPY_NULL;
3513: }
3514: else {
3515: result = vm_map_copyin_page_list(cont_args->map,
3516: cont_args->src_addr, cont_args->src_len, src_destroy,
3517: cont_args->steal_pages, copy_result, TRUE);
3518:
3519: if (src_destroy && !cont_args->steal_pages &&
3520: vm_map_copy_has_cont(*copy_result)) {
3521: vm_map_copyin_args_t new_args;
3522: /*
3523: * Transfer old destroy info.
3524: */
3525: new_args = (vm_map_copyin_args_t)
3526: (*copy_result)->cpy_cont_args;
3527: new_args->destroy_addr = cont_args->destroy_addr;
3528: new_args->destroy_len = cont_args->destroy_len;
3529: }
3530: }
1.1.1.2 root 3531:
1.1 root 3532: vm_map_deallocate(cont_args->map);
3533: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3534:
3535: return(result);
3536: }
3537:
3538: /*
3539: * vm_map_copyin_page_list:
3540: *
3541: * This is a variant of vm_map_copyin that copies in a list of pages.
3542: * If steal_pages is TRUE, the pages are only in the returned list.
3543: * If steal_pages is FALSE, the pages are busy and still in their
3544: * objects. A continuation may be returned if not all the pages fit:
3545: * the recipient of this copy_result must be prepared to deal with it.
3546: */
3547:
1.1.1.4 root 3548: kern_return_t vm_map_copyin_page_list(
3549: vm_map_t src_map,
3550: vm_offset_t src_addr,
3551: vm_size_t len,
3552: boolean_t src_destroy,
3553: boolean_t steal_pages,
3554: vm_map_copy_t *copy_result, /* OUT */
3555: boolean_t is_cont)
1.1 root 3556: {
3557: vm_map_entry_t src_entry;
3558: vm_page_t m;
3559: vm_offset_t src_start;
3560: vm_offset_t src_end;
3561: vm_size_t src_size;
3562: vm_object_t src_object;
3563: vm_offset_t src_offset;
3564: vm_offset_t src_last_offset;
3565: vm_map_copy_t copy; /* Resulting copy */
3566: kern_return_t result = KERN_SUCCESS;
3567: boolean_t need_map_lookup;
3568: vm_map_copyin_args_t cont_args;
3569:
3570: /*
3571: * If steal_pages is FALSE, this leaves busy pages in
3572: * the object. A continuation must be used if src_destroy
3573: * is true in this case (!steal_pages && src_destroy).
3574: *
3575: * XXX Still have a more general problem of what happens
3576: * XXX if the same page occurs twice in a list. Deadlock
3577: * XXX can happen if vm_fault_page was called. A
3578: * XXX possible solution is to use a continuation if vm_fault_page
3579: * XXX is called and we cross a map entry boundary.
3580: */
3581:
3582: /*
3583: * Check for copies of zero bytes.
3584: */
3585:
3586: if (len == 0) {
3587: *copy_result = VM_MAP_COPY_NULL;
3588: return(KERN_SUCCESS);
3589: }
3590:
3591: /*
3592: * Compute start and end of region
3593: */
3594:
3595: src_start = trunc_page(src_addr);
3596: src_end = round_page(src_addr + len);
3597:
3598: /*
3599: * Check that the end address doesn't overflow
3600: */
3601:
3602: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3603: return KERN_INVALID_ADDRESS;
3604: }
3605:
3606: /*
3607: * Allocate a header element for the page list.
3608: *
3609: * Record original offset and size, as caller may not
3610: * be page-aligned.
3611: */
3612:
1.1.1.3 root 3613: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3614: copy->type = VM_MAP_COPY_PAGE_LIST;
3615: copy->cpy_npages = 0;
3616: copy->offset = src_addr;
3617: copy->size = len;
3618: copy->cpy_cont = ((kern_return_t (*)()) 0);
3619: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
1.1.1.2 root 3620:
1.1 root 3621: /*
3622: * Find the beginning of the region.
3623: */
3624:
3625: do_map_lookup:
3626:
3627: vm_map_lock(src_map);
3628:
3629: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3630: result = KERN_INVALID_ADDRESS;
3631: goto error;
3632: }
3633: need_map_lookup = FALSE;
3634:
3635: /*
3636: * Go through entries until we get to the end.
3637: */
3638:
3639: while (TRUE) {
3640:
3641: if (! (src_entry->protection & VM_PROT_READ)) {
3642: result = KERN_PROTECTION_FAILURE;
3643: goto error;
3644: }
3645:
3646: if (src_end > src_entry->vme_end)
3647: src_size = src_entry->vme_end - src_start;
3648: else
3649: src_size = src_end - src_start;
3650:
3651: src_object = src_entry->object.vm_object;
3652: src_offset = src_entry->offset +
3653: (src_start - src_entry->vme_start);
3654:
3655: /*
3656: * If src_object is NULL, allocate it now;
3657: * we're going to fault on it shortly.
3658: */
3659: if (src_object == VM_OBJECT_NULL) {
3660: src_object = vm_object_allocate((vm_size_t)
3661: src_entry->vme_end -
3662: src_entry->vme_start);
3663: src_entry->object.vm_object = src_object;
3664: }
3665:
3666: /*
3667: * Iterate over pages. Fault in ones that aren't present.
3668: */
3669: src_last_offset = src_offset + src_size;
3670: for (; (src_offset < src_last_offset && !need_map_lookup);
3671: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3672:
3673: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3674: make_continuation:
3675: /*
3676: * At this point we have the max number of
3677: * pages busy for this thread that we're
3678: * willing to allow. Stop here and record
3679: * arguments for the remainder. Note:
3680: * this means that this routine isn't atomic,
3681: * but that's the breaks. Note that only
3682: * the first vm_map_copy_t that comes back
3683: * from this routine has the right offset
3684: * and size; those from continuations are
3685: * page rounded, and short by the amount
3686: * already done.
3687: *
3688: * Reset src_end so the src_destroy
3689: * code at the bottom doesn't do
3690: * something stupid.
3691: */
3692:
1.1.1.2 root 3693: cont_args = (vm_map_copyin_args_t)
1.1 root 3694: kalloc(sizeof(vm_map_copyin_args_data_t));
3695: cont_args->map = src_map;
3696: vm_map_reference(src_map);
3697: cont_args->src_addr = src_start;
3698: cont_args->src_len = len - (src_start - src_addr);
3699: if (src_destroy) {
3700: cont_args->destroy_addr = cont_args->src_addr;
3701: cont_args->destroy_len = cont_args->src_len;
3702: }
3703: else {
3704: cont_args->destroy_addr = (vm_offset_t) 0;
3705: cont_args->destroy_len = (vm_offset_t) 0;
3706: }
3707: cont_args->steal_pages = steal_pages;
3708:
3709: copy->cpy_cont_args = (char *) cont_args;
3710: copy->cpy_cont = vm_map_copyin_page_list_cont;
3711:
3712: src_end = src_start;
3713: vm_map_clip_end(src_map, src_entry, src_end);
3714: break;
3715: }
3716:
3717: /*
3718: * Try to find the page of data.
3719: */
3720: vm_object_lock(src_object);
3721: vm_object_paging_begin(src_object);
3722: if (((m = vm_page_lookup(src_object, src_offset)) !=
3723: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3724: !m->absent && !m->error) {
3725:
3726: /*
3727: * This is the page. Mark it busy
3728: * and keep the paging reference on
3729: * the object whilst we do our thing.
3730: */
3731: m->busy = TRUE;
3732:
3733: /*
3734: * Also write-protect the page, so
3735: * that the map`s owner cannot change
3736: * the data. The busy bit will prevent
3737: * faults on the page from succeeding
3738: * until the copy is released; after
3739: * that, the page can be re-entered
3740: * as writable, since we didn`t alter
3741: * the map entry. This scheme is a
3742: * cheap copy-on-write.
3743: *
3744: * Don`t forget the protection and
3745: * the page_lock value!
3746: *
3747: * If the source is being destroyed
3748: * AND not shared writable, we don`t
3749: * have to protect the page, since
3750: * we will destroy the (only)
3751: * writable mapping later.
3752: */
3753: if (!src_destroy ||
3754: src_object->use_shared_copy)
3755: {
3756: pmap_page_protect(m->phys_addr,
3757: src_entry->protection
3758: & ~m->page_lock
3759: & ~VM_PROT_WRITE);
3760: }
3761:
3762: }
3763: else {
3764: vm_prot_t result_prot;
3765: vm_page_t top_page;
3766: kern_return_t kr;
1.1.1.2 root 3767:
1.1 root 3768: /*
3769: * Have to fault the page in; must
3770: * unlock the map to do so. While
3771: * the map is unlocked, anything
3772: * can happen, we must lookup the
3773: * map entry before continuing.
3774: */
3775: vm_map_unlock(src_map);
3776: need_map_lookup = TRUE;
3777: retry:
3778: result_prot = VM_PROT_READ;
1.1.1.2 root 3779:
1.1 root 3780: kr = vm_fault_page(src_object, src_offset,
3781: VM_PROT_READ, FALSE, FALSE,
3782: &result_prot, &m, &top_page,
3783: FALSE, (void (*)()) 0);
3784: /*
3785: * Cope with what happened.
3786: */
3787: switch (kr) {
3788: case VM_FAULT_SUCCESS:
3789: break;
3790: case VM_FAULT_INTERRUPTED: /* ??? */
3791: case VM_FAULT_RETRY:
3792: vm_object_lock(src_object);
3793: vm_object_paging_begin(src_object);
3794: goto retry;
3795: case VM_FAULT_MEMORY_SHORTAGE:
3796: VM_PAGE_WAIT((void (*)()) 0);
3797: vm_object_lock(src_object);
3798: vm_object_paging_begin(src_object);
3799: goto retry;
3800: case VM_FAULT_FICTITIOUS_SHORTAGE:
3801: vm_page_more_fictitious();
3802: vm_object_lock(src_object);
3803: vm_object_paging_begin(src_object);
3804: goto retry;
3805: case VM_FAULT_MEMORY_ERROR:
3806: /*
3807: * Something broke. If this
3808: * is a continuation, return
3809: * a partial result if possible,
3810: * else fail the whole thing.
3811: * In the continuation case, the
3812: * next continuation call will
3813: * get this error if it persists.
3814: */
3815: vm_map_lock(src_map);
3816: if (is_cont &&
3817: copy->cpy_npages != 0)
3818: goto make_continuation;
3819:
3820: result = KERN_MEMORY_ERROR;
3821: goto error;
3822: }
1.1.1.2 root 3823:
1.1 root 3824: if (top_page != VM_PAGE_NULL) {
3825: vm_object_lock(src_object);
3826: VM_PAGE_FREE(top_page);
3827: vm_object_paging_end(src_object);
3828: vm_object_unlock(src_object);
3829: }
3830:
3831: /*
3832: * We do not need to write-protect
3833: * the page, since it cannot have
3834: * been in the pmap (and we did not
3835: * enter it above). The busy bit
3836: * will protect the page from being
3837: * entered as writable until it is
3838: * unlocked.
3839: */
3840:
3841: }
3842:
3843: /*
3844: * The page is busy, its object is locked, and
3845: * we have a paging reference on it. Either
3846: * the map is locked, or need_map_lookup is
3847: * TRUE.
1.1.1.2 root 3848: *
1.1 root 3849: * Put the page in the page list.
3850: */
3851: copy->cpy_page_list[copy->cpy_npages++] = m;
3852: vm_object_unlock(m->object);
3853: }
1.1.1.2 root 3854:
1.1 root 3855: /*
3856: * DETERMINE whether the entire region
3857: * has been copied.
3858: */
3859: if (src_start >= src_end && src_end != 0) {
3860: if (need_map_lookup)
3861: vm_map_lock(src_map);
3862: break;
3863: }
3864:
3865: /*
3866: * If need_map_lookup is TRUE, have to start over with
3867: * another map lookup. Note that we dropped the map
3868: * lock (to call vm_fault_page) above only in this case.
3869: */
3870: if (need_map_lookup)
3871: goto do_map_lookup;
3872:
3873: /*
3874: * Verify that there are no gaps in the region
3875: */
3876:
3877: src_start = src_entry->vme_end;
3878: src_entry = src_entry->vme_next;
3879: if (src_entry->vme_start != src_start) {
3880: result = KERN_INVALID_ADDRESS;
3881: goto error;
3882: }
3883: }
3884:
3885: /*
3886: * If steal_pages is true, make sure all
3887: * pages in the copy are not in any object
3888: * We try to remove them from the original
3889: * object, but we may have to copy them.
3890: *
3891: * At this point every page in the list is busy
3892: * and holds a paging reference to its object.
3893: * When we're done stealing, every page is busy,
3894: * and in no object (m->tabled == FALSE).
3895: */
3896: src_start = trunc_page(src_addr);
3897: if (steal_pages) {
1.1.1.4 root 3898: int i;
1.1 root 3899: vm_offset_t unwire_end;
3900:
3901: unwire_end = src_start;
3902: for (i = 0; i < copy->cpy_npages; i++) {
3903:
3904: /*
3905: * Remove the page from its object if it
3906: * can be stolen. It can be stolen if:
3907: *
1.1.1.2 root 3908: * (1) The source is being destroyed,
1.1 root 3909: * the object is temporary, and
3910: * not shared.
3911: * (2) The page is not precious.
3912: *
3913: * The not shared check consists of two
3914: * parts: (a) there are no objects that
3915: * shadow this object. (b) it is not the
3916: * object in any shared map entries (i.e.,
3917: * use_shared_copy is not set).
3918: *
3919: * The first check (a) means that we can't
3920: * steal pages from objects that are not
3921: * at the top of their shadow chains. This
3922: * should not be a frequent occurrence.
3923: *
3924: * Stealing wired pages requires telling the
3925: * pmap module to let go of them.
1.1.1.2 root 3926: *
1.1 root 3927: * NOTE: stealing clean pages from objects
3928: * whose mappings survive requires a call to
3929: * the pmap module. Maybe later.
3930: */
3931: m = copy->cpy_page_list[i];
3932: src_object = m->object;
3933: vm_object_lock(src_object);
3934:
3935: if (src_destroy &&
3936: src_object->temporary &&
3937: (!src_object->shadowed) &&
3938: (!src_object->use_shared_copy) &&
3939: !m->precious) {
3940: vm_offset_t page_vaddr;
1.1.1.2 root 3941:
1.1 root 3942: page_vaddr = src_start + (i * PAGE_SIZE);
3943: if (m->wire_count > 0) {
3944:
3945: assert(m->wire_count == 1);
3946: /*
3947: * In order to steal a wired
3948: * page, we have to unwire it
3949: * first. We do this inline
3950: * here because we have the page.
3951: *
3952: * Step 1: Unwire the map entry.
3953: * Also tell the pmap module
3954: * that this piece of the
3955: * pmap is pageable.
3956: */
3957: vm_object_unlock(src_object);
3958: if (page_vaddr >= unwire_end) {
3959: if (!vm_map_lookup_entry(src_map,
3960: page_vaddr, &src_entry))
3961: panic("vm_map_copyin_page_list: missing wired map entry");
3962:
3963: vm_map_clip_start(src_map, src_entry,
3964: page_vaddr);
3965: vm_map_clip_end(src_map, src_entry,
3966: src_start + src_size);
3967:
3968: assert(src_entry->wired_count > 0);
3969: src_entry->wired_count = 0;
1.1.1.5 ! root 3970: if (src_entry->user_wired_count)
! 3971: src_map->user_wired -= src_entry->vme_end - src_entry->vme_start;
1.1 root 3972: src_entry->user_wired_count = 0;
3973: unwire_end = src_entry->vme_end;
3974: pmap_pageable(vm_map_pmap(src_map),
3975: page_vaddr, unwire_end, TRUE);
3976: }
3977:
3978: /*
3979: * Step 2: Unwire the page.
3980: * pmap_remove handles this for us.
3981: */
3982: vm_object_lock(src_object);
3983: }
3984:
3985: /*
3986: * Don't need to remove the mapping;
3987: * vm_map_delete will handle it.
3988: *
3989: * Steal the page. Setting the wire count
3990: * to zero is vm_page_unwire without
3991: * activating the page.
3992: */
3993: vm_page_lock_queues();
3994: vm_page_remove(m);
3995: if (m->wire_count > 0) {
3996: m->wire_count = 0;
3997: vm_page_wire_count--;
3998: } else {
3999: VM_PAGE_QUEUES_REMOVE(m);
4000: }
4001: vm_page_unlock_queues();
4002: }
4003: else {
4004: /*
4005: * Have to copy this page. Have to
4006: * unlock the map while copying,
4007: * hence no further page stealing.
4008: * Hence just copy all the pages.
4009: * Unlock the map while copying;
4010: * This means no further page stealing.
4011: */
4012: vm_object_unlock(src_object);
4013: vm_map_unlock(src_map);
4014:
4015: vm_map_copy_steal_pages(copy);
4016:
4017: vm_map_lock(src_map);
4018: break;
4019: }
4020:
4021: vm_object_paging_end(src_object);
4022: vm_object_unlock(src_object);
4023: }
4024:
4025: /*
4026: * If the source should be destroyed, do it now, since the
4027: * copy was successful.
4028: */
4029:
4030: if (src_destroy) {
4031: (void) vm_map_delete(src_map, src_start, src_end);
4032: }
4033: }
4034: else {
4035: /*
4036: * !steal_pages leaves busy pages in the map.
4037: * This will cause src_destroy to hang. Use
4038: * a continuation to prevent this.
4039: */
4040: if (src_destroy && !vm_map_copy_has_cont(copy)) {
1.1.1.2 root 4041: cont_args = (vm_map_copyin_args_t)
1.1 root 4042: kalloc(sizeof(vm_map_copyin_args_data_t));
4043: vm_map_reference(src_map);
4044: cont_args->map = src_map;
4045: cont_args->src_addr = (vm_offset_t) 0;
4046: cont_args->src_len = (vm_size_t) 0;
4047: cont_args->destroy_addr = src_start;
4048: cont_args->destroy_len = src_end - src_start;
4049: cont_args->steal_pages = FALSE;
4050:
4051: copy->cpy_cont_args = (char *) cont_args;
4052: copy->cpy_cont = vm_map_copyin_page_list_cont;
4053: }
1.1.1.2 root 4054:
1.1 root 4055: }
4056:
4057: vm_map_unlock(src_map);
4058:
4059: *copy_result = copy;
4060: return(result);
4061:
4062: error:
4063: vm_map_unlock(src_map);
4064: vm_map_copy_discard(copy);
4065: return(result);
4066: }
4067:
4068: /*
4069: * vm_map_fork:
4070: *
4071: * Create and return a new map based on the old
4072: * map, according to the inheritance values on the
4073: * regions in that map.
4074: *
4075: * The source map must not be locked.
4076: */
1.1.1.4 root 4077: vm_map_t vm_map_fork(vm_map_t old_map)
1.1 root 4078: {
4079: vm_map_t new_map;
4080: vm_map_entry_t old_entry;
4081: vm_map_entry_t new_entry;
4082: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4083: vm_size_t new_size = 0;
4084: vm_size_t entry_size;
4085: vm_object_t object;
4086:
4087: vm_map_lock(old_map);
4088:
4089: new_map = vm_map_create(new_pmap,
4090: old_map->min_offset,
4091: old_map->max_offset,
4092: old_map->hdr.entries_pageable);
4093:
4094: for (
4095: old_entry = vm_map_first_entry(old_map);
4096: old_entry != vm_map_to_entry(old_map);
4097: ) {
4098: if (old_entry->is_sub_map)
4099: panic("vm_map_fork: encountered a submap");
4100:
4101: entry_size = (old_entry->vme_end - old_entry->vme_start);
4102:
4103: switch (old_entry->inheritance) {
4104: case VM_INHERIT_NONE:
4105: break;
4106:
4107: case VM_INHERIT_SHARE:
4108: /*
4109: * New sharing code. New map entry
4110: * references original object. Temporary
4111: * objects use asynchronous copy algorithm for
4112: * future copies. First make sure we have
4113: * the right object. If we need a shadow,
4114: * or someone else already has one, then
4115: * make a new shadow and share it.
4116: */
4117:
4118: object = old_entry->object.vm_object;
4119: if (object == VM_OBJECT_NULL) {
4120: object = vm_object_allocate(
4121: (vm_size_t)(old_entry->vme_end -
4122: old_entry->vme_start));
4123: old_entry->offset = 0;
4124: old_entry->object.vm_object = object;
4125: assert(!old_entry->needs_copy);
4126: }
4127: else if (old_entry->needs_copy || object->shadowed ||
4128: (object->temporary && !old_entry->is_shared &&
4129: object->size > (vm_size_t)(old_entry->vme_end -
4130: old_entry->vme_start))) {
4131:
4132: assert(object->temporary);
4133: assert(!(object->shadowed && old_entry->is_shared));
4134: vm_object_shadow(
4135: &old_entry->object.vm_object,
4136: &old_entry->offset,
4137: (vm_size_t) (old_entry->vme_end -
4138: old_entry->vme_start));
1.1.1.2 root 4139:
1.1 root 4140: /*
4141: * If we're making a shadow for other than
4142: * copy on write reasons, then we have
4143: * to remove write permission.
4144: */
4145:
4146: if (!old_entry->needs_copy &&
4147: (old_entry->protection & VM_PROT_WRITE)) {
4148: pmap_protect(vm_map_pmap(old_map),
4149: old_entry->vme_start,
4150: old_entry->vme_end,
4151: old_entry->protection &
4152: ~VM_PROT_WRITE);
4153: }
4154: old_entry->needs_copy = FALSE;
4155: object = old_entry->object.vm_object;
4156: }
4157:
4158: /*
4159: * Set use_shared_copy to indicate that
4160: * object must use shared (delayed) copy-on
4161: * write. This is ignored for permanent objects.
4162: * Bump the reference count for the new entry
4163: */
4164:
4165: vm_object_lock(object);
4166: object->use_shared_copy = TRUE;
4167: object->ref_count++;
4168: vm_object_unlock(object);
4169:
1.1.1.3 root 4170: new_entry = vm_map_entry_create(new_map);
4171:
1.1 root 4172: if (old_entry->projected_on != 0) {
4173: /*
4174: * If entry is projected buffer, clone the
4175: * entry exactly.
4176: */
4177:
4178: vm_map_entry_copy_full(new_entry, old_entry);
4179:
4180: } else {
4181: /*
4182: * Clone the entry, using object ref from above.
4183: * Mark both entries as shared.
4184: */
4185:
4186: vm_map_entry_copy(new_entry, old_entry);
4187: old_entry->is_shared = TRUE;
4188: new_entry->is_shared = TRUE;
4189: }
4190:
4191: /*
4192: * Insert the entry into the new map -- we
4193: * know we're inserting at the end of the new
4194: * map.
4195: */
4196:
4197: vm_map_entry_link(
4198: new_map,
4199: vm_map_last_entry(new_map),
4200: new_entry);
4201:
4202: /*
4203: * Update the physical map
4204: */
4205:
4206: pmap_copy(new_map->pmap, old_map->pmap,
4207: new_entry->vme_start,
4208: entry_size,
4209: old_entry->vme_start);
4210:
4211: new_size += entry_size;
4212: break;
4213:
4214: case VM_INHERIT_COPY:
4215: if (old_entry->wired_count == 0) {
4216: boolean_t src_needs_copy;
4217: boolean_t new_entry_needs_copy;
4218:
4219: new_entry = vm_map_entry_create(new_map);
4220: vm_map_entry_copy(new_entry, old_entry);
4221:
4222: if (vm_object_copy_temporary(
4223: &new_entry->object.vm_object,
4224: &new_entry->offset,
4225: &src_needs_copy,
4226: &new_entry_needs_copy)) {
4227:
4228: /*
4229: * Handle copy-on-write obligations
4230: */
4231:
4232: if (src_needs_copy && !old_entry->needs_copy) {
4233: vm_object_pmap_protect(
4234: old_entry->object.vm_object,
4235: old_entry->offset,
4236: entry_size,
4237: (old_entry->is_shared ?
4238: PMAP_NULL :
4239: old_map->pmap),
4240: old_entry->vme_start,
4241: old_entry->protection &
4242: ~VM_PROT_WRITE);
4243:
4244: old_entry->needs_copy = TRUE;
4245: }
4246:
4247: new_entry->needs_copy = new_entry_needs_copy;
4248:
4249: /*
4250: * Insert the entry at the end
4251: * of the map.
4252: */
4253:
4254: vm_map_entry_link(new_map,
4255: vm_map_last_entry(new_map),
4256: new_entry);
4257:
4258:
4259: new_size += entry_size;
4260: break;
4261: }
4262:
4263: vm_map_entry_dispose(new_map, new_entry);
4264: }
4265:
4266: /* INNER BLOCK (copy cannot be optimized) */ {
4267:
4268: vm_offset_t start = old_entry->vme_start;
4269: vm_map_copy_t copy;
4270: vm_map_entry_t last = vm_map_last_entry(new_map);
4271:
4272: vm_map_unlock(old_map);
4273: if (vm_map_copyin(old_map,
4274: start,
4275: entry_size,
4276: FALSE,
1.1.1.2 root 4277: ©)
1.1 root 4278: != KERN_SUCCESS) {
4279: vm_map_lock(old_map);
4280: if (!vm_map_lookup_entry(old_map, start, &last))
4281: last = last->vme_next;
4282: old_entry = last;
4283: /*
4284: * For some error returns, want to
4285: * skip to the next element.
4286: */
4287:
4288: continue;
4289: }
4290:
4291: /*
4292: * Insert the copy into the new map
4293: */
4294:
4295: vm_map_copy_insert(new_map, last, copy);
4296: new_size += entry_size;
4297:
4298: /*
4299: * Pick up the traversal at the end of
4300: * the copied region.
4301: */
4302:
4303: vm_map_lock(old_map);
4304: start += entry_size;
4305: if (!vm_map_lookup_entry(old_map, start, &last))
4306: last = last->vme_next;
4307: else
4308: vm_map_clip_start(old_map, last, start);
4309: old_entry = last;
4310:
4311: continue;
4312: /* INNER BLOCK (copy cannot be optimized) */ }
4313: }
4314: old_entry = old_entry->vme_next;
4315: }
4316:
4317: new_map->size = new_size;
4318: vm_map_unlock(old_map);
4319:
4320: return(new_map);
4321: }
4322:
4323: /*
4324: * vm_map_lookup:
4325: *
4326: * Finds the VM object, offset, and
4327: * protection for a given virtual address in the
4328: * specified map, assuming a page fault of the
4329: * type specified.
4330: *
4331: * Returns the (object, offset, protection) for
4332: * this address, whether it is wired down, and whether
4333: * this map has the only reference to the data in question.
4334: * In order to later verify this lookup, a "version"
4335: * is returned.
4336: *
4337: * The map should not be locked; it will not be
4338: * locked on exit. In order to guarantee the
4339: * existence of the returned object, it is returned
4340: * locked.
4341: *
4342: * If a lookup is requested with "write protection"
4343: * specified, the map may be changed to perform virtual
4344: * copying operations, although the data referenced will
4345: * remain the same.
4346: */
1.1.1.4 root 4347: kern_return_t vm_map_lookup(
4348: vm_map_t *var_map, /* IN/OUT */
4349: vm_offset_t vaddr,
4350: vm_prot_t fault_type,
4351:
4352: vm_map_version_t *out_version, /* OUT */
4353: vm_object_t *object, /* OUT */
4354: vm_offset_t *offset, /* OUT */
4355: vm_prot_t *out_prot, /* OUT */
4356: boolean_t *wired) /* OUT */
4357: {
4358: vm_map_entry_t entry;
4359: vm_map_t map = *var_map;
4360: vm_prot_t prot;
1.1 root 4361:
4362: RetryLookup: ;
4363:
4364: /*
4365: * Lookup the faulting address.
4366: */
4367:
4368: vm_map_lock_read(map);
4369:
4370: #define RETURN(why) \
4371: { \
4372: vm_map_unlock_read(map); \
4373: return(why); \
4374: }
4375:
4376: /*
4377: * If the map has an interesting hint, try it before calling
4378: * full blown lookup routine.
4379: */
4380:
4381: simple_lock(&map->hint_lock);
4382: entry = map->hint;
4383: simple_unlock(&map->hint_lock);
4384:
4385: if ((entry == vm_map_to_entry(map)) ||
4386: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4387: vm_map_entry_t tmp_entry;
4388:
4389: /*
4390: * Entry was either not a valid hint, or the vaddr
4391: * was not contained in the entry, so do a full lookup.
4392: */
4393: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4394: RETURN(KERN_INVALID_ADDRESS);
4395:
4396: entry = tmp_entry;
4397: }
4398:
4399: /*
4400: * Handle submaps.
4401: */
4402:
4403: if (entry->is_sub_map) {
4404: vm_map_t old_map = map;
4405:
4406: *var_map = map = entry->object.sub_map;
4407: vm_map_unlock_read(old_map);
4408: goto RetryLookup;
4409: }
1.1.1.2 root 4410:
1.1 root 4411: /*
4412: * Check whether this task is allowed to have
4413: * this page.
4414: */
4415:
4416: prot = entry->protection;
4417:
1.1.1.3 root 4418: if ((fault_type & (prot)) != fault_type) {
1.1 root 4419: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4420: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4421: } else {
4422: RETURN(KERN_PROTECTION_FAILURE);
4423: }
1.1.1.3 root 4424: }
1.1 root 4425:
4426: /*
4427: * If this page is not pageable, we have to get
4428: * it for all possible accesses.
4429: */
4430:
1.1.1.3 root 4431: if ((*wired = (entry->wired_count != 0)))
1.1 root 4432: prot = fault_type = entry->protection;
4433:
4434: /*
4435: * If the entry was copy-on-write, we either ...
4436: */
4437:
4438: if (entry->needs_copy) {
4439: /*
4440: * If we want to write the page, we may as well
4441: * handle that now since we've got the map locked.
4442: *
4443: * If we don't need to write the page, we just
4444: * demote the permissions allowed.
4445: */
4446:
4447: if (fault_type & VM_PROT_WRITE) {
4448: /*
4449: * Make a new object, and place it in the
4450: * object chain. Note that no new references
4451: * have appeared -- one just moved from the
4452: * map to the new object.
4453: */
4454:
4455: if (vm_map_lock_read_to_write(map)) {
4456: goto RetryLookup;
4457: }
4458: map->timestamp++;
4459:
4460: vm_object_shadow(
4461: &entry->object.vm_object,
4462: &entry->offset,
4463: (vm_size_t) (entry->vme_end - entry->vme_start));
1.1.1.2 root 4464:
1.1 root 4465: entry->needs_copy = FALSE;
1.1.1.2 root 4466:
1.1 root 4467: vm_map_lock_write_to_read(map);
4468: }
4469: else {
4470: /*
4471: * We're attempting to read a copy-on-write
4472: * page -- don't allow writes.
4473: */
4474:
4475: prot &= (~VM_PROT_WRITE);
4476: }
4477: }
4478:
4479: /*
4480: * Create an object if necessary.
4481: */
4482: if (entry->object.vm_object == VM_OBJECT_NULL) {
4483:
4484: if (vm_map_lock_read_to_write(map)) {
4485: goto RetryLookup;
4486: }
4487:
4488: entry->object.vm_object = vm_object_allocate(
4489: (vm_size_t)(entry->vme_end - entry->vme_start));
4490: entry->offset = 0;
4491: vm_map_lock_write_to_read(map);
4492: }
4493:
4494: /*
4495: * Return the object/offset from this entry. If the entry
4496: * was copy-on-write or empty, it has been fixed up. Also
4497: * return the protection.
4498: */
4499:
4500: *offset = (vaddr - entry->vme_start) + entry->offset;
4501: *object = entry->object.vm_object;
4502: *out_prot = prot;
4503:
4504: /*
4505: * Lock the object to prevent it from disappearing
4506: */
4507:
4508: vm_object_lock(*object);
4509:
4510: /*
4511: * Save the version number and unlock the map.
4512: */
4513:
4514: out_version->main_timestamp = map->timestamp;
4515:
4516: RETURN(KERN_SUCCESS);
1.1.1.2 root 4517:
1.1 root 4518: #undef RETURN
4519: }
4520:
4521: /*
4522: * vm_map_verify:
4523: *
4524: * Verifies that the map in question has not changed
4525: * since the given version. If successful, the map
4526: * will not change until vm_map_verify_done() is called.
4527: */
1.1.1.4 root 4528: boolean_t vm_map_verify(
4529: vm_map_t map,
4530: vm_map_version_t *version) /* REF */
1.1 root 4531: {
4532: boolean_t result;
4533:
4534: vm_map_lock_read(map);
4535: result = (map->timestamp == version->main_timestamp);
4536:
4537: if (!result)
4538: vm_map_unlock_read(map);
4539:
4540: return(result);
4541: }
4542:
4543: /*
4544: * vm_map_verify_done:
4545: *
4546: * Releases locks acquired by a vm_map_verify.
4547: *
4548: * This is now a macro in vm/vm_map.h. It does a
4549: * vm_map_unlock_read on the map.
4550: */
4551:
4552: /*
4553: * vm_region:
4554: *
4555: * User call to obtain information about a region in
4556: * a task's address map.
4557: */
4558:
1.1.1.4 root 4559: kern_return_t vm_region(
4560: vm_map_t map,
4561: vm_offset_t *address, /* IN/OUT */
4562: vm_size_t *size, /* OUT */
4563: vm_prot_t *protection, /* OUT */
4564: vm_prot_t *max_protection, /* OUT */
4565: vm_inherit_t *inheritance, /* OUT */
4566: boolean_t *is_shared, /* OUT */
4567: ipc_port_t *object_name, /* OUT */
4568: vm_offset_t *offset_in_object) /* OUT */
1.1 root 4569: {
4570: vm_map_entry_t tmp_entry;
4571: vm_map_entry_t entry;
4572: vm_offset_t tmp_offset;
4573: vm_offset_t start;
4574:
4575: if (map == VM_MAP_NULL)
4576: return(KERN_INVALID_ARGUMENT);
4577:
4578: start = *address;
4579:
4580: vm_map_lock_read(map);
4581: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4582: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4583: vm_map_unlock_read(map);
4584: return(KERN_NO_SPACE);
4585: }
4586: } else {
4587: entry = tmp_entry;
4588: }
4589:
4590: start = entry->vme_start;
4591: *protection = entry->protection;
4592: *max_protection = entry->max_protection;
4593: *inheritance = entry->inheritance;
4594: *address = start;
4595: *size = (entry->vme_end - start);
4596:
4597: tmp_offset = entry->offset;
4598:
4599:
4600: if (entry->is_sub_map) {
4601: *is_shared = FALSE;
4602: *object_name = IP_NULL;
4603: *offset_in_object = tmp_offset;
4604: } else {
4605: *is_shared = entry->is_shared;
4606: *object_name = vm_object_name(entry->object.vm_object);
4607: *offset_in_object = tmp_offset;
4608: }
4609:
4610: vm_map_unlock_read(map);
4611:
4612: return(KERN_SUCCESS);
4613: }
4614:
4615: /*
4616: * Routine: vm_map_simplify
4617: *
4618: * Description:
4619: * Attempt to simplify the map representation in
4620: * the vicinity of the given starting address.
4621: * Note:
4622: * This routine is intended primarily to keep the
4623: * kernel maps more compact -- they generally don't
4624: * benefit from the "expand a map entry" technology
4625: * at allocation time because the adjacent entry
4626: * is often wired down.
4627: */
1.1.1.4 root 4628: void vm_map_simplify(
4629: vm_map_t map,
4630: vm_offset_t start)
1.1 root 4631: {
4632: vm_map_entry_t this_entry;
4633: vm_map_entry_t prev_entry;
4634:
4635: vm_map_lock(map);
4636: if (
4637: (vm_map_lookup_entry(map, start, &this_entry)) &&
4638: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4639:
4640: (prev_entry->vme_end == start) &&
4641:
4642: (prev_entry->is_shared == FALSE) &&
4643: (prev_entry->is_sub_map == FALSE) &&
4644:
4645: (this_entry->is_shared == FALSE) &&
4646: (this_entry->is_sub_map == FALSE) &&
4647:
4648: (prev_entry->inheritance == this_entry->inheritance) &&
4649: (prev_entry->protection == this_entry->protection) &&
4650: (prev_entry->max_protection == this_entry->max_protection) &&
4651: (prev_entry->wired_count == this_entry->wired_count) &&
4652: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4653:
4654: (prev_entry->needs_copy == this_entry->needs_copy) &&
4655:
4656: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4657: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4658: == this_entry->offset) &&
4659: (prev_entry->projected_on == 0) &&
1.1.1.2 root 4660: (this_entry->projected_on == 0)
1.1 root 4661: ) {
4662: if (map->first_free == this_entry)
4663: map->first_free = prev_entry;
4664:
4665: SAVE_HINT(map, prev_entry);
4666: vm_map_entry_unlink(map, this_entry);
4667: prev_entry->vme_end = this_entry->vme_end;
4668: vm_object_deallocate(this_entry->object.vm_object);
4669: vm_map_entry_dispose(map, this_entry);
4670: }
4671: vm_map_unlock(map);
4672: }
4673:
4674:
4675: /*
4676: * Routine: vm_map_machine_attribute
4677: * Purpose:
4678: * Provide machine-specific attributes to mappings,
4679: * such as cachability etc. for machines that provide
4680: * them. NUMA architectures and machines with big/strange
4681: * caches will use this.
4682: * Note:
4683: * Responsibilities for locking and checking are handled here,
4684: * everything else in the pmap module. If any non-volatile
4685: * information must be kept, the pmap module should handle
4686: * it itself. [This assumes that attributes do not
4687: * need to be inherited, which seems ok to me]
4688: */
1.1.1.4 root 4689: kern_return_t vm_map_machine_attribute(
4690: vm_map_t map,
4691: vm_offset_t address,
4692: vm_size_t size,
4693: vm_machine_attribute_t attribute,
4694: vm_machine_attribute_val_t* value) /* IN/OUT */
1.1 root 4695: {
4696: kern_return_t ret;
4697:
4698: if (address < vm_map_min(map) ||
4699: (address + size) > vm_map_max(map))
4700: return KERN_INVALID_ARGUMENT;
4701:
4702: vm_map_lock(map);
4703:
4704: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4705:
4706: vm_map_unlock(map);
4707:
4708: return ret;
4709: }
4710:
4711:
4712: #if MACH_KDB
4713:
4714: #define printf kdbprintf
4715:
4716: /*
4717: * vm_map_print: [ debug ]
4718: */
1.1.1.4 root 4719: void vm_map_print(vm_map_t map)
1.1 root 4720: {
1.1.1.4 root 4721: vm_map_entry_t entry;
1.1 root 4722:
4723: iprintf("Task map 0x%X: pmap=0x%X,",
4724: (vm_offset_t) map, (vm_offset_t) (map->pmap));
4725: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4726: printf("version=%d\n", map->timestamp);
4727: indent += 2;
4728: for (entry = vm_map_first_entry(map);
4729: entry != vm_map_to_entry(map);
4730: entry = entry->vme_next) {
4731: static char *inheritance_name[3] = { "share", "copy", "none"};
4732:
4733: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4734: printf("start=0x%X, end=0x%X, ",
4735: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
4736: printf("prot=%X/%X/%s, ",
4737: entry->protection,
4738: entry->max_protection,
4739: inheritance_name[entry->inheritance]);
4740: if (entry->wired_count != 0) {
4741: printf("wired(");
4742: if (entry->user_wired_count != 0)
4743: printf("u");
4744: if (entry->wired_count >
4745: ((entry->user_wired_count == 0) ? 0 : 1))
4746: printf("k");
4747: printf(") ");
4748: }
4749: if (entry->in_transition) {
4750: printf("in transition");
4751: if (entry->needs_wakeup)
4752: printf("(wake request)");
4753: printf(", ");
4754: }
4755: if (entry->is_sub_map) {
4756: printf("submap=0x%X, offset=0x%X\n",
4757: (vm_offset_t) entry->object.sub_map,
4758: (vm_offset_t) entry->offset);
4759: } else {
4760: printf("object=0x%X, offset=0x%X",
4761: (vm_offset_t) entry->object.vm_object,
4762: (vm_offset_t) entry->offset);
4763: if (entry->is_shared)
4764: printf(", shared");
4765: if (entry->needs_copy)
4766: printf(", copy needed");
4767: printf("\n");
4768:
4769: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4770: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
4771: indent += 2;
4772: vm_object_print(entry->object.vm_object);
4773: indent -= 2;
4774: }
4775: }
4776: }
4777: indent -= 2;
4778: }
4779:
4780: /*
4781: * Routine: vm_map_copy_print
4782: * Purpose:
4783: * Pretty-print a copy object for ddb.
4784: */
4785:
4786: void vm_map_copy_print(copy)
1.1.1.4 root 4787: const vm_map_copy_t copy;
1.1 root 4788: {
4789: int i, npages;
4790:
4791: printf("copy object 0x%x\n", copy);
4792:
4793: indent += 2;
4794:
4795: iprintf("type=%d", copy->type);
4796: switch (copy->type) {
4797: case VM_MAP_COPY_ENTRY_LIST:
4798: printf("[entry_list]");
4799: break;
1.1.1.2 root 4800:
1.1 root 4801: case VM_MAP_COPY_OBJECT:
4802: printf("[object]");
4803: break;
1.1.1.2 root 4804:
1.1 root 4805: case VM_MAP_COPY_PAGE_LIST:
4806: printf("[page_list]");
4807: break;
4808:
4809: default:
4810: printf("[bad type]");
4811: break;
4812: }
4813: printf(", offset=0x%x", copy->offset);
4814: printf(", size=0x%x\n", copy->size);
4815:
4816: switch (copy->type) {
4817: case VM_MAP_COPY_ENTRY_LIST:
4818: /* XXX add stuff here */
4819: break;
4820:
4821: case VM_MAP_COPY_OBJECT:
4822: iprintf("object=0x%x\n", copy->cpy_object);
4823: break;
4824:
4825: case VM_MAP_COPY_PAGE_LIST:
4826: iprintf("npages=%d", copy->cpy_npages);
4827: printf(", cont=%x", copy->cpy_cont);
4828: printf(", cont_args=%x\n", copy->cpy_cont_args);
4829: if (copy->cpy_npages < 0) {
4830: npages = 0;
4831: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4832: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4833: } else {
4834: npages = copy->cpy_npages;
4835: }
4836: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4837: for (i = 0; i < npages - 1; i++) {
4838: printf("0x%x, ", copy->cpy_page_list[i]);
4839: }
4840: if (npages > 0) {
4841: printf("0x%x", copy->cpy_page_list[npages - 1]);
4842: }
4843: printf("}\n");
4844: break;
4845: }
4846:
1.1.1.4 root 4847: indent -= 2;
1.1 root 4848: }
1.1.1.2 root 4849: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.