![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_map.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52: /*
53: * File: vm/vm_map.c
54: * Author: Avadis Tevanian, Jr., Michael Wayne Young
55: * Date: 1985
56: *
57: * Virtual memory mapping module.
58: */
59:
60: #include <cpus.h>
61: #include <task_swapper.h>
62: #include <mach_assert.h>
63: #include <dipc.h>
64:
65: #include <mach/kern_return.h>
66: #include <mach/port.h>
67: #include <mach/vm_attributes.h>
68: #include <mach/vm_param.h>
69: #include <mach/vm_behavior.h>
70: #include <kern/assert.h>
71: #include <kern/counters.h>
72: #include <kern/zalloc.h>
73: #include <vm/vm_init.h>
74: #include <vm/vm_fault.h>
75: #include <vm/vm_map.h>
76: #include <vm/vm_object.h>
77: #include <vm/vm_page.h>
78: #include <vm/vm_kern.h>
79: #include <ipc/ipc_port.h>
80: #include <kern/sched_prim.h>
81: #include <kern/misc_protos.h>
82: #include <mach/vm_task_server.h>
83: #include <mach/mach_host_server.h>
84: #include <ddb/tr.h>
85: #include <kern/xpr.h>
86:
87: /* Internal prototypes
88: */
89: extern boolean_t vm_map_range_check(
90: vm_map_t map,
91: vm_offset_t start,
92: vm_offset_t end,
93: vm_map_entry_t *entry);
94:
95: extern vm_map_entry_t _vm_map_entry_create(
96: struct vm_map_header *map_header);
97:
98: extern void _vm_map_entry_dispose(
99: struct vm_map_header *map_header,
100: vm_map_entry_t entry);
101:
102: extern void vm_map_pmap_enter(
103: vm_map_t map,
104: vm_offset_t addr,
105: vm_offset_t end_addr,
106: vm_object_t object,
107: vm_offset_t offset,
108: vm_prot_t protection);
109:
110: extern void _vm_map_clip_end(
111: struct vm_map_header *map_header,
112: vm_map_entry_t entry,
113: vm_offset_t end);
114:
115: extern void vm_map_entry_delete(
116: vm_map_t map,
117: vm_map_entry_t entry);
118:
119: extern kern_return_t vm_map_delete(
120: vm_map_t map,
121: vm_offset_t start,
122: vm_offset_t end,
123: int flags);
124:
125: extern void vm_map_copy_steal_pages(
126: vm_map_copy_t copy);
127:
128: extern kern_return_t vm_map_copy_overwrite_unaligned(
129: vm_map_t dst_map,
130: vm_map_entry_t entry,
131: vm_map_copy_t copy,
132: vm_offset_t start);
133:
134: extern kern_return_t vm_map_copy_overwrite_aligned(
135: vm_map_t dst_map,
136: vm_map_entry_t tmp_entry,
137: vm_map_copy_t copy,
138: vm_offset_t start);
139:
140: extern kern_return_t vm_map_copyout_kernel_buffer(
141: vm_map_t map,
142: vm_offset_t *addr, /* IN/OUT */
143: vm_map_copy_t copy,
144: boolean_t overwrite);
145:
146: extern kern_return_t vm_map_copyin_page_list_cont(
147: vm_map_copyin_args_t cont_args,
148: vm_map_copy_t *copy_result); /* OUT */
149:
150: extern void vm_map_fork_share(
151: vm_map_t old_map,
152: vm_map_entry_t old_entry,
153: vm_map_t new_map);
154:
155: extern boolean_t vm_map_fork_copy(
156: vm_map_t old_map,
157: vm_map_entry_t *old_entry_p,
158: vm_map_t new_map);
159:
160: extern kern_return_t vm_remap_range_allocate(
161: vm_map_t map,
162: vm_offset_t *address, /* IN/OUT */
163: vm_size_t size,
164: vm_offset_t mask,
165: boolean_t anywhere,
166: vm_map_entry_t *map_entry); /* OUT */
167:
168: extern void _vm_map_clip_start(
169: struct vm_map_header *map_header,
170: vm_map_entry_t entry,
171: vm_offset_t start);
172:
173: /*
174: * Macros to copy a vm_map_entry. We must be careful to correctly
175: * manage the wired page count. vm_map_entry_copy() creates a new
176: * map entry to the same memory - the wired count in the new entry
177: * must be set to zero. vm_map_entry_copy_full() creates a new
178: * entry that is identical to the old entry. This preserves the
179: * wire count; it's used for map splitting and zone changing in
180: * vm_map_copyout.
181: */
182: #define vm_map_entry_copy(NEW,OLD) \
183: MACRO_BEGIN \
184: *(NEW) = *(OLD); \
185: (NEW)->is_shared = FALSE; \
186: (NEW)->needs_wakeup = FALSE; \
187: (NEW)->in_transition = FALSE; \
188: (NEW)->wired_count = 0; \
189: (NEW)->user_wired_count = 0; \
190: MACRO_END
191:
192: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
193:
194: /*
195: * Virtual memory maps provide for the mapping, protection,
196: * and sharing of virtual memory objects. In addition,
197: * this module provides for an efficient virtual copy of
198: * memory from one map to another.
199: *
200: * Synchronization is required prior to most operations.
201: *
202: * Maps consist of an ordered doubly-linked list of simple
203: * entries; a single hint is used to speed up lookups.
204: *
205: * Sharing maps have been deleted from this version of Mach.
206: * All shared objects are now mapped directly into the respective
207: * maps. This requires a change in the copy on write strategy;
208: * the asymmetric (delayed) strategy is used for shared temporary
209: * objects instead of the symmetric (shadow) strategy. All maps
210: * are now "top level" maps (either task map, kernel map or submap
211: * of the kernel map).
212: *
213: * Since portions of maps are specified by start/end addreses,
214: * which may not align with existing map entries, all
215: * routines merely "clip" entries to these start/end values.
216: * [That is, an entry is split into two, bordering at a
217: * start or end value.] Note that these clippings may not
218: * always be necessary (as the two resulting entries are then
219: * not changed); however, the clipping is done for convenience.
220: * No attempt is currently made to "glue back together" two
221: * abutting entries.
222: *
223: * The symmetric (shadow) copy strategy implements virtual copy
224: * by copying VM object references from one map to
225: * another, and then marking both regions as copy-on-write.
226: * It is important to note that only one writeable reference
227: * to a VM object region exists in any map when this strategy
228: * is used -- this means that shadow object creation can be
229: * delayed until a write operation occurs. The symmetric (delayed)
230: * strategy allows multiple maps to have writeable references to
231: * the same region of a vm object, and hence cannot delay creating
232: * its copy objects. See vm_object_copy_quickly() in vm_object.c.
233: * Copying of permanent objects is completely different; see
234: * vm_object_copy_strategically() in vm_object.c.
235: */
236:
237: zone_t vm_map_zone; /* zone for vm_map structures */
238: zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */
239: zone_t vm_map_kentry_zone; /* zone for kernel entry structures */
240: zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */
241:
242:
243: /*
244: * Placeholder object for submap operations. This object is dropped
245: * into the range by a call to vm_map_find, and removed when
246: * vm_map_submap creates the submap.
247: */
248:
249: vm_object_t vm_submap_object;
250:
251: /*
252: * vm_map_init:
253: *
254: * Initialize the vm_map module. Must be called before
255: * any other vm_map routines.
256: *
257: * Map and entry structures are allocated from zones -- we must
258: * initialize those zones.
259: *
260: * There are three zones of interest:
261: *
262: * vm_map_zone: used to allocate maps.
263: * vm_map_entry_zone: used to allocate map entries.
264: * vm_map_kentry_zone: used to allocate map entries for the kernel.
265: *
266: * The kernel allocates map entries from a special zone that is initially
267: * "crammed" with memory. It would be difficult (perhaps impossible) for
268: * the kernel to allocate more memory to a entry zone when it became
269: * empty since the very act of allocating memory implies the creation
270: * of a new entry.
271: */
272:
273: vm_offset_t map_data;
274: vm_size_t map_data_size;
275: vm_offset_t kentry_data;
276: vm_size_t kentry_data_size;
277: int kentry_count = 2048; /* to init kentry_data_size */
278:
279: /*
280: * Threshold for aggressive (eager) page map entering for vm copyout
281: * operations. Any copyout larger will NOT be aggressively entered.
282: */
283: vm_size_t vm_map_aggressive_enter_max; /* set by bootstrap */
284:
285: void
286: vm_map_init(
287: void)
288: {
289: vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024,
290: PAGE_SIZE, "maps");
291:
292: vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
293: 1024*1024, PAGE_SIZE*5,
294: "non-kernel map entries");
295:
296: vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
297: kentry_data_size, kentry_data_size,
298: "kernel map entries");
299:
300: vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy),
301: 16*1024, PAGE_SIZE, "map copies");
302:
303: /*
304: * Cram the map and kentry zones with initial data.
305: * Set kentry_zone non-collectible to aid zone_gc().
306: */
307: zone_change(vm_map_zone, Z_COLLECT, FALSE);
308: zone_change(vm_map_kentry_zone, Z_COLLECT, FALSE);
309: zone_change(vm_map_kentry_zone, Z_EXPAND, FALSE);
310: zcram(vm_map_zone, map_data, map_data_size);
311: zcram(vm_map_kentry_zone, kentry_data, kentry_data_size);
312: }
313:
314: void
315: vm_map_steal_memory(
316: void)
317: {
318: map_data_size = round_page(10 * sizeof(struct vm_map));
319: map_data = pmap_steal_memory(map_data_size);
320:
321: /*
322: * Limiting worst case: vm_map_kentry_zone needs to map each "available"
323: * physical page (i.e. that beyond the kernel image and page tables)
324: * individually; we fudge this slightly and guess at most one entry per
325: * two pages. This works out to roughly .4 of 1% of physical memory,
326: * or roughly 1800 entries (56K) for a 16M machine with 4K pages.
327: */
328:
329: kentry_count = pmap_free_pages() / 2;
330:
331: kentry_data_size =
332: round_page(kentry_count * sizeof(struct vm_map_entry));
333: kentry_data = pmap_steal_memory(kentry_data_size);
334: }
335:
336: /*
337: * vm_map_create:
338: *
339: * Creates and returns a new empty VM map with
340: * the given physical map structure, and having
341: * the given lower and upper address bounds.
342: */
343: vm_map_t
344: vm_map_create(
345: pmap_t pmap,
346: vm_offset_t min,
347: vm_offset_t max,
348: boolean_t pageable)
349: {
350: register vm_map_t result;
351:
352: result = (vm_map_t) zalloc(vm_map_zone);
353: if (result == VM_MAP_NULL)
354: panic("vm_map_create");
355:
356: vm_map_first_entry(result) = vm_map_to_entry(result);
357: vm_map_last_entry(result) = vm_map_to_entry(result);
358: result->hdr.nentries = 0;
359: result->hdr.entries_pageable = pageable;
360:
361: result->size = 0;
362: result->ref_count = 1;
363: #if TASK_SWAPPER
364: result->res_count = 1;
365: result->sw_state = MAP_SW_IN;
366: #endif /* TASK_SWAPPER */
367: result->pmap = pmap;
368: result->min_offset = min;
369: result->max_offset = max;
370: result->wiring_required = FALSE;
371: result->no_zero_fill = FALSE;
372: result->wait_for_space = FALSE;
373: result->first_free = vm_map_to_entry(result);
374: result->hint = vm_map_to_entry(result);
375: vm_map_lock_init(result);
376: mutex_init(&result->s_lock, ETAP_VM_RESULT);
377:
378: return(result);
379: }
380:
381: /*
382: * vm_map_entry_create: [ internal use only ]
383: *
384: * Allocates a VM map entry for insertion in the
385: * given map (or map copy). No fields are filled.
386: */
387: #define vm_map_entry_create(map) \
388: _vm_map_entry_create(&(map)->hdr)
389:
390: #define vm_map_copy_entry_create(copy) \
391: _vm_map_entry_create(&(copy)->cpy_hdr)
392:
393: vm_map_entry_t
394: _vm_map_entry_create(
395: register struct vm_map_header *map_header)
396: {
397: register zone_t zone;
398: register vm_map_entry_t entry;
399:
400: if (map_header->entries_pageable)
401: zone = vm_map_entry_zone;
402: else
403: zone = vm_map_kentry_zone;
404:
405: entry = (vm_map_entry_t) zalloc(zone);
406: if (entry == VM_MAP_ENTRY_NULL)
407: panic("vm_map_entry_create");
408:
409: return(entry);
410: }
411:
412: /*
413: * vm_map_entry_dispose: [ internal use only ]
414: *
415: * Inverse of vm_map_entry_create.
416: */
417: #define vm_map_entry_dispose(map, entry) \
418: MACRO_BEGIN \
419: assert((entry) != (map)->first_free && \
420: (entry) != (map)->hint); \
421: _vm_map_entry_dispose(&(map)->hdr, (entry)); \
422: MACRO_END
423:
424: #define vm_map_copy_entry_dispose(map, entry) \
425: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
426:
427: void
428: _vm_map_entry_dispose(
429: register struct vm_map_header *map_header,
430: register vm_map_entry_t entry)
431: {
432: register zone_t zone;
433:
434: if (map_header->entries_pageable)
435: zone = vm_map_entry_zone;
436: else
437: zone = vm_map_kentry_zone;
438:
439: zfree(zone, (vm_offset_t) entry);
440: }
441:
442: boolean_t first_free_is_valid(vm_map_t map); /* forward */
443: boolean_t first_free_check = FALSE;
444: boolean_t
445: first_free_is_valid(
446: vm_map_t map)
447: {
448: vm_map_entry_t entry, next;
449:
450: if (!first_free_check)
451: return TRUE;
452:
453: entry = vm_map_to_entry(map);
454: next = entry->vme_next;
455: while (trunc_page(next->vme_start) == trunc_page(entry->vme_end) ||
456: (trunc_page(next->vme_start) == trunc_page(entry->vme_start) &&
457: next != vm_map_to_entry(map))) {
458: entry = next;
459: next = entry->vme_next;
460: if (entry == vm_map_to_entry(map))
461: break;
462: }
463: if (map->first_free != entry) {
464: printf("Bad first_free for map 0x%x: 0x%x should be 0x%x\n",
465: map, map->first_free, entry);
466: return FALSE;
467: }
468: return TRUE;
469: }
470:
471: /*
472: * UPDATE_FIRST_FREE:
473: *
474: * Updates the map->first_free pointer to the
475: * entry immediately before the first hole in the map.
476: * The map should be locked.
477: */
478: #define UPDATE_FIRST_FREE(map, new_first_free) \
479: MACRO_BEGIN \
480: vm_map_t UFF_map; \
481: vm_map_entry_t UFF_first_free; \
482: vm_map_entry_t UFF_next_entry; \
483: UFF_map = (map); \
484: UFF_first_free = (new_first_free); \
485: UFF_next_entry = UFF_first_free->vme_next; \
486: while (trunc_page(UFF_next_entry->vme_start) == \
487: trunc_page(UFF_first_free->vme_end) || \
488: (trunc_page(UFF_next_entry->vme_start) == \
489: trunc_page(UFF_first_free->vme_start) && \
490: UFF_next_entry != vm_map_to_entry(UFF_map))) { \
491: UFF_first_free = UFF_next_entry; \
492: UFF_next_entry = UFF_first_free->vme_next; \
493: if (UFF_first_free == vm_map_to_entry(UFF_map)) \
494: break; \
495: } \
496: UFF_map->first_free = UFF_first_free; \
497: assert(first_free_is_valid(UFF_map)); \
498: MACRO_END
499:
500: /*
501: * vm_map_entry_{un,}link:
502: *
503: * Insert/remove entries from maps (or map copies).
504: */
505: #define vm_map_entry_link(map, after_where, entry) \
506: MACRO_BEGIN \
507: vm_map_t VMEL_map; \
508: vm_map_entry_t VMEL_entry; \
509: VMEL_map = (map); \
510: VMEL_entry = (entry); \
511: _vm_map_entry_link(&VMEL_map->hdr, after_where, VMEL_entry); \
512: UPDATE_FIRST_FREE(VMEL_map, VMEL_map->first_free); \
513: MACRO_END
514:
515:
516: #define vm_map_copy_entry_link(copy, after_where, entry) \
517: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, (entry))
518:
519: #define _vm_map_entry_link(hdr, after_where, entry) \
520: MACRO_BEGIN \
521: (hdr)->nentries++; \
522: (entry)->vme_prev = (after_where); \
523: (entry)->vme_next = (after_where)->vme_next; \
524: (entry)->vme_prev->vme_next = (entry)->vme_next->vme_prev = (entry); \
525: MACRO_END
526:
527: #define vm_map_entry_unlink(map, entry) \
528: MACRO_BEGIN \
529: vm_map_t VMEU_map; \
530: vm_map_entry_t VMEU_entry; \
531: vm_map_entry_t VMEU_first_free; \
532: VMEU_map = (map); \
533: VMEU_entry = (entry); \
534: if (VMEU_entry->vme_start <= VMEU_map->first_free->vme_start) \
535: VMEU_first_free = VMEU_entry->vme_prev; \
536: else \
537: VMEU_first_free = VMEU_map->first_free; \
538: _vm_map_entry_unlink(&VMEU_map->hdr, VMEU_entry); \
539: UPDATE_FIRST_FREE(VMEU_map, VMEU_first_free); \
540: MACRO_END
541:
542: #define vm_map_copy_entry_unlink(copy, entry) \
543: _vm_map_entry_unlink(&(copy)->cpy_hdr, (entry))
544:
545: #define _vm_map_entry_unlink(hdr, entry) \
546: MACRO_BEGIN \
547: (hdr)->nentries--; \
548: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
549: (entry)->vme_prev->vme_next = (entry)->vme_next; \
550: MACRO_END
551:
552: #if MACH_ASSERT && TASK_SWAPPER
553: /*
554: * vm_map_reference:
555: *
556: * Adds valid reference and residence counts to the given map.
557: * The map must be in memory (i.e. non-zero residence count).
558: *
559: */
560: void
561: vm_map_reference(
562: register vm_map_t map)
563: {
564: if (map == VM_MAP_NULL)
565: return;
566:
567: mutex_lock(&map->s_lock);
568: assert(map->res_count > 0);
569: assert(map->ref_count >= map->res_count);
570: map->ref_count++;
571: map->res_count++;
572: mutex_unlock(&map->s_lock);
573: }
574:
575: /*
576: * vm_map_res_reference:
577: *
578: * Adds another valid residence count to the given map.
579: *
580: * Map is locked so this function can be called from
581: * vm_map_swapin.
582: *
583: */
584: void vm_map_res_reference(register vm_map_t map)
585: {
586: /* assert map is locked */
587: assert(map->res_count >= 0);
588: assert(map->ref_count >= map->res_count);
589: if (map->res_count == 0) {
590: mutex_unlock(&map->s_lock);
591: vm_map_lock(map);
592: vm_map_swapin(map);
593: mutex_lock(&map->s_lock);
594: ++map->res_count;
595: vm_map_unlock(map);
596: } else
597: ++map->res_count;
598: }
599:
600: /*
601: * vm_map_reference_swap:
602: *
603: * Adds valid reference and residence counts to the given map.
604: *
605: * The map may not be in memory (i.e. zero residence count).
606: *
607: */
608: void vm_map_reference_swap(register vm_map_t map)
609: {
610: assert(map != VM_MAP_NULL);
611: mutex_lock(&map->s_lock);
612: assert(map->res_count >= 0);
613: assert(map->ref_count >= map->res_count);
614: map->ref_count++;
615: vm_map_res_reference(map);
616: mutex_unlock(&map->s_lock);
617: }
618:
619: /*
620: * vm_map_res_deallocate:
621: *
622: * Decrement residence count on a map; possibly causing swapout.
623: *
624: * The map must be in memory (i.e. non-zero residence count).
625: *
626: * The map is locked, so this function is callable from vm_map_deallocate.
627: *
628: */
629: void vm_map_res_deallocate(register vm_map_t map)
630: {
631: assert(map->res_count > 0);
632: if (--map->res_count == 0) {
633: mutex_unlock(&map->s_lock);
634: vm_map_lock(map);
635: vm_map_swapout(map);
636: vm_map_unlock(map);
637: mutex_lock(&map->s_lock);
638: }
639: assert(map->ref_count >= map->res_count);
640: }
641: #endif /* MACH_ASSERT && TASK_SWAPPER */
642:
643: /*
644: * vm_map_deallocate:
645: *
646: * Removes a reference from the specified map,
647: * destroying it if no references remain.
648: * The map should not be locked.
649: */
650: void
651: vm_map_deallocate(
652: register vm_map_t map)
653: {
654: unsigned int ref;
655:
656: if (map == VM_MAP_NULL)
657: return;
658:
659: mutex_lock(&map->s_lock);
660: ref = --map->ref_count;
661: if (ref > 0) {
662: vm_map_res_deallocate(map);
663: mutex_unlock(&map->s_lock);
664: /*
665: * Someone may be waiting in task_halt_wait. They
666: * wait there for the map reference count to reach
667: * two (one for the task and one for their act). Wake
668: * them up if need be.
669: */
670: if (ref == 2)
671: thread_wakeup(&map->ref_count);
672: return;
673: }
674: assert(map->ref_count == 0);
675: mutex_unlock(&map->s_lock);
676:
677: #if TASK_SWAPPER
678: /*
679: * The map residence count isn't decremented here because
680: * the vm_map_delete below will traverse the entire map,
681: * deleting entries, and the residence counts on objects
682: * and sharing maps will go away then.
683: */
684: #endif
685:
686: vm_map_destroy(map);
687: }
688:
689: /*
690: * vm_map_destroy:
691: *
692: * Actually destroy a map.
693: */
694: void
695: vm_map_destroy(
696: register vm_map_t map)
697: {
698: vm_map_lock(map);
699: (void) vm_map_delete(map, map->min_offset,
700: map->max_offset, VM_MAP_NO_FLAGS);
701: vm_map_unlock(map);
702:
703: pmap_destroy(map->pmap);
704:
705: zfree(vm_map_zone, (vm_offset_t) map);
706: }
707:
708: #if TASK_SWAPPER
709: /*
710: * vm_map_swapin/vm_map_swapout
711: *
712: * Swap a map in and out, either referencing or releasing its resources.
713: * These functions are internal use only; however, they must be exported
714: * because they may be called from macros, which are exported.
715: *
716: * In the case of swapout, there could be races on the residence count,
717: * so if the residence count is up, we return, assuming that a
718: * vm_map_deallocate() call in the near future will bring us back.
719: *
720: * Locking:
721: * -- We use the map write lock for synchronization among races.
722: * -- The map write lock, and not the simple s_lock, protects the
723: * swap state of the map.
724: * -- If a map entry is a share map, then we hold both locks, in
725: * hierarchical order.
726: *
727: * Synchronization Notes:
728: * 1) If a vm_map_swapin() call happens while swapout in progress, it
729: * will block on the map lock and proceed when swapout is through.
730: * 2) A vm_map_reference() call at this time is illegal, and will
731: * cause a panic. vm_map_reference() is only allowed on resident
732: * maps, since it refuses to block.
733: * 3) A vm_map_swapin() call during a swapin will block, and
734: * proceeed when the first swapin is done, turning into a nop.
735: * This is the reason the res_count is not incremented until
736: * after the swapin is complete.
737: * 4) There is a timing hole after the checks of the res_count, before
738: * the map lock is taken, during which a swapin may get the lock
739: * before a swapout about to happen. If this happens, the swapin
740: * will detect the state and increment the reference count, causing
741: * the swapout to be a nop, thereby delaying it until a later
742: * vm_map_deallocate. If the swapout gets the lock first, then
743: * the swapin will simply block until the swapout is done, and
744: * then proceed.
745: *
746: * Because vm_map_swapin() is potentially an expensive operation, it
747: * should be used with caution.
748: *
749: * Invariants:
750: * 1) A map with a residence count of zero is either swapped, or
751: * being swapped.
752: * 2) A map with a non-zero residence count is either resident,
753: * or being swapped in.
754: */
755:
756: int vm_map_swap_enable = 1;
757:
758: void vm_map_swapin (vm_map_t map)
759: {
760: register vm_map_entry_t entry;
761:
762: if (!vm_map_swap_enable) /* debug */
763: return;
764:
765: /*
766: * Map is locked
767: * First deal with various races.
768: */
769: if (map->sw_state == MAP_SW_IN)
770: /*
771: * we raced with swapout and won. Returning will incr.
772: * the res_count, turning the swapout into a nop.
773: */
774: return;
775:
776: /*
777: * The residence count must be zero. If we raced with another
778: * swapin, the state would have been IN; if we raced with a
779: * swapout (after another competing swapin), we must have lost
780: * the race to get here (see above comment), in which case
781: * res_count is still 0.
782: */
783: assert(map->res_count == 0);
784:
785: /*
786: * There are no intermediate states of a map going out or
787: * coming in, since the map is locked during the transition.
788: */
789: assert(map->sw_state == MAP_SW_OUT);
790:
791: /*
792: * We now operate upon each map entry. If the entry is a sub-
793: * or share-map, we call vm_map_res_reference upon it.
794: * If the entry is an object, we call vm_object_res_reference
795: * (this may iterate through the shadow chain).
796: * Note that we hold the map locked the entire time,
797: * even if we get back here via a recursive call in
798: * vm_map_res_reference.
799: */
800: entry = vm_map_first_entry(map);
801:
802: while (entry != vm_map_to_entry(map)) {
803: if (entry->object.vm_object != VM_OBJECT_NULL) {
804: if (entry->is_sub_map) {
805: vm_map_t lmap = entry->object.sub_map;
806: mutex_lock(&lmap->s_lock);
807: vm_map_res_reference(lmap);
808: mutex_unlock(&lmap->s_lock);
809: } else {
810: vm_object_t object = entry->object.vm_object;
811: vm_object_lock(object);
812: /*
813: * This call may iterate through the
814: * shadow chain.
815: */
816: vm_object_res_reference(object);
817: vm_object_unlock(object);
818: }
819: }
820: entry = entry->vme_next;
821: }
822: assert(map->sw_state == MAP_SW_OUT);
823: map->sw_state = MAP_SW_IN;
824: }
825:
826: void vm_map_swapout(vm_map_t map)
827: {
828: register vm_map_entry_t entry;
829:
830: /*
831: * Map is locked
832: * First deal with various races.
833: * If we raced with a swapin and lost, the residence count
834: * will have been incremented to 1, and we simply return.
835: */
836: mutex_lock(&map->s_lock);
837: if (map->res_count != 0) {
838: mutex_unlock(&map->s_lock);
839: return;
840: }
841: mutex_unlock(&map->s_lock);
842:
843: /*
844: * There are no intermediate states of a map going out or
845: * coming in, since the map is locked during the transition.
846: */
847: assert(map->sw_state == MAP_SW_IN);
848:
849: if (!vm_map_swap_enable)
850: return;
851:
852: /*
853: * We now operate upon each map entry. If the entry is a sub-
854: * or share-map, we call vm_map_res_deallocate upon it.
855: * If the entry is an object, we call vm_object_res_deallocate
856: * (this may iterate through the shadow chain).
857: * Note that we hold the map locked the entire time,
858: * even if we get back here via a recursive call in
859: * vm_map_res_deallocate.
860: */
861: entry = vm_map_first_entry(map);
862:
863: while (entry != vm_map_to_entry(map)) {
864: if (entry->object.vm_object != VM_OBJECT_NULL) {
865: if (entry->is_sub_map) {
866: vm_map_t lmap = entry->object.sub_map;
867: mutex_lock(&lmap->s_lock);
868: vm_map_res_deallocate(lmap);
869: mutex_unlock(&lmap->s_lock);
870: } else {
871: vm_object_t object = entry->object.vm_object;
872: vm_object_lock(object);
873: /*
874: * This call may take a long time,
875: * since it could actively push
876: * out pages (if we implement it
877: * that way).
878: */
879: vm_object_res_deallocate(object);
880: vm_object_unlock(object);
881: }
882: }
883: entry = entry->vme_next;
884: }
885: assert(map->sw_state == MAP_SW_IN);
886: map->sw_state = MAP_SW_OUT;
887: }
888:
889: #endif /* TASK_SWAPPER */
890:
891:
892: /*
893: * SAVE_HINT:
894: *
895: * Saves the specified entry as the hint for
896: * future lookups. Performs necessary interlocks.
897: */
898: #define SAVE_HINT(map,value) \
899: mutex_lock(&(map)->s_lock); \
900: (map)->hint = (value); \
901: mutex_unlock(&(map)->s_lock);
902:
903: /*
904: * vm_map_lookup_entry: [ internal use only ]
905: *
906: * Finds the map entry containing (or
907: * immediately preceding) the specified address
908: * in the given map; the entry is returned
909: * in the "entry" parameter. The boolean
910: * result indicates whether the address is
911: * actually contained in the map.
912: */
913: boolean_t
914: vm_map_lookup_entry(
915: register vm_map_t map,
916: register vm_offset_t address,
917: vm_map_entry_t *entry) /* OUT */
918: {
919: register vm_map_entry_t cur;
920: register vm_map_entry_t last;
921:
922: /*
923: * Start looking either from the head of the
924: * list, or from the hint.
925: */
926:
927: mutex_lock(&map->s_lock);
928: cur = map->hint;
929: mutex_unlock(&map->s_lock);
930:
931: if (cur == vm_map_to_entry(map))
932: cur = cur->vme_next;
933:
934: if (address >= cur->vme_start) {
935: /*
936: * Go from hint to end of list.
937: *
938: * But first, make a quick check to see if
939: * we are already looking at the entry we
940: * want (which is usually the case).
941: * Note also that we don't need to save the hint
942: * here... it is the same hint (unless we are
943: * at the header, in which case the hint didn't
944: * buy us anything anyway).
945: */
946: last = vm_map_to_entry(map);
947: if ((cur != last) && (cur->vme_end > address)) {
948: *entry = cur;
949: return(TRUE);
950: }
951: }
952: else {
953: /*
954: * Go from start to hint, *inclusively*
955: */
956: last = cur->vme_next;
957: cur = vm_map_first_entry(map);
958: }
959:
960: /*
961: * Search linearly
962: */
963:
964: while (cur != last) {
965: if (cur->vme_end > address) {
966: if (address >= cur->vme_start) {
967: /*
968: * Save this lookup for future
969: * hints, and return
970: */
971:
972: *entry = cur;
973: SAVE_HINT(map, cur);
974: return(TRUE);
975: }
976: break;
977: }
978: cur = cur->vme_next;
979: }
980: *entry = cur->vme_prev;
981: SAVE_HINT(map, *entry);
982: return(FALSE);
983: }
984:
985: /*
986: * Routine: vm_map_find_space
987: * Purpose:
988: * Allocate a range in the specified virtual address map,
989: * returning the entry allocated for that range.
990: * Used by kmem_alloc, etc.
991: *
992: * The map must be NOT be locked. It will be returned locked
993: * on KERN_SUCCESS, unlocked on failure.
994: *
995: * If an entry is allocated, the object/offset fields
996: * are initialized to zero.
997: */
998: kern_return_t
999: vm_map_find_space(
1000: register vm_map_t map,
1001: vm_offset_t *address, /* OUT */
1002: vm_size_t size,
1003: vm_offset_t mask,
1004: vm_map_entry_t *o_entry) /* OUT */
1005: {
1006: register vm_map_entry_t entry, new_entry;
1007: register vm_offset_t start;
1008: register vm_offset_t end;
1009:
1010: new_entry = vm_map_entry_create(map);
1011:
1012: /*
1013: * Look for the first possible address; if there's already
1014: * something at this address, we have to start after it.
1015: */
1016:
1017: vm_map_lock(map);
1018:
1019: assert(first_free_is_valid(map));
1020: if ((entry = map->first_free) == vm_map_to_entry(map))
1021: start = map->min_offset;
1022: else
1023: start = entry->vme_end;
1024:
1025: /*
1026: * In any case, the "entry" always precedes
1027: * the proposed new region throughout the loop:
1028: */
1029:
1030: while (TRUE) {
1031: register vm_map_entry_t next;
1032:
1033: /*
1034: * Find the end of the proposed new region.
1035: * Be sure we didn't go beyond the end, or
1036: * wrap around the address.
1037: */
1038:
1039: end = ((start + mask) & ~mask);
1040: if (end < start) {
1041: vm_map_entry_dispose(map, new_entry);
1042: vm_map_unlock(map);
1043: return(KERN_NO_SPACE);
1044: }
1045: start = end;
1046: end += size;
1047:
1048: if ((end > map->max_offset) || (end < start)) {
1049: vm_map_entry_dispose(map, new_entry);
1050: vm_map_unlock(map);
1051: return(KERN_NO_SPACE);
1052: }
1053:
1054: /*
1055: * If there are no more entries, we must win.
1056: */
1057:
1058: next = entry->vme_next;
1059: if (next == vm_map_to_entry(map))
1060: break;
1061:
1062: /*
1063: * If there is another entry, it must be
1064: * after the end of the potential new region.
1065: */
1066:
1067: if (next->vme_start >= end)
1068: break;
1069:
1070: /*
1071: * Didn't fit -- move to the next entry.
1072: */
1073:
1074: entry = next;
1075: start = entry->vme_end;
1076: }
1077:
1078: /*
1079: * At this point,
1080: * "start" and "end" should define the endpoints of the
1081: * available new range, and
1082: * "entry" should refer to the region before the new
1083: * range, and
1084: *
1085: * the map should be locked.
1086: */
1087:
1088: *address = start;
1089:
1090: new_entry->vme_start = start;
1091: new_entry->vme_end = end;
1092: assert(page_aligned(new_entry->vme_start));
1093: assert(page_aligned(new_entry->vme_end));
1094:
1095: new_entry->is_shared = FALSE;
1096: new_entry->is_sub_map = FALSE;
1097: new_entry->object.vm_object = VM_OBJECT_NULL;
1098: new_entry->offset = (vm_offset_t) 0;
1099:
1100: new_entry->needs_copy = FALSE;
1101:
1102: new_entry->inheritance = VM_INHERIT_DEFAULT;
1103: new_entry->protection = VM_PROT_DEFAULT;
1104: new_entry->max_protection = VM_PROT_ALL;
1105: new_entry->behavior = VM_BEHAVIOR_DEFAULT;
1106: new_entry->wired_count = 0;
1107: new_entry->user_wired_count = 0;
1108:
1109: new_entry->in_transition = FALSE;
1110: new_entry->needs_wakeup = FALSE;
1111:
1112: /*
1113: * Insert the new entry into the list
1114: */
1115:
1116: vm_map_entry_link(map, entry, new_entry);
1117:
1118: map->size += size;
1119:
1120: /*
1121: * Update the lookup hint
1122: */
1123: SAVE_HINT(map, new_entry);
1124:
1125: *o_entry = new_entry;
1126: return(KERN_SUCCESS);
1127: }
1128:
1129: int vm_map_pmap_enter_print = FALSE;
1130: int vm_map_pmap_enter_enable = FALSE;
1131:
1132: /*
1133: * Routine: vm_map_pmap_enter
1134: *
1135: * Description:
1136: * Force pages from the specified object to be entered into
1137: * the pmap at the specified address if they are present.
1138: * As soon as a page not found in the object the scan ends.
1139: *
1140: * Returns:
1141: * Nothing.
1142: *
1143: * In/out conditions:
1144: * The source map should not be locked on entry.
1145: */
1146: void
1147: vm_map_pmap_enter(
1148: vm_map_t map,
1149: register vm_offset_t addr,
1150: register vm_offset_t end_addr,
1151: register vm_object_t object,
1152: vm_offset_t offset,
1153: vm_prot_t protection)
1154: {
1155: while (addr < end_addr) {
1156: register vm_page_t m;
1157:
1158: vm_object_lock(object);
1159: vm_object_paging_begin(object);
1160:
1161: m = vm_page_lookup(object, offset);
1162: if (m == VM_PAGE_NULL || m->busy ||
1163: (m->unusual && ( m->error || m->restart || m->absent ||
1164: protection & m->page_lock))) {
1165:
1166: vm_object_paging_end(object);
1167: vm_object_unlock(object);
1168: return;
1169: }
1170:
1171: assert(!m->fictitious); /* XXX is this possible ??? */
1172:
1173: if (vm_map_pmap_enter_print) {
1174: printf("vm_map_pmap_enter:");
1175: printf("map: %x, addr: %x, object: %x, offset: %x\n",
1176: map, addr, object, offset);
1177: }
1178:
1179: m->busy = TRUE;
1180: vm_object_unlock(object);
1181:
1182: PMAP_ENTER(map->pmap, addr, m,
1183: protection, FALSE);
1184:
1185: vm_object_lock(object);
1186: PAGE_WAKEUP_DONE(m);
1187: vm_page_lock_queues();
1188: if (!m->active && !m->inactive)
1189: vm_page_activate(m);
1190: vm_page_unlock_queues();
1191: vm_object_paging_end(object);
1192: vm_object_unlock(object);
1193:
1194: offset += PAGE_SIZE;
1195: addr += PAGE_SIZE;
1196: }
1197: }
1198:
1199: /*
1200: * Routine: vm_map_enter
1201: *
1202: * Description:
1203: * Allocate a range in the specified virtual address map.
1204: * The resulting range will refer to memory defined by
1205: * the given memory object and offset into that object.
1206: *
1207: * Arguments are as defined in the vm_map call.
1208: */
1209: kern_return_t
1210: vm_map_enter(
1211: register vm_map_t map,
1212: vm_offset_t *address, /* IN/OUT */
1213: vm_size_t size,
1214: vm_offset_t mask,
1215: int flags,
1216: vm_object_t object,
1217: vm_offset_t offset,
1218: boolean_t needs_copy,
1219: vm_prot_t cur_protection,
1220: vm_prot_t max_protection,
1221: vm_inherit_t inheritance)
1222: {
1223: vm_map_entry_t entry;
1224: register vm_offset_t start;
1225: register vm_offset_t end;
1226: kern_return_t result = KERN_SUCCESS;
1227:
1228: boolean_t anywhere = VM_FLAGS_ANYWHERE & flags;
1229: char alias;
1230:
1231: VM_GET_FLAGS_ALIAS(flags, alias);
1232:
1233: #define RETURN(value) { result = value; goto BailOut; }
1234:
1235: assert(page_aligned(*address));
1236: assert(page_aligned(size));
1237: StartAgain: ;
1238:
1239: start = *address;
1240:
1241: if (anywhere) {
1242: vm_map_lock(map);
1243:
1244: /*
1245: * Calculate the first possible address.
1246: */
1247:
1248: if (start < map->min_offset)
1249: start = map->min_offset;
1250: if (start > map->max_offset)
1251: RETURN(KERN_NO_SPACE);
1252:
1253: /*
1254: * Look for the first possible address;
1255: * if there's already something at this
1256: * address, we have to start after it.
1257: */
1258:
1259: assert(first_free_is_valid(map));
1260: if (start == map->min_offset) {
1261: if ((entry = map->first_free) != vm_map_to_entry(map))
1262: start = entry->vme_end;
1263: } else {
1264: vm_map_entry_t tmp_entry;
1265: if (vm_map_lookup_entry(map, start, &tmp_entry))
1266: start = tmp_entry->vme_end;
1267: entry = tmp_entry;
1268: }
1269:
1270: /*
1271: * In any case, the "entry" always precedes
1272: * the proposed new region throughout the
1273: * loop:
1274: */
1275:
1276: while (TRUE) {
1277: register vm_map_entry_t next;
1278:
1279: /*
1280: * Find the end of the proposed new region.
1281: * Be sure we didn't go beyond the end, or
1282: * wrap around the address.
1283: */
1284:
1285: end = ((start + mask) & ~mask);
1286: if (end < start)
1287: RETURN(KERN_NO_SPACE);
1288: start = end;
1289: end += size;
1290:
1291: if ((end > map->max_offset) || (end < start)) {
1292: if (map->wait_for_space) {
1293: if (size <= (map->max_offset -
1294: map->min_offset)) {
1295: assert_wait((event_t)map,
1296: THREAD_ABORTSAFE);
1297: vm_map_unlock(map);
1298: thread_block((void (*)(void))0);
1299: goto StartAgain;
1300: }
1301: }
1302: RETURN(KERN_NO_SPACE);
1303: }
1304:
1305: /*
1306: * If there are no more entries, we must win.
1307: */
1308:
1309: next = entry->vme_next;
1310: if (next == vm_map_to_entry(map))
1311: break;
1312:
1313: /*
1314: * If there is another entry, it must be
1315: * after the end of the potential new region.
1316: */
1317:
1318: if (next->vme_start >= end)
1319: break;
1320:
1321: /*
1322: * Didn't fit -- move to the next entry.
1323: */
1324:
1325: entry = next;
1326: start = entry->vme_end;
1327: }
1328: *address = start;
1329: } else {
1330: vm_map_entry_t temp_entry;
1331:
1332: /*
1333: * Verify that:
1334: * the address doesn't itself violate
1335: * the mask requirement.
1336: */
1337:
1338: vm_map_lock(map);
1339: if ((start & mask) != 0)
1340: RETURN(KERN_NO_SPACE);
1341:
1342: /*
1343: * ... the address is within bounds
1344: */
1345:
1346: end = start + size;
1347:
1348: if ((start < map->min_offset) ||
1349: (end > map->max_offset) ||
1350: (start >= end)) {
1351: RETURN(KERN_INVALID_ADDRESS);
1352: }
1353:
1354: /*
1355: * ... the starting address isn't allocated
1356: */
1357:
1358: if (vm_map_lookup_entry(map, start, &temp_entry))
1359: RETURN(KERN_NO_SPACE);
1360:
1361: entry = temp_entry;
1362:
1363: /*
1364: * ... the next region doesn't overlap the
1365: * end point.
1366: */
1367:
1368: if ((entry->vme_next != vm_map_to_entry(map)) &&
1369: (entry->vme_next->vme_start < end))
1370: RETURN(KERN_NO_SPACE);
1371: }
1372:
1373: /*
1374: * At this point,
1375: * "start" and "end" should define the endpoints of the
1376: * available new range, and
1377: * "entry" should refer to the region before the new
1378: * range, and
1379: *
1380: * the map should be locked.
1381: */
1382:
1383: /*
1384: * See whether we can avoid creating a new entry (and object) by
1385: * extending one of our neighbors. [So far, we only attempt to
1386: * extend from below.]
1387: */
1388:
1389: if ((object == VM_OBJECT_NULL) &&
1390: (entry != vm_map_to_entry(map)) &&
1391: (entry->vme_end == start) &&
1392: (!entry->is_shared) &&
1393: (!entry->is_sub_map) &&
1394: (entry->alias == alias) &&
1395: (entry->inheritance == inheritance) &&
1396: (entry->protection == cur_protection) &&
1397: (entry->max_protection == max_protection) &&
1398: (entry->behavior == VM_BEHAVIOR_DEFAULT) &&
1399: (entry->in_transition == 0) &&
1400: (entry->wired_count == 0)) { /* implies user_wired_count == 0 */
1401: if (vm_object_coalesce(entry->object.vm_object,
1402: VM_OBJECT_NULL,
1403: entry->offset,
1404: (vm_offset_t) 0,
1405: (vm_size_t)(entry->vme_end - entry->vme_start),
1406: (vm_size_t)(end - entry->vme_end))) {
1407:
1408: /*
1409: * Coalesced the two objects - can extend
1410: * the previous map entry to include the
1411: * new range.
1412: */
1413: map->size += (end - entry->vme_end);
1414: entry->vme_end = end;
1415: UPDATE_FIRST_FREE(map, map->first_free);
1416: RETURN(KERN_SUCCESS);
1417: }
1418: }
1419:
1420: /*
1421: * Create a new entry
1422: */
1423:
1424: { /**/
1425: register vm_map_entry_t new_entry;
1426:
1427: new_entry = vm_map_entry_insert(map, entry, start, end, object,
1428: offset, needs_copy, FALSE, FALSE,
1429: cur_protection, max_protection,
1430: VM_BEHAVIOR_DEFAULT, inheritance, 0);
1431: new_entry->alias = alias;
1432: vm_map_unlock(map);
1433:
1434: /* Wire down the new entry if the user
1435: * requested all new map entries be wired.
1436: */
1437: if (map->wiring_required) {
1438: result = vm_map_wire(map, start, end,
1439: new_entry->protection, TRUE);
1440: return(result);
1441: }
1442:
1443: if ((object != VM_OBJECT_NULL) &&
1444: (vm_map_pmap_enter_enable) &&
1445: (!anywhere) &&
1446: (!needs_copy) &&
1447: (size < (128*1024))) {
1448: vm_map_pmap_enter(map, start, end,
1449: object, offset, cur_protection);
1450: }
1451:
1452: return(result);
1453: } /**/
1454:
1455: BailOut: ;
1456: vm_map_unlock(map);
1457: return(result);
1458:
1459: #undef RETURN
1460: }
1461:
1462: /*
1463: * vm_map_clip_start: [ internal use only ]
1464: *
1465: * Asserts that the given entry begins at or after
1466: * the specified address; if necessary,
1467: * it splits the entry into two.
1468: */
1469: #define vm_map_clip_start(map, entry, startaddr) \
1470: MACRO_BEGIN \
1471: vm_map_t VMCS_map; \
1472: vm_map_entry_t VMCS_entry; \
1473: vm_offset_t VMCS_startaddr; \
1474: VMCS_map = (map); \
1475: VMCS_entry = (entry); \
1476: VMCS_startaddr = (startaddr); \
1477: if (VMCS_startaddr > VMCS_entry->vme_start) \
1478: _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
1479: UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free); \
1480: MACRO_END
1481:
1482: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1483: MACRO_BEGIN \
1484: if ((startaddr) > (entry)->vme_start) \
1485: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1486: MACRO_END
1487:
1488: /*
1489: * This routine is called only when it is known that
1490: * the entry must be split.
1491: */
1492: void
1493: _vm_map_clip_start(
1494: register struct vm_map_header *map_header,
1495: register vm_map_entry_t entry,
1496: register vm_offset_t start)
1497: {
1498: register vm_map_entry_t new_entry;
1499:
1500: /*
1501: * Split off the front portion --
1502: * note that we must insert the new
1503: * entry BEFORE this one, so that
1504: * this entry has the specified starting
1505: * address.
1506: */
1507:
1508: new_entry = _vm_map_entry_create(map_header);
1509: vm_map_entry_copy_full(new_entry, entry);
1510:
1511: new_entry->vme_end = start;
1512: entry->offset += (start - entry->vme_start);
1513: entry->vme_start = start;
1514:
1515: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1516:
1517: if (entry->is_sub_map)
1518: vm_map_reference(new_entry->object.sub_map);
1519: else
1520: vm_object_reference(new_entry->object.vm_object);
1521: }
1522:
1523: /*
1524: * vm_map_clip_end: [ internal use only ]
1525: *
1526: * Asserts that the given entry ends at or before
1527: * the specified address; if necessary,
1528: * it splits the entry into two.
1529: */
1530: #define vm_map_clip_end(map, entry, endaddr) \
1531: MACRO_BEGIN \
1532: vm_map_t VMCE_map; \
1533: vm_map_entry_t VMCE_entry; \
1534: vm_offset_t VMCE_endaddr; \
1535: VMCE_map = (map); \
1536: VMCE_entry = (entry); \
1537: VMCE_endaddr = (endaddr); \
1538: if (VMCE_endaddr < VMCE_entry->vme_end) \
1539: _vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
1540: UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free); \
1541: MACRO_END
1542:
1543: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1544: MACRO_BEGIN \
1545: if ((endaddr) < (entry)->vme_end) \
1546: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1547: MACRO_END
1548:
1549: /*
1550: * This routine is called only when it is known that
1551: * the entry must be split.
1552: */
1553: void
1554: _vm_map_clip_end(
1555: register struct vm_map_header *map_header,
1556: register vm_map_entry_t entry,
1557: register vm_offset_t end)
1558: {
1559: register vm_map_entry_t new_entry;
1560:
1561: /*
1562: * Create a new entry and insert it
1563: * AFTER the specified entry
1564: */
1565:
1566: new_entry = _vm_map_entry_create(map_header);
1567: vm_map_entry_copy_full(new_entry, entry);
1568:
1569: new_entry->vme_start = entry->vme_end = end;
1570: new_entry->offset += (end - entry->vme_start);
1571:
1572: _vm_map_entry_link(map_header, entry, new_entry);
1573:
1574: if (entry->is_sub_map)
1575: vm_map_reference(new_entry->object.sub_map);
1576: else
1577: vm_object_reference(new_entry->object.vm_object);
1578: }
1579:
1580:
1581: /*
1582: * VM_MAP_RANGE_CHECK: [ internal use only ]
1583: *
1584: * Asserts that the starting and ending region
1585: * addresses fall within the valid range of the map.
1586: */
1587: #define VM_MAP_RANGE_CHECK(map, start, end) \
1588: { \
1589: if (start < vm_map_min(map)) \
1590: start = vm_map_min(map); \
1591: if (end > vm_map_max(map)) \
1592: end = vm_map_max(map); \
1593: if (start > end) \
1594: start = end; \
1595: }
1596:
1597: /*
1598: * vm_map_range_check: [ internal use only ]
1599: *
1600: * Check that the region defined by the specified start and
1601: * end addresses are wholly contained within a single map
1602: * entry or set of adjacent map entries of the spacified map,
1603: * i.e. the specified region contains no unmapped space.
1604: * If any or all of the region is unmapped, FALSE is returned.
1605: * Otherwise, TRUE is returned and if the output argument 'entry'
1606: * is not NULL it points to the map entry containing the start
1607: * of the region.
1608: *
1609: * The map is locked for reading on entry and is left locked.
1610: */
1611: boolean_t
1612: vm_map_range_check(
1613: register vm_map_t map,
1614: register vm_offset_t start,
1615: register vm_offset_t end,
1616: vm_map_entry_t *entry)
1617: {
1618: vm_map_entry_t cur;
1619: register vm_offset_t prev;
1620:
1621: /*
1622: * Basic sanity checks first
1623: */
1624: if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
1625: return (FALSE);
1626:
1627: /*
1628: * Check first if the region starts within a valid
1629: * mapping for the map.
1630: */
1631: if (!vm_map_lookup_entry(map, start, &cur))
1632: return (FALSE);
1633:
1634: /*
1635: * Optimize for the case that the region is contained
1636: * in a single map entry.
1637: */
1638: if (entry != (vm_map_entry_t *) NULL)
1639: *entry = cur;
1640: if (end <= cur->vme_end)
1641: return (TRUE);
1642:
1643: /*
1644: * If the region is not wholly contained within a
1645: * single entry, walk the entries looking for holes.
1646: */
1647: prev = cur->vme_end;
1648: cur = cur->vme_next;
1649: while ((cur != vm_map_to_entry(map)) && (prev == cur->vme_start)) {
1650: if (end <= cur->vme_end)
1651: return (TRUE);
1652: prev = cur->vme_end;
1653: cur = cur->vme_next;
1654: }
1655: return (FALSE);
1656: }
1657:
1658: /*
1659: * vm_map_submap: [ kernel use only ]
1660: *
1661: * Mark the given range as handled by a subordinate map.
1662: *
1663: * This range must have been created with vm_map_find using
1664: * the vm_submap_object, and no other operations may have been
1665: * performed on this range prior to calling vm_map_submap.
1666: *
1667: * Only a limited number of operations can be performed
1668: * within this rage after calling vm_map_submap:
1669: * vm_fault
1670: * [Don't try vm_map_copyin!]
1671: *
1672: * To remove a submapping, one must first remove the
1673: * range from the superior map, and then destroy the
1674: * submap (if desired). [Better yet, don't try it.]
1675: */
1676: kern_return_t
1677: vm_map_submap(
1678: register vm_map_t map,
1679: register vm_offset_t start,
1680: register vm_offset_t end,
1681: vm_map_t submap,
1682: vm_offset_t offset)
1683: {
1684: vm_map_entry_t entry;
1685: register kern_return_t result = KERN_INVALID_ARGUMENT;
1686: register vm_object_t object;
1687:
1688: vm_map_lock(map);
1689:
1690: VM_MAP_RANGE_CHECK(map, start, end);
1691:
1692: if (vm_map_lookup_entry(map, start, &entry)) {
1693: vm_map_clip_start(map, entry, start);
1694: }
1695: else
1696: entry = entry->vme_next;
1697:
1698: vm_map_clip_end(map, entry, end);
1699:
1700: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1701: (!entry->is_sub_map) &&
1702: ((object = entry->object.vm_object) == vm_submap_object) &&
1703: (object->resident_page_count == 0) &&
1704: (object->copy == VM_OBJECT_NULL) &&
1705: (object->shadow == VM_OBJECT_NULL) &&
1706: (!object->pager_created)) {
1707: entry->offset = offset;
1708: entry->object.vm_object = VM_OBJECT_NULL;
1709: vm_object_deallocate(object);
1710: entry->is_sub_map = TRUE;
1711: vm_map_reference(entry->object.sub_map = submap);
1712: result = KERN_SUCCESS;
1713: }
1714: vm_map_unlock(map);
1715:
1716: return(result);
1717: }
1718:
1719: /*
1720: * vm_map_protect:
1721: *
1722: * Sets the protection of the specified address
1723: * region in the target map. If "set_max" is
1724: * specified, the maximum protection is to be set;
1725: * otherwise, only the current protection is affected.
1726: */
1727: kern_return_t
1728: vm_map_protect(
1729: register vm_map_t map,
1730: register vm_offset_t start,
1731: register vm_offset_t end,
1732: register vm_prot_t new_prot,
1733: register boolean_t set_max)
1734: {
1735: register vm_map_entry_t current;
1736: register vm_offset_t prev;
1737: vm_map_entry_t entry;
1738: vm_prot_t new_max;
1739: boolean_t clip;
1740:
1741: XPR(XPR_VM_MAP,
1742: "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
1743: (integer_t)map, start, end, new_prot, set_max);
1744:
1745: vm_map_lock(map);
1746:
1747: /*
1748: * Lookup the entry. If it doesn't start in a valid
1749: * entry, return an error. Remember if we need to
1750: * clip the entry. We don't do it here because we don't
1751: * want to make any changes until we've scanned the
1752: * entire range below for address and protection
1753: * violations.
1754: */
1755: if (!(clip = vm_map_lookup_entry(map, start, &entry))) {
1756: vm_map_unlock(map);
1757: return(KERN_INVALID_ADDRESS);
1758: }
1759:
1760: /*
1761: * Make a first pass to check for protection and address
1762: * violations.
1763: */
1764:
1765: current = entry;
1766: prev = current->vme_start;
1767: while ((current != vm_map_to_entry(map)) &&
1768: (current->vme_start < end)) {
1769:
1770: /*
1771: * If there is a hole, return an error.
1772: */
1773: if (current->vme_start != prev) {
1774: vm_map_unlock(map);
1775: return(KERN_INVALID_ADDRESS);
1776: }
1777:
1778: new_max = current->max_protection;
1779: if(new_prot & VM_PROT_COPY) {
1780: new_max |= VM_PROT_WRITE;
1781: }
1782: if ((new_prot & new_max) != new_prot) {
1783: vm_map_unlock(map);
1784: return(KERN_PROTECTION_FAILURE);
1785: }
1786:
1787: prev = current->vme_end;
1788: current = current->vme_next;
1789: }
1790: if (end > prev) {
1791: vm_map_unlock(map);
1792: return(KERN_INVALID_ADDRESS);
1793: }
1794:
1795: /*
1796: * Go back and fix up protections.
1797: * Clip to start here if the range starts within
1798: * the entry.
1799: */
1800:
1801: current = entry;
1802: if (clip) {
1803: vm_map_clip_start(map, entry, start);
1804: }
1805: while ((current != vm_map_to_entry(map)) &&
1806: (current->vme_start < end)) {
1807:
1808: vm_prot_t old_prot;
1809:
1810: vm_map_clip_end(map, current, end);
1811:
1812: old_prot = current->protection;
1813:
1814: if(new_prot & VM_PROT_COPY) {
1815: /* caller is asking specifically to copy the */
1816: /* mapped data, this implies that max protection */
1817: /* will include write. Caller must be prepared */
1818: /* for loss of shared memory communication in the */
1819: /* target area after taking this step */
1820: current->needs_copy = TRUE;
1821: current->max_protection |= VM_PROT_WRITE;
1822: }
1823:
1824: if (set_max)
1825: current->protection =
1826: (current->max_protection = new_prot) &
1827: old_prot;
1828: else
1829: current->protection = new_prot;
1830:
1831: /*
1832: * Update physical map if necessary.
1833: * If the request is to turn off write protection,
1834: * we won't do it for real (in pmap). This is because
1835: * it would cause copy-on-write to fail. We've already
1836: * set, the new protection in the map, so if a
1837: * write-protect fault occurred, it will be fixed up
1838: * properly, COW or not.
1839: */
1840:
1841: if ((current->protection != old_prot) && !(current->protection & VM_PROT_WRITE)) {
1842: pmap_protect(map->pmap, current->vme_start,
1843: current->vme_end,
1844: current->protection);
1845: }
1846: current = current->vme_next;
1847: }
1848:
1849: vm_map_unlock(map);
1850: return(KERN_SUCCESS);
1851: }
1852:
1853:
1854: /*
1855: * vm_map_inherit:
1856: *
1857: * Sets the inheritance of the specified address
1858: * range in the target map. Inheritance
1859: * affects how the map will be shared with
1860: * child maps at the time of vm_map_fork.
1861: */
1862: kern_return_t
1863: vm_map_inherit(
1864: register vm_map_t map,
1865: register vm_offset_t start,
1866: register vm_offset_t end,
1867: register vm_inherit_t new_inheritance)
1868: {
1869: register vm_map_entry_t entry;
1870: vm_map_entry_t temp_entry;
1871:
1872: vm_map_lock(map);
1873:
1874: VM_MAP_RANGE_CHECK(map, start, end);
1875:
1876: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1877: entry = temp_entry;
1878: vm_map_clip_start(map, entry, start);
1879: }
1880: else {
1881: temp_entry = temp_entry->vme_next;
1882: entry = temp_entry;
1883: }
1884:
1885: /* first check entire range for submaps which can't support the */
1886: /* given inheritance. */
1887: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1888: if(entry->is_sub_map) {
1889: if(new_inheritance == VM_INHERIT_COPY)
1890: return(KERN_INVALID_ARGUMENT);
1891: }
1892:
1893: entry = entry->vme_next;
1894: }
1895:
1896: entry = temp_entry;
1897:
1898: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1899: vm_map_clip_end(map, entry, end);
1900:
1901: entry->inheritance = new_inheritance;
1902:
1903: entry = entry->vme_next;
1904: }
1905:
1906: vm_map_unlock(map);
1907: return(KERN_SUCCESS);
1908: }
1909:
1910: /*
1911: * vm_map_wire:
1912: *
1913: * Sets the pageability of the specified address range in the
1914: * target map as wired. Regions specified as not pageable require
1915: * locked-down physical memory and physical page maps. The
1916: * access_type variable indicates types of accesses that must not
1917: * generate page faults. This is checked against protection of
1918: * memory being locked-down.
1919: *
1920: * The map must not be locked, but a reference must remain to the
1921: * map throughout the call.
1922: */
1923: kern_return_t
1924: vm_map_wire(
1925: register vm_map_t map,
1926: register vm_offset_t start,
1927: register vm_offset_t end,
1928: register vm_prot_t access_type,
1929: boolean_t user_wire)
1930: {
1931: register vm_map_entry_t entry;
1932: struct vm_map_entry *first_entry, tmp_entry;
1933: register vm_offset_t s,e;
1934: kern_return_t rc;
1935: boolean_t need_wakeup;
1936: unsigned int last_timestamp;
1937: vm_size_t size;
1938:
1939: vm_map_lock(map);
1940: last_timestamp = map->timestamp;
1941:
1942: VM_MAP_RANGE_CHECK(map, start, end);
1943: assert(page_aligned(start));
1944: assert(page_aligned(end));
1945:
1946: if (vm_map_lookup_entry(map, start, &first_entry)) {
1947: entry = first_entry;
1948: /* vm_map_clip_start will be done later. */
1949: } else {
1950: /* Start address is not in map */
1951: vm_map_unlock(map);
1952: return(KERN_INVALID_ADDRESS);
1953: }
1954:
1955: s=start;
1956: need_wakeup = FALSE;
1957: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1958: if(entry->is_sub_map) {
1959: vm_offset_t sub_start;
1960: vm_offset_t sub_end;
1961: vm_offset_t local_end;
1962:
1963: if(entry->vme_start < start)
1964: sub_start = start;
1965: else
1966: sub_start = entry->vme_start;
1967: sub_start -= entry->vme_start;
1968: sub_start += entry->offset;
1969:
1970: if(entry->vme_end < end)
1971: sub_end = entry->vme_end;
1972: else
1973: sub_end = end;
1974: sub_end -= entry->vme_start;
1975: sub_end += entry->offset;
1976: local_end = entry->vme_end;
1977: vm_map_unlock(map);
1978: if (vm_map_wire(entry->object.sub_map,
1979: sub_start, sub_end,
1980: access_type, user_wire)
1981: != KERN_SUCCESS) {
1982: vm_map_unwire(map, start, s, user_wire);
1983: return(KERN_FAILURE);
1984: }
1985: vm_map_lock(map);
1986: if (last_timestamp+1 != map->timestamp) {
1987: /*
1988: * Find the entry again. It could have been clipped
1989: * after we unlocked the map.
1990: */
1991: if (!vm_map_lookup_entry(map, local_end,
1992: &first_entry))
1993: panic("vm_map_wire: re-lookup failed");
1994:
1995: entry = first_entry;
1996: } else
1997: entry = entry->vme_next;
1998:
1999: last_timestamp = map->timestamp;
2000: continue;
2001: }
2002:
2003: /*
2004: * If another thread is wiring/unwiring this entry then
2005: * block after informing other thread to wake us up.
2006: */
2007: if (entry->in_transition) {
2008: /*
2009: * We have not clipped the entry. Make sure that
2010: * the start address is in range so that the lookup
2011: * below will succeed.
2012: */
2013: s = entry->vme_start < start? start: entry->vme_start;
2014:
2015: entry->needs_wakeup = TRUE;
2016:
2017: /*
2018: * wake up anybody waiting on entries that we have
2019: * already wired.
2020: */
2021: if (need_wakeup) {
2022: vm_map_entry_wakeup(map);
2023: need_wakeup = FALSE;
2024: }
2025: /*
2026: * User wiring is interruptible
2027: */
2028: vm_map_entry_wait(map,
2029: (user_wire) ? THREAD_ABORTSAFE :
2030: THREAD_UNINT);
2031: if (user_wire && current_thread()->wait_result ==
2032: THREAD_INTERRUPTED) {
2033: /*
2034: * undo the wirings we have done so far
2035: * We do not clear the needs_wakeup flag,
2036: * because we cannot tell if we were the
2037: * only one waiting.
2038: */
2039: vm_map_unwire(map, start, s, user_wire);
2040: return(KERN_FAILURE);
2041: }
2042:
2043: vm_map_lock(map);
2044: /*
2045: * Cannot avoid a lookup here. reset timestamp.
2046: */
2047: last_timestamp = map->timestamp;
2048:
2049: /*
2050: * The entry could have been clipped, look it up again.
2051: * Worse that can happen is, it may not exist anymore.
2052: */
2053: if (!vm_map_lookup_entry(map, s, &first_entry)) {
2054: if (!user_wire)
2055: panic("vm_map_wire: re-lookup failed");
2056:
2057: /*
2058: * User: undo everything upto the previous
2059: * entry. let vm_map_unwire worry about
2060: * checking the validity of the range.
2061: */
2062: vm_map_unlock(map);
2063: vm_map_unwire(map, start, s, user_wire);
2064: return(KERN_FAILURE);
2065: }
2066: entry = first_entry;
2067: continue;
2068: }
2069:
2070: /*
2071: * If this entry is already wired then increment
2072: * the appropriate wire reference count.
2073: */
2074: if (entry->wired_count) {
2075: /* sanity check: wired_count is a short */
2076: if (entry->wired_count >= MAX_WIRE_COUNT)
2077: panic("vm_map_wire: too many wirings");
2078:
2079: if (user_wire &&
2080: entry->user_wired_count >= MAX_WIRE_COUNT) {
2081: vm_map_unlock(map);
2082: vm_map_unwire(map, start,
2083: entry->vme_start, user_wire);
2084: return(KERN_FAILURE);
2085: }
2086: /*
2087: * entry is already wired down, get our reference
2088: * after clipping to our range.
2089: */
2090: vm_map_clip_start(map, entry, start);
2091: vm_map_clip_end(map, entry, end);
2092: if (!user_wire || (entry->user_wired_count++ == 0))
2093: entry->wired_count++;
2094:
2095: entry = entry->vme_next;
2096: continue;
2097: }
2098:
2099: /*
2100: * Unwired entry
2101: */
2102:
2103:
2104: /*
2105: * Perform actions of vm_map_lookup that need the write
2106: * lock on the map: create a shadow object for a
2107: * copy-on-write region, or an object for a zero-fill
2108: * region.
2109: */
2110: size = entry->vme_end - entry->vme_start;
2111: /*
2112: * If wiring a copy-on-write page, we need to copy it now
2113: * even if we're only (currently) requesting read access.
2114: * This is aggressive, but once it's wired we can't move it.
2115: */
2116: if (entry->needs_copy) {
2117: vm_object_shadow(&entry->object.vm_object,
2118: &entry->offset, size);
2119: entry->needs_copy = FALSE;
2120: } else if (entry->object.vm_object == VM_OBJECT_NULL) {
2121: entry->object.vm_object = vm_object_allocate(size);
2122: entry->offset = (vm_offset_t)0;
2123: }
2124:
2125: vm_map_clip_start(map, entry, start);
2126: vm_map_clip_end(map, entry, end);
2127:
2128: s = entry->vme_start;
2129: e = entry->vme_end;
2130:
2131: /*
2132: * Check for holes and protection mismatch.
2133: * Holes: Next entry should be contiguous unless this
2134: * is the end of the region.
2135: * Protection: Access requested must be allowed, unless
2136: * wiring is by protection class
2137: */
2138: if ((((entry->vme_end < end) &&
2139: ((entry->vme_next == vm_map_to_entry(map)) ||
2140: (entry->vme_next->vme_start > entry->vme_end))) ||
2141: ((entry->protection & access_type) != access_type))) {
2142: /*
2143: * Found a hole or protection problem.
2144: * Unwire the region we wired so far.
2145: */
2146: if (start != entry->vme_start) {
2147: vm_map_unlock(map);
2148: vm_map_unwire(map, start, s, user_wire);
2149: } else {
2150: vm_map_unlock(map);
2151: }
2152: return((entry->protection&access_type) != access_type?
2153: KERN_PROTECTION_FAILURE: KERN_INVALID_ADDRESS);
2154: }
2155:
2156: assert(entry->wired_count == 0 && entry->user_wired_count == 0);
2157:
2158: if (user_wire)
2159: entry->user_wired_count++;
2160: entry->wired_count++;
2161:
2162: entry->in_transition = TRUE;
2163:
2164: /*
2165: * This entry might get split once we unlock the map.
2166: * In vm_fault_wire(), we need the current range as
2167: * defined by this entry. In order for this to work
2168: * along with a simultaneous clip operation, we make a
2169: * temporary copy of this entry and use that for the
2170: * wiring. Note that the underlying objects do not
2171: * change during a clip.
2172: */
2173: tmp_entry = *entry;
2174:
2175: /*
2176: * The in_transition state guarentees that the entry
2177: * (or entries for this range, if split occured) will be
2178: * there when the map lock is acquired for the second time.
2179: */
2180: vm_map_unlock(map);
2181: rc = vm_fault_wire(map, &tmp_entry);
2182: vm_map_lock(map);
2183:
2184: if (last_timestamp+1 != map->timestamp) {
2185: /*
2186: * Find the entry again. It could have been clipped
2187: * after we unlocked the map.
2188: */
2189: if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
2190: &first_entry))
2191: panic("vm_map_wire: re-lookup failed");
2192:
2193: entry = first_entry;
2194: }
2195:
2196: last_timestamp = map->timestamp;
2197:
2198: while ((entry != vm_map_to_entry(map)) &&
2199: (entry->vme_start < tmp_entry.vme_end)) {
2200: assert(entry->in_transition);
2201: entry->in_transition = FALSE;
2202: if (entry->needs_wakeup) {
2203: entry->needs_wakeup = FALSE;
2204: need_wakeup = TRUE;
2205: }
2206: if (rc != KERN_SUCCESS) { /* from vm_*_wire */
2207: if (user_wire)
2208: entry->user_wired_count--;
2209: entry->wired_count--;
2210: }
2211: entry = entry->vme_next;
2212: }
2213:
2214: if (rc != KERN_SUCCESS) { /* from vm_*_wire */
2215: vm_map_unlock(map);
2216: if (need_wakeup)
2217: vm_map_entry_wakeup(map);
2218: /*
2219: * undo everything upto the previous entry.
2220: */
2221: (void)vm_map_unwire(map, start, s, user_wire);
2222: return rc;
2223: }
2224: } /* end while loop through map entries */
2225: vm_map_unlock(map);
2226:
2227: /*
2228: * wake up anybody waiting on entries we wired.
2229: */
2230: if (need_wakeup)
2231: vm_map_entry_wakeup(map);
2232:
2233: return(KERN_SUCCESS);
2234:
2235: }
2236:
2237:
2238: /*
2239: * vm_map_unwire:
2240: *
2241: * Sets the pageability of the specified address range in the target
2242: * as pageable. Regions specified must have been wired previously.
2243: *
2244: * The map must not be locked, but a reference must remain to the map
2245: * throughout the call.
2246: *
2247: * Kernel will panic on failures. User unwire ignores holes and
2248: * unwired and intransition entries to avoid losing memory by leaving
2249: * it unwired.
2250: */
2251: kern_return_t
2252: vm_map_unwire(
2253: register vm_map_t map,
2254: register vm_offset_t start,
2255: register vm_offset_t end,
2256: boolean_t user_wire)
2257: {
2258: register vm_map_entry_t entry;
2259: struct vm_map_entry *first_entry, tmp_entry;
2260: boolean_t need_wakeup;
2261: unsigned int last_timestamp;
2262:
2263: vm_map_lock(map);
2264: last_timestamp = map->timestamp;
2265:
2266: VM_MAP_RANGE_CHECK(map, start, end);
2267: assert(page_aligned(start));
2268: assert(page_aligned(end));
2269:
2270: if (vm_map_lookup_entry(map, start, &first_entry)) {
2271: entry = first_entry;
2272: /* vm_map_clip_start will be done later. */
2273: }
2274: else {
2275: /* Start address is not in map. */
2276: vm_map_unlock(map);
2277: return(KERN_INVALID_ADDRESS);
2278: }
2279:
2280: need_wakeup = FALSE;
2281: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
2282: if(entry->is_sub_map) {
2283: vm_offset_t sub_start;
2284: vm_offset_t sub_end;
2285: vm_offset_t local_end;
2286:
2287: if(entry->vme_start < start)
2288: sub_start = start;
2289: else
2290: sub_start = entry->vme_start;
2291: sub_start -= entry->vme_start;
2292: sub_start += entry->offset;
2293:
2294: if(entry->vme_end < end)
2295: sub_end = entry->vme_end;
2296: else
2297: sub_end = end;
2298: sub_end -= entry->vme_start;
2299: sub_end += entry->offset;
2300: local_end = entry->vme_end;
2301: vm_map_unlock(map);
2302: vm_map_unwire(entry->object.sub_map,
2303: sub_start, sub_end,
2304: user_wire);
2305:
2306: vm_map_lock(map);
2307: if (last_timestamp+1 != map->timestamp) {
2308: /*
2309: * Find the entry again. It could have been clipped
2310: * after we unlocked the map.
2311: */
2312: if (!vm_map_lookup_entry(map, local_end,
2313: &first_entry))
2314: panic("vm_map_unwire: re-lookup failed");
2315:
2316: entry = first_entry;
2317: } else
2318: entry = entry->vme_next;
2319:
2320: last_timestamp = map->timestamp;
2321: continue;
2322: }
2323:
2324:
2325: if (entry->in_transition) {
2326: /*
2327: * 1)
2328: * Another thread is wiring down this entry. Note
2329: * that if it is not for the other thread we would
2330: * be unwiring an unwired entry. This is not
2331: * permitted. If we wait, we will be unwiring memory
2332: * we did not wire.
2333: *
2334: * 2)
2335: * Another thread is unwiring this entry. We did not
2336: * have a reference to it, because if we did, this
2337: * entry will not be getting unwired now.
2338: */
2339: if (!user_wire)
2340: panic("vm_map_unwire: in_transition entry");
2341:
2342: entry = entry->vme_next;
2343: continue;
2344: }
2345:
2346: if (entry->wired_count == 0 ||
2347: (user_wire && entry->user_wired_count == 0)) {
2348: if (!user_wire)
2349: panic("vm_map_unwire: entry is unwired");
2350:
2351: entry = entry->vme_next;
2352: continue;
2353: }
2354:
2355: assert(entry->wired_count > 0 &&
2356: (!user_wire || entry->user_wired_count > 0));
2357:
2358: vm_map_clip_start(map, entry, start);
2359: vm_map_clip_end(map, entry, end);
2360:
2361: /*
2362: * Check for holes
2363: * Holes: Next entry should be contiguous unless
2364: * this is the end of the region.
2365: */
2366: if (((entry->vme_end < end) &&
2367: ((entry->vme_next == vm_map_to_entry(map)) ||
2368: (entry->vme_next->vme_start > entry->vme_end)))) {
2369:
2370: if (!user_wire)
2371: panic("vm_map_unwire: non-contiguous region");
2372: entry = entry->vme_next;
2373: continue;
2374: }
2375:
2376: if (!user_wire || (--entry->user_wired_count == 0))
2377: entry->wired_count--;
2378:
2379: if (entry->wired_count != 0) {
2380: entry = entry->vme_next;
2381: continue;
2382: }
2383:
2384: entry->in_transition = TRUE;
2385: tmp_entry = *entry; /* see comment in vm_map_wire() */
2386:
2387: /*
2388: * We can unlock the map now. The in_transition state
2389: * guarantees existance of the entry.
2390: */
2391: vm_map_unlock(map);
2392: vm_fault_unwire(map, &tmp_entry, FALSE);
2393: vm_map_lock(map);
2394:
2395: if (last_timestamp+1 != map->timestamp) {
2396: /*
2397: * Find the entry again. It could have been clipped
2398: * or deleted after we unlocked the map.
2399: */
2400: if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
2401: &first_entry)) {
2402: if (!user_wire)
2403: panic("vm_map_unwire: re-lookup failed");
2404: entry = first_entry->vme_next;
2405: } else
2406: entry = first_entry;
2407: }
2408: last_timestamp = map->timestamp;
2409:
2410: /*
2411: * clear transition bit for all constituent entries that
2412: * were in the original entry (saved in tmp_entry). Also
2413: * check for waiters.
2414: */
2415: while ((entry != vm_map_to_entry(map)) &&
2416: (entry->vme_start < tmp_entry.vme_end)) {
2417: assert(entry->in_transition);
2418: entry->in_transition = FALSE;
2419: if (entry->needs_wakeup) {
2420: entry->needs_wakeup = FALSE;
2421: need_wakeup = TRUE;
2422: }
2423: entry = entry->vme_next;
2424: }
2425: }
2426: vm_map_unlock(map);
2427: /*
2428: * wake up anybody waiting on entries that we have unwired.
2429: */
2430: if (need_wakeup)
2431: vm_map_entry_wakeup(map);
2432: return(KERN_SUCCESS);
2433:
2434: }
2435:
2436: /*
2437: * vm_map_entry_delete: [ internal use only ]
2438: *
2439: * Deallocate the given entry from the target map.
2440: */
2441: void
2442: vm_map_entry_delete(
2443: register vm_map_t map,
2444: register vm_map_entry_t entry)
2445: {
2446: register vm_offset_t s, e;
2447: register vm_object_t object;
2448: extern vm_object_t kernel_object;
2449:
2450: s = entry->vme_start;
2451: e = entry->vme_end;
2452: assert(page_aligned(s));
2453: assert(page_aligned(e));
2454: assert(entry->wired_count == 0);
2455: assert(entry->user_wired_count == 0);
2456:
2457:
2458: /*
2459: * Deallocate the object only after removing all
2460: * pmap entries pointing to its pages.
2461: */
2462: if (entry->is_sub_map)
2463: vm_map_deallocate(entry->object.sub_map);
2464: else
2465: vm_object_deallocate(entry->object.vm_object);
2466:
2467: vm_map_entry_unlink(map, entry);
2468: map->size -= e - s;
2469:
2470: vm_map_entry_dispose(map, entry);
2471: }
2472:
2473:
2474: /*
2475: * vm_map_delete: [ internal use only ]
2476: *
2477: * Deallocates the given address range from the target map.
2478: * Removes all user wirings. Unwires one kernel wiring if
2479: * VM_MAP_REMOVE_KUNWIRE is set. Waits for kernel wirings to go
2480: * away if VM_MAP_REMOVE_WAIT_FOR_KWIRE is set. Sleeps
2481: * interruptibly if VM_MAP_REMOVE_INTERRUPTIBLE is set.
2482: *
2483: * This routine is called with map locked and leaves map locked.
2484: */
2485: kern_return_t
2486: vm_map_delete(
2487: register vm_map_t map,
2488: vm_offset_t start,
2489: register vm_offset_t end,
2490: int flags)
2491: {
2492: register vm_map_entry_t entry, next;
2493: struct vm_map_entry *first_entry, tmp_entry;
2494: register vm_offset_t s, e;
2495: register vm_object_t object;
2496: boolean_t need_wakeup;
2497: unsigned int last_timestamp = ~0; /* unlikely value */
2498: int interruptible;
2499: extern vm_map_t kernel_map;
2500:
2501: interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ?
2502: THREAD_ABORTSAFE : THREAD_UNINT;
2503:
2504: /*
2505: * Find the start of the region, and clip it
2506: */
2507: if (vm_map_lookup_entry(map, start, &first_entry)) {
2508: entry = first_entry;
2509: vm_map_clip_start(map, entry, start);
2510:
2511: /*
2512: * Fix the lookup hint now, rather than each
2513: * time through the loop.
2514: */
2515: SAVE_HINT(map, entry->vme_prev);
2516: } else {
2517: entry = first_entry->vme_next;
2518: }
2519:
2520: need_wakeup = FALSE;
2521: /*
2522: * Step through all entries in this region
2523: */
2524: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
2525:
2526: vm_map_clip_end(map, entry, end);
2527: retry_entry:
2528: if (entry->in_transition) {
2529: /*
2530: * Another thread is wiring/unwiring this entry.
2531: * Let the other thread know we are waiting.
2532: */
2533: s = entry->vme_start;
2534: entry->needs_wakeup = TRUE;
2535:
2536: /*
2537: * wake up anybody waiting on entries that we have
2538: * already unwired/deleted.
2539: */
2540: if (need_wakeup) {
2541: vm_map_entry_wakeup(map);
2542: need_wakeup = FALSE;
2543: }
2544:
2545: vm_map_entry_wait(map, interruptible);
2546:
2547: if (interruptible &&
2548: current_thread()->wait_result == THREAD_INTERRUPTED)
2549: /*
2550: * We do not clear the needs_wakeup flag,
2551: * since we cannot tell if we were the only one.
2552: */
2553: return KERN_ABORTED;
2554:
2555: vm_map_lock(map);
2556: /*
2557: * Cannot avoid a lookup here. reset timestamp.
2558: */
2559: last_timestamp = map->timestamp;
2560:
2561: /*
2562: * The entry could have been clipped or it
2563: * may not exist anymore. Look it up again.
2564: */
2565: if (!vm_map_lookup_entry(map, s, &first_entry)) {
2566: assert((map != kernel_map) &&
2567: (!entry->is_sub_map));
2568: /*
2569: * User: use the next entry
2570: */
2571: entry = first_entry->vme_next;
2572: } else {
2573: entry = first_entry;
2574: SAVE_HINT(map, entry->vme_prev);
2575: }
2576: goto retry_entry;
2577: } /* end in_transition */
2578:
2579: if (entry->wired_count) {
2580: /*
2581: * Remove a kernel wiring if requested or if
2582: * there are user wirings.
2583: */
2584: if ((flags & VM_MAP_REMOVE_KUNWIRE) ||
2585: (entry->user_wired_count > 0))
2586: entry->wired_count--;
2587:
2588: /* remove all user wire references */
2589: entry->user_wired_count = 0;
2590:
2591: if (entry->wired_count != 0) {
2592: assert((map != kernel_map) &&
2593: (!entry->is_sub_map));
2594: /*
2595: * Cannot continue. Typical case is when
2596: * a user thread has physical io pending on
2597: * on this page. Either wait for the
2598: * kernel wiring to go away or return an
2599: * error.
2600: */
2601: if (flags & VM_MAP_REMOVE_WAIT_FOR_KWIRE) {
2602:
2603: s = entry->vme_start;
2604: entry->needs_wakeup = TRUE;
2605: vm_map_entry_wait(map, interruptible);
2606:
2607: if (interruptible &&
2608: current_thread()->wait_result ==
2609: THREAD_INTERRUPTED)
2610: /*
2611: * We do not clear the
2612: * needs_wakeup flag, since we
2613: * cannot tell if we were the
2614: * only one.
2615: */
2616: return KERN_ABORTED;
2617:
2618: vm_map_lock(map);
2619: /*
2620: * Cannot avoid a lookup here. reset
2621: * timestamp.
2622: */
2623: last_timestamp = map->timestamp;
2624:
2625: /*
2626: * The entry could have been clipped or
2627: * it may not exist anymore. Look it
2628: * up again.
2629: */
2630: if (!vm_map_lookup_entry(map, s,
2631: &first_entry)) {
2632: assert((map != kernel_map) &&
2633: (!entry->is_sub_map));
2634: /*
2635: * User: use the next entry
2636: */
2637: entry = first_entry->vme_next;
2638: } else {
2639: entry = first_entry;
2640: SAVE_HINT(map, entry->vme_prev);
2641: }
2642: goto retry_entry;
2643: }
2644: else {
2645: return KERN_FAILURE;
2646: }
2647: }
2648:
2649: entry->in_transition = TRUE;
2650: /*
2651: * copy current entry. see comment in vm_map_wire()
2652: */
2653: tmp_entry = *entry;
2654: s = entry->vme_start;
2655: e = entry->vme_end;
2656:
2657: /*
2658: * We can unlock the map now. The in_transition
2659: * state guarentees existance of the entry.
2660: */
2661: vm_map_unlock(map);
2662: vm_fault_unwire(map, &tmp_entry,
2663: tmp_entry.object.vm_object == kernel_object);
2664: vm_map_lock(map);
2665:
2666: if (last_timestamp+1 != map->timestamp) {
2667: /*
2668: * Find the entry again. It could have
2669: * been clipped after we unlocked the map.
2670: */
2671: if (!vm_map_lookup_entry(map, s, &first_entry)){
2672: assert((map != kernel_map) &&
2673: (!entry->is_sub_map));
2674: first_entry = first_entry->vme_next;
2675: } else {
2676: SAVE_HINT(map, entry->vme_prev);
2677: }
2678: } else {
2679: SAVE_HINT(map, entry->vme_prev);
2680: first_entry = entry;
2681: }
2682:
2683: last_timestamp = map->timestamp;
2684:
2685: entry = first_entry;
2686: while ((entry != vm_map_to_entry(map)) &&
2687: (entry->vme_start < tmp_entry.vme_end)) {
2688: assert(entry->in_transition);
2689: entry->in_transition = FALSE;
2690: if (entry->needs_wakeup) {
2691: entry->needs_wakeup = FALSE;
2692: need_wakeup = TRUE;
2693: }
2694: entry = entry->vme_next;
2695: }
2696: /*
2697: * We have unwired the entry(s). Go back and
2698: * delete them.
2699: */
2700: entry = first_entry;
2701: goto retry_entry;
2702: }
2703:
2704: /* entry is unwired */
2705: assert(entry->wired_count == 0);
2706: assert(entry->user_wired_count == 0);
2707:
2708: if ((!entry->is_sub_map &&
2709: entry->object.vm_object != kernel_object) ||
2710: entry->is_sub_map) {
2711: pmap_remove(map->pmap,
2712: entry->vme_start, entry->vme_end);
2713: }
2714:
2715: next = entry->vme_next;
2716: vm_map_entry_delete(map, entry);
2717: entry = next;
2718: }
2719:
2720: if (map->wait_for_space)
2721: thread_wakeup((event_t) map);
2722: /*
2723: * wake up anybody waiting on entries that we have already deleted.
2724: */
2725: if (need_wakeup)
2726: vm_map_entry_wakeup(map);
2727:
2728: return KERN_SUCCESS;
2729: }
2730:
2731: /*
2732: * vm_map_remove:
2733: *
2734: * Remove the given address range from the target map.
2735: * This is the exported form of vm_map_delete.
2736: */
2737: kern_return_t
2738: vm_map_remove(
2739: register vm_map_t map,
2740: register vm_offset_t start,
2741: register vm_offset_t end,
2742: register boolean_t flags)
2743: {
2744: register kern_return_t result;
2745:
2746: vm_map_lock(map);
2747: VM_MAP_RANGE_CHECK(map, start, end);
2748: result = vm_map_delete(map, start, end, flags);
2749: vm_map_unlock(map);
2750:
2751: return(result);
2752: }
2753:
2754:
2755: /*
2756: * vm_map_copy_steal_pages:
2757: *
2758: * Steal all the pages from a vm_map_copy page_list by copying ones
2759: * that have not already been stolen.
2760: */
2761: void
2762: vm_map_copy_steal_pages(
2763: vm_map_copy_t copy)
2764: {
2765: register vm_page_t m, new_m;
2766: register int i;
2767: vm_object_t object;
2768:
2769: assert(copy->type == VM_MAP_COPY_PAGE_LIST);
2770: for (i = 0; i < copy->cpy_npages; i++) {
2771:
2772: /*
2773: * If the page is not tabled, then it's already stolen.
2774: */
2775: m = copy->cpy_page_list[i];
2776: if (!m->tabled)
2777: continue;
2778:
2779: /*
2780: * Page was not stolen, get a new
2781: * one and do the copy now.
2782: */
2783: while ((new_m = vm_page_grab()) == VM_PAGE_NULL) {
2784: VM_PAGE_WAIT();
2785: }
2786:
2787: vm_page_gobble(new_m); /* mark as consumed internally */
2788: vm_page_copy(m, new_m);
2789:
2790: object = m->object;
2791: vm_object_lock(object);
2792: vm_page_lock_queues();
2793: if (!m->active && !m->inactive)
2794: vm_page_activate(m);
2795: vm_page_unlock_queues();
2796: PAGE_WAKEUP_DONE(m);
2797: vm_object_paging_end(object);
2798: vm_object_unlock(object);
2799:
2800: copy->cpy_page_list[i] = new_m;
2801: }
2802: copy->cpy_page_loose = TRUE;
2803: }
2804:
2805: /*
2806: * vm_map_copy_page_discard:
2807: *
2808: * Get rid of the pages in a page_list copy. If the pages are
2809: * stolen, they are freed. If the pages are not stolen, they
2810: * are unbusied, and associated state is cleaned up.
2811: */
2812: void
2813: vm_map_copy_page_discard(
2814: vm_map_copy_t copy)
2815: {
2816: assert(copy->type == VM_MAP_COPY_PAGE_LIST);
2817: while (copy->cpy_npages > 0) {
2818: vm_page_t m;
2819:
2820: if ((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
2821: VM_PAGE_NULL) {
2822:
2823: /*
2824: * If it's not in the table, then it's
2825: * a stolen page that goes back
2826: * to the free list. Else it belongs
2827: * to some object, and we hold a
2828: * paging reference on that object.
2829: */
2830: if (!m->tabled) {
2831: VM_PAGE_FREE(m);
2832: }
2833: else {
2834: vm_object_t object;
2835:
2836: object = m->object;
2837:
2838: vm_object_lock(object);
2839: vm_page_lock_queues();
2840: if (!m->active && !m->inactive)
2841: vm_page_activate(m);
2842: vm_page_unlock_queues();
2843:
2844: if ((!m->busy)) {
2845: kern_return_t kr;
2846: kr = vm_page_unpin(m);
2847: assert(kr == KERN_SUCCESS);
2848: } else {
2849: PAGE_WAKEUP_DONE(m);
2850: }
2851: vm_object_paging_end(object);
2852: vm_object_unlock(object);
2853: }
2854: }
2855: }
2856: }
2857:
2858: /*
2859: * Routine: vm_map_copy_discard
2860: *
2861: * Description:
2862: * Dispose of a map copy object (returned by
2863: * vm_map_copyin).
2864: */
2865: void
2866: vm_map_copy_discard(
2867: vm_map_copy_t copy)
2868: {
2869: TR_DECL("vm_map_copy_discard");
2870:
2871: /* tr3("enter: copy 0x%x type %d", copy, copy->type);*/
2872: free_next_copy:
2873: if (copy == VM_MAP_COPY_NULL)
2874: return;
2875:
2876: switch (copy->type) {
2877: case VM_MAP_COPY_ENTRY_LIST:
2878: while (vm_map_copy_first_entry(copy) !=
2879: vm_map_copy_to_entry(copy)) {
2880: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
2881:
2882: vm_map_copy_entry_unlink(copy, entry);
2883: vm_object_deallocate(entry->object.vm_object);
2884: vm_map_copy_entry_dispose(copy, entry);
2885: }
2886: break;
2887: case VM_MAP_COPY_OBJECT:
2888: vm_object_deallocate(copy->cpy_object);
2889: break;
2890: case VM_MAP_COPY_PAGE_LIST:
2891:
2892: /*
2893: * To clean this up, we have to unbusy all the pages
2894: * and release the paging references in their objects.
2895: */
2896: if (copy->cpy_npages > 0)
2897: vm_map_copy_page_discard(copy);
2898:
2899: /*
2900: * If there's a continuation, abort it. The
2901: * abort routine releases any storage.
2902: */
2903: if (vm_map_copy_has_cont(copy)) {
2904:
2905: assert(vm_map_copy_cont_is_valid(copy));
2906: /*
2907: * Special case: recognize
2908: * vm_map_copy_discard_cont and optimize
2909: * here to avoid tail recursion.
2910: */
2911: if (copy->cpy_cont == vm_map_copy_discard_cont) {
2912: register vm_map_copy_t new_copy;
2913:
2914: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
2915: zfree(vm_map_copy_zone, (vm_offset_t) copy);
2916: copy = new_copy;
2917: goto free_next_copy;
2918: } else {
2919: vm_map_copy_abort_cont(copy);
2920: }
2921: }
2922:
2923: break;
2924:
2925: case VM_MAP_COPY_KERNEL_BUFFER:
2926:
2927: /*
2928: * The vm_map_copy_t and possibly the data buffer were
2929: * allocated by a single call to kalloc(), i.e. the
2930: * vm_map_copy_t was not allocated out of the zone.
2931: */
2932: kfree((vm_offset_t) copy, copy->cpy_kalloc_size);
2933: return;
2934: }
2935: zfree(vm_map_copy_zone, (vm_offset_t) copy);
2936: }
2937:
2938: /*
2939: * Routine: vm_map_copy_copy
2940: *
2941: * Description:
2942: * Move the information in a map copy object to
2943: * a new map copy object, leaving the old one
2944: * empty.
2945: *
2946: * This is used by kernel routines that need
2947: * to look at out-of-line data (in copyin form)
2948: * before deciding whether to return SUCCESS.
2949: * If the routine returns FAILURE, the original
2950: * copy object will be deallocated; therefore,
2951: * these routines must make a copy of the copy
2952: * object and leave the original empty so that
2953: * deallocation will not fail.
2954: */
2955: vm_map_copy_t
2956: vm_map_copy_copy(
2957: vm_map_copy_t copy)
2958: {
2959: vm_map_copy_t new_copy;
2960:
2961: if (copy == VM_MAP_COPY_NULL)
2962: return VM_MAP_COPY_NULL;
2963:
2964: /*
2965: * Allocate a new copy object, and copy the information
2966: * from the old one into it.
2967: */
2968:
2969: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
2970: *new_copy = *copy;
2971:
2972: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
2973: /*
2974: * The links in the entry chain must be
2975: * changed to point to the new copy object.
2976: */
2977: vm_map_copy_first_entry(copy)->vme_prev
2978: = vm_map_copy_to_entry(new_copy);
2979: vm_map_copy_last_entry(copy)->vme_next
2980: = vm_map_copy_to_entry(new_copy);
2981: }
2982:
2983: /*
2984: * Change the old copy object into one that contains
2985: * nothing to be deallocated.
2986: */
2987: copy->type = VM_MAP_COPY_OBJECT;
2988: copy->cpy_object = VM_OBJECT_NULL;
2989:
2990: /*
2991: * Return the new object.
2992: */
2993: return new_copy;
2994: }
2995:
2996: /*
2997: * Routine: vm_map_copy_discard_cont
2998: *
2999: * Description:
3000: * A version of vm_map_copy_discard that can be called
3001: * as a continuation from a vm_map_copy page list.
3002: */
3003: kern_return_t
3004: vm_map_copy_discard_cont(
3005: vm_map_copyin_args_t cont_args,
3006: vm_map_copy_t *copy_result) /* OUT */
3007: {
3008: vm_map_copy_discard((vm_map_copy_t) cont_args);
3009: if (copy_result != (vm_map_copy_t *)0)
3010: *copy_result = VM_MAP_COPY_NULL;
3011: return(KERN_SUCCESS);
3012: }
3013:
3014: kern_return_t
3015: vm_map_overwrite_submap_recurse(
3016: vm_map_t dst_map,
3017: vm_offset_t dst_addr,
3018: vm_size_t dst_size)
3019: {
3020: vm_offset_t dst_end;
3021: vm_map_entry_t tmp_entry;
3022: vm_map_entry_t entry;
3023: kern_return_t result;
3024: boolean_t encountered_sub_map = FALSE;
3025:
3026:
3027:
3028: /*
3029: * Verify that the destination is all writeable
3030: * initially. We have to trunc the destination
3031: * address and round the copy size or we'll end up
3032: * splitting entries in strange ways.
3033: */
3034:
3035: dst_end = round_page(dst_addr + dst_size);
3036:
3037: start_pass_1:
3038: vm_map_lock(dst_map);
3039: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
3040: vm_map_unlock(dst_map);
3041: return(KERN_INVALID_ADDRESS);
3042: }
3043:
3044: vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));
3045:
3046: for (entry = tmp_entry;;) {
3047: vm_map_entry_t next;
3048:
3049: next = entry->vme_next;
3050: while(entry->is_sub_map) {
3051: vm_offset_t sub_start;
3052: vm_offset_t sub_end;
3053: vm_offset_t local_end;
3054:
3055: if (entry->in_transition) {
3056: /*
3057: * Say that we are waiting, and wait for entry.
3058: */
3059: entry->needs_wakeup = TRUE;
3060: vm_map_entry_wait(dst_map, THREAD_UNINT);
3061:
3062: goto start_pass_1;
3063: }
3064:
3065: encountered_sub_map = TRUE;
3066: sub_start = entry->offset;
3067:
3068: if(entry->vme_end < dst_end)
3069: sub_end = entry->vme_end;
3070: else
3071: sub_end = dst_end;
3072: sub_end -= entry->vme_start;
3073: sub_end += entry->offset;
3074: local_end = entry->vme_end;
3075: vm_map_unlock(dst_map);
3076:
3077: result = vm_map_overwrite_submap_recurse(
3078: entry->object.sub_map,
3079: sub_start,
3080: sub_end - sub_start);
3081:
3082: if(result != KERN_SUCCESS)
3083: return result;
3084: if (dst_end <= entry->vme_end)
3085: return KERN_SUCCESS;
3086: vm_map_lock(dst_map);
3087: if(!vm_map_lookup_entry(dst_map, local_end,
3088: &tmp_entry)) {
3089: vm_map_unlock(dst_map);
3090: return(KERN_INVALID_ADDRESS);
3091: }
3092: entry = tmp_entry;
3093: next = entry->vme_next;
3094: }
3095:
3096: if ( ! (entry->protection & VM_PROT_WRITE)) {
3097: vm_map_unlock(dst_map);
3098: return(KERN_PROTECTION_FAILURE);
3099: }
3100:
3101: /*
3102: * If the entry is in transition, we must wait
3103: * for it to exit that state. Anything could happen
3104: * when we unlock the map, so start over.
3105: */
3106: if (entry->in_transition) {
3107:
3108: /*
3109: * Say that we are waiting, and wait for entry.
3110: */
3111: entry->needs_wakeup = TRUE;
3112: vm_map_entry_wait(dst_map, THREAD_UNINT);
3113:
3114: goto start_pass_1;
3115: }
3116:
3117: /*
3118: * our range is contained completely within this map entry
3119: */
3120: if (dst_end <= entry->vme_end) {
3121: vm_map_unlock(dst_map);
3122: return KERN_SUCCESS;
3123: }
3124: /*
3125: * check that range specified is contiguous region
3126: */
3127: if ((next == vm_map_to_entry(dst_map)) ||
3128: (next->vme_start != entry->vme_end)) {
3129: vm_map_unlock(dst_map);
3130: return(KERN_INVALID_ADDRESS);
3131: }
3132:
3133: /*
3134: * Check for permanent objects in the destination.
3135: */
3136: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
3137: ((!entry->object.vm_object->internal) ||
3138: (entry->object.vm_object->true_share))) {
3139: if(encountered_sub_map) {
3140: vm_map_unlock(dst_map);
3141: return(KERN_FAILURE);
3142: }
3143: }
3144:
3145:
3146: entry = next;
3147: }/* for */
3148: vm_map_unlock(dst_map);
3149: return(KERN_SUCCESS);
3150: }
3151:
3152: /*
3153: * Routine: vm_map_copy_overwrite
3154: *
3155: * Description:
3156: * Copy the memory described by the map copy
3157: * object (copy; returned by vm_map_copyin) onto
3158: * the specified destination region (dst_map, dst_addr).
3159: * The destination must be writeable.
3160: *
3161: * Unlike vm_map_copyout, this routine actually
3162: * writes over previously-mapped memory. If the
3163: * previous mapping was to a permanent (user-supplied)
3164: * memory object, it is preserved.
3165: *
3166: * The attributes (protection and inheritance) of the
3167: * destination region are preserved.
3168: *
3169: * If successful, consumes the copy object.
3170: * Otherwise, the caller is responsible for it.
3171: *
3172: * Implementation notes:
3173: * To overwrite aligned temporary virtual memory, it is
3174: * sufficient to remove the previous mapping and insert
3175: * the new copy. This replacement is done either on
3176: * the whole region (if no permanent virtual memory
3177: * objects are embedded in the destination region) or
3178: * in individual map entries.
3179: *
3180: * To overwrite permanent virtual memory , it is necessary
3181: * to copy each page, as the external memory management
3182: * interface currently does not provide any optimizations.
3183: *
3184: * Unaligned memory also has to be copied. It is possible
3185: * to use 'vm_trickery' to copy the aligned data. This is
3186: * not done but not hard to implement.
3187: *
3188: * Once a page of permanent memory has been overwritten,
3189: * it is impossible to interrupt this function; otherwise,
3190: * the call would be neither atomic nor location-independent.
3191: * The kernel-state portion of a user thread must be
3192: * interruptible.
3193: *
3194: * It may be expensive to forward all requests that might
3195: * overwrite permanent memory (vm_write, vm_copy) to
3196: * uninterruptible kernel threads. This routine may be
3197: * called by interruptible threads; however, success is
3198: * not guaranteed -- if the request cannot be performed
3199: * atomically and interruptibly, an error indication is
3200: * returned.
3201: */
3202:
3203: kern_return_t
3204: vm_map_copy_overwrite(
3205: vm_map_t dst_map,
3206: vm_offset_t dst_addr,
3207: vm_map_copy_t copy,
3208: boolean_t interruptible)
3209: {
3210: vm_offset_t dst_end;
3211: vm_map_entry_t tmp_entry;
3212: vm_map_entry_t entry;
3213: kern_return_t kr;
3214: boolean_t aligned = TRUE;
3215: boolean_t contains_permanent_objects = FALSE;
3216: boolean_t encountered_sub_map = FALSE;
3217: vm_offset_t base_addr;
3218: vm_size_t copy_size;
3219: vm_size_t total_size;
3220:
3221:
3222: /*
3223: * Check for null copy object.
3224: */
3225:
3226: if (copy == VM_MAP_COPY_NULL)
3227: return(KERN_SUCCESS);
3228:
3229: /*
3230: * Check for special kernel buffer allocated
3231: * by new_ipc_kmsg_copyin.
3232: */
3233:
3234: if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
3235: return(vm_map_copyout_kernel_buffer(dst_map, &dst_addr,
3236: copy, TRUE));
3237: }
3238:
3239: /*
3240: * Only works for entry lists at the moment. Will
3241: * support page lists later.
3242: */
3243:
3244: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
3245:
3246: if (copy->size == 0) {
3247: vm_map_copy_discard(copy);
3248: return(KERN_SUCCESS);
3249: }
3250:
3251: /*
3252: * Verify that the destination is all writeable
3253: * initially. We have to trunc the destination
3254: * address and round the copy size or we'll end up
3255: * splitting entries in strange ways.
3256: */
3257:
3258: if (!page_aligned(copy->size) ||
3259: !page_aligned (copy->offset) ||
3260: !page_aligned (dst_addr))
3261: {
3262: aligned = FALSE;
3263: dst_end = round_page(dst_addr + copy->size);
3264: } else {
3265: dst_end = dst_addr + copy->size;
3266: }
3267:
3268: start_pass_1:
3269: vm_map_lock(dst_map);
3270: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
3271: vm_map_unlock(dst_map);
3272: return(KERN_INVALID_ADDRESS);
3273: }
3274: vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));
3275: for (entry = tmp_entry;;) {
3276: vm_map_entry_t next = entry->vme_next;
3277:
3278: while(entry->is_sub_map) {
3279: vm_offset_t sub_start;
3280: vm_offset_t sub_end;
3281: vm_offset_t local_end;
3282:
3283: if (entry->in_transition) {
3284:
3285: /*
3286: * Say that we are waiting, and wait for entry.
3287: */
3288: entry->needs_wakeup = TRUE;
3289: vm_map_entry_wait(dst_map, THREAD_UNINT);
3290:
3291: goto start_pass_1;
3292: }
3293:
3294: local_end = entry->vme_end;
3295: if (!(entry->needs_copy)) {
3296: /* if needs_copy we are a COW submap */
3297: /* in such a case we just replace so */
3298: /* there is no need for the follow- */
3299: /* ing check. */
3300: encountered_sub_map = TRUE;
3301: sub_start = entry->offset;
3302:
3303: if(entry->vme_end < dst_end)
3304: sub_end = entry->vme_end;
3305: else
3306: sub_end = dst_end;
3307: sub_end -= entry->vme_start;
3308: sub_end += entry->offset;
3309: vm_map_unlock(dst_map);
3310:
3311: kr = vm_map_overwrite_submap_recurse(
3312: entry->object.sub_map,
3313: sub_start,
3314: sub_end - sub_start);
3315: if(kr != KERN_SUCCESS)
3316: return kr;
3317: vm_map_lock(dst_map);
3318: }
3319:
3320: if (dst_end <= entry->vme_end)
3321: goto start_overwrite;
3322: if(!vm_map_lookup_entry(dst_map, local_end,
3323: &entry)) {
3324: vm_map_unlock(dst_map);
3325: return(KERN_INVALID_ADDRESS);
3326: }
3327: next = entry->vme_next;
3328: }
3329:
3330: if ( ! (entry->protection & VM_PROT_WRITE)) {
3331: vm_map_unlock(dst_map);
3332: return(KERN_PROTECTION_FAILURE);
3333: }
3334:
3335: /*
3336: * If the entry is in transition, we must wait
3337: * for it to exit that state. Anything could happen
3338: * when we unlock the map, so start over.
3339: */
3340: if (entry->in_transition) {
3341:
3342: /*
3343: * Say that we are waiting, and wait for entry.
3344: */
3345: entry->needs_wakeup = TRUE;
3346: vm_map_entry_wait(dst_map, THREAD_UNINT);
3347:
3348: goto start_pass_1;
3349: }
3350:
3351: /*
3352: * our range is contained completely within this map entry
3353: */
3354: if (dst_end <= entry->vme_end)
3355: break;
3356: /*
3357: * check that range specified is contiguous region
3358: */
3359: if ((next == vm_map_to_entry(dst_map)) ||
3360: (next->vme_start != entry->vme_end)) {
3361: vm_map_unlock(dst_map);
3362: return(KERN_INVALID_ADDRESS);
3363: }
3364:
3365:
3366: /*
3367: * Check for permanent objects in the destination.
3368: */
3369: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
3370: ((!entry->object.vm_object->internal) ||
3371: (entry->object.vm_object->true_share))) {
3372: if(encountered_sub_map) {
3373: vm_map_unlock(dst_map);
3374: return(KERN_INVALID_ADDRESS);
3375: }
3376: contains_permanent_objects = TRUE;
3377: }
3378:
3379: entry = next;
3380: }/* for */
3381:
3382: start_overwrite:
3383: /*
3384: * If there are permanent objects in the destination, then
3385: * the copy cannot be interrupted.
3386: */
3387:
3388: if (interruptible && contains_permanent_objects) {
3389: vm_map_unlock(dst_map);
3390: return(KERN_FAILURE); /* XXX */
3391: }
3392:
3393: /*
3394: *
3395: * Make a second pass, overwriting the data
3396: * At the beginning of each loop iteration,
3397: * the next entry to be overwritten is "tmp_entry"
3398: * (initially, the value returned from the lookup above),
3399: * and the starting address expected in that entry
3400: * is "start".
3401: */
3402:
3403: total_size = copy->size;
3404: if(encountered_sub_map) {
3405: copy_size = 0;
3406: /* re-calculate tmp_entry since we've had the map */
3407: /* unlocked */
3408: if (!vm_map_lookup_entry( dst_map, dst_addr, &tmp_entry)) {
3409: vm_map_unlock(dst_map);
3410: return(KERN_INVALID_ADDRESS);
3411: }
3412: } else {
3413: copy_size = copy->size;
3414: }
3415:
3416: base_addr = dst_addr;
3417: while(TRUE) {
3418: /* deconstruct the copy object and do in parts */
3419: /* only in sub_map, interruptable case */
3420: vm_map_entry_t copy_entry;
3421: vm_map_entry_t previous_prev;
3422: vm_map_entry_t next_copy;
3423: int nentries;
3424: int remaining_entries;
3425: int new_offset;
3426:
3427: for (entry = tmp_entry; copy_size == 0;) {
3428: vm_map_entry_t next;
3429:
3430: next = entry->vme_next;
3431:
3432: /* tmp_entry and base address are moved along */
3433: /* each time we encounter a sub-map. Otherwise */
3434: /* entry can outpase tmp_entry, and the copy_size */
3435: /* may reflect the distance between them */
3436: /* if the current entry is found to be in transition */
3437: /* we will start over at the beginning or the last */
3438: /* encounter of a submap as dictated by base_addr */
3439: /* we will zero copy_size accordingly. */
3440: if (entry->in_transition) {
3441: /*
3442: * Say that we are waiting, and wait for entry.
3443: */
3444: entry->needs_wakeup = TRUE;
3445: vm_map_entry_wait(dst_map, THREAD_UNINT);
3446:
3447: vm_map_lock(dst_map);
3448: if(!vm_map_lookup_entry(dst_map, base_addr,
3449: &tmp_entry)) {
3450: vm_map_unlock(dst_map);
3451: return(KERN_INVALID_ADDRESS);
3452: }
3453: copy_size = 0;
3454: entry = tmp_entry;
3455: continue;
3456: }
3457: if(entry->is_sub_map) {
3458: vm_offset_t sub_start;
3459: vm_offset_t sub_end;
3460: vm_offset_t local_end;
3461:
3462: if (entry->needs_copy) {
3463: /* if this is a COW submap */
3464: /* just back the range with a */
3465: /* anonymous entry */
3466: if(entry->vme_end < dst_end)
3467: sub_end = entry->vme_end;
3468: else
3469: sub_end = dst_end;
3470: if(entry->vme_start < base_addr)
3471: sub_start = base_addr;
3472: else
3473: sub_start = entry->vme_start;
3474: vm_map_clip_end(
3475: dst_map, entry, sub_end);
3476: vm_map_clip_start(
3477: dst_map, entry, sub_start);
3478: entry->is_sub_map = FALSE;
3479: vm_map_deallocate(
3480: entry->object.sub_map);
3481: entry->object.sub_map = NULL;
3482: entry->is_shared = FALSE;
3483: entry->needs_copy = FALSE;
3484: entry->offset = 0;
3485: entry->protection = VM_PROT_ALL;
3486: entry->max_protection = VM_PROT_ALL;
3487: entry->wired_count = 0;
3488: entry->user_wired_count = 0;
3489: if(entry->inheritance
3490: == VM_INHERIT_SHARE)
3491: entry->inheritance = VM_INHERIT_COPY;
3492: continue;
3493: }
3494: /* first take care of any non-sub_map */
3495: /* entries to send */
3496: if(base_addr < entry->vme_start) {
3497: /* stuff to send */
3498: copy_size =
3499: entry->vme_start - base_addr;
3500: break;
3501: }
3502: sub_start = entry->offset;
3503:
3504: if(entry->vme_end < dst_end)
3505: sub_end = entry->vme_end;
3506: else
3507: sub_end = dst_end;
3508: sub_end -= entry->vme_start;
3509: sub_end += entry->offset;
3510: local_end = entry->vme_end;
3511: vm_map_unlock(dst_map);
3512: copy_size = sub_end - sub_start;
3513:
3514: /* adjust the copy object */
3515: if (total_size > copy_size) {
3516: vm_size_t local_size = 0;
3517: vm_size_t entry_size;
3518:
3519: nentries = 1;
3520: new_offset = copy->offset;
3521: copy_entry = vm_map_copy_first_entry(copy);
3522: while(copy_entry !=
3523: vm_map_copy_to_entry(copy)){
3524: entry_size = copy_entry->vme_end -
3525: copy_entry->vme_start;
3526: if((local_size < copy_size) &&
3527: ((local_size + entry_size)
3528: >= copy_size)) {
3529: vm_map_copy_clip_end(copy,
3530: copy_entry,
3531: copy_entry->vme_start +
3532: (copy_size - local_size));
3533: entry_size = copy_entry->vme_end -
3534: copy_entry->vme_start;
3535: local_size += entry_size;
3536: new_offset += entry_size;
3537: }
3538: if(local_size >= copy_size) {
3539: next_copy = copy_entry->vme_next;
3540: copy_entry->vme_next =
3541: vm_map_copy_to_entry(copy);
3542: previous_prev =
3543: copy->cpy_hdr.links.prev;
3544: copy->cpy_hdr.links.prev = copy_entry;
3545: copy->size = copy_size;
3546: remaining_entries =
3547: copy->cpy_hdr.nentries;
3548: remaining_entries -= nentries;
3549: copy->cpy_hdr.nentries = nentries;
3550: break;
3551: } else {
3552: local_size += entry_size;
3553: new_offset += entry_size;
3554: nentries++;
3555: }
3556: copy_entry = copy_entry->vme_next;
3557: }
3558: }
3559:
3560: kr = vm_map_copy_overwrite(
3561: entry->object.sub_map,
3562: sub_start,
3563: copy,
3564: interruptible);
3565: if(kr != KERN_SUCCESS) {
3566: if(next_copy != NULL) {
3567: copy->cpy_hdr.nentries +=
3568: remaining_entries;
3569: copy->cpy_hdr.links.prev->vme_next =
3570: next_copy;
3571: copy->cpy_hdr.links.prev
3572: = previous_prev;
3573: copy->size = total_size;
3574: }
3575: return kr;
3576: }
3577: if (dst_end <= local_end) {
3578: return(KERN_SUCCESS);
3579: }
3580: /* otherwise copy no longer exists, it was */
3581: /* destroyed after successful copy_overwrite */
3582: copy = (vm_map_copy_t)
3583: zalloc(vm_map_copy_zone);
3584: vm_map_copy_first_entry(copy) =
3585: vm_map_copy_last_entry(copy) =
3586: vm_map_copy_to_entry(copy);
3587: copy->type = VM_MAP_COPY_ENTRY_LIST;
3588: copy->offset = new_offset;
3589:
3590: total_size -= copy_size;
3591: copy_size = 0;
3592: /* put back remainder of copy in container */
3593: if(next_copy != NULL) {
3594: copy->cpy_hdr.nentries = remaining_entries;
3595: copy->cpy_hdr.links.next = next_copy;
3596: copy->cpy_hdr.links.prev = previous_prev;
3597: copy->size = total_size;
3598: next_copy->vme_prev =
3599: vm_map_copy_to_entry(copy);
3600: next_copy = NULL;
3601: }
3602: base_addr = local_end;
3603: vm_map_lock(dst_map);
3604: if(!vm_map_lookup_entry(dst_map,
3605: local_end, &tmp_entry)) {
3606: vm_map_unlock(dst_map);
3607: return(KERN_INVALID_ADDRESS);
3608: }
3609: entry = tmp_entry;
3610: continue;
3611: }
3612: if (dst_end <= entry->vme_end) {
3613: copy_size = dst_end - base_addr;
3614: break;
3615: }
3616:
3617: if ((next == vm_map_to_entry(dst_map)) ||
3618: (next->vme_start != entry->vme_end)) {
3619: vm_map_unlock(dst_map);
3620: return(KERN_INVALID_ADDRESS);
3621: }
3622:
3623: entry = next;
3624: }/* for */
3625:
3626: next_copy = NULL;
3627: nentries = 1;
3628:
3629: /* adjust the copy object */
3630: if (total_size > copy_size) {
3631: vm_size_t local_size = 0;
3632: vm_size_t entry_size;
3633:
3634: new_offset = copy->offset;
3635: copy_entry = vm_map_copy_first_entry(copy);
3636: while(copy_entry != vm_map_copy_to_entry(copy)) {
3637: entry_size = copy_entry->vme_end -
3638: copy_entry->vme_start;
3639: if((local_size < copy_size) &&
3640: ((local_size + entry_size)
3641: >= copy_size)) {
3642: vm_map_copy_clip_end(copy, copy_entry,
3643: copy_entry->vme_start +
3644: (copy_size - local_size));
3645: entry_size = copy_entry->vme_end -
3646: copy_entry->vme_start;
3647: local_size += entry_size;
3648: new_offset += entry_size;
3649: }
3650: if(local_size >= copy_size) {
3651: next_copy = copy_entry->vme_next;
3652: copy_entry->vme_next =
3653: vm_map_copy_to_entry(copy);
3654: previous_prev =
3655: copy->cpy_hdr.links.prev;
3656: copy->cpy_hdr.links.prev = copy_entry;
3657: copy->size = copy_size;
3658: remaining_entries =
3659: copy->cpy_hdr.nentries;
3660: remaining_entries -= nentries;
3661: copy->cpy_hdr.nentries = nentries;
3662: break;
3663: } else {
3664: local_size += entry_size;
3665: new_offset += entry_size;
3666: nentries++;
3667: }
3668: copy_entry = copy_entry->vme_next;
3669: }
3670: }
3671:
3672: if (aligned) {
3673: if ((kr = vm_map_copy_overwrite_aligned(
3674: dst_map, tmp_entry,
3675: copy, base_addr)) != KERN_SUCCESS) {
3676: if(next_copy != NULL) {
3677: copy->cpy_hdr.nentries +=
3678: remaining_entries;
3679: copy->cpy_hdr.links.prev->vme_next =
3680: next_copy;
3681: copy->cpy_hdr.links.prev =
3682: previous_prev;
3683: copy->size += copy_size;
3684: }
3685: return kr;
3686: }
3687: vm_map_unlock(dst_map);
3688: } else {
3689: /*
3690: * Performance gain:
3691: *
3692: * if the copy and dst address are misaligned but the same
3693: * offset within the page we can copy_not_aligned the
3694: * misaligned parts and copy aligned the rest. If they are
3695: * aligned but len is unaligned we simply need to copy
3696: * the end bit unaligned. We'll need to split the misaligned
3697: * bits of the region in this case !
3698: */
3699: /* ALWAYS UNLOCKS THE dst_map MAP */
3700: if ((kr = vm_map_copy_overwrite_unaligned( dst_map,
3701: tmp_entry, copy, base_addr)) != KERN_SUCCESS) {
3702: if(next_copy != NULL) {
3703: copy->cpy_hdr.nentries +=
3704: remaining_entries;
3705: copy->cpy_hdr.links.prev->vme_next =
3706: next_copy;
3707: copy->cpy_hdr.links.prev =
3708: previous_prev;
3709: copy->size += copy_size;
3710: }
3711: return kr;
3712: }
3713: }
3714: total_size -= copy_size;
3715: if(total_size == 0)
3716: break;
3717: base_addr += copy_size;
3718: copy_size = 0;
3719: copy->offset = new_offset;
3720: if(next_copy != NULL) {
3721: copy->cpy_hdr.nentries = remaining_entries;
3722: copy->cpy_hdr.links.next = next_copy;
3723: copy->cpy_hdr.links.prev = previous_prev;
3724: next_copy->vme_prev = vm_map_copy_to_entry(copy);
3725: copy->size = total_size;
3726: }
3727: vm_map_lock(dst_map);
3728: while(TRUE) {
3729: if (!vm_map_lookup_entry(dst_map,
3730: base_addr, &tmp_entry)) {
3731: vm_map_unlock(dst_map);
3732: return(KERN_INVALID_ADDRESS);
3733: }
3734: if (tmp_entry->in_transition) {
3735: entry->needs_wakeup = TRUE;
3736: vm_map_entry_wait(dst_map, THREAD_UNINT);
3737: } else {
3738: break;
3739: }
3740: }
3741: vm_map_clip_start(dst_map, tmp_entry, trunc_page(base_addr));
3742:
3743: entry = tmp_entry;
3744: } /* while */
3745:
3746: /*
3747: * Throw away the vm_map_copy object
3748: */
3749: vm_map_copy_discard(copy);
3750:
3751: return(KERN_SUCCESS);
3752: }/* vm_map_copy_overwrite */
3753:
3754: /*
3755: * Routine: vm_map_copy_overwrite_unaligned
3756: *
3757: * Decription:
3758: * Physically copy unaligned data
3759: *
3760: * Implementation:
3761: * Unaligned parts of pages have to be physically copied. We use
3762: * a modified form of vm_fault_copy (which understands none-aligned
3763: * page offsets and sizes) to do the copy. We attempt to copy as
3764: * much memory in one go as possibly, however vm_fault_copy copies
3765: * within 1 memory object so we have to find the smaller of "amount left"
3766: * "source object data size" and "target object data size". With
3767: * unaligned data we don't need to split regions, therefore the source
3768: * (copy) object should be one map entry, the target range may be split
3769: * over multiple map entries however. In any event we are pessimistic
3770: * about these assumptions.
3771: *
3772: * Assumptions:
3773: * dst_map is locked on entry and is return locked on success,
3774: * unlocked on error.
3775: */
3776:
3777: kern_return_t
3778: vm_map_copy_overwrite_unaligned(
3779: vm_map_t dst_map,
3780: vm_map_entry_t entry,
3781: vm_map_copy_t copy,
3782: vm_offset_t start)
3783: {
3784: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
3785: vm_map_version_t version;
3786: vm_object_t dst_object;
3787: vm_offset_t dst_offset;
3788: vm_offset_t src_offset;
3789: vm_offset_t entry_offset;
3790: vm_offset_t entry_end;
3791: vm_size_t src_size,
3792: dst_size,
3793: copy_size,
3794: amount_left;
3795: kern_return_t kr = KERN_SUCCESS;
3796:
3797: vm_map_lock_write_to_read(dst_map);
3798:
3799: src_offset = copy->offset - trunc_page(copy->offset);
3800: amount_left = copy->size;
3801: /*
3802: * unaligned so we never clipped this entry, we need the offset into
3803: * the vm_object not just the data.
3804: */
3805: while (amount_left > 0) {
3806:
3807: /* "start" must be within the current map entry */
3808: assert ((start>=entry->vme_start) && (start<entry->vme_end));
3809:
3810: dst_offset = start - entry->vme_start;
3811:
3812: dst_size = entry->vme_end - start;
3813:
3814: src_size = copy_entry->vme_end -
3815: (copy_entry->vme_start + src_offset);
3816:
3817: if (dst_size < src_size) {
3818: /*
3819: * we can only copy dst_size bytes before
3820: * we have to get the next destination entry
3821: */
3822: copy_size = dst_size;
3823: } else {
3824: /*
3825: * we can only copy src_size bytes before
3826: * we have to get the next source copy entry
3827: */
3828: copy_size = src_size;
3829: }
3830:
3831: if (copy_size > amount_left) {
3832: copy_size = amount_left;
3833: }
3834: /*
3835: * Entry needs copy, create a shadow shadow object for
3836: * copy on write region.
3837: */
3838: if (entry->needs_copy &&
3839: ((entry->protection & VM_PROT_WRITE) != 0))
3840: {
3841: if (vm_map_lock_read_to_write(dst_map)) {
3842: vm_map_lock_read(dst_map);
3843: goto RetryLookup;
3844: }
3845: vm_object_shadow(&entry->object.vm_object,
3846: &entry->offset,
3847: (vm_size_t)(entry->vme_end
3848: - entry->vme_start));
3849: entry->needs_copy = FALSE;
3850: vm_map_lock_write_to_read(dst_map);
3851: }
3852: dst_object = entry->object.vm_object;
3853: /*
3854: * unlike with the virtual (aligned) copy we're going
3855: * to fault on it therefore we need a target object.
3856: */
3857: if (dst_object == VM_OBJECT_NULL) {
3858: if (vm_map_lock_read_to_write(dst_map)) {
3859: vm_map_lock_read(dst_map);
3860: goto RetryLookup;
3861: }
3862: dst_object = vm_object_allocate((vm_size_t)
3863: entry->vme_end - entry->vme_start);
3864: entry->object.vm_object = dst_object;
3865: entry->offset = 0;
3866: vm_map_lock_write_to_read(dst_map);
3867: }
3868: /*
3869: * Take an object reference and unlock map. The "entry" may
3870: * disappear or change when the map is unlocked.
3871: */
3872: vm_object_reference(dst_object);
3873: version.main_timestamp = dst_map->timestamp;
3874: entry_offset = entry->offset;
3875: entry_end = entry->vme_end;
3876: vm_map_unlock_read(dst_map);
3877: /*
3878: * Copy as much as possible in one pass
3879: */
3880: kr = vm_fault_copy(
3881: copy_entry->object.vm_object,
3882: copy_entry->offset + src_offset,
3883: ©_size,
3884: dst_object,
3885: entry_offset + dst_offset,
3886: dst_map,
3887: &version,
3888: THREAD_UNINT );
3889:
3890: start += copy_size;
3891: src_offset += copy_size;
3892: amount_left -= copy_size;
3893: /*
3894: * Release the object reference
3895: */
3896: vm_object_deallocate(dst_object);
3897: /*
3898: * If a hard error occurred, return it now
3899: */
3900: if (kr != KERN_SUCCESS)
3901: return kr;
3902:
3903: if ((copy_entry->vme_start + src_offset) == copy_entry->vme_end
3904: || amount_left == 0)
3905: {
3906: /*
3907: * all done with this copy entry, dispose.
3908: */
3909: vm_map_copy_entry_unlink(copy, copy_entry);
3910: vm_object_deallocate(copy_entry->object.vm_object);
3911: vm_map_copy_entry_dispose(copy, copy_entry);
3912:
3913: if ((copy_entry = vm_map_copy_first_entry(copy))
3914: == vm_map_copy_to_entry(copy) && amount_left) {
3915: /*
3916: * not finished copying but run out of source
3917: */
3918: return KERN_INVALID_ADDRESS;
3919: }
3920: src_offset = 0;
3921: }
3922:
3923: if (amount_left == 0)
3924: return KERN_SUCCESS;
3925:
3926: vm_map_lock_read(dst_map);
3927: if (version.main_timestamp == dst_map->timestamp) {
3928: if (start == entry_end) {
3929: /*
3930: * destination region is split. Use the version
3931: * information to avoid a lookup in the normal
3932: * case.
3933: */
3934: entry = entry->vme_next;
3935: /*
3936: * should be contiguous. Fail if we encounter
3937: * a hole in the destination.
3938: */
3939: if (start != entry->vme_start) {
3940: vm_map_unlock_read(dst_map);
3941: return KERN_INVALID_ADDRESS ;
3942: }
3943: }
3944: } else {
3945: /*
3946: * Map version check failed.
3947: * we must lookup the entry because somebody
3948: * might have changed the map behind our backs.
3949: */
3950: RetryLookup:
3951: if (!vm_map_lookup_entry(dst_map, start, &entry))
3952: {
3953: vm_map_unlock_read(dst_map);
3954: return KERN_INVALID_ADDRESS ;
3955: }
3956: }
3957: }/* while */
3958:
3959: /* NOTREACHED ?? */
3960: vm_map_unlock_read(dst_map);
3961:
3962: return KERN_SUCCESS;
3963: }/* vm_map_copy_overwrite_unaligned */
3964:
3965: /*
3966: * Routine: vm_map_copy_overwrite_aligned
3967: *
3968: * Description:
3969: * Does all the vm_trickery possible for whole pages.
3970: *
3971: * Implementation:
3972: *
3973: * If there are no permanent objects in the destination,
3974: * and the source and destination map entry zones match,
3975: * and the destination map entry is not shared,
3976: * then the map entries can be deleted and replaced
3977: * with those from the copy. The following code is the
3978: * basic idea of what to do, but there are lots of annoying
3979: * little details about getting protection and inheritance
3980: * right. Should add protection, inheritance, and sharing checks
3981: * to the above pass and make sure that no wiring is involved.
3982: */
3983:
3984: kern_return_t
3985: vm_map_copy_overwrite_aligned(
3986: vm_map_t dst_map,
3987: vm_map_entry_t tmp_entry,
3988: vm_map_copy_t copy,
3989: vm_offset_t start)
3990: {
3991: vm_object_t object;
3992: vm_map_entry_t copy_entry;
3993: vm_size_t copy_size;
3994: vm_size_t size;
3995: vm_map_entry_t entry;
3996:
3997: while ((copy_entry = vm_map_copy_first_entry(copy))
3998: != vm_map_copy_to_entry(copy))
3999: {
4000: copy_size = (copy_entry->vme_end - copy_entry->vme_start);
4001:
4002: entry = tmp_entry;
4003: size = (entry->vme_end - entry->vme_start);
4004: /*
4005: * Make sure that no holes popped up in the
4006: * address map, and that the protection is
4007: * still valid, in case the map was unlocked
4008: * earlier.
4009: */
4010:
4011: if ((entry->vme_start != start) || (entry->is_sub_map)) {
4012: vm_map_unlock(dst_map);
4013: return(KERN_INVALID_ADDRESS);
4014: }
4015: assert(entry != vm_map_to_entry(dst_map));
4016:
4017: /*
4018: * Check protection again
4019: */
4020:
4021: if ( ! (entry->protection & VM_PROT_WRITE)) {
4022: vm_map_unlock(dst_map);
4023: return(KERN_PROTECTION_FAILURE);
4024: }
4025:
4026: /*
4027: * Adjust to source size first
4028: */
4029:
4030: if (copy_size < size) {
4031: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
4032: size = copy_size;
4033: }
4034:
4035: /*
4036: * Adjust to destination size
4037: */
4038:
4039: if (size < copy_size) {
4040: vm_map_copy_clip_end(copy, copy_entry,
4041: copy_entry->vme_start + size);
4042: copy_size = size;
4043: }
4044:
4045: assert((entry->vme_end - entry->vme_start) == size);
4046: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
4047: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
4048:
4049: /*
4050: * If the destination contains temporary unshared memory,
4051: * we can perform the copy by throwing it away and
4052: * installing the source data.
4053: */
4054:
4055: object = entry->object.vm_object;
4056: if ((!entry->is_shared &&
4057: ((object == VM_OBJECT_NULL) ||
4058: (object->internal && !object->true_share))) ||
4059: entry->needs_copy) {
4060: vm_object_t old_object = entry->object.vm_object;
4061: vm_offset_t old_offset = entry->offset;
4062: vm_offset_t offset;
4063:
4064: /*
4065: * Ensure that the source and destination aren't
4066: * identical
4067: */
4068: if (old_object == copy_entry->object.vm_object &&
4069: old_offset == copy_entry->offset) {
4070: vm_map_copy_entry_unlink(copy, copy_entry);
4071: vm_map_copy_entry_dispose(copy, copy_entry);
4072:
4073: if (old_object != VM_OBJECT_NULL)
4074: vm_object_deallocate(old_object);
4075:
4076: start = tmp_entry->vme_end;
4077: tmp_entry = tmp_entry->vme_next;
4078: continue;
4079: }
4080:
4081: entry->object = copy_entry->object;
4082: object = entry->object.vm_object;
4083: offset = entry->offset = copy_entry->offset;
4084: entry->needs_copy = copy_entry->needs_copy;
4085: entry->wired_count = 0;
4086: entry->user_wired_count = 0;
4087:
4088: vm_map_copy_entry_unlink(copy, copy_entry);
4089: vm_map_copy_entry_dispose(copy, copy_entry);
4090:
4091: if (old_object != VM_OBJECT_NULL) {
4092: vm_object_pmap_protect(
4093: old_object,
4094: old_offset,
4095: size,
4096: dst_map->pmap,
4097: tmp_entry->vme_start,
4098: VM_PROT_NONE);
4099:
4100: vm_object_deallocate(old_object);
4101: }
4102:
4103: /*
4104: * Try to aggressively enter physical mappings
4105: * (but avoid uninstantiated objects)
4106: */
4107: if (object != VM_OBJECT_NULL) {
4108: vm_offset_t va = entry->vme_start;
4109:
4110: while (va < entry->vme_end) {
4111: register vm_page_t m;
4112: vm_prot_t prot;
4113:
4114: /*
4115: * Look for the page in the top object
4116: */
4117: prot = entry->protection;
4118: vm_object_lock(object);
4119: vm_object_paging_begin(object);
4120:
4121: if ((m = vm_page_lookup(object,offset)) !=
4122: VM_PAGE_NULL && !m->busy &&
4123: !m->fictitious &&
4124: (!m->unusual || (!m->error &&
4125: !m->restart && !m->absent &&
4126: (prot & m->page_lock) == 0))) {
4127:
4128: m->busy = TRUE;
4129: vm_object_unlock(object);
4130:
4131: /*
4132: * Honor COW obligations
4133: */
4134: if (entry->needs_copy)
4135: prot &= ~VM_PROT_WRITE;
4136:
4137: PMAP_ENTER(dst_map->pmap, va, m,
4138: prot, FALSE);
4139:
4140: vm_object_lock(object);
4141: vm_page_lock_queues();
4142: if (!m->active && !m->inactive)
4143: vm_page_activate(m);
4144: vm_page_unlock_queues();
4145: PAGE_WAKEUP_DONE(m);
4146: }
4147: vm_object_paging_end(object);
4148: vm_object_unlock(object);
4149:
4150: offset += PAGE_SIZE;
4151: va += PAGE_SIZE;
4152: } /* end while (va < entry->vme_end) */
4153: } /* end if (object) */
4154:
4155: /*
4156: * Set up for the next iteration. The map
4157: * has not been unlocked, so the next
4158: * address should be at the end of this
4159: * entry, and the next map entry should be
4160: * the one following it.
4161: */
4162:
4163: start = tmp_entry->vme_end;
4164: tmp_entry = tmp_entry->vme_next;
4165: } else {
4166: vm_map_version_t version;
4167: vm_object_t dst_object = entry->object.vm_object;
4168: vm_offset_t dst_offset = entry->offset;
4169: kern_return_t r;
4170:
4171: /*
4172: * Take an object reference, and record
4173: * the map version information so that the
4174: * map can be safely unlocked.
4175: */
4176:
4177: vm_object_reference(dst_object);
4178:
4179: version.main_timestamp = dst_map->timestamp;
4180:
4181: vm_map_unlock(dst_map);
4182:
4183: /*
4184: * Copy as much as possible in one pass
4185: */
4186:
4187: copy_size = size;
4188: r = vm_fault_copy(
4189: copy_entry->object.vm_object,
4190: copy_entry->offset,
4191: ©_size,
4192: dst_object,
4193: dst_offset,
4194: dst_map,
4195: &version,
4196: THREAD_UNINT );
4197:
4198: /*
4199: * Release the object reference
4200: */
4201:
4202: vm_object_deallocate(dst_object);
4203:
4204: /*
4205: * If a hard error occurred, return it now
4206: */
4207:
4208: if (r != KERN_SUCCESS)
4209: return(r);
4210:
4211: if (copy_size != 0) {
4212: /*
4213: * Dispose of the copied region
4214: */
4215:
4216: vm_map_copy_clip_end(copy, copy_entry,
4217: copy_entry->vme_start + copy_size);
4218: vm_map_copy_entry_unlink(copy, copy_entry);
4219: vm_object_deallocate(copy_entry->object.vm_object);
4220: vm_map_copy_entry_dispose(copy, copy_entry);
4221: }
4222:
4223: /*
4224: * Pick up in the destination map where we left off.
4225: *
4226: * Use the version information to avoid a lookup
4227: * in the normal case.
4228: */
4229:
4230: start += copy_size;
4231: vm_map_lock(dst_map);
4232: if ((version.main_timestamp + 1) == dst_map->timestamp) {
4233: /* We can safely use saved tmp_entry value */
4234:
4235: vm_map_clip_end(dst_map, tmp_entry, start);
4236: tmp_entry = tmp_entry->vme_next;
4237: } else {
4238: /* Must do lookup of tmp_entry */
4239:
4240: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
4241: vm_map_unlock(dst_map);
4242: return(KERN_INVALID_ADDRESS);
4243: }
4244: vm_map_clip_start(dst_map, tmp_entry, start);
4245: }
4246: }
4247: }/* while */
4248:
4249: return(KERN_SUCCESS);
4250: }/* vm_map_copy_overwrite_aligned */
4251:
4252: #if DIPC
4253: #include <dipc/dipc_counters.h>
4254: dcntr_decl(unsigned int c_dipc_overwrite_opt = 0;)
4255: dcntr_decl(unsigned int c_dipc_overwrite_nonopt = 0;)
4256: dcntr_decl(unsigned int c_dipc_overwrite_recv_done = 0;)
4257: #else /* DIPC */
4258: #define dstat_decl(foo)
4259: #define dstat(foo)
4260: #define dcntr_decl(foo)
4261: #define dcntr(foo)
4262: #endif /* DIPC */
4263:
4264: #if DIPC
4265: vm_map_copy_t
4266: vm_map_copy_overwrite_recv(
4267: vm_map_t map,
4268: vm_offset_t addr,
4269: vm_size_t size)
4270: {
4271: vm_offset_t end_addr;
4272: vm_map_copy_t copy = VM_MAP_COPY_NULL;
4273: vm_map_entry_t entry, new_entry;
4274: kern_return_t kr;
4275:
4276: if (size == 0)
4277: return VM_MAP_COPY_NULL;
4278:
4279: if (!page_aligned(size) || !page_aligned(addr)) {
4280: end_addr = round_page(addr + size);
4281: } else {
4282: end_addr = addr + size;
4283: }
4284:
4285: /* Do allocations before taking map lock. */
4286:
4287: /* build an ENTRY_LIST vm_map_copy_t */
4288: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
4289: vm_map_copy_first_entry(copy) =
4290: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
4291: copy->type = VM_MAP_COPY_ENTRY_LIST;
4292: copy->offset = addr;
4293: copy->size = size;
4294: copy->cpy_hdr.nentries = 0;
4295: copy->cpy_hdr.entries_pageable = TRUE;
4296: new_entry = vm_map_copy_entry_create(copy);
4297:
4298: vm_map_lock(map);
4299: if (!vm_map_lookup_entry(map, addr, &entry)) {
4300: vm_map_unlock(map);
4301: vm_map_copy_entry_dispose(copy, new_entry);
4302: vm_map_copy_discard(copy);
4303: return VM_MAP_COPY_NULL;
4304: }
4305: if (entry->is_sub_map) {
4306: vm_map_unlock(map);
4307: vm_map_copy_entry_dispose(copy, new_entry);
4308: vm_map_copy_discard(copy);
4309: return VM_MAP_COPY_NULL;
4310: }
4311:
4312: /*
4313: * for now, only do this if the range is contained completely
4314: * within a single entry. Just give up if entry is in_transition:
4315: * it could be another thread wiring/unwiring, but it could be another
4316: * part of this message that set it, in which case we'd deadlock if
4317: * we waited for it to clear. If the entry is not writeable, give up
4318: * and let the ipc_kmsg_copyout code handle the error later.
4319: */
4320: if ((end_addr > entry->vme_end) ||
4321: entry->in_transition ||
4322: !(entry->protection & VM_PROT_WRITE)) {
4323: vm_map_unlock(map);
4324: dcntr(++c_dipc_overwrite_nonopt);
4325: vm_map_copy_entry_dispose(copy, new_entry);
4326: vm_map_copy_discard(copy);
4327: return VM_MAP_COPY_NULL;
4328: }
4329:
4330: vm_map_clip_start(map, entry, trunc_page(addr));
4331:
4332: /* handle COW obligations */
4333: if (entry->needs_copy) {
4334: vm_object_shadow(&entry->object.vm_object, &entry->offset,
4335: (vm_size_t)(entry->vme_end - entry->vme_start));
4336: entry->needs_copy = FALSE;
4337: }
4338:
4339: /* make sure there's a target object to put pages into */
4340: if (entry->object.vm_object == VM_OBJECT_NULL) {
4341: entry->object.vm_object = vm_object_allocate((vm_size_t)
4342: (entry->vme_end - entry->vme_start));
4343: entry->offset = 0;
4344: }
4345:
4346: /*
4347: * set in_transition to keep the entry in the map from moving around
4348: * while DIPC fills the object in with data. This must be cleared
4349: * later on by calling vm_map_copy_overwrite_recv_done (below).
4350: */
4351: entry->in_transition = TRUE;
4352:
4353: /* make a copy of the entry */
4354: vm_map_entry_copy(new_entry, entry);
4355: vm_object_reference(new_entry->object.vm_object);
4356:
4357: vm_map_unlock(map);
4358:
4359: /* link in the new entry */
4360: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry);
4361:
4362: dcntr(++c_dipc_overwrite_opt);
4363: return copy;
4364: }
4365:
4366: /*
4367: * Find the map entry that corresponds to the copy entry, then clear
4368: * in_transition on it. Discard the copy.
4369: */
4370: kern_return_t
4371: vm_map_copy_overwrite_recv_done(
4372: vm_map_t map,
4373: vm_map_copy_t copy)
4374: {
4375: vm_map_entry_t entry, map_entry;
4376: boolean_t need_wakeup = FALSE;
4377:
4378: assert(copy != VM_MAP_COPY_NULL);
4379: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
4380: dcntr(++c_dipc_overwrite_recv_done);
4381:
4382: entry = vm_map_copy_first_entry(copy);
4383: assert(entry != VM_MAP_ENTRY_NULL);
4384:
4385: vm_map_lock(map);
4386:
4387: if (!vm_map_lookup_entry(map, entry->vme_start, &map_entry)) {
4388: vm_map_unlock(map);
4389: return KERN_INVALID_ADDRESS;
4390: }
4391: while ((map_entry != vm_map_to_entry(map)) &&
4392: (map_entry->vme_end <= entry->vme_end)) {
4393: assert(map_entry->in_transition);
4394: map_entry->in_transition = FALSE;
4395: if (map_entry->needs_wakeup) {
4396: map_entry->needs_wakeup = FALSE;
4397: need_wakeup = TRUE;
4398: }
4399: map_entry = map_entry->vme_next;
4400: }
4401: vm_map_unlock(map);
4402: if (need_wakeup)
4403: vm_map_entry_wakeup(map);
4404: vm_map_copy_discard(copy);
4405:
4406: return KERN_SUCCESS;
4407: }
4408:
4409: #endif /* DIPC */
4410:
4411: /*
4412: * Routine: vm_map_copyout_kernel_buffer
4413: *
4414: * Description:
4415: * Copy out data from a kernel buffer into space in the
4416: * destination map. The space may be otpionally dynamically
4417: * allocated.
4418: *
4419: * If successful, consumes the copy object.
4420: * Otherwise, the caller is responsible for it.
4421: */
4422: kern_return_t
4423: vm_map_copyout_kernel_buffer(
4424: vm_map_t map,
4425: vm_offset_t *addr, /* IN/OUT */
4426: vm_map_copy_t copy,
4427: boolean_t overwrite)
4428: {
4429: kern_return_t kr = KERN_SUCCESS;
4430: thread_act_t thr_act = current_act();
4431:
4432: if (!overwrite) {
4433:
4434: /*
4435: * Allocate space in the target map for the data
4436: */
4437: *addr = 0;
4438: kr = vm_map_enter(map,
4439: addr,
4440: round_page(copy->size),
4441: (vm_offset_t) 0,
4442: TRUE,
4443: VM_OBJECT_NULL,
4444: (vm_offset_t) 0,
4445: FALSE,
4446: VM_PROT_DEFAULT,
4447: VM_PROT_ALL,
4448: VM_INHERIT_DEFAULT);
4449: if (kr != KERN_SUCCESS)
4450: return(kr);
4451: }
4452:
4453: /*
4454: * Copyout the data from the kernel buffer to the target map.
4455: */
4456: if (thr_act->map == map) {
4457:
4458: /*
4459: * If the target map is the current map, just do
4460: * the copy.
4461: */
4462: if (copyout((char *)copy->cpy_kdata, (char *)*addr,
4463: copy->size)) {
4464: kr = KERN_INVALID_ADDRESS;
4465: }
4466: }
4467: else {
4468: vm_map_t oldmap;
4469:
4470: /*
4471: * If the target map is another map, assume the
4472: * target's address space identity for the duration
4473: * of the copy.
4474: */
4475: vm_map_reference(map);
4476: oldmap = vm_map_switch(map);
4477:
4478: if (copyout((char *)copy->cpy_kdata, (char *)*addr,
4479: copy->size)) {
4480: kr = KERN_INVALID_ADDRESS;
4481: }
4482:
4483: (void) vm_map_switch(oldmap);
4484: vm_map_deallocate(map);
4485: }
4486:
4487: kfree((vm_offset_t)copy, copy->cpy_kalloc_size);
4488:
4489: return(kr);
4490: }
4491:
4492: /*
4493: * Macro: vm_map_copy_insert
4494: *
4495: * Description:
4496: * Link a copy chain ("copy") into a map at the
4497: * specified location (after "where").
4498: * Side effects:
4499: * The copy chain is destroyed.
4500: * Warning:
4501: * The arguments are evaluated multiple times.
4502: */
4503: #define vm_map_copy_insert(map, where, copy) \
4504: MACRO_BEGIN \
4505: vm_map_t VMCI_map; \
4506: vm_map_entry_t VMCI_where; \
4507: vm_map_copy_t VMCI_copy; \
4508: VMCI_map = (map); \
4509: VMCI_where = (where); \
4510: VMCI_copy = (copy); \
4511: ((VMCI_where->vme_next)->vme_prev = vm_map_copy_last_entry(VMCI_copy))\
4512: ->vme_next = (VMCI_where->vme_next); \
4513: ((VMCI_where)->vme_next = vm_map_copy_first_entry(VMCI_copy)) \
4514: ->vme_prev = VMCI_where; \
4515: VMCI_map->hdr.nentries += VMCI_copy->cpy_hdr.nentries; \
4516: UPDATE_FIRST_FREE(VMCI_map, VMCI_map->first_free); \
4517: zfree(vm_map_copy_zone, (vm_offset_t) VMCI_copy); \
4518: MACRO_END
4519:
4520: /*
4521: * Routine: vm_map_copyout
4522: *
4523: * Description:
4524: * Copy out a copy chain ("copy") into newly-allocated
4525: * space in the destination map.
4526: *
4527: * If successful, consumes the copy object.
4528: * Otherwise, the caller is responsible for it.
4529: */
4530: kern_return_t
4531: vm_map_copyout(
4532: register vm_map_t dst_map,
4533: vm_offset_t *dst_addr, /* OUT */
4534: register vm_map_copy_t copy)
4535: {
4536: vm_size_t size;
4537: vm_size_t adjustment;
4538: vm_offset_t start;
4539: vm_offset_t vm_copy_start;
4540: vm_map_entry_t last;
4541: register
4542: vm_map_entry_t entry;
4543:
4544: /*
4545: * Check for null copy object.
4546: */
4547:
4548: if (copy == VM_MAP_COPY_NULL) {
4549: *dst_addr = 0;
4550: return(KERN_SUCCESS);
4551: }
4552:
4553: /*
4554: * Check for special copy object, created
4555: * by vm_map_copyin_object.
4556: */
4557:
4558: if (copy->type == VM_MAP_COPY_OBJECT) {
4559: vm_object_t object = copy->cpy_object;
4560: kern_return_t kr;
4561: vm_size_t offset;
4562:
4563: offset = trunc_page(copy->offset);
4564: size = round_page(copy->size + copy->offset - offset);
4565: *dst_addr = 0;
4566: kr = vm_map_enter(dst_map, dst_addr, size,
4567: (vm_offset_t) 0, TRUE,
4568: object, offset, FALSE,
4569: VM_PROT_DEFAULT, VM_PROT_ALL,
4570: VM_INHERIT_DEFAULT);
4571: if (kr != KERN_SUCCESS)
4572: return(kr);
4573: /* Account for non-pagealigned copy object */
4574: *dst_addr += copy->offset - offset;
4575: zfree(vm_map_copy_zone, (vm_offset_t) copy);
4576: return(KERN_SUCCESS);
4577: }
4578:
4579: /*
4580: * Check for special kernel buffer allocated
4581: * by new_ipc_kmsg_copyin.
4582: */
4583:
4584: if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
4585: return(vm_map_copyout_kernel_buffer(dst_map, dst_addr,
4586: copy, FALSE));
4587: }
4588:
4589: if (copy->type == VM_MAP_COPY_PAGE_LIST)
4590: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
4591:
4592: /*
4593: * Find space for the data
4594: */
4595:
4596: vm_copy_start = trunc_page(copy->offset);
4597: size = round_page(copy->offset + copy->size) - vm_copy_start;
4598:
4599: StartAgain: ;
4600:
4601: vm_map_lock(dst_map);
4602: assert(first_free_is_valid(dst_map));
4603: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
4604: vm_map_min(dst_map) : last->vme_end;
4605:
4606: while (TRUE) {
4607: vm_map_entry_t next = last->vme_next;
4608: vm_offset_t end = start + size;
4609:
4610: if ((end > dst_map->max_offset) || (end < start)) {
4611: if (dst_map->wait_for_space) {
4612: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
4613: assert_wait((event_t) dst_map,
4614: THREAD_INTERRUPTIBLE);
4615: vm_map_unlock(dst_map);
4616: thread_block((void (*)(void))0);
4617: goto StartAgain;
4618: }
4619: }
4620: vm_map_unlock(dst_map);
4621: return(KERN_NO_SPACE);
4622: }
4623:
4624: if ((next == vm_map_to_entry(dst_map)) ||
4625: (next->vme_start >= end))
4626: break;
4627:
4628: last = next;
4629: start = last->vme_end;
4630: }
4631:
4632: /*
4633: * Since we're going to just drop the map
4634: * entries from the copy into the destination
4635: * map, they must come from the same pool.
4636: */
4637:
4638: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
4639: /*
4640: * Mismatches occur when dealing with the default
4641: * pager.
4642: */
4643: zone_t old_zone;
4644: vm_map_entry_t next, new;
4645:
4646: /*
4647: * Find the zone that the copies were allocated from
4648: */
4649: old_zone = (copy->cpy_hdr.entries_pageable)
4650: ? vm_map_entry_zone
4651: : vm_map_kentry_zone;
4652: entry = vm_map_copy_first_entry(copy);
4653:
4654: /*
4655: * Reinitialize the copy so that vm_map_copy_entry_link
4656: * will work.
4657: */
4658: copy->cpy_hdr.nentries = 0;
4659: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
4660: vm_map_copy_first_entry(copy) =
4661: vm_map_copy_last_entry(copy) =
4662: vm_map_copy_to_entry(copy);
4663:
4664: /*
4665: * Copy each entry.
4666: */
4667: while (entry != vm_map_copy_to_entry(copy)) {
4668: new = vm_map_copy_entry_create(copy);
4669: vm_map_entry_copy_full(new, entry);
4670: vm_map_copy_entry_link(copy,
4671: vm_map_copy_last_entry(copy),
4672: new);
4673: next = entry->vme_next;
4674: zfree(old_zone, (vm_offset_t) entry);
4675: entry = next;
4676: }
4677: }
4678:
4679: /*
4680: * Adjust the addresses in the copy chain, and
4681: * reset the region attributes.
4682: */
4683:
4684: adjustment = start - vm_copy_start;
4685: for (entry = vm_map_copy_first_entry(copy);
4686: entry != vm_map_copy_to_entry(copy);
4687: entry = entry->vme_next) {
4688: entry->vme_start += adjustment;
4689: entry->vme_end += adjustment;
4690:
4691: entry->inheritance = VM_INHERIT_DEFAULT;
4692: entry->protection = VM_PROT_DEFAULT;
4693: entry->max_protection = VM_PROT_ALL;
4694: entry->behavior = VM_BEHAVIOR_DEFAULT;
4695:
4696: /*
4697: * If the entry is now wired,
4698: * map the pages into the destination map.
4699: */
4700: if (entry->wired_count != 0) {
4701: register vm_offset_t va;
4702: vm_offset_t offset;
4703: register vm_object_t object;
4704:
4705: object = entry->object.vm_object;
4706: offset = entry->offset;
4707: va = entry->vme_start;
4708:
4709: pmap_pageable(dst_map->pmap,
4710: entry->vme_start,
4711: entry->vme_end,
4712: TRUE);
4713:
4714: while (va < entry->vme_end) {
4715: register vm_page_t m;
4716:
4717: /*
4718: * Look up the page in the object.
4719: * Assert that the page will be found in the
4720: * top object:
4721: * either
4722: * the object was newly created by
4723: * vm_object_copy_slowly, and has
4724: * copies of all of the pages from
4725: * the source object
4726: * or
4727: * the object was moved from the old
4728: * map entry; because the old map
4729: * entry was wired, all of the pages
4730: * were in the top-level object.
4731: * (XXX not true if we wire pages for
4732: * reading)
4733: */
4734: vm_object_lock(object);
4735: vm_object_paging_begin(object);
4736:
4737: m = vm_page_lookup(object, offset);
4738: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
4739: m->absent)
4740: panic("vm_map_copyout: wiring 0x%x", m);
4741:
4742: m->busy = TRUE;
4743: vm_object_unlock(object);
4744:
4745: PMAP_ENTER(dst_map->pmap, va, m,
4746: entry->protection, TRUE);
4747:
4748: vm_object_lock(object);
4749: PAGE_WAKEUP_DONE(m);
4750: /* the page is wired, so we don't have to activate */
4751: vm_object_paging_end(object);
4752: vm_object_unlock(object);
4753:
4754: offset += PAGE_SIZE;
4755: va += PAGE_SIZE;
4756: }
4757: }
4758: else if (size <= vm_map_aggressive_enter_max) {
4759:
4760: register vm_offset_t va;
4761: vm_offset_t offset;
4762: register vm_object_t object;
4763: vm_prot_t prot;
4764:
4765: object = entry->object.vm_object;
4766: if (object != VM_OBJECT_NULL) {
4767:
4768: offset = entry->offset;
4769: va = entry->vme_start;
4770: while (va < entry->vme_end) {
4771: register vm_page_t m;
4772:
4773: /*
4774: * Look up the page in the object.
4775: * Assert that the page will be found
4776: * in the top object if at all...
4777: */
4778: vm_object_lock(object);
4779: vm_object_paging_begin(object);
4780:
4781: if (((m = vm_page_lookup(object,
4782: offset))
4783: != VM_PAGE_NULL) &&
4784: !m->busy && !m->fictitious &&
4785: !m->absent && !m->error) {
4786: m->busy = TRUE;
4787: vm_object_unlock(object);
4788:
4789: /* honor cow obligations */
4790: prot = entry->protection;
4791: if (entry->needs_copy)
4792: prot &= ~VM_PROT_WRITE;
4793:
4794: PMAP_ENTER(dst_map->pmap, va,
4795: m, prot, FALSE);
4796:
4797: vm_object_lock(object);
4798: vm_page_lock_queues();
4799: if (!m->active && !m->inactive)
4800: vm_page_activate(m);
4801: vm_page_unlock_queues();
4802: PAGE_WAKEUP_DONE(m);
4803: }
4804: vm_object_paging_end(object);
4805: vm_object_unlock(object);
4806:
4807: offset += PAGE_SIZE;
4808: va += PAGE_SIZE;
4809: }
4810: }
4811: }
4812: }
4813:
4814: /*
4815: * Correct the page alignment for the result
4816: */
4817:
4818: *dst_addr = start + (copy->offset - vm_copy_start);
4819:
4820: /*
4821: * Update the hints and the map size
4822: */
4823:
4824: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
4825:
4826: dst_map->size += size;
4827:
4828: /*
4829: * Link in the copy
4830: */
4831:
4832: vm_map_copy_insert(dst_map, last, copy);
4833:
4834: vm_map_unlock(dst_map);
4835:
4836: /*
4837: * XXX If wiring_required, call vm_map_pageable
4838: */
4839:
4840: return(KERN_SUCCESS);
4841: }
4842:
4843: boolean_t vm_map_aggressive_enter; /* not used yet */
4844:
4845: /*
4846: *
4847: * vm_map_copyout_page_list:
4848: *
4849: * Version of vm_map_copyout() for page list vm map copies.
4850: *
4851: */
4852: kern_return_t
4853: vm_map_copyout_page_list(
4854: register vm_map_t dst_map,
4855: vm_offset_t *dst_addr, /* OUT */
4856: register vm_map_copy_t copy)
4857: {
4858: vm_size_t size;
4859: vm_offset_t start;
4860: vm_offset_t end;
4861: vm_offset_t offset;
4862: vm_map_entry_t last;
4863: register
4864: vm_object_t object;
4865: vm_page_t *page_list, m;
4866: vm_map_entry_t entry;
4867: vm_offset_t old_last_offset;
4868: boolean_t cont_invoked, needs_wakeup;
4869: kern_return_t result = KERN_SUCCESS;
4870: vm_map_copy_t orig_copy;
4871: vm_offset_t dst_offset;
4872: boolean_t must_wire;
4873: boolean_t aggressive_enter;
4874:
4875: /*
4876: * Check for null copy object.
4877: */
4878:
4879: if (copy == VM_MAP_COPY_NULL) {
4880: *dst_addr = 0;
4881: return(KERN_SUCCESS);
4882: }
4883:
4884: assert(copy->type == VM_MAP_COPY_PAGE_LIST);
4885:
4886: /*
4887: * Make sure the pages are stolen, because we are
4888: * going to put them in a new object. Assume that
4889: * all pages are identical to first in this regard.
4890: */
4891:
4892: page_list = ©->cpy_page_list[0];
4893: if (!copy->cpy_page_loose)
4894: vm_map_copy_steal_pages(copy);
4895:
4896: /*
4897: * Find space for the data
4898: */
4899:
4900: size = round_page(copy->offset + copy->size) -
4901: trunc_page(copy->offset);
4902: StartAgain:
4903: vm_map_lock(dst_map);
4904: must_wire = dst_map->wiring_required;
4905:
4906: assert(first_free_is_valid(dst_map));
4907: last = dst_map->first_free;
4908: if (last == vm_map_to_entry(dst_map)) {
4909: start = vm_map_min(dst_map);
4910: } else {
4911: start = last->vme_end;
4912: }
4913:
4914: while (TRUE) {
4915: vm_map_entry_t next = last->vme_next;
4916: end = start + size;
4917:
4918: if ((end > dst_map->max_offset) || (end < start)) {
4919: if (dst_map->wait_for_space) {
4920: if (size <= (dst_map->max_offset -
4921: dst_map->min_offset)) {
4922: assert_wait((event_t) dst_map,
4923: THREAD_INTERRUPTIBLE);
4924: vm_map_unlock(dst_map);
4925: thread_block((void (*)(void))0);
4926: goto StartAgain;
4927: }
4928: }
4929: vm_map_unlock(dst_map);
4930: return(KERN_NO_SPACE);
4931: }
4932:
4933: if ((next == vm_map_to_entry(dst_map)) ||
4934: (next->vme_start >= end)) {
4935: break;
4936: }
4937:
4938: last = next;
4939: start = last->vme_end;
4940: }
4941:
4942: /*
4943: * See whether we can avoid creating a new entry (and object) by
4944: * extending one of our neighbors. [So far, we only attempt to
4945: * extend from below.]
4946: *
4947: * The code path below here is a bit twisted. If any of the
4948: * extension checks fails, we branch to create_object. If
4949: * it all works, we fall out the bottom and goto insert_pages.
4950: */
4951: if (last == vm_map_to_entry(dst_map) ||
4952: last->vme_end != start ||
4953: last->is_shared != FALSE ||
4954: last->is_sub_map != FALSE ||
4955: last->inheritance != VM_INHERIT_DEFAULT ||
4956: last->protection != VM_PROT_DEFAULT ||
4957: last->max_protection != VM_PROT_ALL ||
4958: last->behavior != VM_BEHAVIOR_DEFAULT ||
4959: last->in_transition ||
4960: (must_wire ? (last->wired_count != 1 ||
4961: last->user_wired_count != 0) :
4962: (last->wired_count != 0))) {
4963: goto create_object;
4964: }
4965:
4966: /*
4967: * If this entry needs an object, make one.
4968: */
4969: if (last->object.vm_object == VM_OBJECT_NULL) {
4970: object = vm_object_allocate(
4971: (vm_size_t)(last->vme_end - last->vme_start + size));
4972: last->object.vm_object = object;
4973: last->offset = 0;
4974: }
4975: else {
4976: vm_offset_t prev_offset = last->offset;
4977: vm_size_t prev_size = start - last->vme_start;
4978: vm_size_t new_size;
4979:
4980: /*
4981: * This is basically vm_object_coalesce.
4982: */
4983:
4984: object = last->object.vm_object;
4985: vm_object_lock(object);
4986:
4987: /*
4988: * Try to collapse the object first
4989: */
4990: vm_object_collapse(object);
4991:
4992: /*
4993: * Can't coalesce if pages not mapped to
4994: * last may be in use anyway:
4995: * . more than one reference
4996: * . paged out
4997: * . shadows another object
4998: * . has a copy elsewhere
4999: * . paging references (pages might be in page-list)
5000: */
5001:
5002: if ((object->ref_count > 1) ||
5003: object->pager_created ||
5004: (object->shadow != VM_OBJECT_NULL) ||
5005: (object->copy != VM_OBJECT_NULL) ||
5006: (object->paging_in_progress != 0)) {
5007: vm_object_unlock(object);
5008: goto create_object;
5009: }
5010:
5011: /*
5012: * Extend the object if necessary. Don't have to call
5013: * vm_object_page_remove because the pages aren't mapped,
5014: * and vm_page_replace will free up any old ones it encounters.
5015: */
5016: new_size = prev_offset + prev_size + size;
5017: if (new_size > object->size) {
5018: #if MACH_PAGEMAP
5019: /*
5020: * We cannot extend an object that has existence info,
5021: * since the existence info might then fail to cover
5022: * the entire object.
5023: *
5024: * This assertion must be true because the object
5025: * has no pager, and we only create existence info
5026: * for objects with pagers.
5027: */
5028: assert(object->existence_map == VM_EXTERNAL_NULL);
5029: #endif /* MACH_PAGEMAP */
5030: object->size = new_size;
5031: }
5032: vm_object_unlock(object);
5033: }
5034:
5035: /*
5036: * Coalesced the two objects - can extend
5037: * the previous map entry to include the
5038: * new range.
5039: */
5040: dst_map->size += size;
5041: last->vme_end = end;
5042: UPDATE_FIRST_FREE(dst_map, dst_map->first_free);
5043:
5044: SAVE_HINT(dst_map, last);
5045:
5046: goto insert_pages;
5047:
5048: create_object:
5049:
5050: /*
5051: * Create object
5052: */
5053: object = vm_object_allocate(size);
5054:
5055: /*
5056: * Create entry
5057: */
5058: last = vm_map_entry_insert(dst_map, last, start, start + size,
5059: object, 0, FALSE, FALSE, TRUE,
5060: VM_PROT_DEFAULT, VM_PROT_ALL,
5061: VM_BEHAVIOR_DEFAULT,
5062: VM_INHERIT_DEFAULT, (must_wire ? 1 : 0));
5063:
5064: /*
5065: * Transfer pages into new object.
5066: * Scan page list in vm_map_copy.
5067: */
5068: insert_pages:
5069: dst_offset = copy->offset & PAGE_MASK;
5070: cont_invoked = FALSE;
5071: orig_copy = copy;
5072: last->in_transition = TRUE;
5073: old_last_offset = last->offset
5074: + (start - last->vme_start);
5075:
5076: aggressive_enter = (size <= vm_map_aggressive_enter_max);
5077:
5078: for (offset = 0; offset < size; offset += PAGE_SIZE) {
5079: m = *page_list;
5080: assert(m && !m->tabled);
5081:
5082: /*
5083: * Must clear busy bit in page before inserting it.
5084: * Ok to skip wakeup logic because nobody else
5085: * can possibly know about this page. Also set
5086: * dirty bit on the assumption that the page is
5087: * not a page of zeros.
5088: */
5089:
5090: m->busy = FALSE;
5091: m->dirty = TRUE;
5092: vm_object_lock(object);
5093: vm_page_lock_queues();
5094: vm_page_replace(m, object, old_last_offset + offset);
5095: if (must_wire) {
5096: vm_page_wire(m);
5097: } else if (aggressive_enter) {
5098: vm_page_activate(m);
5099: }
5100: vm_page_unlock_queues();
5101: vm_object_unlock(object);
5102:
5103: if (aggressive_enter || must_wire) {
5104: PMAP_ENTER(dst_map->pmap,
5105: last->vme_start + m->offset - last->offset,
5106: m, last->protection, must_wire);
5107: }
5108:
5109: *page_list++ = VM_PAGE_NULL;
5110: assert(copy != VM_MAP_COPY_NULL);
5111: assert(copy->type == VM_MAP_COPY_PAGE_LIST);
5112: if (--(copy->cpy_npages) == 0 &&
5113: vm_map_copy_has_cont(copy)) {
5114: vm_map_copy_t new_copy;
5115:
5116: /*
5117: * Ok to unlock map because entry is
5118: * marked in_transition.
5119: */
5120: cont_invoked = TRUE;
5121: vm_map_unlock(dst_map);
5122: vm_map_copy_invoke_cont(copy, &new_copy, &result);
5123:
5124: if (result == KERN_SUCCESS) {
5125:
5126: /*
5127: * If we got back a copy with real pages,
5128: * steal them now. Either all of the
5129: * pages in the list are tabled or none
5130: * of them are; mixtures are not possible.
5131: *
5132: * Save original copy for consume on
5133: * success logic at end of routine.
5134: */
5135: if (copy != orig_copy)
5136: vm_map_copy_discard(copy);
5137:
5138: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
5139: page_list = ©->cpy_page_list[0];
5140: if (!copy->cpy_page_loose)
5141: vm_map_copy_steal_pages(copy);
5142: }
5143: }
5144: else {
5145: /*
5146: * Continuation failed.
5147: */
5148: vm_map_lock(dst_map);
5149: goto error;
5150: }
5151:
5152: vm_map_lock(dst_map);
5153: }
5154: }
5155:
5156: *dst_addr = start + dst_offset;
5157:
5158: /*
5159: * Clear the in transition bits. This is easy if we
5160: * didn't have a continuation.
5161: */
5162: error:
5163: needs_wakeup = FALSE;
5164: if (!cont_invoked) {
5165: /*
5166: * We didn't unlock the map, so nobody could
5167: * be waiting.
5168: */
5169: last->in_transition = FALSE;
5170: assert(!last->needs_wakeup);
5171: }
5172: else {
5173: if (!vm_map_lookup_entry(dst_map, start, &entry))
5174: panic("vm_map_copyout_page_list: missing entry");
5175:
5176: /*
5177: * Clear transition bit for all constituent entries that
5178: * were in the original entry. Also check for waiters.
5179: */
5180: while ((entry != vm_map_to_entry(dst_map)) &&
5181: (entry->vme_start < end)) {
5182: assert(entry->in_transition);
5183: entry->in_transition = FALSE;
5184: if (entry->needs_wakeup) {
5185: entry->needs_wakeup = FALSE;
5186: needs_wakeup = TRUE;
5187: }
5188: entry = entry->vme_next;
5189: }
5190: }
5191:
5192: if (result != KERN_SUCCESS)
5193: (void) vm_map_delete(dst_map, start, end, VM_MAP_NO_FLAGS);
5194:
5195: vm_map_unlock(dst_map);
5196:
5197: if (needs_wakeup)
5198: vm_map_entry_wakeup(dst_map);
5199:
5200: /*
5201: * Consume on success logic.
5202: */
5203: if (copy != VM_MAP_COPY_NULL && copy != orig_copy) {
5204: zfree(vm_map_copy_zone, (vm_offset_t) copy);
5205: }
5206: if (result == KERN_SUCCESS) {
5207: assert(orig_copy != VM_MAP_COPY_NULL);
5208: assert(orig_copy->type == VM_MAP_COPY_PAGE_LIST);
5209: zfree(vm_map_copy_zone, (vm_offset_t) orig_copy);
5210: }
5211:
5212: return(result);
5213: }
5214:
5215: /*
5216: * Routine: vm_map_copyin
5217: *
5218: * Description:
5219: * Copy the specified region (src_addr, len) from the
5220: * source address space (src_map), possibly removing
5221: * the region from the source address space (src_destroy).
5222: *
5223: * Returns:
5224: * A vm_map_copy_t object (copy_result), suitable for
5225: * insertion into another address space (using vm_map_copyout),
5226: * copying over another address space region (using
5227: * vm_map_copy_overwrite). If the copy is unused, it
5228: * should be destroyed (using vm_map_copy_discard).
5229: *
5230: * In/out conditions:
5231: * The source map should not be locked on entry.
5232: */
5233: #if DIPC
5234: dstat_decl(unsigned int c_vmcc_volatile = 0;)
5235: dstat_decl(unsigned int c_vmcc_null_entry = 0;)
5236: dstat_decl(unsigned int c_vmcc_null_entry_continue = 0;)
5237: dstat_decl(unsigned int c_vmcc_src_destroy_opt = 0;)
5238: dstat_decl(unsigned int c_vmcc_wasnt_wired = 0;)
5239: dstat_decl(unsigned int c_vmcc_vocq = 0;)
5240: dstat_decl(unsigned int c_vmcc_vopp = 0;)
5241: dstat_decl(unsigned int c_vmcc_voept = 0;)
5242: dstat_decl(unsigned int c_vmcc_vocs = 0;)
5243: dstat_decl(unsigned int c_vmcc_vocstrat = 0;)
5244: dstat_decl(unsigned int c_vmcc_vmle = 0;)
5245: dstat_decl(unsigned int c_vmcc_reloop = 0;)
5246: dstat_decl(unsigned int c_vmcc_vmd = 0;)
5247: #endif /* DIPC */
5248:
5249:
5250: typedef struct submap_map {
5251: vm_map_t parent_map;
5252: vm_offset_t base_start;
5253: vm_offset_t base_end;
5254: struct submap_map *next;
5255: } submap_map_t;
5256:
5257: kern_return_t
5258: vm_map_copyin_common(
5259: vm_map_t src_map,
5260: vm_offset_t src_addr,
5261: vm_size_t len,
5262: boolean_t src_destroy,
5263: boolean_t src_volatile,
5264: vm_map_copy_t *copy_result, /* OUT */
5265: boolean_t use_maxprot)
5266: {
5267: vm_map_entry_t tmp_entry; /* Result of last map lookup --
5268: * in multi-level lookup, this
5269: * entry contains the actual
5270: * vm_object/offset.
5271: */
5272: register
5273: vm_map_entry_t new_entry = VM_MAP_ENTRY_NULL; /* Map entry for copy */
5274:
5275: vm_offset_t src_start; /* Start of current entry --
5276: * where copy is taking place now
5277: */
5278: vm_offset_t src_end; /* End of entire region to be
5279: * copied */
5280: vm_offset_t base_start; /* submap fields to save offsets */
5281: /* in original map */
5282: vm_offset_t base_end;
5283: vm_map_t base_map=src_map;
5284: vm_map_entry_t base_entry;
5285: boolean_t map_share=FALSE;
5286: submap_map_t *parent_maps = NULL;
5287:
5288: register
5289: vm_map_copy_t copy; /* Resulting copy */
5290:
5291: /*
5292: * Check for copies of zero bytes.
5293: */
5294:
5295: if (len == 0) {
5296: *copy_result = VM_MAP_COPY_NULL;
5297: return(KERN_SUCCESS);
5298: }
5299:
5300: /*
5301: * Compute start and end of region
5302: */
5303:
5304: src_start = trunc_page(src_addr);
5305: src_end = round_page(src_addr + len);
5306:
5307: #if DIPC
5308: XPR(XPR_VM_MAP,
5309: "vm_map_copyin_common map 0x%x addr 0x%x len 0x%x dest %d volatile %d\n",
5310: (natural_t)src_map, src_addr, len, src_destroy, src_volatile);
5311: #else /* DIPC */
5312: XPR(XPR_VM_MAP, "vm_map_copyin_common map 0x%x addr 0x%x len 0x%x dest %d\n",
5313: (natural_t)src_map, src_addr, len, src_destroy, 0);
5314: #endif /* DIPC */
5315:
5316: /*
5317: * Check that the end address doesn't overflow
5318: */
5319:
5320: if (src_end <= src_start)
5321: if ((src_end < src_start) || (src_start != 0))
5322: return(KERN_INVALID_ADDRESS);
5323:
5324: /*
5325: * Allocate a header element for the list.
5326: *
5327: * Use the start and end in the header to
5328: * remember the endpoints prior to rounding.
5329: */
5330:
5331: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
5332: vm_map_copy_first_entry(copy) =
5333: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
5334: copy->type = VM_MAP_COPY_ENTRY_LIST;
5335: copy->cpy_hdr.nentries = 0;
5336: copy->cpy_hdr.entries_pageable = TRUE;
5337:
5338: copy->offset = src_addr;
5339: copy->size = len;
5340:
5341: new_entry = vm_map_copy_entry_create(copy);
5342:
5343: #define RETURN(x) \
5344: MACRO_BEGIN \
5345: vm_map_unlock(src_map); \
5346: if (new_entry != VM_MAP_ENTRY_NULL) \
5347: vm_map_copy_entry_dispose(copy,new_entry); \
5348: vm_map_copy_discard(copy); \
5349: { \
5350: submap_map_t *ptr; \
5351: \
5352: for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) { \
5353: parent_maps=parent_maps->next; \
5354: kfree((vm_offset_t)ptr, sizeof(submap_map_t)); \
5355: } \
5356: } \
5357: MACRO_RETURN(x); \
5358: MACRO_END
5359:
5360: /*
5361: * Find the beginning of the region.
5362: */
5363:
5364: vm_map_lock(src_map);
5365:
5366: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
5367: RETURN(KERN_INVALID_ADDRESS);
5368: vm_map_clip_start(src_map, tmp_entry, src_start);
5369:
5370: /*
5371: * Go through entries until we get to the end.
5372: */
5373:
5374: while (TRUE) {
5375: register
5376: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
5377: vm_size_t src_size; /* Size of source
5378: * map entry (in both
5379: * maps)
5380: */
5381:
5382: register
5383: vm_object_t src_object; /* Object to copy */
5384: vm_offset_t src_offset;
5385:
5386: boolean_t src_needs_copy; /* Should source map
5387: * be made read-only
5388: * for copy-on-write?
5389: */
5390:
5391: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
5392:
5393: boolean_t was_wired; /* Was source wired? */
5394: vm_map_version_t version; /* Version before locks
5395: * dropped to make copy
5396: */
5397: kern_return_t result; /* Return value from
5398: * copy_strategically.
5399: */
5400: while(tmp_entry->is_sub_map) {
5401: vm_size_t submap_len;
5402: submap_map_t *ptr;
5403:
5404: ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
5405: ptr->next = parent_maps;
5406: parent_maps = ptr;
5407: ptr->parent_map = src_map;
5408: ptr->base_start = src_start;
5409: ptr->base_end = src_end;
5410: submap_len = tmp_entry->vme_end - tmp_entry->vme_start;
5411: if(submap_len > (src_end-src_start))
5412: submap_len = src_end-src_start;
5413: ptr->base_start += submap_len;
5414:
5415: src_start -= tmp_entry->vme_start;
5416: src_start += tmp_entry->offset;
5417: src_end = src_start + submap_len;
5418: src_map = tmp_entry->object.sub_map;
5419: vm_map_lock(src_map);
5420: vm_map_unlock(ptr->parent_map);
5421: if (!vm_map_lookup_entry(
5422: src_map, src_start, &tmp_entry))
5423: RETURN(KERN_INVALID_ADDRESS);
5424: map_share = TRUE;
5425: vm_map_clip_start(src_map, tmp_entry, src_start);
5426: src_entry = tmp_entry;
5427: }
5428: /*
5429: * Create a new address map entry to hold the result.
5430: * Fill in the fields from the appropriate source entries.
5431: * We must unlock the source map to do this if we need
5432: * to allocate a map entry.
5433: */
5434: if (new_entry == VM_MAP_ENTRY_NULL) {
5435: version.main_timestamp = src_map->timestamp;
5436: vm_map_unlock(src_map);
5437:
5438: dstat(++c_vmcc_null_entry);
5439: new_entry = vm_map_copy_entry_create(copy);
5440:
5441: vm_map_lock(src_map);
5442: if ((version.main_timestamp + 1) != src_map->timestamp) {
5443: if (!vm_map_lookup_entry(src_map, src_start,
5444: &tmp_entry)) {
5445: RETURN(KERN_INVALID_ADDRESS);
5446: }
5447: vm_map_clip_start(src_map, tmp_entry, src_start);
5448: dstat(++c_vmcc_null_entry_continue);
5449: continue; /* restart w/ new tmp_entry */
5450: }
5451: }
5452:
5453: /*
5454: * Verify that the region can be read.
5455: */
5456: if (((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE &&
5457: !use_maxprot) ||
5458: (src_entry->max_protection & VM_PROT_READ) == 0)
5459: RETURN(KERN_PROTECTION_FAILURE);
5460:
5461: /*
5462: * Clip against the endpoints of the entire region.
5463: */
5464:
5465: vm_map_clip_end(src_map, src_entry, src_end);
5466:
5467: src_size = src_entry->vme_end - src_start;
5468: src_object = src_entry->object.vm_object;
5469: src_offset = src_entry->offset;
5470: was_wired = (src_entry->wired_count != 0);
5471:
5472: vm_map_entry_copy(new_entry, src_entry);
5473:
5474: /*
5475: * Attempt non-blocking copy-on-write optimizations.
5476: */
5477:
5478: if (src_destroy &&
5479: (src_object == VM_OBJECT_NULL ||
5480: (src_object->internal && !src_object->true_share
5481: && !map_share))) {
5482: /*
5483: * If we are destroying the source, and the object
5484: * is internal, we can move the object reference
5485: * from the source to the copy. The copy is
5486: * copy-on-write only if the source is.
5487: * We make another reference to the object, because
5488: * destroying the source entry will deallocate it.
5489: */
5490: vm_object_reference(src_object);
5491:
5492: /*
5493: * Copy is always unwired. vm_map_copy_entry
5494: * set its wired count to zero.
5495: */
5496:
5497: dstat(++c_vmcc_src_destroy_opt);
5498: goto CopySuccessful;
5499: }
5500:
5501: #if DIPC
5502: /*
5503: * If the caller promises not to modify the data,
5504: * we don't have to apply copy-on-write processing
5505: * to it. This works best in the distributed case.
5506: * In the local case, we can wind up with fully
5507: * shared data between sender and receiver -- a
5508: * behavior that we aren't entirely sure we want
5509: * at this point. So for now, we'll only do this
5510: * on remote data transfers.
5511: */
5512: if (src_volatile == TRUE) {
5513: XPR(XPR_VM_MAP,
5514: "vmcc src_obj 0x%x ent 0x%x obj 0x%x VOLATILE\n",
5515: src_object, new_entry, new_entry->object.vm_object,
5516: 0, 0);
5517: assert(src_destroy == FALSE);
5518: dstat(++c_vmcc_volatile);
5519: vm_object_reference(src_object);
5520: goto CopySuccessful;
5521: }
5522: #endif /* DIPC */
5523:
5524: RestartCopy:
5525: XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",
5526: src_object, new_entry, new_entry->object.vm_object,
5527: was_wired, 0);
5528: dstat(!was_wired ? ++c_vmcc_wasnt_wired : 0);
5529: if (!was_wired &&
5530: vm_object_copy_quickly(
5531: &new_entry->object.vm_object,
5532: src_offset,
5533: src_size,
5534: &src_needs_copy,
5535: &new_entry_needs_copy)) {
5536:
5537: dstat(++c_vmcc_vocq);
5538: new_entry->needs_copy = new_entry_needs_copy;
5539:
5540: /*
5541: * Handle copy-on-write obligations
5542: */
5543:
5544: if (src_needs_copy && !tmp_entry->needs_copy) {
5545: if (tmp_entry->is_shared ||
5546: tmp_entry->object.vm_object->true_share ||
5547: map_share) {
5548: /* dec ref gained in copy_quickly */
5549: vm_object_lock(src_object);
5550: src_object->ref_count--;
5551: vm_object_res_deallocate(src_object);
5552: vm_object_unlock(src_object);
5553: new_entry->object.vm_object =
5554: vm_object_copy_delayed(
5555: src_object,
5556: src_offset,
5557: src_size);
5558: } else {
5559: dstat(++c_vmcc_vopp);
5560: vm_object_pmap_protect(
5561: src_object,
5562: src_offset,
5563: src_size,
5564: (src_entry->is_shared ?
5565: PMAP_NULL
5566: : src_map->pmap),
5567: src_entry->vme_start,
5568: src_entry->protection &
5569: ~VM_PROT_WRITE);
5570:
5571: tmp_entry->needs_copy = TRUE;
5572: }
5573: }
5574:
5575: /*
5576: * The map has never been unlocked, so it's safe
5577: * to move to the next entry rather than doing
5578: * another lookup.
5579: */
5580:
5581: goto CopySuccessful;
5582: }
5583:
5584: new_entry->needs_copy = FALSE;
5585:
5586: /*
5587: * Take an object reference, so that we may
5588: * release the map lock(s).
5589: */
5590:
5591: assert(src_object != VM_OBJECT_NULL);
5592: vm_object_reference(src_object);
5593:
5594: /*
5595: * Record the timestamp for later verification.
5596: * Unlock the map.
5597: */
5598:
5599: version.main_timestamp = src_map->timestamp;
5600: vm_map_unlock(src_map);
5601:
5602: /*
5603: * Perform the copy
5604: */
5605:
5606: if (was_wired) {
5607: dstat(++c_vmcc_vocs);
5608: vm_object_lock(src_object);
5609: result = vm_object_copy_slowly(
5610: src_object,
5611: src_offset,
5612: src_size,
5613: THREAD_UNINT,
5614: &new_entry->object.vm_object);
5615: new_entry->offset = 0;
5616: new_entry->needs_copy = FALSE;
5617: } else {
5618: dstat(++c_vmcc_vocstrat);
5619: result = vm_object_copy_strategically(src_object,
5620: src_offset,
5621: src_size,
5622: &new_entry->object.vm_object,
5623: &new_entry->offset,
5624: &new_entry_needs_copy);
5625:
5626: new_entry->needs_copy = new_entry_needs_copy;
5627:
5628: }
5629:
5630: if (result != KERN_SUCCESS &&
5631: result != KERN_MEMORY_RESTART_COPY) {
5632: vm_map_lock(src_map);
5633: RETURN(result);
5634: }
5635:
5636: /*
5637: * Throw away the extra reference
5638: */
5639:
5640: vm_object_deallocate(src_object);
5641:
5642: /*
5643: * Verify that the map has not substantially
5644: * changed while the copy was being made.
5645: */
5646:
5647: vm_map_lock(src_map); /* Increments timestamp once! */
5648:
5649: if ((version.main_timestamp + 1) == src_map->timestamp)
5650: goto VerificationSuccessful;
5651:
5652: /*
5653: * Simple version comparison failed.
5654: *
5655: * Retry the lookup and verify that the
5656: * same object/offset are still present.
5657: *
5658: * [Note: a memory manager that colludes with
5659: * the calling task can detect that we have
5660: * cheated. While the map was unlocked, the
5661: * mapping could have been changed and restored.]
5662: */
5663:
5664: dstat(++c_vmcc_vmle);
5665: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
5666: RETURN(KERN_INVALID_ADDRESS);
5667: }
5668:
5669: src_entry = tmp_entry;
5670: vm_map_clip_start(src_map, src_entry, src_start);
5671:
5672: if ((src_entry->protection & VM_PROT_READ == VM_PROT_NONE &&
5673: !use_maxprot) ||
5674: src_entry->max_protection & VM_PROT_READ == 0)
5675: goto VerificationFailed;
5676:
5677: if (src_entry->vme_end < new_entry->vme_end)
5678: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
5679:
5680: if ((src_entry->object.vm_object != src_object) ||
5681: (src_entry->offset != src_offset) ) {
5682:
5683: /*
5684: * Verification failed.
5685: *
5686: * Start over with this top-level entry.
5687: */
5688:
5689: VerificationFailed: ;
5690:
5691: vm_object_deallocate(new_entry->object.vm_object);
5692: tmp_entry = src_entry;
5693: continue;
5694: }
5695:
5696: /*
5697: * Verification succeeded.
5698: */
5699:
5700: VerificationSuccessful: ;
5701:
5702: if (result == KERN_MEMORY_RESTART_COPY)
5703: goto RestartCopy;
5704:
5705: /*
5706: * Copy succeeded.
5707: */
5708:
5709: CopySuccessful: ;
5710:
5711: /*
5712: * Link in the new copy entry.
5713: */
5714:
5715: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
5716: new_entry);
5717:
5718: /*
5719: * Determine whether the entire region
5720: * has been copied.
5721: */
5722: src_start = new_entry->vme_end;
5723: new_entry = VM_MAP_ENTRY_NULL;
5724: while ((src_start >= src_end) && (src_end != 0)) {
5725: if (src_map != base_map) {
5726: submap_map_t *ptr;
5727:
5728: ptr = parent_maps;
5729: assert(ptr != NULL);
5730: parent_maps = parent_maps->next;
5731: vm_map_lock(ptr->parent_map);
5732: vm_map_unlock(src_map);
5733: src_map = ptr->parent_map;
5734: src_start = ptr->base_start;
5735: src_end = ptr->base_end;
5736: if ((src_end > src_start) &&
5737: !vm_map_lookup_entry(
5738: src_map, src_start, &tmp_entry))
5739: RETURN(KERN_INVALID_ADDRESS);
5740: kfree((vm_offset_t)ptr, sizeof(submap_map_t));
5741: if(parent_maps == NULL)
5742: map_share = FALSE;
5743: src_entry = tmp_entry->vme_prev;
5744: } else
5745: break;
5746: }
5747: if ((src_start >= src_end) && (src_end != 0))
5748: break;
5749:
5750: /*
5751: * Verify that there are no gaps in the region
5752: */
5753:
5754: tmp_entry = src_entry->vme_next;
5755: dstat(++c_vmcc_reloop);
5756: if (tmp_entry->vme_start != src_start)
5757: RETURN(KERN_INVALID_ADDRESS);
5758: }
5759:
5760: /*
5761: * If the source should be destroyed, do it now, since the
5762: * copy was successful.
5763: */
5764: if (src_destroy) {
5765: dstat(++c_vmcc_vmd);
5766: (void) vm_map_delete(src_map,
5767: trunc_page(src_addr),
5768: src_end,
5769: (src_map == kernel_map) ?
5770: VM_MAP_REMOVE_KUNWIRE :
5771: VM_MAP_NO_FLAGS);
5772: }
5773:
5774: vm_map_unlock(src_map);
5775:
5776: *copy_result = copy;
5777: return(KERN_SUCCESS);
5778:
5779: #undef RETURN
5780: }
5781:
5782: /*
5783: * vm_map_copyin_object:
5784: *
5785: * Create a copy object from an object.
5786: * Our caller donates an object reference.
5787: */
5788:
5789: kern_return_t
5790: vm_map_copyin_object(
5791: vm_object_t object,
5792: vm_offset_t offset, /* offset of region in object */
5793: vm_size_t size, /* size of region in object */
5794: vm_map_copy_t *copy_result) /* OUT */
5795: {
5796: vm_map_copy_t copy; /* Resulting copy */
5797:
5798: /*
5799: * We drop the object into a special copy object
5800: * that contains the object directly.
5801: */
5802:
5803: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
5804: copy->type = VM_MAP_COPY_OBJECT;
5805: copy->cpy_object = object;
5806: copy->cpy_index = 0;
5807: copy->offset = offset;
5808: copy->size = size;
5809:
5810: *copy_result = copy;
5811: return(KERN_SUCCESS);
5812: }
5813:
5814: /*
5815: * vm_map_copyin_page_list_cont:
5816: *
5817: * Continuation routine for vm_map_copyin_page_list.
5818: *
5819: * If vm_map_copyin_page_list can't fit the entire vm range
5820: * into a single page list object, it creates a continuation.
5821: * When the target of the operation has used the pages in the
5822: * initial page list, it invokes the continuation, which calls
5823: * this routine. If an error happens, the continuation is aborted
5824: * (abort arg to this routine is TRUE). To avoid deadlocks, the
5825: * pages are discarded from the initial page list before invoking
5826: * the continuation.
5827: *
5828: * NOTE: This is not the same sort of continuation used by
5829: * the scheduler.
5830: */
5831:
5832: kern_return_t
5833: vm_map_copyin_page_list_cont(
5834: vm_map_copyin_args_t cont_args,
5835: vm_map_copy_t *copy_result) /* OUT */
5836: {
5837: kern_return_t result = KERN_SUCCESS;
5838: register boolean_t abort, src_destroy, src_destroy_only;
5839:
5840: /*
5841: * Check for cases that only require memory destruction.
5842: */
5843: abort = (copy_result == (vm_map_copy_t *) 0);
5844: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
5845: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
5846:
5847: if (abort || src_destroy_only) {
5848: if (src_destroy)
5849: result = vm_map_remove(cont_args->map,
5850: cont_args->destroy_addr,
5851: cont_args->destroy_addr + cont_args->destroy_len,
5852: VM_MAP_NO_FLAGS);
5853: if (!abort)
5854: *copy_result = VM_MAP_COPY_NULL;
5855: }
5856: else {
5857: result = vm_map_copyin_page_list(cont_args->map,
5858: cont_args->src_addr, cont_args->src_len,
5859: cont_args->options, copy_result, TRUE);
5860:
5861: if (src_destroy &&
5862: (cont_args->options & VM_MAP_COPYIN_OPT_STEAL_PAGES) &&
5863: vm_map_copy_has_cont(*copy_result)) {
5864: vm_map_copyin_args_t new_args;
5865: /*
5866: * Transfer old destroy info.
5867: */
5868: new_args = (vm_map_copyin_args_t)
5869: (*copy_result)->cpy_cont_args;
5870: new_args->destroy_addr = cont_args->destroy_addr;
5871: new_args->destroy_len = cont_args->destroy_len;
5872: }
5873: }
5874:
5875: vm_map_deallocate(cont_args->map);
5876: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
5877:
5878: return(result);
5879: }
5880:
5881: /*
5882: * vm_map_copyin_page_list:
5883: *
5884: * This is a variant of vm_map_copyin that copies in a list of pages.
5885: * If steal_pages is TRUE, the pages are only in the returned list.
5886: * If steal_pages is FALSE, the pages are busy and still in their
5887: * objects. A continuation may be returned if not all the pages fit:
5888: * the recipient of this copy_result must be prepared to deal with it.
5889: */
5890:
5891: kern_return_t
5892: vm_map_copyin_page_list(
5893: vm_map_t src_map,
5894: vm_offset_t src_addr,
5895: vm_size_t len,
5896: int options,
5897: vm_map_copy_t *copy_result, /* OUT */
5898: boolean_t is_cont)
5899: {
5900: vm_map_entry_t src_entry;
5901: vm_page_t m;
5902: vm_offset_t src_start;
5903: vm_offset_t src_end;
5904: vm_size_t src_size;
5905: register vm_object_t src_object;
5906: register vm_offset_t src_offset;
5907: vm_offset_t src_last_offset;
5908: register vm_map_copy_t copy; /* Resulting copy */
5909: kern_return_t result = KERN_SUCCESS;
5910: boolean_t need_map_lookup;
5911: vm_map_copyin_args_t cont_args;
5912: kern_return_t error_code;
5913: vm_prot_t prot;
5914: boolean_t wired;
5915: boolean_t no_zero_fill;
5916:
5917: submap_map_t *parent_maps = NULL;
5918: vm_map_t base_map = src_map;
5919:
5920: prot = (options & VM_MAP_COPYIN_OPT_VM_PROT);
5921: no_zero_fill = (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL);
5922:
5923: /*
5924: * If steal_pages is FALSE, this leaves busy pages in
5925: * the object. A continuation must be used if src_destroy
5926: * is true in this case (!steal_pages && src_destroy).
5927: *
5928: * XXX Still have a more general problem of what happens
5929: * XXX if the same page occurs twice in a list. Deadlock
5930: * XXX can happen if vm_fault_page was called. A
5931: * XXX possible solution is to use a continuation if vm_fault_page
5932: * XXX is called and we cross a map entry boundary.
5933: */
5934:
5935: /*
5936: * Check for copies of zero bytes.
5937: */
5938:
5939: if (len == 0) {
5940: *copy_result = VM_MAP_COPY_NULL;
5941: return(KERN_SUCCESS);
5942: }
5943:
5944: /*
5945: * Compute start and end of region
5946: */
5947:
5948: src_start = trunc_page(src_addr);
5949: src_end = round_page(src_addr + len);
5950:
5951: /*
5952: * If the region is not page aligned, override the no_zero_fill
5953: * argument.
5954: */
5955:
5956: if (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL) {
5957: if (!page_aligned(src_addr) || !page_aligned(src_addr +len))
5958: options &= ~VM_MAP_COPYIN_OPT_NO_ZERO_FILL;
5959: }
5960:
5961: /*
5962: * Check that the end address doesn't overflow
5963: */
5964:
5965: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
5966: return KERN_INVALID_ADDRESS;
5967: }
5968:
5969: /*
5970: * Allocate a header element for the page list.
5971: *
5972: * Record original offset and size, as caller may not
5973: * be page-aligned.
5974: */
5975:
5976: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
5977: copy->type = VM_MAP_COPY_PAGE_LIST;
5978: copy->cpy_npages = 0;
5979: copy->cpy_page_loose = FALSE;
5980: copy->offset = src_addr;
5981: copy->size = len;
5982: copy->cpy_cont = VM_MAP_COPY_CONT_NULL;
5983: copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL;
5984:
5985: /*
5986: * Find the beginning of the region.
5987: */
5988:
5989: do_map_lookup:
5990:
5991: vm_map_lock(src_map);
5992:
5993: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
5994: result = KERN_INVALID_ADDRESS;
5995: goto error;
5996: }
5997: need_map_lookup = FALSE;
5998:
5999: /*
6000: * Go through entries until we get to the end.
6001: */
6002:
6003: while (TRUE) {
6004: if ((src_entry->protection & prot) != prot) {
6005: result = KERN_PROTECTION_FAILURE;
6006: goto error;
6007: }
6008:
6009: /* translate down through submaps to find the target entry */
6010: while(src_entry->is_sub_map) {
6011: vm_size_t submap_len;
6012: submap_map_t *ptr;
6013:
6014: ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
6015: ptr->next = parent_maps;
6016: parent_maps = ptr;
6017: ptr->parent_map = src_map;
6018: ptr->base_start = src_start;
6019: ptr->base_end = src_end;
6020: submap_len = src_entry->vme_end - src_entry->vme_start;
6021: if(submap_len > (src_end-src_start))
6022: submap_len = src_end-src_start;
6023: ptr->base_start += submap_len;
6024:
6025: src_start -= src_entry->vme_start;
6026: src_start += src_entry->offset;
6027: src_end = src_start + submap_len;
6028: src_map = src_entry->object.sub_map;
6029: vm_map_lock(src_map);
6030: vm_map_unlock(ptr->parent_map);
6031: if (!vm_map_lookup_entry(
6032: src_map, src_start, &src_entry)) {
6033: result = KERN_INVALID_ADDRESS;
6034: goto error;
6035: }
6036: vm_map_clip_start(src_map, src_entry, src_start);
6037: }
6038:
6039: wired = (src_entry->wired_count != 0);
6040:
6041: if (src_end > src_entry->vme_end)
6042: src_size = src_entry->vme_end - src_start;
6043: else
6044: src_size = src_end - src_start;
6045:
6046: src_object = src_entry->object.vm_object;
6047:
6048: /*
6049: * If src_object is NULL, allocate it now;
6050: * we're going to fault on it shortly.
6051: */
6052: if (src_object == VM_OBJECT_NULL) {
6053: src_object = vm_object_allocate((vm_size_t)
6054: src_entry->vme_end -
6055: src_entry->vme_start);
6056: src_entry->object.vm_object = src_object;
6057: }
6058: else if (src_entry->needs_copy && (prot & VM_PROT_WRITE)) {
6059: vm_object_shadow(
6060: &src_entry->object.vm_object,
6061: &src_entry->offset,
6062: (vm_size_t) (src_entry->vme_end -
6063: src_entry->vme_start));
6064:
6065: src_entry->needs_copy = FALSE;
6066:
6067: /* reset src_object */
6068: src_object = src_entry->object.vm_object;
6069: }
6070:
6071: /*
6072: * calculate src_offset now, since vm_object_shadow
6073: * may have changed src_entry->offset.
6074: */
6075: src_offset = src_entry->offset + (src_start - src_entry->vme_start);
6076:
6077: /*
6078: * Iterate over pages. Fault in ones that aren't present.
6079: */
6080: src_last_offset = src_offset + src_size;
6081: for (; (src_offset < src_last_offset);
6082: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
6083:
6084: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
6085: vm_offset_t src_delta;
6086: make_continuation:
6087: /*
6088: * At this point we have the max number of
6089: * pages busy for this thread that we're
6090: * willing to allow. Stop here and record
6091: * arguments for the remainder. Note:
6092: * this means that this routine isn't atomic,
6093: * but that's the breaks. Note that only
6094: * the first vm_map_copy_t that comes back
6095: * from this routine has the right offset
6096: * and size; those from continuations are
6097: * page rounded, and short by the amount
6098: * already done.
6099: *
6100: * Reset src_end so the src_destroy
6101: * code at the bottom doesn't do
6102: * something stupid.
6103: */
6104:
6105: src_delta = src_end - src_start;
6106: while (src_map != base_map) {
6107: submap_map_t *ptr;
6108:
6109: if(!need_map_lookup) {
6110: vm_map_unlock(src_map);
6111: }
6112: ptr = parent_maps;
6113: assert(ptr != NULL);
6114: parent_maps = parent_maps->next;
6115: src_map = ptr->parent_map;
6116: src_start = ptr->base_start - src_delta;
6117: src_delta = ptr->base_end - src_start;
6118: kfree((vm_offset_t)ptr, sizeof(submap_map_t));
6119:
6120: need_map_lookup = TRUE;
6121: }
6122: src_end = src_start;
6123:
6124:
6125: cont_args = (vm_map_copyin_args_t)
6126: kalloc(sizeof(vm_map_copyin_args_data_t));
6127: cont_args->map = src_map;
6128: vm_map_reference(src_map);
6129: cont_args->src_addr = src_start;
6130: cont_args->src_len = len - (src_start - src_addr);
6131: if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
6132: cont_args->destroy_addr = cont_args->src_addr;
6133: cont_args->destroy_len = cont_args->src_len;
6134: } else {
6135: cont_args->destroy_addr = (vm_offset_t) 0;
6136: cont_args->destroy_len = (vm_offset_t) 0;
6137: }
6138: cont_args->options = options;
6139:
6140: copy->cpy_cont_args = cont_args;
6141: copy->cpy_cont = vm_map_copyin_page_list_cont;
6142:
6143: break;
6144: }
6145:
6146: /*
6147: * Try to find the page of data. Have to
6148: * fault it in if there's no page, or something
6149: * going on with the page, or the object has
6150: * a copy object.
6151: */
6152: vm_object_lock(src_object);
6153: vm_object_paging_begin(src_object);
6154: if (((m = vm_page_lookup(src_object, src_offset)) !=
6155: VM_PAGE_NULL) &&
6156: !m->busy && !m->fictitious && !m->unusual &&
6157: ((prot & VM_PROT_WRITE) == 0 ||
6158: (m->object->copy == VM_OBJECT_NULL))) {
6159:
6160: if (!m->absent &&
6161: !(options & VM_MAP_COPYIN_OPT_STEAL_PAGES)) {
6162:
6163: /*
6164: * The page is present and will not be
6165: * replaced, prep it. Thus allowing
6166: * mutiple access on this page
6167: */
6168: kern_return_t kr;
6169:
6170: kr = vm_page_prep(m);
6171: assert(kr == KERN_SUCCESS);
6172: kr = vm_page_pin(m);
6173: assert(kr == KERN_SUCCESS);
6174: } else {
6175: /*
6176: * This is the page. Mark it busy
6177: * and keep the paging reference on
6178: * the object whilst we do our thing.
6179: */
6180:
6181: m->busy = TRUE;
6182: }
6183: } else {
6184: vm_prot_t result_prot;
6185: vm_page_t top_page;
6186: kern_return_t kr;
6187: boolean_t data_supply;
6188:
6189: /*
6190: * Have to fault the page in; must
6191: * unlock the map to do so. While
6192: * the map is unlocked, anything
6193: * can happen, we must lookup the
6194: * map entry before continuing.
6195: */
6196: vm_map_unlock(src_map);
6197: need_map_lookup = TRUE;
6198: data_supply = src_object->silent_overwrite &&
6199: (prot & VM_PROT_WRITE) &&
6200: src_start >= src_addr &&
6201: src_start + PAGE_SIZE <=
6202: src_addr + len;
6203:
6204: retry:
6205: result_prot = prot;
6206:
6207: XPR(XPR_VM_FAULT,
6208: "vm_map_copyin_page_list -> vm_fault_page\n",
6209: 0,0,0,0,0);
6210: kr = vm_fault_page(src_object, src_offset,
6211: prot, FALSE, THREAD_UNINT,
6212: src_entry->offset,
6213: src_entry->offset +
6214: (src_entry->vme_end -
6215: src_entry->vme_start),
6216: VM_BEHAVIOR_SEQUENTIAL,
6217: &result_prot, &m, &top_page,
6218: (int *)0,
6219: &error_code,
6220: options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL,
6221: data_supply);
6222: /*
6223: * Cope with what happened.
6224: */
6225: switch (kr) {
6226: case VM_FAULT_SUCCESS:
6227:
6228: /*
6229: * If we lost write access,
6230: * try again.
6231: */
6232: if ((prot & VM_PROT_WRITE) &&
6233: !(result_prot & VM_PROT_WRITE)) {
6234: vm_object_lock(src_object);
6235: vm_object_paging_begin(src_object);
6236: goto retry;
6237: }
6238: break;
6239: case VM_FAULT_INTERRUPTED: /* ??? */
6240: case VM_FAULT_RETRY:
6241: vm_object_lock(src_object);
6242: vm_object_paging_begin(src_object);
6243: goto retry;
6244: case VM_FAULT_MEMORY_SHORTAGE:
6245: VM_PAGE_WAIT();
6246: vm_object_lock(src_object);
6247: vm_object_paging_begin(src_object);
6248: goto retry;
6249: case VM_FAULT_FICTITIOUS_SHORTAGE:
6250: vm_page_more_fictitious();
6251: vm_object_lock(src_object);
6252: vm_object_paging_begin(src_object);
6253: goto retry;
6254: case VM_FAULT_MEMORY_ERROR:
6255: /*
6256: * Something broke. If this
6257: * is a continuation, return
6258: * a partial result if possible,
6259: * else fail the whole thing.
6260: * In the continuation case, the
6261: * next continuation call will
6262: * get this error if it persists.
6263: */
6264: vm_map_lock(src_map);
6265: if (is_cont &&
6266: copy->cpy_npages != 0)
6267: goto make_continuation;
6268:
6269: result = error_code ? error_code : KERN_MEMORY_ERROR;
6270: goto error;
6271: }
6272:
6273: if (top_page != VM_PAGE_NULL) {
6274: vm_object_lock(src_object);
6275: VM_PAGE_FREE(top_page);
6276: vm_object_paging_end(src_object);
6277: vm_object_unlock(src_object);
6278: }
6279:
6280: }
6281:
6282: /*
6283: * The page is busy, its object is locked, and
6284: * we have a paging reference on it. Either
6285: * the map is locked, or need_map_lookup is
6286: * TRUE.
6287: */
6288:
6289: /*
6290: * Put the page in the page list.
6291: */
6292: copy->cpy_page_list[copy->cpy_npages++] = m;
6293: vm_object_unlock(m->object);
6294:
6295: /*
6296: * Pmap enter support. Only used for
6297: * device I/O for colocated server.
6298: *
6299: * WARNING: This code assumes that this
6300: * option is only used for well behaved
6301: * memory. If the mapping has changed,
6302: * the following code will make mistakes.
6303: *
6304: * XXXO probably ought to do pmap_extract first,
6305: * XXXO to avoid needless pmap_enter, but this
6306: * XXXO can't detect protection mismatch??
6307: */
6308:
6309: if (options & VM_MAP_COPYIN_OPT_PMAP_ENTER) {
6310: /*
6311: * XXX Only used on kernel map.
6312: * XXX Must not remove VM_PROT_WRITE on
6313: * XXX an I/O only requiring VM_PROT_READ
6314: * XXX as another I/O may be active on same page
6315: * XXX assume that if mapping exists, it must
6316: * XXX have the equivalent of at least VM_PROT_READ,
6317: * XXX but don't assume it has VM_PROT_WRITE as the
6318: * XXX pmap might not all the rights of the object
6319: */
6320: assert(vm_map_pmap(src_map) == kernel_pmap);
6321:
6322: if ((prot & VM_PROT_WRITE) ||
6323: (pmap_extract(vm_map_pmap(src_map),
6324: src_start) != m->phys_addr))
6325:
6326: PMAP_ENTER(vm_map_pmap(src_map), src_start,
6327: m, prot, wired);
6328: }
6329: if(need_map_lookup) {
6330: need_map_lookup = FALSE;
6331: vm_map_lock(src_map);
6332: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
6333: result = KERN_INVALID_ADDRESS;
6334: goto error;
6335: }
6336: }
6337: }
6338:
6339: /*
6340: * Verify that there are no gaps in the region
6341: */
6342: src_start = src_entry->vme_end;
6343: if (src_start < src_end) {
6344: src_entry = src_entry->vme_next;
6345: if (need_map_lookup) {
6346: need_map_lookup = FALSE;
6347: vm_map_lock(src_map);
6348: if(!vm_map_lookup_entry(src_map,
6349: src_start, &src_entry)) {
6350: result = KERN_INVALID_ADDRESS;
6351: goto error;
6352: }
6353: } else if (src_entry->vme_start != src_start) {
6354: result = KERN_INVALID_ADDRESS;
6355: goto error;
6356: }
6357: }
6358:
6359: /*
6360: * DETERMINE whether the entire region
6361: * has been copied.
6362: */
6363:
6364: while ((src_start >= src_end) && (src_end != 0)) {
6365: if (src_map != base_map) {
6366: submap_map_t *ptr;
6367:
6368: ptr = parent_maps;
6369: assert(ptr != NULL);
6370: parent_maps = parent_maps->next;
6371: src_start = ptr->base_start;
6372: src_end = ptr->base_end;
6373: if(need_map_lookup) {
6374: need_map_lookup = FALSE;
6375: }
6376: else {
6377: vm_map_unlock(src_map);
6378: }
6379: src_map = ptr->parent_map;
6380: vm_map_lock(src_map);
6381: if((src_start < src_end) &&
6382: (!vm_map_lookup_entry(ptr->parent_map,
6383: src_start, &src_entry))) {
6384: result = KERN_INVALID_ADDRESS;
6385: kfree((vm_offset_t)ptr, sizeof(submap_map_t));
6386: goto error;
6387: }
6388: kfree((vm_offset_t)ptr, sizeof(submap_map_t));
6389: } else
6390: break;
6391: }
6392: if ((src_start >= src_end) && (src_end != 0)) {
6393: if (need_map_lookup)
6394: vm_map_lock(src_map);
6395: break;
6396: }
6397:
6398: }
6399:
6400: /*
6401: * If steal_pages is true, make sure all
6402: * pages in the copy are not in any object
6403: * We try to remove them from the original
6404: * object, but we may have to copy them.
6405: *
6406: * At this point every page in the list is busy
6407: * and holds a paging reference to its object.
6408: * When we're done stealing, every page is busy,
6409: * and in no object (m->tabled == FALSE).
6410: */
6411: src_start = trunc_page(src_addr);
6412: if (options & VM_MAP_COPYIN_OPT_STEAL_PAGES) {
6413: register int i;
6414: vm_offset_t page_vaddr;
6415: vm_offset_t unwire_end;
6416: vm_offset_t map_entry_end;
6417: boolean_t share_map = FALSE;
6418:
6419: unwire_end = src_start;
6420: map_entry_end = src_start;
6421: for (i = 0; i < copy->cpy_npages; i++) {
6422:
6423:
6424: /*
6425: * Remove the page from its object if it
6426: * can be stolen. It can be stolen if:
6427: *
6428: * (1) The source is being destroyed,
6429: * the object is internal (hence
6430: * temporary), and not shared.
6431: * (2) The page is not precious.
6432: *
6433: * The not shared check consists of two
6434: * parts: (a) there are no objects that
6435: * shadow this object. (b) it is not the
6436: * object in any shared map entries (i.e.,
6437: * use_shared_copy is not set).
6438: *
6439: * The first check (a) means that we can't
6440: * steal pages from objects that are not
6441: * at the top of their shadow chains. This
6442: * should not be a frequent occurrence.
6443: *
6444: * Stealing wired pages requires telling the
6445: * pmap module to let go of them.
6446: *
6447: * NOTE: stealing clean pages from objects
6448: * whose mappings survive requires a call to
6449: * the pmap module. Maybe later.
6450: */
6451: m = copy->cpy_page_list[i];
6452: src_object = m->object;
6453: vm_object_lock(src_object);
6454:
6455: page_vaddr = src_start + (i * PAGE_SIZE);
6456: if(page_vaddr > map_entry_end) {
6457: if (!vm_map_lookup_entry(src_map, page_vaddr, &src_entry))
6458: share_map = TRUE;
6459: else if (src_entry->is_sub_map) {
6460: map_entry_end = src_entry->vme_end;
6461: share_map = TRUE;
6462: } else {
6463: map_entry_end = src_entry->vme_end;
6464: share_map = FALSE;
6465: }
6466: }
6467:
6468:
6469: if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
6470: src_object->internal &&
6471: !src_object->true_share &&
6472: (!src_object->shadowed) &&
6473: (src_object->copy_strategy ==
6474: MEMORY_OBJECT_COPY_SYMMETRIC) &&
6475: !m->precious &&
6476: !share_map) {
6477:
6478: if (m->wire_count > 0) {
6479:
6480: assert(m->wire_count == 1);
6481: /*
6482: * In order to steal a wired
6483: * page, we have to unwire it
6484: * first. We do this inline
6485: * here because we have the page.
6486: *
6487: * Step 1: Unwire the map entry.
6488: * Also tell the pmap module
6489: * that this piece of the
6490: * pmap is pageable.
6491: */
6492: vm_object_unlock(src_object);
6493: if (page_vaddr >= unwire_end) {
6494: if (!vm_map_lookup_entry(src_map,
6495: page_vaddr, &src_entry))
6496: panic("vm_map_copyin_page_list: missing wired map entry");
6497:
6498: vm_map_clip_start(src_map, src_entry,
6499: page_vaddr);
6500: vm_map_clip_end(src_map, src_entry,
6501: src_start + src_size);
6502:
6503: /* revisit why this assert fails CDY
6504: assert(src_entry->wired_count > 0);
6505: */
6506: src_entry->wired_count = 0;
6507: src_entry->user_wired_count = 0;
6508: unwire_end = src_entry->vme_end;
6509: pmap_pageable(vm_map_pmap(src_map),
6510: page_vaddr, unwire_end, TRUE);
6511: }
6512:
6513: /*
6514: * Step 2: Unwire the page.
6515: * pmap_remove handles this for us.
6516: */
6517: vm_object_lock(src_object);
6518: }
6519:
6520: /*
6521: * Don't need to remove the mapping;
6522: * vm_map_delete will handle it.
6523: *
6524: * Steal the page. Setting the wire count
6525: * to zero is vm_page_unwire without
6526: * activating the page.
6527: */
6528: vm_page_lock_queues();
6529: vm_page_remove(m);
6530: if (m->wire_count > 0) {
6531: m->wire_count = 0;
6532: vm_page_wire_count--;
6533: } else {
6534: VM_PAGE_QUEUES_REMOVE(m);
6535: }
6536: vm_page_unlock_queues();
6537: } else {
6538: /*
6539: * Have to copy this page. Have to
6540: * unlock the map while copying,
6541: * hence no further page stealing.
6542: * Hence just copy all the pages.
6543: * Unlock the map while copying;
6544: * This means no further page stealing.
6545: */
6546: vm_object_unlock(src_object);
6547: vm_map_unlock(src_map);
6548: vm_map_copy_steal_pages(copy);
6549: vm_map_lock(src_map);
6550: break;
6551: }
6552:
6553: vm_object_paging_end(src_object);
6554: vm_object_unlock(src_object);
6555: }
6556:
6557: copy->cpy_page_loose = TRUE;
6558:
6559: /*
6560: * If the source should be destroyed, do it now, since the
6561: * copy was successful.
6562: */
6563:
6564: if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
6565: (void) vm_map_delete(src_map, src_start,
6566: src_end, VM_MAP_NO_FLAGS);
6567: }
6568: } else {
6569: /*
6570: * Not stealing pages leaves busy or prepped pages in the map.
6571: * This will cause source destruction to hang. Use
6572: * a continuation to prevent this.
6573: */
6574: if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
6575: !vm_map_copy_has_cont(copy)) {
6576: cont_args = (vm_map_copyin_args_t)
6577: kalloc(sizeof(vm_map_copyin_args_data_t));
6578: vm_map_reference(src_map);
6579: cont_args->map = src_map;
6580: cont_args->src_addr = (vm_offset_t) 0;
6581: cont_args->src_len = (vm_size_t) 0;
6582: cont_args->destroy_addr = src_start;
6583: cont_args->destroy_len = src_end - src_start;
6584: cont_args->options = options;
6585:
6586: copy->cpy_cont_args = cont_args;
6587: copy->cpy_cont = vm_map_copyin_page_list_cont;
6588: }
6589: }
6590:
6591: vm_map_unlock(src_map);
6592:
6593: *copy_result = copy;
6594: return(result);
6595:
6596: error:
6597: {
6598: submap_map_t *ptr;
6599:
6600: vm_map_unlock(src_map);
6601: vm_map_copy_discard(copy);
6602:
6603: for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) {
6604: parent_maps=parent_maps->next;
6605: kfree((vm_offset_t)ptr, sizeof(submap_map_t));
6606: }
6607: return(result);
6608: }
6609: }
6610:
6611: void
6612: vm_map_fork_share(
6613: vm_map_t old_map,
6614: vm_map_entry_t old_entry,
6615: vm_map_t new_map)
6616: {
6617: vm_object_t object;
6618: vm_map_entry_t new_entry;
6619:
6620: /*
6621: * New sharing code. New map entry
6622: * references original object. Internal
6623: * objects use asynchronous copy algorithm for
6624: * future copies. First make sure we have
6625: * the right object. If we need a shadow,
6626: * or someone else already has one, then
6627: * make a new shadow and share it.
6628: */
6629:
6630: object = old_entry->object.vm_object;
6631: if (old_entry->is_sub_map) {
6632: assert(old_entry->wired_count == 0);
6633: } else if (object == VM_OBJECT_NULL) {
6634: object = vm_object_allocate((vm_size_t)(old_entry->vme_end -
6635: old_entry->vme_start));
6636: old_entry->offset = 0;
6637: old_entry->object.vm_object = object;
6638: assert(!old_entry->needs_copy);
6639: } else if (object->copy_strategy !=
6640: MEMORY_OBJECT_COPY_SYMMETRIC) {
6641:
6642: /*
6643: * We are already using an asymmetric
6644: * copy, and therefore we already have
6645: * the right object.
6646: */
6647:
6648: assert(! old_entry->needs_copy);
6649: }
6650: else if (old_entry->needs_copy || /* case 1 */
6651: object->shadowed || /* case 2 */
6652: (!object->true_share && /* case 3 */
6653: !old_entry->is_shared &&
6654: (object->size >
6655: (vm_size_t)(old_entry->vme_end -
6656: old_entry->vme_start)))) {
6657:
6658: /*
6659: * We need to create a shadow.
6660: * There are three cases here.
6661: * In the first case, we need to
6662: * complete a deferred symmetrical
6663: * copy that we participated in.
6664: * In the second and third cases,
6665: * we need to create the shadow so
6666: * that changes that we make to the
6667: * object do not interfere with
6668: * any symmetrical copies which
6669: * have occured (case 2) or which
6670: * might occur (case 3).
6671: *
6672: * The first case is when we had
6673: * deferred shadow object creation
6674: * via the entry->needs_copy mechanism.
6675: * This mechanism only works when
6676: * only one entry points to the source
6677: * object, and we are about to create
6678: * a second entry pointing to the
6679: * same object. The problem is that
6680: * there is no way of mapping from
6681: * an object to the entries pointing
6682: * to it. (Deferred shadow creation
6683: * works with one entry because occurs
6684: * at fault time, and we walk from the
6685: * entry to the object when handling
6686: * the fault.)
6687: *
6688: * The second case is when the object
6689: * to be shared has already been copied
6690: * with a symmetric copy, but we point
6691: * directly to the object without
6692: * needs_copy set in our entry. (This
6693: * can happen because different ranges
6694: * of an object can be pointed to by
6695: * different entries. In particular,
6696: * a single entry pointing to an object
6697: * can be split by a call to vm_inherit,
6698: * which, combined with task_create, can
6699: * result in the different entries
6700: * having different needs_copy values.)
6701: * The shadowed flag in the object allows
6702: * us to detect this case. The problem
6703: * with this case is that if this object
6704: * has or will have shadows, then we
6705: * must not perform an asymmetric copy
6706: * of this object, since such a copy
6707: * allows the object to be changed, which
6708: * will break the previous symmetrical
6709: * copies (which rely upon the object
6710: * not changing). In a sense, the shadowed
6711: * flag says "don't change this object".
6712: * We fix this by creating a shadow
6713: * object for this object, and sharing
6714: * that. This works because we are free
6715: * to change the shadow object (and thus
6716: * to use an asymmetric copy strategy);
6717: * this is also semantically correct,
6718: * since this object is temporary, and
6719: * therefore a copy of the object is
6720: * as good as the object itself. (This
6721: * is not true for permanent objects,
6722: * since the pager needs to see changes,
6723: * which won't happen if the changes
6724: * are made to a copy.)
6725: *
6726: * The third case is when the object
6727: * to be shared has parts sticking
6728: * outside of the entry we're working
6729: * with, and thus may in the future
6730: * be subject to a symmetrical copy.
6731: * (This is a preemptive version of
6732: * case 2.)
6733: */
6734:
6735: assert(!(object->shadowed && old_entry->is_shared));
6736: vm_object_shadow(&old_entry->object.vm_object,
6737: &old_entry->offset,
6738: (vm_size_t) (old_entry->vme_end -
6739: old_entry->vme_start));
6740:
6741: /*
6742: * If we're making a shadow for other than
6743: * copy on write reasons, then we have
6744: * to remove write permission.
6745: */
6746:
6747: if (!old_entry->needs_copy &&
6748: (old_entry->protection & VM_PROT_WRITE)) {
6749: pmap_protect(vm_map_pmap(old_map),
6750: old_entry->vme_start,
6751: old_entry->vme_end,
6752: old_entry->protection & ~VM_PROT_WRITE);
6753: }
6754: old_entry->needs_copy = FALSE;
6755: object = old_entry->object.vm_object;
6756: }
6757:
6758: /*
6759: * If object was using a symmetric copy strategy,
6760: * change its copy strategy to the default
6761: * asymmetric copy strategy, which is copy_delay
6762: * in the non-norma case and copy_call in the
6763: * norma case. Bump the reference count for the
6764: * new entry.
6765: */
6766:
6767: if(old_entry->is_sub_map) {
6768: vm_map_lock(old_entry->object.sub_map);
6769: vm_map_reference(old_entry->object.sub_map);
6770: vm_map_unlock(old_entry->object.sub_map);
6771: } else {
6772: vm_object_lock(object);
6773: object->ref_count++;
6774: vm_object_res_reference(object);
6775: if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
6776: object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
6777: }
6778: vm_object_unlock(object);
6779: }
6780:
6781: /*
6782: * Clone the entry, using object ref from above.
6783: * Mark both entries as shared.
6784: */
6785:
6786: new_entry = vm_map_entry_create(new_map);
6787: vm_map_entry_copy(new_entry, old_entry);
6788: old_entry->is_shared = TRUE;
6789: new_entry->is_shared = TRUE;
6790:
6791: /*
6792: * Insert the entry into the new map -- we
6793: * know we're inserting at the end of the new
6794: * map.
6795: */
6796:
6797: vm_map_entry_link(new_map, vm_map_last_entry(new_map), new_entry);
6798:
6799: /*
6800: * Update the physical map
6801: */
6802:
6803: if (old_entry->is_sub_map) {
6804: /* Bill Angell pmap support goes here */
6805: } else {
6806: pmap_copy(new_map->pmap, old_map->pmap, new_entry->vme_start,
6807: old_entry->vme_end - old_entry->vme_start,
6808: old_entry->vme_start);
6809: }
6810: }
6811:
6812: boolean_t
6813: vm_map_fork_copy(
6814: vm_map_t old_map,
6815: vm_map_entry_t *old_entry_p,
6816: vm_map_t new_map)
6817: {
6818: vm_map_entry_t old_entry = *old_entry_p;
6819: vm_size_t entry_size = old_entry->vme_end - old_entry->vme_start;
6820: vm_offset_t start = old_entry->vme_start;
6821: vm_map_copy_t copy;
6822: vm_map_entry_t last = vm_map_last_entry(new_map);
6823:
6824: vm_map_unlock(old_map);
6825: /*
6826: * Use maxprot version of copyin because we
6827: * care about whether this memory can ever
6828: * be accessed, not just whether it's accessible
6829: * right now.
6830: */
6831: if (vm_map_copyin_maxprot(old_map, start, entry_size, FALSE, ©)
6832: != KERN_SUCCESS) {
6833: /*
6834: * The map might have changed while it
6835: * was unlocked, check it again. Skip
6836: * any blank space or permanently
6837: * unreadable region.
6838: */
6839: vm_map_lock(old_map);
6840: if (!vm_map_lookup_entry(old_map, start, &last) ||
6841: last->max_protection & VM_PROT_READ ==
6842: VM_PROT_NONE) {
6843: last = last->vme_next;
6844: }
6845: *old_entry_p = last;
6846:
6847: /*
6848: * XXX For some error returns, want to
6849: * XXX skip to the next element. Note
6850: * that INVALID_ADDRESS and
6851: * PROTECTION_FAILURE are handled above.
6852: */
6853:
6854: return FALSE;
6855: }
6856:
6857: /*
6858: * Insert the copy into the new map
6859: */
6860:
6861: vm_map_copy_insert(new_map, last, copy);
6862:
6863: /*
6864: * Pick up the traversal at the end of
6865: * the copied region.
6866: */
6867:
6868: vm_map_lock(old_map);
6869: start += entry_size;
6870: if (! vm_map_lookup_entry(old_map, start, &last)) {
6871: last = last->vme_next;
6872: } else {
6873: vm_map_clip_start(old_map, last, start);
6874: }
6875: *old_entry_p = last;
6876:
6877: return TRUE;
6878: }
6879:
6880: /*
6881: * vm_map_fork:
6882: *
6883: * Create and return a new map based on the old
6884: * map, according to the inheritance values on the
6885: * regions in that map.
6886: *
6887: * The source map must not be locked.
6888: */
6889: vm_map_t
6890: vm_map_fork(
6891: vm_map_t old_map)
6892: {
6893: pmap_t new_pmap = pmap_create((vm_size_t) 0);
6894: vm_map_t new_map;
6895: vm_map_entry_t old_entry;
6896: vm_size_t new_size = 0, entry_size;
6897: vm_map_entry_t new_entry;
6898: boolean_t src_needs_copy;
6899: boolean_t new_entry_needs_copy;
6900:
6901: vm_map_reference_swap(old_map);
6902: vm_map_lock(old_map);
6903:
6904: new_map = vm_map_create(new_pmap,
6905: old_map->min_offset,
6906: old_map->max_offset,
6907: old_map->hdr.entries_pageable);
6908:
6909: for (
6910: old_entry = vm_map_first_entry(old_map);
6911: old_entry != vm_map_to_entry(old_map);
6912: ) {
6913:
6914: entry_size = old_entry->vme_end - old_entry->vme_start;
6915:
6916: switch (old_entry->inheritance) {
6917: case VM_INHERIT_NONE:
6918: break;
6919:
6920: case VM_INHERIT_SHARE:
6921: vm_map_fork_share(old_map, old_entry, new_map);
6922: new_size += entry_size;
6923: break;
6924:
6925: case VM_INHERIT_COPY:
6926:
6927: /*
6928: * Inline the copy_quickly case;
6929: * upon failure, fall back on call
6930: * to vm_map_fork_copy.
6931: */
6932:
6933: if(old_entry->is_sub_map)
6934: break;
6935: if (old_entry->wired_count != 0) {
6936: goto slow_vm_map_fork_copy;
6937: }
6938:
6939: new_entry = vm_map_entry_create(new_map);
6940: vm_map_entry_copy(new_entry, old_entry);
6941:
6942: if (! vm_object_copy_quickly(
6943: &new_entry->object.vm_object,
6944: old_entry->offset,
6945: (old_entry->vme_end -
6946: old_entry->vme_start),
6947: &src_needs_copy,
6948: &new_entry_needs_copy)) {
6949: vm_map_entry_dispose(new_map, new_entry);
6950: goto slow_vm_map_fork_copy;
6951: }
6952:
6953: /*
6954: * Handle copy-on-write obligations
6955: */
6956:
6957: if (src_needs_copy && !old_entry->needs_copy) {
6958: vm_object_pmap_protect(
6959: old_entry->object.vm_object,
6960: old_entry->offset,
6961: (old_entry->vme_end -
6962: old_entry->vme_start),
6963: (old_entry->is_shared ? PMAP_NULL :
6964: old_map->pmap),
6965: old_entry->vme_start,
6966: old_entry->protection & ~VM_PROT_WRITE);
6967:
6968: old_entry->needs_copy = TRUE;
6969: }
6970: new_entry->needs_copy = new_entry_needs_copy;
6971:
6972: /*
6973: * Insert the entry at the end
6974: * of the map.
6975: */
6976:
6977: vm_map_entry_link(new_map, vm_map_last_entry(new_map),
6978: new_entry);
6979: new_size += entry_size;
6980: break;
6981:
6982: slow_vm_map_fork_copy:
6983: if (vm_map_fork_copy(old_map, &old_entry, new_map)) {
6984: new_size += entry_size;
6985: }
6986: continue;
6987: }
6988: old_entry = old_entry->vme_next;
6989: }
6990:
6991: new_map->size = new_size;
6992: vm_map_unlock(old_map);
6993: vm_map_deallocate(old_map);
6994:
6995: return(new_map);
6996: }
6997:
6998: /*
6999: * vm_map_lookup_locked:
7000: *
7001: * Finds the VM object, offset, and
7002: * protection for a given virtual address in the
7003: * specified map, assuming a page fault of the
7004: * type specified.
7005: *
7006: * Returns the (object, offset, protection) for
7007: * this address, whether it is wired down, and whether
7008: * this map has the only reference to the data in question.
7009: * In order to later verify this lookup, a "version"
7010: * is returned.
7011: *
7012: * The map MUST be locked by the caller and WILL be
7013: * locked on exit. In order to guarantee the
7014: * existence of the returned object, it is returned
7015: * locked.
7016: *
7017: * If a lookup is requested with "write protection"
7018: * specified, the map may be changed to perform virtual
7019: * copying operations, although the data referenced will
7020: * remain the same.
7021: */
7022: kern_return_t
7023: vm_map_lookup_locked(
7024: vm_map_t *var_map, /* IN/OUT */
7025: register vm_offset_t vaddr,
7026: register vm_prot_t fault_type,
7027: vm_map_version_t *out_version, /* OUT */
7028: vm_object_t *object, /* OUT */
7029: vm_offset_t *offset, /* OUT */
7030: vm_prot_t *out_prot, /* OUT */
7031: boolean_t *wired, /* OUT */
7032: int *behavior, /* OUT */
7033: vm_offset_t *lo_offset, /* OUT */
7034: vm_offset_t *hi_offset) /* OUT */
7035: {
7036: vm_map_entry_t entry;
7037: register vm_map_t map = *var_map;
7038: vm_map_t old_map = *var_map;
7039: vm_offset_t old_vaddr = vaddr;
7040: vm_map_t cow_sub_map;
7041: vm_offset_t old_start;
7042: vm_offset_t old_end;
7043: register vm_prot_t prot;
7044:
7045: RetryLookup: ;
7046:
7047: /*
7048: * If the map has an interesting hint, try it before calling
7049: * full blown lookup routine.
7050: */
7051:
7052: mutex_lock(&map->s_lock);
7053: entry = map->hint;
7054: mutex_unlock(&map->s_lock);
7055:
7056: if ((entry == vm_map_to_entry(map)) ||
7057: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
7058: vm_map_entry_t tmp_entry;
7059:
7060: /*
7061: * Entry was either not a valid hint, or the vaddr
7062: * was not contained in the entry, so do a full lookup.
7063: */
7064: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
7065: return KERN_INVALID_ADDRESS;
7066:
7067: entry = tmp_entry;
7068: }
7069: if(map == old_map) {
7070: old_start = entry->vme_start;
7071: old_end = entry->vme_end;
7072: }
7073:
7074: /*
7075: * Handle submaps. Drop lock on upper map, submap is
7076: * returned locked.
7077: */
7078:
7079: submap_recurse:
7080: if (entry->is_sub_map) {
7081: vm_map_entry_t our_entry;
7082: vm_offset_t local_vaddr;
7083: vm_offset_t end_delta;
7084: vm_offset_t start_delta;
7085: vm_map_entry_t submap_entry;
7086: boolean_t mapped_needs_copy=FALSE;
7087:
7088: local_vaddr = vaddr;
7089: our_entry = entry;
7090:
7091: vm_map_lock_read(entry->object.sub_map);
7092: vm_map_unlock_read(map);
7093: *var_map = map = entry->object.sub_map;
7094:
7095: /* calculate the offset in the submap for vaddr */
7096: local_vaddr = (local_vaddr - entry->vme_start) + entry->offset;
7097:
7098: if(entry->needs_copy) {
7099: if(!mapped_needs_copy) {
7100: cow_sub_map = entry->object.sub_map;
7101: mapped_needs_copy = TRUE;
7102: }
7103: }
7104:
7105: if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {
7106: return KERN_INVALID_ADDRESS;
7107: }
7108: /* find the attenuated shadow of the underlying object */
7109: /* on our target map */
7110:
7111: /* in english the submap object may extend beyond the */
7112: /* region mapped by the entry or, may only fill a portion */
7113: /* of it. For our purposes, we only care if the object */
7114: /* doesn't fill. In this case the area which will */
7115: /* ultimately be clipped in the top map will only need */
7116: /* to be as big as the portion of the underlying entry */
7117: /* which is mapped */
7118: start_delta = submap_entry->vme_start > entry->offset ?
7119: submap_entry->vme_start - entry->offset : 0;
7120:
7121: end_delta =
7122: (entry->offset + start_delta + (old_end - old_start)) <=
7123: submap_entry->vme_end ?
7124: 0 : (entry->offset +
7125: start_delta + (old_end - old_start))
7126: - submap_entry->vme_end;
7127:
7128: old_start += start_delta;
7129: old_end -= end_delta;
7130:
7131: vaddr = local_vaddr;
7132: entry = submap_entry;
7133: if(submap_entry->is_sub_map) {
7134: goto submap_recurse;
7135: }
7136: if((fault_type & VM_PROT_WRITE) &&
7137: !(submap_entry->protection & VM_PROT_WRITE)) {
7138: if(mapped_needs_copy) {
7139:
7140: vm_object_t copy_object;
7141: vm_offset_t object_offset;
7142: vm_offset_t local_start;
7143: vm_offset_t local_end;
7144:
7145: object_offset = submap_entry->offset;
7146:
7147: /* This is the COW case, lets connect */
7148: /* an entry in our space to the underlying */
7149: /* object in the submap, bypassing the */
7150: /* submap. */
7151:
7152: /* set up shadow object */
7153: copy_object =
7154: vm_object_copy_delayed(
7155: submap_entry->object.vm_object,
7156: submap_entry->offset,
7157: old_end-old_start);
7158:
7159: /* This case works opposite of the */
7160: /* normal submap case. We go back */
7161: /* to the top of the map tree and */
7162: /* clip out the target portion of */
7163: /* the sub_map, substituting the */
7164: /* new copy object, we do this for */
7165: /* all maps in the tree, down to */
7166: /* the COW one. */
7167:
7168: vm_map_unlock_read(map);
7169: vm_map_lock(old_map);
7170: local_start = old_start;
7171: local_end = old_end;
7172: local_vaddr = vaddr = old_vaddr;
7173: map = old_map;
7174: while(TRUE) {
7175: vm_map_t new_map;
7176:
7177: if(!vm_map_lookup_entry(map,
7178: local_vaddr, &entry)) {
7179: vm_object_deallocate(
7180: copy_object);
7181: vm_map_unlock(map);
7182: vm_map_lock_read(*var_map);
7183: return KERN_INVALID_ADDRESS;
7184: }
7185:
7186: /* clip out the portion of space */
7187: /* mapped by the sub map which */
7188: /* corresponds to the underlying */
7189: /* object */
7190: vm_map_clip_start(map, entry,
7191: local_start);
7192: vm_map_clip_end(map, entry,
7193: local_end);
7194: new_map = entry->object.sub_map;
7195:
7196: /* update local start and end */
7197: /* in possible underlying submap */
7198: local_start = entry->offset;
7199: local_end = (entry->vme_end -
7200: entry->vme_start) +
7201: entry->offset;
7202: local_vaddr -= entry->vme_start;
7203: local_vaddr += entry->offset;
7204:
7205: /* substitute copy object for */
7206: /* shared map entry */
7207: vm_map_deallocate(
7208: entry->object.sub_map);
7209: entry->is_sub_map = FALSE;
7210: vm_object_reference(copy_object);
7211: entry->object.vm_object = copy_object;
7212: entry->protection |= VM_PROT_WRITE;
7213: entry->max_protection |= VM_PROT_WRITE;
7214: entry->needs_copy = FALSE;
7215: if(entry->inheritance
7216: == VM_INHERIT_SHARE)
7217: entry->inheritance = VM_INHERIT_COPY;
7218: entry->offset = object_offset;
7219: if (map != old_map)
7220: entry->is_shared = TRUE;
7221:
7222: vm_map_unlock(map);
7223: if(new_map == cow_sub_map)
7224: break;
7225: map = new_map;
7226: vm_map_lock(map);
7227: }
7228: map = old_map;
7229: vm_map_lock_read(map);
7230: vaddr = old_vaddr;
7231: /* get rid of extra reference */
7232: vm_object_deallocate(copy_object);
7233: if(!vm_map_lookup_entry(map, vaddr, &entry)) {
7234: return KERN_INVALID_ADDRESS;
7235: }
7236: } else {
7237: /* write fault! */
7238: return KERN_INVALID_ADDRESS;
7239: }
7240: }
7241: vm_map_lock_read(old_map);
7242: vm_map_unlock_read(map);
7243: *var_map = map = old_map;
7244: }
7245:
7246: /*
7247: * Check whether this task is allowed to have
7248: * this page.
7249: */
7250:
7251: prot = entry->protection;
7252: if ((fault_type & (prot)) != fault_type)
7253: return KERN_PROTECTION_FAILURE;
7254:
7255: /*
7256: * If this page is not pageable, we have to get
7257: * it for all possible accesses.
7258: */
7259:
7260: if (*wired = (entry->wired_count != 0))
7261: prot = fault_type = entry->protection;
7262:
7263: /*
7264: * If the entry was copy-on-write, we either ...
7265: */
7266:
7267: if (entry->needs_copy) {
7268: /*
7269: * If we want to write the page, we may as well
7270: * handle that now since we've got the map locked.
7271: *
7272: * If we don't need to write the page, we just
7273: * demote the permissions allowed.
7274: */
7275:
7276: if (fault_type & VM_PROT_WRITE || *wired) {
7277: /*
7278: * Make a new object, and place it in the
7279: * object chain. Note that no new references
7280: * have appeared -- one just moved from the
7281: * map to the new object.
7282: */
7283:
7284: if (vm_map_lock_read_to_write(map)) {
7285: vm_map_lock_read(map);
7286: goto RetryLookup;
7287: }
7288: vm_object_shadow(&entry->object.vm_object,
7289: &entry->offset,
7290: (vm_size_t) (entry->vme_end -
7291: entry->vme_start));
7292:
7293: entry->needs_copy = FALSE;
7294: vm_map_lock_write_to_read(map);
7295: }
7296: else {
7297: /*
7298: * We're attempting to read a copy-on-write
7299: * page -- don't allow writes.
7300: */
7301:
7302: prot &= (~VM_PROT_WRITE);
7303: }
7304: }
7305:
7306: /*
7307: * Create an object if necessary.
7308: */
7309: if (entry->object.vm_object == VM_OBJECT_NULL) {
7310:
7311: if (vm_map_lock_read_to_write(map)) {
7312: vm_map_lock_read(map);
7313: goto RetryLookup;
7314: }
7315:
7316: entry->object.vm_object = vm_object_allocate(
7317: (vm_size_t)(entry->vme_end - entry->vme_start));
7318: entry->offset = 0;
7319: vm_map_lock_write_to_read(map);
7320: }
7321:
7322: /*
7323: * Return the object/offset from this entry. If the entry
7324: * was copy-on-write or empty, it has been fixed up. Also
7325: * return the protection.
7326: */
7327:
7328: *offset = (vaddr - entry->vme_start) + entry->offset;
7329: *object = entry->object.vm_object;
7330: *out_prot = prot;
7331: *behavior = entry->behavior;
7332: *lo_offset = entry->offset;
7333: *hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;
7334:
7335: /*
7336: * Lock the object to prevent it from disappearing
7337: */
7338:
7339: vm_object_lock(*object);
7340:
7341: /*
7342: * Save the version number
7343: */
7344:
7345: out_version->main_timestamp = map->timestamp;
7346:
7347: return KERN_SUCCESS;
7348: }
7349:
7350:
7351: /*
7352: * vm_map_verify:
7353: *
7354: * Verifies that the map in question has not changed
7355: * since the given version. If successful, the map
7356: * will not change until vm_map_verify_done() is called.
7357: */
7358: boolean_t
7359: vm_map_verify(
7360: register vm_map_t map,
7361: register vm_map_version_t *version) /* REF */
7362: {
7363: boolean_t result;
7364:
7365: vm_map_lock_read(map);
7366: result = (map->timestamp == version->main_timestamp);
7367:
7368: if (!result)
7369: vm_map_unlock_read(map);
7370:
7371: return(result);
7372: }
7373:
7374: /*
7375: * vm_map_verify_done:
7376: *
7377: * Releases locks acquired by a vm_map_verify.
7378: *
7379: * This is now a macro in vm/vm_map.h. It does a
7380: * vm_map_unlock_read on the map.
7381: */
7382:
7383:
7384: /*
7385: * vm_region:
7386: *
7387: * User call to obtain information about a region in
7388: * a task's address map. Currently, only one flavor is
7389: * supported.
7390: *
7391: * XXX The reserved and behavior fields cannot be filled
7392: * in until the vm merge from the IK is completed, and
7393: * vm_reserve is implemented.
7394: *
7395: * XXX Dependency: syscall_vm_region() also supports only one flavor.
7396: */
7397:
7398: kern_return_t
7399: vm_region(
7400: vm_map_t map,
7401: vm_offset_t *address, /* IN/OUT */
7402: vm_size_t *size, /* OUT */
7403: vm_region_flavor_t flavor, /* IN */
7404: vm_region_info_t info, /* OUT */
7405: mach_msg_type_number_t *count, /* IN/OUT */
7406: ipc_port_t *object_name) /* OUT */
7407: {
7408: vm_map_entry_t tmp_entry;
7409: register
7410: vm_map_entry_t entry;
7411: register
7412: vm_offset_t start;
7413: vm_region_basic_info_t basic;
7414: vm_region_extended_info_t extended;
7415: vm_region_top_info_t top;
7416:
7417: if (map == VM_MAP_NULL)
7418: return(KERN_INVALID_ARGUMENT);
7419:
7420: switch (flavor) {
7421:
7422: case VM_REGION_BASIC_INFO:
7423: {
7424: if (*count < VM_REGION_BASIC_INFO_COUNT)
7425: return(KERN_INVALID_ARGUMENT);
7426:
7427: basic = (vm_region_basic_info_t) info;
7428: *count = VM_REGION_BASIC_INFO_COUNT;
7429:
7430: vm_map_lock_read(map);
7431:
7432: start = *address;
7433: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
7434: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
7435: vm_map_unlock_read(map);
7436: return(KERN_INVALID_ADDRESS);
7437: }
7438: } else {
7439: entry = tmp_entry;
7440: }
7441:
7442: start = entry->vme_start;
7443:
7444: basic->offset = entry->offset;
7445: basic->protection = entry->protection;
7446: basic->inheritance = entry->inheritance;
7447: basic->max_protection = entry->max_protection;
7448: basic->behavior = entry->behavior;
7449: basic->user_wired_count = entry->user_wired_count;
7450: basic->reserved = FALSE; /* XXX when vm_reserve fini */
7451: *address = start;
7452: *size = (entry->vme_end - start);
7453:
7454: if (object_name) *object_name = IP_NULL;
7455: if (entry->is_sub_map) {
7456: basic->shared = FALSE;
7457: } else {
7458: basic->shared = entry->is_shared;
7459: }
7460:
7461: vm_map_unlock_read(map);
7462: return(KERN_SUCCESS);
7463: }
7464: case VM_REGION_EXTENDED_INFO:
7465: { void vm_region_walk();
7466:
7467: if (*count < VM_REGION_EXTENDED_INFO_COUNT)
7468: return(KERN_INVALID_ARGUMENT);
7469:
7470: extended = (vm_region_extended_info_t) info;
7471: *count = VM_REGION_EXTENDED_INFO_COUNT;
7472:
7473: vm_map_lock_read(map);
7474:
7475: start = *address;
7476: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
7477: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
7478: vm_map_unlock_read(map);
7479: return(KERN_INVALID_ADDRESS);
7480: }
7481: } else {
7482: entry = tmp_entry;
7483: }
7484: start = entry->vme_start;
7485:
7486: extended->protection = entry->protection;
7487: extended->user_tag = entry->alias;
7488: extended->pages_resident = 0;
7489: extended->pages_swapped_out = 0;
7490: extended->pages_shared_now_private = 0;
7491: extended->pages_referenced = 0;
7492: extended->external_pager = 0;
7493: extended->shadow_depth = 0;
7494:
7495: vm_region_walk(entry, extended, entry->offset, entry->vme_end - start, map, start);
7496:
7497: if (object_name)
7498: *object_name = IP_NULL;
7499: *address = start;
7500: *size = (entry->vme_end - start);
7501:
7502: vm_map_unlock_read(map);
7503: return(KERN_SUCCESS);
7504: }
7505: case VM_REGION_TOP_INFO:
7506: { void vm_region_top_walk();
7507:
7508: if (*count < VM_REGION_TOP_INFO_COUNT)
7509: return(KERN_INVALID_ARGUMENT);
7510:
7511: top = (vm_region_top_info_t) info;
7512: *count = VM_REGION_TOP_INFO_COUNT;
7513:
7514: vm_map_lock_read(map);
7515:
7516: start = *address;
7517: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
7518: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
7519: vm_map_unlock_read(map);
7520: return(KERN_INVALID_ADDRESS);
7521: }
7522: } else {
7523: entry = tmp_entry;
7524:
7525: }
7526: start = entry->vme_start;
7527:
7528: top->private_pages_resident = 0;
7529: top->shared_pages_resident = 0;
7530:
7531: vm_region_top_walk(entry, top);
7532:
7533: if (object_name)
7534: *object_name = IP_NULL;
7535: *address = start;
7536: *size = (entry->vme_end - start);
7537:
7538: vm_map_unlock_read(map);
7539: return(KERN_SUCCESS);
7540: }
7541: default:
7542: return(KERN_INVALID_ARGUMENT);
7543: }
7544: }
7545:
7546:
7547: void
7548: vm_region_top_walk(
7549: vm_map_entry_t entry,
7550: vm_region_top_info_t top)
7551: {
7552: register struct vm_object *obj, *tmp_obj;
7553:
7554: if (entry->object.vm_object == 0) {
7555: top->share_mode = SM_EMPTY;
7556: top->ref_count = 0;
7557: top->obj_id = 0;
7558: return;
7559: }
7560: if (entry->is_sub_map)
7561: vm_region_top_walk((vm_map_entry_t)entry->object.sub_map, top);
7562: else {
7563: obj = entry->object.vm_object;
7564:
7565: vm_object_lock(obj);
7566:
7567: if (obj->shadow) {
7568: if (obj->ref_count == 1)
7569: top->private_pages_resident = obj->resident_page_count;
7570: else
7571: top->shared_pages_resident = obj->resident_page_count;
7572: top->ref_count = obj->ref_count;
7573: top->share_mode = SM_COW;
7574:
7575: while (tmp_obj = obj->shadow) {
7576: vm_object_lock(tmp_obj);
7577: vm_object_unlock(obj);
7578: obj = tmp_obj;
7579:
7580: top->shared_pages_resident += obj->resident_page_count;
7581: top->ref_count += obj->resident_page_count - 1;
7582: }
7583: } else {
7584: if (entry->needs_copy) {
7585: top->share_mode = SM_COW;
7586: top->shared_pages_resident = obj->resident_page_count;
7587: } else {
7588: if (obj->ref_count == 1) {
7589: top->share_mode = SM_PRIVATE;
7590: top->private_pages_resident = obj->resident_page_count;
7591: } else {
7592: top->share_mode = SM_SHARED;
7593: top->shared_pages_resident = obj->resident_page_count;
7594: }
7595: }
7596: top->ref_count = obj->ref_count;
7597: }
7598: top->obj_id = (int)obj;
7599:
7600: vm_object_unlock(obj);
7601: }
7602: }
7603:
7604: void
7605: vm_region_walk(
7606: vm_map_entry_t entry,
7607: vm_region_extended_info_t extended,
7608: vm_offset_t offset,
7609: vm_offset_t range,
7610: vm_map_t map,
7611: vm_offset_t va)
7612: {
7613: register struct vm_object *obj, *tmp_obj;
7614: register vm_offset_t last_offset;
7615: register int i;
7616: void vm_region_look_for_page();
7617:
7618: if (entry->object.vm_object == 0) {
7619: extended->share_mode = SM_EMPTY;
7620: extended->ref_count = 0;
7621: return;
7622: }
7623: if (entry->is_sub_map)
7624: vm_region_walk((vm_map_entry_t)entry->object.sub_map, extended, offset + entry->offset,
7625: range, map, va);
7626: else {
7627: obj = entry->object.vm_object;
7628:
7629: vm_object_lock(obj);
7630:
7631: for (last_offset = offset + range; offset < last_offset; offset += PAGE_SIZE, va += PAGE_SIZE)
7632: vm_region_look_for_page(obj, extended, offset, obj->ref_count, 0, map, va);
7633:
7634: if (extended->shadow_depth || entry->needs_copy)
7635: extended->share_mode = SM_COW;
7636: else {
7637: if (obj->ref_count == 1)
7638: extended->share_mode = SM_PRIVATE;
7639: else {
7640: if (obj->true_share)
7641: extended->share_mode = SM_TRUESHARED;
7642: else
7643: extended->share_mode = SM_SHARED;
7644: }
7645: }
7646: extended->ref_count = obj->ref_count - extended->shadow_depth;
7647:
7648: for (i = 0; i < extended->shadow_depth; i++) {
7649: if ((tmp_obj = obj->shadow) == 0)
7650: break;
7651: vm_object_lock(tmp_obj);
7652: vm_object_unlock(obj);
7653: extended->ref_count += tmp_obj->ref_count;
7654: obj = tmp_obj;
7655: }
7656: vm_object_unlock(obj);
7657:
7658: if (extended->share_mode == SM_SHARED) {
7659: register vm_map_entry_t cur;
7660: register vm_map_entry_t last;
7661: int my_refs;
7662:
7663: obj = entry->object.vm_object;
7664: last = vm_map_to_entry(map);
7665: my_refs = 0;
7666:
7667: for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
7668: my_refs += vm_region_count_obj_refs(cur, obj);
7669:
7670: if (my_refs == obj->ref_count)
7671: extended->share_mode = SM_PRIVATE_ALIASED;
7672: else if (my_refs > 1)
7673: extended->share_mode = SM_SHARED_ALIASED;
7674: }
7675: }
7676: }
7677:
7678:
7679:
7680: void
7681: vm_region_look_for_page(
7682: vm_object_t object,
7683: vm_region_extended_info_t extended,
7684: vm_offset_t offset,
7685: int max_refcnt,
7686: int depth,
7687: vm_map_t map,
7688: vm_offset_t va)
7689: {
7690: register vm_page_t p;
7691: register vm_object_t shadow;
7692:
7693: shadow = object->shadow;
7694:
7695: queue_iterate(&object->memq, p, vm_page_t, listq) {
7696: if (p->offset == offset) {
7697: if (shadow && (max_refcnt == 1))
7698: extended->pages_shared_now_private++;
7699:
7700: if (pmap_extract(vm_map_pmap(map), va))
7701: extended->pages_referenced++;
7702: extended->pages_resident++;
7703:
7704: return;
7705: }
7706: }
7707: if (object->existence_map) {
7708: if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_EXISTS) {
7709: if (shadow && (max_refcnt == 1))
7710: extended->pages_shared_now_private++;
7711: extended->pages_swapped_out++;
7712:
7713: return;
7714: }
7715: }
7716: if (shadow) {
7717: vm_object_lock(shadow);
7718:
7719: if (++depth > extended->shadow_depth)
7720: extended->shadow_depth = depth;
7721:
7722: if (shadow->ref_count > max_refcnt)
7723: max_refcnt = shadow->ref_count;
7724:
7725: vm_region_look_for_page(shadow, extended, offset + object->shadow_offset,
7726: max_refcnt, depth, map, va);
7727: vm_object_unlock(shadow);
7728:
7729: return;
7730: }
7731: if ( !(object->pager_trusted) && !(object->internal))
7732: extended->external_pager = 1;
7733: }
7734:
7735:
7736: vm_region_count_obj_refs(
7737: vm_map_entry_t entry,
7738: vm_object_t object)
7739: {
7740: register int ref_count;
7741: register vm_object_t chk_obj;
7742: register vm_object_t tmp_obj;
7743:
7744: if (entry->object.vm_object == 0)
7745: return(0);
7746:
7747: if (entry->is_sub_map)
7748: ref_count = vm_region_count_obj_refs((vm_map_entry_t)entry->object.sub_map, object);
7749: else {
7750: ref_count = 0;
7751:
7752: chk_obj = entry->object.vm_object;
7753: vm_object_lock(chk_obj);
7754:
7755: while (chk_obj) {
7756: if (chk_obj == object)
7757: ref_count++;
7758: if (tmp_obj = chk_obj->shadow)
7759: vm_object_lock(tmp_obj);
7760: vm_object_unlock(chk_obj);
7761:
7762: chk_obj = tmp_obj;
7763: }
7764: }
7765: return(ref_count);
7766: }
7767:
7768:
7769: /*
7770: * Routine: vm_map_simplify
7771: *
7772: * Description:
7773: * Attempt to simplify the map representation in
7774: * the vicinity of the given starting address.
7775: * Note:
7776: * This routine is intended primarily to keep the
7777: * kernel maps more compact -- they generally don't
7778: * benefit from the "expand a map entry" technology
7779: * at allocation time because the adjacent entry
7780: * is often wired down.
7781: */
7782: void
7783: vm_map_simplify(
7784: vm_map_t map,
7785: vm_offset_t start)
7786: {
7787: vm_map_entry_t this_entry;
7788: vm_map_entry_t prev_entry;
7789: vm_map_entry_t next_entry;
7790:
7791: vm_map_lock(map);
7792: if (
7793: (vm_map_lookup_entry(map, start, &this_entry)) &&
7794: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
7795:
7796: (prev_entry->vme_end == this_entry->vme_start) &&
7797:
7798: (prev_entry->is_shared == FALSE) &&
7799: (prev_entry->is_sub_map == FALSE) &&
7800:
7801: (this_entry->is_shared == FALSE) &&
7802: (this_entry->is_sub_map == FALSE) &&
7803:
7804: (prev_entry->inheritance == this_entry->inheritance) &&
7805: (prev_entry->protection == this_entry->protection) &&
7806: (prev_entry->max_protection == this_entry->max_protection) &&
7807: (prev_entry->behavior == this_entry->behavior) &&
7808: (prev_entry->wired_count == this_entry->wired_count) &&
7809: (prev_entry->user_wired_count == this_entry->user_wired_count)&&
7810: (prev_entry->in_transition == FALSE) &&
7811: (this_entry->in_transition == FALSE) &&
7812:
7813: (prev_entry->needs_copy == this_entry->needs_copy) &&
7814:
7815: (prev_entry->object.vm_object == this_entry->object.vm_object)&&
7816: ((prev_entry->offset +
7817: (prev_entry->vme_end - prev_entry->vme_start))
7818: == this_entry->offset)
7819: ) {
7820: SAVE_HINT(map, prev_entry);
7821: vm_map_entry_unlink(map, this_entry);
7822: prev_entry->vme_end = this_entry->vme_end;
7823: UPDATE_FIRST_FREE(map, map->first_free);
7824: vm_object_deallocate(this_entry->object.vm_object);
7825: vm_map_entry_dispose(map, this_entry);
7826: counter(c_vm_map_simplified_lower++);
7827: }
7828: if (
7829: (vm_map_lookup_entry(map, start, &this_entry)) &&
7830: ((next_entry = this_entry->vme_next) != vm_map_to_entry(map)) &&
7831:
7832: (next_entry->vme_start == this_entry->vme_end) &&
7833:
7834: (next_entry->is_shared == FALSE) &&
7835: (next_entry->is_sub_map == FALSE) &&
7836:
7837: (next_entry->is_shared == FALSE) &&
7838: (next_entry->is_sub_map == FALSE) &&
7839:
7840: (next_entry->inheritance == this_entry->inheritance) &&
7841: (next_entry->protection == this_entry->protection) &&
7842: (next_entry->max_protection == this_entry->max_protection) &&
7843: (next_entry->behavior == this_entry->behavior) &&
7844: (next_entry->wired_count == this_entry->wired_count) &&
7845: (next_entry->user_wired_count == this_entry->user_wired_count)&&
7846: (this_entry->in_transition == FALSE) &&
7847: (next_entry->in_transition == FALSE) &&
7848:
7849: (next_entry->needs_copy == this_entry->needs_copy) &&
7850:
7851: (next_entry->object.vm_object == this_entry->object.vm_object)&&
7852: ((this_entry->offset +
7853: (this_entry->vme_end - this_entry->vme_start))
7854: == next_entry->offset)
7855: ) {
7856: vm_map_entry_unlink(map, next_entry);
7857: this_entry->vme_end = next_entry->vme_end;
7858: UPDATE_FIRST_FREE(map, map->first_free);
7859: vm_object_deallocate(next_entry->object.vm_object);
7860: vm_map_entry_dispose(map, next_entry);
7861: counter(c_vm_map_simplified_upper++);
7862: }
7863: counter(c_vm_map_simplify_called++);
7864: vm_map_unlock(map);
7865: }
7866:
7867:
7868: /*
7869: * Routine: vm_map_machine_attribute
7870: * Purpose:
7871: * Provide machine-specific attributes to mappings,
7872: * such as cachability etc. for machines that provide
7873: * them. NUMA architectures and machines with big/strange
7874: * caches will use this.
7875: * Note:
7876: * Responsibilities for locking and checking are handled here,
7877: * everything else in the pmap module. If any non-volatile
7878: * information must be kept, the pmap module should handle
7879: * it itself. [This assumes that attributes do not
7880: * need to be inherited, which seems ok to me]
7881: */
7882: kern_return_t
7883: vm_map_machine_attribute(
7884: vm_map_t map,
7885: vm_offset_t address,
7886: vm_size_t size,
7887: vm_machine_attribute_t attribute,
7888: vm_machine_attribute_val_t* value) /* IN/OUT */
7889: {
7890: kern_return_t ret;
7891:
7892: if (address < vm_map_min(map) ||
7893: (address + size) > vm_map_max(map))
7894: return KERN_INVALID_ADDRESS;
7895:
7896: vm_map_lock(map);
7897:
7898: ret = pmap_attribute(map->pmap, address, size, attribute, value);
7899:
7900: vm_map_unlock(map);
7901:
7902: return ret;
7903: }
7904:
7905: /*
7906: * vm_map_behavior_set:
7907: *
7908: * Sets the paging reference behavior of the specified address
7909: * range in the target map. Paging reference behavior affects
7910: * how pagein operations resulting from faults on the map will be
7911: * clustered.
7912: */
7913: kern_return_t
7914: vm_map_behavior_set(
7915: vm_map_t map,
7916: vm_offset_t start,
7917: vm_offset_t end,
7918: vm_behavior_t new_behavior)
7919: {
7920: register vm_map_entry_t entry;
7921: vm_map_entry_t temp_entry;
7922:
7923: XPR(XPR_VM_MAP,
7924: "vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
7925: (integer_t)map, start, end, new_behavior, 0);
7926:
7927: switch (new_behavior) {
7928: case VM_BEHAVIOR_DEFAULT:
7929: case VM_BEHAVIOR_RANDOM:
7930: case VM_BEHAVIOR_SEQUENTIAL:
7931: case VM_BEHAVIOR_RSEQNTL:
7932: break;
7933: default:
7934: return(KERN_INVALID_ARGUMENT);
7935: }
7936:
7937: vm_map_lock(map);
7938:
7939: /*
7940: * The entire address range must be valid for the map.
7941: * Note that vm_map_range_check() does a
7942: * vm_map_lookup_entry() internally and returns the
7943: * entry containing the start of the address range if
7944: * the entire range is valid.
7945: */
7946: if (vm_map_range_check(map, start, end, &temp_entry)) {
7947: entry = temp_entry;
7948: vm_map_clip_start(map, entry, start);
7949: }
7950: else {
7951: vm_map_unlock(map);
7952: return(KERN_INVALID_ADDRESS);
7953: }
7954:
7955: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
7956: vm_map_clip_end(map, entry, end);
7957:
7958: entry->behavior = new_behavior;
7959:
7960: entry = entry->vme_next;
7961: }
7962:
7963: vm_map_unlock(map);
7964: return(KERN_SUCCESS);
7965: }
7966:
7967: #if DIPC
7968:
7969: /*
7970: * This should one day be eliminated;
7971: * we should always construct the right flavor of copy object
7972: * the first time. Troublesome areas include vm_read, where vm_map_copyin
7973: * is called without knowing whom the copy object is for.
7974: * There are also situations where we do want a lazy data structure
7975: * even if we are sending to a remote port...
7976: */
7977:
7978: extern kern_return_t vm_map_object_to_page_list(
7979: vm_map_copy_t *caller_copy);
7980:
7981: /*
7982: * Convert a copy to a page list. The copy argument is in/out
7983: * because we probably have to allocate a new vm_map_copy structure.
7984: * We take responsibility for discarding the old structure and
7985: * use a continuation to do so. Postponing this discard ensures
7986: * that the objects containing the pages we've marked busy will stick
7987: * around.
7988: *
7989: * N.B. For the entry list case, be warned that this routine steals
7990: * the pages from the entry list's objects!
7991: */
7992: kern_return_t
7993: vm_map_convert_to_page_list(
7994: vm_map_copy_t *caller_copy)
7995: {
7996: vm_map_entry_t entry;
7997: vm_offset_t va;
7998: vm_offset_t offset;
7999: vm_object_t object;
8000: vm_map_copy_t copy, new_copy;
8001: vm_size_t vm_copy_size;
8002: vm_prot_t result_prot;
8003: vm_page_t m, top_page;
8004: kern_return_t result;
8005: kern_return_t kr;
8006:
8007: copy = *caller_copy;
8008:
8009: /*
8010:
8011: * We may not have to do anything,
8012: * or may not be able to do anything.
8013: */
8014: if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) {
8015: return KERN_SUCCESS;
8016: }
8017: if (copy->type == VM_MAP_COPY_OBJECT) {
8018: return vm_map_object_to_page_list(caller_copy);
8019: }
8020: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
8021:
8022: /*
8023: * Check size. If this is really big, copy it out
8024: * to the kernel map, and copy in using src_destroy
8025: */
8026: vm_copy_size = round_page(copy->offset + copy->size) -
8027: trunc_page(copy->offset);
8028:
8029: if (vm_copy_size > (VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE)) {
8030: vm_size_t copy_size;
8031:
8032: offset = copy->offset;
8033: copy_size = copy->size;
8034: result = vm_map_copyout(kernel_map, &va, copy);
8035: assert(result == KERN_SUCCESS);
8036: va += offset - trunc_page(offset);
8037: result = vm_map_copyin_page_list(kernel_map, va, copy_size,
8038: (VM_MAP_COPYIN_OPT_SRC_DESTROY|
8039: VM_MAP_COPYIN_OPT_STEAL_PAGES|
8040: VM_PROT_READ),
8041: caller_copy, FALSE);
8042: return(result);
8043: }
8044:
8045: /*
8046: * Allocate the new copy. Set its continuation to
8047: * discard the old one.
8048: */
8049: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
8050: new_copy->type = VM_MAP_COPY_PAGE_LIST;
8051: new_copy->cpy_npages = 0;
8052: new_copy->cpy_page_loose = FALSE;
8053: new_copy->offset = copy->offset;
8054: new_copy->size = copy->size;
8055: new_copy->cpy_cont = vm_map_copy_discard_cont;
8056: new_copy->cpy_cont_args = (vm_map_copyin_args_t) copy;
8057:
8058: /*
8059: * Iterate over entries.
8060: */
8061: for (entry = vm_map_copy_first_entry(copy);
8062: entry != vm_map_copy_to_entry(copy);
8063: entry = entry->vme_next) {
8064:
8065: object = entry->object.vm_object;
8066: offset = entry->offset;
8067: /*
8068: * Iterate over pages.
8069: */
8070: for (va = entry->vme_start;
8071: va < entry->vme_end;
8072: va += PAGE_SIZE, offset += PAGE_SIZE) {
8073:
8074: assert(new_copy->cpy_npages !=
8075: VM_MAP_COPY_PAGE_LIST_MAX);
8076:
8077: /*
8078: * If the object is null, this is
8079: * zero fill data.
8080: */
8081: if (object == VM_OBJECT_NULL) {
8082: while ((m = vm_page_grab()) == VM_PAGE_NULL) {
8083: VM_PAGE_WAIT();
8084: }
8085: vm_page_zero_fill(m);
8086: vm_page_gobble(m);
8087: new_copy->cpy_page_list[
8088: new_copy->cpy_npages++] = m;
8089: continue;
8090: }
8091:
8092: /*
8093: * Try to find the page of data.
8094: */
8095:
8096: vm_object_lock(object);
8097: if (((m = vm_page_lookup(object, offset)) !=
8098: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
8099: !m->unusual) {
8100:
8101: /*
8102: * This is the page. Mark it busy.
8103: * Remove it from its old object.
8104: */
8105: m->busy = TRUE;
8106: vm_page_lock_queues();
8107: VM_PAGE_QUEUES_REMOVE(m);
8108: vm_page_remove(m);
8109: vm_page_unlock_queues();
8110: vm_object_unlock(object);
8111: new_copy->cpy_page_list[new_copy->cpy_npages++]
8112: = m;
8113: continue;
8114: }
8115:
8116: retry:
8117: result_prot = VM_PROT_READ;
8118: vm_object_paging_begin(object);
8119: XPR(XPR_VM_FAULT,
8120: "vm_map_convert_to_page_list -> vm_fault_page\n",
8121: 0,0,0,0,0);
8122: kr = vm_fault_page(object, offset,
8123: VM_PROT_READ, FALSE, THREAD_UNINT,
8124: entry->offset,
8125: entry->offset +
8126: (entry->vme_end -
8127: entry->vme_start),
8128: VM_BEHAVIOR_SEQUENTIAL,
8129: &result_prot, &m, &top_page,
8130: (int *)0,
8131: FALSE, FALSE, FALSE);
8132: if (kr == VM_FAULT_MEMORY_SHORTAGE) {
8133: VM_PAGE_WAIT();
8134: vm_object_lock(object);
8135: goto retry;
8136: }
8137: if (kr == VM_FAULT_FICTITIOUS_SHORTAGE) {
8138: vm_page_more_fictitious();
8139: vm_object_lock(object);
8140: goto retry;
8141: }
8142: if (kr != VM_FAULT_SUCCESS) {
8143: /* XXX what about data_error? */
8144: vm_object_lock(object);
8145: goto retry;
8146: }
8147:
8148: assert(m != VM_PAGE_NULL);
8149: m->busy = TRUE;
8150: vm_page_lock_queues();
8151: VM_PAGE_QUEUES_REMOVE(m);
8152: vm_page_remove(m);
8153: vm_page_unlock_queues();
8154: vm_object_paging_end(object);
8155: vm_object_unlock(object);
8156: new_copy->cpy_page_list[new_copy->cpy_npages++] = m;
8157:
8158: if (top_page != VM_PAGE_NULL) {
8159: assert(top_page->object == object);
8160: vm_object_lock(object);
8161: VM_PAGE_FREE(top_page);
8162: vm_object_paging_end(object);
8163: vm_object_unlock(object);
8164: }
8165: }
8166: }
8167:
8168: *caller_copy = new_copy;
8169: return KERN_SUCCESS;
8170: }
8171:
8172:
8173: /*
8174: * Continue converting pages from an object-flavor
8175: * copy object into a page-list flavor copy object.
8176: * The parameters in the copy object were updated
8177: * by the last invocation of vm_map_object_to_page_list
8178: * so there's no work to be done other than calling
8179: * back into that routine. Note that the contents
8180: * of the underlying object itself never change
8181: * during the conversion process.
8182: */
8183: kern_return_t
8184: vm_map_object_to_page_list_cont(
8185: vm_map_copy_t cont_args,
8186: vm_map_copy_t *copy_result) /* OUT */
8187: {
8188: boolean_t abort;
8189:
8190: abort = (copy_result == (vm_map_copy_t *) 0);
8191: if (abort) {
8192: printf("vm_map_object_to_page_list_cont: abort\n");
8193: vm_map_copy_discard(cont_args);
8194: }
8195:
8196: assert(cont_args->type == VM_MAP_COPY_OBJECT);
8197: *copy_result = cont_args;
8198: return vm_map_object_to_page_list(copy_result);
8199: }
8200:
8201:
8202: /*
8203: * Convert a copy object from object flavor to
8204: * page list flavor. For objects containing more
8205: * than the allowable maximum number of pages, we
8206: * perform the conversion a piece at a time.
8207: *
8208: * The object flavor copy object contains a vm_object_t
8209: * that has all the pages; the type, offset and size
8210: * fields always present in a vm_map_copy_t; and an
8211: * index field unique to the object flavor. This index
8212: * field MUST be zero when first calling this routine.
8213: * It is used to record progress converting pages from
8214: * the object to the page list.
8215: *
8216: * Assumptions:
8217: * - This routine always returns KERN_SUCCESS.
8218: */
8219: kern_return_t
8220: vm_map_object_to_page_list(
8221: vm_map_copy_t *caller_copy)
8222: {
8223: vm_object_t object;
8224: vm_offset_t offset;
8225: vm_map_copy_t copy, new_copy;
8226: vm_size_t index;
8227: vm_size_t copy_size;
8228: vm_page_t m;
8229: vm_prot_t result_prot;
8230: vm_page_t top_page;
8231: kern_return_t kr;
8232:
8233: assert(caller_copy != (vm_map_copy_t *) 0);
8234: assert(*caller_copy != VM_MAP_COPY_NULL);
8235: assert((*caller_copy)->type == VM_MAP_COPY_OBJECT);
8236:
8237: copy = *caller_copy;
8238: object = copy->cpy_object;
8239:
8240: /*
8241: * Allocate the new copy. Set its continuation to
8242: * discard the old one.
8243: */
8244: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
8245: new_copy->type = VM_MAP_COPY_PAGE_LIST;
8246: new_copy->cpy_npages = 0;
8247: new_copy->cpy_page_loose = FALSE;
8248: new_copy->offset = copy->offset - trunc_page(copy->offset);
8249: new_copy->size = copy->size;
8250: assert((long)new_copy->size >= 0);
8251: if (new_copy->size == 0)
8252: printf("vm_object_to_page_list: zero size\n");
8253: new_copy->cpy_cont = vm_map_copy_discard_cont;
8254: new_copy->cpy_cont_args = (vm_map_copyin_args_t) copy;
8255:
8256: /*
8257: * Compute range of object to extract.
8258: */
8259: index = (vm_size_t)copy->cpy_index; /* XXX!XXX! */
8260: assert(copy->offset < page_size);
8261: copy_size = round_page(copy->offset + copy->size);
8262: assert(page_aligned(copy_size));
8263:
8264: /*
8265: * Detect an object larger than the maximum permitted
8266: * for an individual page list block. Save the rest
8267: * for later.
8268: */
8269: if (copy_size > VM_MAP_COPY_PAGE_LIST_MAX_SIZE) {
8270: /*
8271: * Only the first copy object in the chain
8272: * has the unaligned offset and total size.
8273: * Each succeeding copy object has an offset
8274: * of zero and a size decreased by the amount
8275: * contained in the previous copy object.
8276: * These values will be used when creating the
8277: * next page list copy object.
8278: *
8279: * Reset the continuation because there are
8280: * more pages left to extract.
8281: */
8282: assert(trunc_page(copy->offset) == 0);
8283: copy->size = copy->size -
8284: (VM_MAP_COPY_PAGE_LIST_MAX_SIZE - copy->offset);
8285: copy->offset = 0;
8286: copy_size = VM_MAP_COPY_PAGE_LIST_MAX_SIZE;
8287: new_copy->cpy_cont = (vm_map_copy_cont_t)
8288: vm_map_object_to_page_list_cont;
8289: assert(vm_map_copy_cont_is_valid(new_copy));
8290: }
8291:
8292: /*
8293: * Fault in all pages that aren't present.
8294: * Modelled on vm_map_copyin_page_list.
8295: */
8296: for (offset = index; offset < index + copy_size; offset += PAGE_SIZE) {
8297: vm_object_lock(object);
8298: vm_object_paging_begin(object);
8299:
8300: m = vm_page_lookup(object, offset);
8301: if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious &&
8302: !m->unusual) {
8303: m->busy = TRUE;
8304: } else {
8305: retry:
8306: result_prot = VM_PROT_READ;
8307: XPR(XPR_VM_FAULT,
8308: "vm_object_to_page_list -> vm_fault_page\n",
8309: 0,0,0,0,0);
8310: kr = vm_fault_page(object, offset,
8311: VM_PROT_READ, FALSE, THREAD_UNINT,
8312: offset, index + copy_size,/* XXX ? */
8313: VM_BEHAVIOR_SEQUENTIAL,
8314: &result_prot, &m, &top_page,
8315: (int *)0,
8316: 0, FALSE, FALSE);
8317: switch (kr) {
8318: case VM_FAULT_SUCCESS:
8319: break;
8320: case VM_FAULT_MEMORY_SHORTAGE:
8321: VM_PAGE_WAIT();
8322: vm_object_lock(object);
8323: vm_object_paging_begin(object);
8324: goto retry;
8325: case VM_FAULT_FICTITIOUS_SHORTAGE:
8326: vm_page_more_fictitious();
8327: /* fall through... */
8328: case VM_FAULT_INTERRUPTED:
8329: case VM_FAULT_RETRY:
8330: vm_object_lock(object);
8331: vm_object_paging_begin(object);
8332: goto retry;
8333: case VM_FAULT_MEMORY_ERROR:
8334: panic("vm_map_object_to_page_list");
8335: break;
8336: }
8337:
8338: if (top_page != VM_PAGE_NULL) {
8339: VM_PAGE_FREE(top_page);
8340: vm_object_paging_end(object);
8341: }
8342: }
8343:
8344: assert(m);
8345: assert(m->busy);
8346: /* assert(m->wire_count == 0); XXX */
8347:
8348: /*
8349: * Got the page. Save it in the page list
8350: * and rip it away from the object.
8351: */
8352: new_copy->cpy_page_list[new_copy->cpy_npages++] = m;
8353: vm_page_lock_queues();
8354: VM_PAGE_QUEUES_REMOVE(m);
8355: vm_page_remove(m);
8356: vm_page_unlock_queues();
8357:
8358: vm_object_paging_end(object);
8359: vm_object_unlock(object);
8360: }
8361:
8362: /*
8363: * Update index for next pass through the object.
8364: */
8365: copy->cpy_index += copy_size;
8366:
8367: *caller_copy = new_copy;
8368: assert(vm_map_copy_cont_is_valid(new_copy));
8369: return (KERN_SUCCESS);
8370: }
8371:
8372: /*
8373: * When allocating a new entry, vm_map_entry_list_from_object must
8374: * use the pageable v. non-pageable entry zone based on the
8375: * attribute of the map into which the entry will be pasted.
8376: * This information must be supplied by the caller, as there is
8377: * no way to obtain it from the object.
8378: *
8379: * N.B. Caller donates a reference.
8380: */
8381: vm_map_copy_t
8382: vm_map_entry_list_from_object(
8383: vm_object_t object,
8384: vm_offset_t offset,
8385: vm_size_t size,
8386: boolean_t pageable)
8387: {
8388: vm_map_entry_t new_entry;
8389: vm_map_copy_t copy;
8390:
8391: assert(object != VM_OBJECT_NULL);
8392: assert(size != 0);
8393:
8394: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
8395: assert(copy != VM_MAP_COPY_NULL);
8396:
8397: copy->type = VM_MAP_COPY_ENTRY_LIST;
8398: vm_map_copy_first_entry(copy) =
8399: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
8400: copy->cpy_hdr.nentries = 0;
8401: copy->cpy_hdr.entries_pageable = pageable;
8402: copy->offset = offset;
8403: copy->size = size;
8404:
8405: /*
8406: * Allocate and initialize an entry for the object.
8407: */
8408: new_entry = vm_map_copy_entry_create(copy);
8409: new_entry->vme_start = trunc_page(copy->offset);
8410: new_entry->vme_end = round_page(copy->offset + copy->size);
8411: new_entry->object.vm_object = object;
8412: new_entry->offset = offset;
8413: new_entry->is_shared = FALSE;
8414: new_entry->is_sub_map = FALSE;
8415: new_entry->needs_copy = FALSE;
8416: new_entry->protection = VM_PROT_DEFAULT;
8417: new_entry->max_protection = VM_PROT_ALL;
8418: new_entry->inheritance = VM_INHERIT_DEFAULT;
8419: new_entry->wired_count = 0;
8420: new_entry->user_wired_count = 0;
8421: new_entry->in_transition = FALSE;
8422:
8423: /*
8424: * Insert entry into copy object, and return.
8425: */
8426: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry);
8427: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
8428: return (copy);
8429: }
8430:
8431: /*
8432: * Convert a page list copy object to an entry list
8433: * flavor copy object.
8434: */
8435: kern_return_t
8436: vm_map_convert_to_entry_list(
8437: vm_map_copy_t copy,
8438: boolean_t pageable)
8439: {
8440: vm_object_t object;
8441: int i;
8442: vm_map_entry_t new_entry;
8443: vm_page_t *page_list;
8444: vm_map_copy_t cur_copy, new_copy;
8445: vm_offset_t offset;
8446: kern_return_t result;
8447:
8448: /*
8449: * Check type of copy object.
8450: */
8451: if (copy->type != VM_MAP_COPY_PAGE_LIST) {
8452: panic("vm_map_convert_to_entry_list 0x%x %d", copy, copy->type);
8453: }
8454:
8455: /*
8456: * Insert all the pages into a new object.
8457: */
8458: object = vm_object_allocate(round_page(copy->offset + copy->size) -
8459: trunc_page(copy->offset));
8460: offset = (vm_offset_t) 0;
8461: cur_copy = copy;
8462:
8463: while (cur_copy) {
8464:
8465: /*
8466: * Make sure the pages are loose. This may be
8467: * a "Can't Happen", but just to be safe ...
8468: */
8469: page_list = &cur_copy->cpy_page_list[0];
8470: if (!copy->cpy_page_loose)
8471: vm_map_copy_steal_pages(cur_copy);
8472:
8473: /*
8474: * Stuff this set of pages into the object, removing
8475: * them from the page list.
8476: */
8477: vm_object_lock(object);
8478: vm_page_lock_queues();
8479: for (i = 0; i < cur_copy->cpy_npages;
8480: i++, offset += PAGE_SIZE) {
8481: register vm_page_t m = *page_list;
8482:
8483: vm_page_insert(m, object, offset);
8484: m->busy = FALSE;
8485: m->dirty = TRUE;
8486: vm_page_activate(m);
8487: *page_list++ = VM_PAGE_NULL;
8488: }
8489: vm_page_unlock_queues();
8490: vm_object_unlock(object);
8491:
8492: /*
8493: * Invoke continuation if present.
8494: */
8495: if (vm_map_copy_has_cont(cur_copy)) {
8496: vm_map_copy_invoke_cont(cur_copy, &new_copy, &result);
8497:
8498: if (result != KERN_SUCCESS)
8499: panic("%s: %s",
8500: "vm_map_convert_to_entry_list",
8501: "continuation failure");
8502: } else {
8503: new_copy = VM_MAP_COPY_NULL;
8504: }
8505: if (cur_copy != copy)
8506: vm_map_copy_discard(cur_copy);
8507:
8508: cur_copy = new_copy;
8509: }
8510:
8511: /*
8512: * Change type of copy object
8513: */
8514: vm_map_copy_first_entry(copy) =
8515: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
8516: copy->type = VM_MAP_COPY_ENTRY_LIST;
8517: copy->cpy_hdr.nentries = 0;
8518: copy->cpy_hdr.entries_pageable = pageable;
8519:
8520: /*
8521: * Allocate and initialize an entry for object
8522: */
8523: new_entry = vm_map_copy_entry_create(copy);
8524: new_entry->vme_start = trunc_page(copy->offset);
8525: new_entry->vme_end = round_page(copy->offset + copy->size);
8526: new_entry->object.vm_object = object;
8527: new_entry->offset = 0;
8528: new_entry->is_shared = FALSE;
8529: new_entry->is_sub_map = FALSE;
8530: new_entry->needs_copy = FALSE;
8531: new_entry->protection = VM_PROT_DEFAULT;
8532: new_entry->max_protection = VM_PROT_ALL;
8533: new_entry->behavior = VM_BEHAVIOR_DEFAULT;
8534: new_entry->inheritance = VM_INHERIT_DEFAULT;
8535: new_entry->wired_count = 0;
8536: new_entry->user_wired_count = 0;
8537:
8538: /*
8539: * Insert entry into copy object, and return.
8540: */
8541: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry);
8542: return(KERN_SUCCESS);
8543: }
8544:
8545: #endif /* DIPC */
8546:
8547: int
8548: vm_map_copy_cont_is_valid(
8549: vm_map_copy_t copy)
8550: {
8551: vm_map_copy_cont_t cont;
8552:
8553: assert(copy->type == VM_MAP_COPY_PAGE_LIST);
8554: cont = copy->cpy_cont;
8555: if (
8556: cont != vm_map_copy_discard_cont &&
8557: cont != vm_map_copyin_page_list_cont ) {
8558: printf("vm_map_copy_cont_is_valid: bogus cont 0x%x\n", cont);
8559: assert((integer_t) cont == 0xdeadbeef);
8560: }
8561: return 1;
8562: }
8563:
8564: #include <mach_kdb.h>
8565: #if MACH_KDB
8566: #include <ddb/db_output.h>
8567: #include <vm/vm_print.h>
8568:
8569: #define printf db_printf
8570:
8571: /*
8572: * Forward declarations for internal functions.
8573: */
8574: extern void vm_map_links_print(
8575: struct vm_map_links *links);
8576:
8577: extern void vm_map_header_print(
8578: struct vm_map_header *header);
8579:
8580: extern void vm_map_entry_print(
8581: vm_map_entry_t entry);
8582:
8583: extern void vm_follow_entry(
8584: vm_map_entry_t entry);
8585:
8586: extern void vm_follow_map(
8587: vm_map_t map);
8588:
8589: /*
8590: * vm_map_links_print: [ debug ]
8591: */
8592: void
8593: vm_map_links_print(
8594: struct vm_map_links *links)
8595: {
8596: iprintf("prev=0x%x, next=0x%x, start=0x%x, end=0x%x\n",
8597: links->prev,
8598: links->next,
8599: links->start,
8600: links->end);
8601: }
8602:
8603: /*
8604: * vm_map_header_print: [ debug ]
8605: */
8606: void
8607: vm_map_header_print(
8608: struct vm_map_header *header)
8609: {
8610: vm_map_links_print(&header->links);
8611: iprintf("nentries=0x%x, %sentries_pageable\n",
8612: header->nentries,
8613: (header->entries_pageable ? "" : "!"));
8614: }
8615:
8616: /*
8617: * vm_follow_entry: [ debug ]
8618: */
8619: void
8620: vm_follow_entry(
8621: vm_map_entry_t entry)
8622: {
8623: extern int db_indent;
8624: int shadows;
8625:
8626: iprintf("map entry 0x%x:\n", entry);
8627:
8628: db_indent += 2;
8629:
8630: shadows = vm_follow_object(entry->object.vm_object);
8631: iprintf("Total objects : %d\n",shadows);
8632:
8633: db_indent -= 2;
8634: }
8635:
8636: /*
8637: * vm_map_entry_print: [ debug ]
8638: */
8639: void
8640: vm_map_entry_print(
8641: register vm_map_entry_t entry)
8642: {
8643: extern int db_indent;
8644: static char *inheritance_name[4] = { "share", "copy", "none", "?"};
8645: static char *behavior_name[4] = { "dflt", "rand", "seqtl", "rseqntl" };
8646:
8647: iprintf("map entry 0x%x:\n", entry);
8648:
8649: db_indent += 2;
8650:
8651: vm_map_links_print(&entry->links);
8652:
8653: iprintf("start=0x%x, end=0x%x, prot=%x/%x/%s\n",
8654: entry->vme_start,
8655: entry->vme_end,
8656: entry->protection,
8657: entry->max_protection,
8658: inheritance_name[(entry->inheritance & 0x3)]);
8659:
8660: iprintf("behavior=%s, wired_count=%d, user_wired_count=%d\n",
8661: behavior_name[(entry->behavior & 0x3)],
8662: entry->wired_count,
8663: entry->user_wired_count);
8664: iprintf("%sin_transition, %sneeds_wakeup\n",
8665: (entry->in_transition ? "" : "!"),
8666: (entry->needs_wakeup ? "" : "!"));
8667:
8668: if (entry->is_sub_map) {
8669: iprintf("submap=0x%x, offset=0x%x\n",
8670: entry->object.sub_map,
8671: entry->offset);
8672: } else {
8673: iprintf("object=0x%x, offset=0x%x, ",
8674: entry->object.vm_object,
8675: entry->offset);
8676: printf("%sis_shared, %sneeds_copy\n",
8677: (entry->is_shared ? "" : "!"),
8678: (entry->needs_copy ? "" : "!"));
8679: }
8680:
8681: db_indent -= 2;
8682: }
8683:
8684: /*
8685: * vm_follow_map: [ debug ]
8686: */
8687: void
8688: vm_follow_map(
8689: vm_map_t map)
8690: {
8691: register vm_map_entry_t entry;
8692: extern int db_indent;
8693:
8694: iprintf("task map 0x%x:\n", map);
8695:
8696: db_indent += 2;
8697:
8698: for (entry = vm_map_first_entry(map);
8699: entry && entry != vm_map_to_entry(map);
8700: entry = entry->vme_next) {
8701: vm_follow_entry(entry);
8702: }
8703:
8704: db_indent -= 2;
8705: }
8706:
8707: /*
8708: * vm_map_print: [ debug ]
8709: */
8710: void
8711: vm_map_print(
8712: register vm_map_t map)
8713: {
8714: register vm_map_entry_t entry;
8715: extern int db_indent;
8716: char *swstate;
8717:
8718: iprintf("task map 0x%x:\n", map);
8719:
8720: db_indent += 2;
8721:
8722: vm_map_header_print(&map->hdr);
8723:
8724: iprintf("pmap=0x%x, size=%d, ref=%d, hint=0x%x, first_free=0x%x\n",
8725: map->pmap,
8726: map->size,
8727: map->ref_count,
8728: map->hint,
8729: map->first_free);
8730:
8731: iprintf("%swait_for_space, %swiring_required, timestamp=%d\n",
8732: (map->wait_for_space ? "" : "!"),
8733: (map->wiring_required ? "" : "!"),
8734: map->timestamp);
8735:
8736: #if TASK_SWAPPER
8737: switch (map->sw_state) {
8738: case MAP_SW_IN:
8739: swstate = "SW_IN";
8740: break;
8741: case MAP_SW_OUT:
8742: swstate = "SW_OUT";
8743: break;
8744: default:
8745: swstate = "????";
8746: break;
8747: }
8748: iprintf("res=%d, sw_state=%s\n", map->res_count, swstate);
8749: #endif /* TASK_SWAPPER */
8750:
8751: for (entry = vm_map_first_entry(map);
8752: entry && entry != vm_map_to_entry(map);
8753: entry = entry->vme_next) {
8754: vm_map_entry_print(entry);
8755: }
8756:
8757: db_indent -= 2;
8758: }
8759:
8760: /*
8761: * Routine: vm_map_copy_print
8762: * Purpose:
8763: * Pretty-print a copy object for ddb.
8764: */
8765:
8766: void
8767: vm_map_copy_print(
8768: vm_map_copy_t copy)
8769: {
8770: extern int db_indent;
8771: int i, npages;
8772: vm_map_entry_t entry;
8773:
8774: printf("copy object 0x%x\n", copy);
8775:
8776: db_indent += 2;
8777:
8778: iprintf("type=%d", copy->type);
8779: switch (copy->type) {
8780: case VM_MAP_COPY_ENTRY_LIST:
8781: printf("[entry_list]");
8782: break;
8783:
8784: case VM_MAP_COPY_OBJECT:
8785: printf("[object]");
8786: break;
8787:
8788: case VM_MAP_COPY_PAGE_LIST:
8789: printf("[page_list]");
8790: break;
8791:
8792: case VM_MAP_COPY_KERNEL_BUFFER:
8793: printf("[kernel_buffer]");
8794: break;
8795:
8796: default:
8797: printf("[bad type]");
8798: break;
8799: }
8800: printf(", offset=0x%x", copy->offset);
8801: printf(", size=0x%x\n", copy->size);
8802:
8803: switch (copy->type) {
8804: case VM_MAP_COPY_ENTRY_LIST:
8805: vm_map_header_print(©->cpy_hdr);
8806: for (entry = vm_map_copy_first_entry(copy);
8807: entry && entry != vm_map_copy_to_entry(copy);
8808: entry = entry->vme_next) {
8809: vm_map_entry_print(entry);
8810: }
8811: break;
8812:
8813: case VM_MAP_COPY_OBJECT:
8814: iprintf("object=0x%x\n", copy->cpy_object);
8815: break;
8816:
8817: case VM_MAP_COPY_KERNEL_BUFFER:
8818: iprintf("kernel buffer=0x%x", copy->cpy_kdata);
8819: printf(", kalloc_size=0x%x\n", copy->cpy_kalloc_size);
8820: break;
8821:
8822: case VM_MAP_COPY_PAGE_LIST:
8823: iprintf("npages=%d", copy->cpy_npages);
8824: printf(", cont=%x", copy->cpy_cont);
8825: printf(", cont_args=%x\n", copy->cpy_cont_args);
8826: if (copy->cpy_npages < 0) {
8827: npages = 0;
8828: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
8829: npages = VM_MAP_COPY_PAGE_LIST_MAX;
8830: } else {
8831: npages = copy->cpy_npages;
8832: }
8833: iprintf("copy->cpy_page_list[0..%d] = {", npages);
8834: for (i = 0; i < npages - 1; i++) {
8835: printf("0x%x, ", copy->cpy_page_list[i]);
8836: }
8837: if (npages > 0) {
8838: printf("0x%x", copy->cpy_page_list[npages - 1]);
8839: }
8840: printf("}\n");
8841: break;
8842: }
8843:
8844: db_indent -=2;
8845: }
8846:
8847: /*
8848: * db_vm_map_total_size(map) [ debug ]
8849: *
8850: * return the total virtual size (in bytes) of the map
8851: */
8852: vm_size_t
8853: db_vm_map_total_size(
8854: vm_map_t map)
8855: {
8856: vm_map_entry_t entry;
8857: vm_size_t total;
8858:
8859: total = 0;
8860: for (entry = vm_map_first_entry(map);
8861: entry != vm_map_to_entry(map);
8862: entry = entry->vme_next) {
8863: total += entry->vme_end - entry->vme_start;
8864: }
8865:
8866: return total;
8867: }
8868:
8869: #endif /* MACH_KDB */
8870:
8871: /*
8872: * Routine: vm_map_entry_insert
8873: *
8874: * Descritpion: This routine inserts a new vm_entry in a locked map.
8875: */
8876: vm_map_entry_t
8877: vm_map_entry_insert(
8878: vm_map_t map,
8879: vm_map_entry_t insp_entry,
8880: vm_offset_t start,
8881: vm_offset_t end,
8882: vm_object_t object,
8883: vm_offset_t offset,
8884: boolean_t needs_copy,
8885: boolean_t is_shared,
8886: boolean_t in_transition,
8887: vm_prot_t cur_protection,
8888: vm_prot_t max_protection,
8889: vm_behavior_t behavior,
8890: vm_inherit_t inheritance,
8891: unsigned wired_count)
8892: {
8893: vm_map_entry_t new_entry;
8894:
8895: assert(insp_entry != (vm_map_entry_t)0);
8896:
8897: new_entry = vm_map_entry_create(map);
8898:
8899: new_entry->vme_start = start;
8900: new_entry->vme_end = end;
8901: assert(page_aligned(new_entry->vme_start));
8902: assert(page_aligned(new_entry->vme_end));
8903:
8904: new_entry->object.vm_object = object;
8905: new_entry->offset = offset;
8906: new_entry->is_shared = is_shared;
8907: new_entry->is_sub_map = FALSE;
8908: new_entry->needs_copy = needs_copy;
8909: new_entry->in_transition = in_transition;
8910: new_entry->needs_wakeup = FALSE;
8911: new_entry->inheritance = inheritance;
8912: new_entry->protection = cur_protection;
8913: new_entry->max_protection = max_protection;
8914: new_entry->behavior = behavior;
8915: new_entry->wired_count = wired_count;
8916: new_entry->user_wired_count = 0;
8917:
8918: /*
8919: * Insert the new entry into the list.
8920: */
8921:
8922: vm_map_entry_link(map, insp_entry, new_entry);
8923: map->size += end - start;
8924:
8925: /*
8926: * Update the free space hint and the lookup hint.
8927: */
8928:
8929: SAVE_HINT(map, new_entry);
8930: return new_entry;
8931: }
8932:
8933: /*
8934: * Routine: vm_remap_extract
8935: *
8936: * Descritpion: This routine returns a vm_entry list from a map.
8937: */
8938: kern_return_t
8939: vm_remap_extract(
8940: vm_map_t map,
8941: vm_offset_t addr,
8942: vm_size_t size,
8943: boolean_t copy,
8944: struct vm_map_header *map_header,
8945: vm_prot_t *cur_protection,
8946: vm_prot_t *max_protection,
8947: /* What, no behavior? */
8948: vm_inherit_t inheritance,
8949: boolean_t pageable)
8950: {
8951: kern_return_t result;
8952: vm_size_t mapped_size;
8953: vm_size_t tmp_size;
8954: vm_map_entry_t src_entry; /* result of last map lookup */
8955: vm_map_entry_t new_entry;
8956: vm_offset_t offset;
8957: vm_offset_t map_address;
8958: vm_offset_t src_start; /* start of entry to map */
8959: vm_offset_t src_end; /* end of region to be mapped */
8960: vm_object_t object;
8961: vm_map_version_t version;
8962: boolean_t src_needs_copy;
8963: boolean_t new_entry_needs_copy;
8964:
8965: assert(map != VM_MAP_NULL);
8966: assert(size != 0 && size == round_page(size));
8967: assert(inheritance == VM_INHERIT_NONE ||
8968: inheritance == VM_INHERIT_COPY ||
8969: inheritance == VM_INHERIT_SHARE);
8970:
8971: /*
8972: * Compute start and end of region.
8973: */
8974: src_start = trunc_page(addr);
8975: src_end = round_page(src_start + size);
8976:
8977: /*
8978: * Initialize map_header.
8979: */
8980: map_header->links.next = (struct vm_map_entry *)&map_header->links;
8981: map_header->links.prev = (struct vm_map_entry *)&map_header->links;
8982: map_header->nentries = 0;
8983: map_header->entries_pageable = pageable;
8984:
8985: *cur_protection = VM_PROT_ALL;
8986: *max_protection = VM_PROT_ALL;
8987:
8988: map_address = 0;
8989: mapped_size = 0;
8990: result = KERN_SUCCESS;
8991:
8992: /*
8993: * The specified source virtual space might correspond to
8994: * multiple map entries, need to loop on them.
8995: */
8996: vm_map_lock(map);
8997: while (mapped_size != size) {
8998: vm_size_t entry_size;
8999:
9000: /*
9001: * Find the beginning of the region.
9002: */
9003: if (! vm_map_lookup_entry(map, src_start, &src_entry)) {
9004: result = KERN_INVALID_ADDRESS;
9005: break;
9006: }
9007:
9008: if (src_start < src_entry->vme_start ||
9009: (mapped_size && src_start != src_entry->vme_start)) {
9010: result = KERN_INVALID_ADDRESS;
9011: break;
9012: }
9013:
9014: if(src_entry->is_sub_map) {
9015: result = KERN_INVALID_ADDRESS;
9016: break;
9017: }
9018:
9019: tmp_size = size - mapped_size;
9020: if (src_end > src_entry->vme_end)
9021: tmp_size -= (src_end - src_entry->vme_end);
9022:
9023: entry_size = (vm_size_t)(src_entry->vme_end -
9024: src_entry->vme_start);
9025:
9026: if(src_entry->is_sub_map) {
9027: vm_map_reference(src_entry->object.sub_map);
9028: } else {
9029: object = src_entry->object.vm_object;
9030:
9031: if (object == VM_OBJECT_NULL) {
9032: object = vm_object_allocate(entry_size);
9033: src_entry->offset = 0;
9034: src_entry->object.vm_object = object;
9035: } else if (object->copy_strategy !=
9036: MEMORY_OBJECT_COPY_SYMMETRIC) {
9037: /*
9038: * We are already using an asymmetric
9039: * copy, and therefore we already have
9040: * the right object.
9041: */
9042: assert(!src_entry->needs_copy);
9043: } else if (src_entry->needs_copy || object->shadowed ||
9044: (object->internal && !object->true_share &&
9045: !src_entry->is_shared &&
9046: object->size > entry_size)) {
9047:
9048: vm_object_shadow(&src_entry->object.vm_object,
9049: &src_entry->offset,
9050: entry_size);
9051:
9052: if (!src_entry->needs_copy &&
9053: (src_entry->protection & VM_PROT_WRITE)) {
9054: pmap_protect(vm_map_pmap(map),
9055: src_entry->vme_start,
9056: src_entry->vme_end,
9057: src_entry->protection &
9058: ~VM_PROT_WRITE);
9059: }
9060:
9061: object = src_entry->object.vm_object;
9062: src_entry->needs_copy = FALSE;
9063: }
9064:
9065:
9066: vm_object_lock(object);
9067: object->ref_count++; /* object ref. for new entry */
9068: VM_OBJ_RES_INCR(object);
9069: if (object->copy_strategy ==
9070: MEMORY_OBJECT_COPY_SYMMETRIC) {
9071: object->copy_strategy =
9072: MEMORY_OBJECT_COPY_DELAY;
9073: }
9074: vm_object_unlock(object);
9075: }
9076:
9077: offset = src_entry->offset + (src_start - src_entry->vme_start);
9078:
9079: new_entry = _vm_map_entry_create(map_header);
9080: vm_map_entry_copy(new_entry, src_entry);
9081:
9082: new_entry->vme_start = map_address;
9083: new_entry->vme_end = map_address + tmp_size;
9084: new_entry->inheritance = inheritance;
9085: new_entry->offset = offset;
9086:
9087: /*
9088: * The new region has to be copied now if required.
9089: */
9090: RestartCopy:
9091: if (!copy) {
9092: src_entry->is_shared = TRUE;
9093: new_entry->is_shared = TRUE;
9094: if (!(new_entry->is_sub_map))
9095: new_entry->needs_copy = FALSE;
9096:
9097: } else if (src_entry->is_sub_map) {
9098: /* make this a COW sub_map if not already */
9099: new_entry->needs_copy = TRUE;
9100: } else if (src_entry->wired_count == 0 &&
9101: vm_object_copy_quickly(&new_entry->object.vm_object,
9102: new_entry->offset,
9103: (new_entry->vme_end -
9104: new_entry->vme_start),
9105: &src_needs_copy,
9106: &new_entry_needs_copy)) {
9107:
9108: new_entry->needs_copy = new_entry_needs_copy;
9109: new_entry->is_shared = FALSE;
9110:
9111: /*
9112: * Handle copy_on_write semantics.
9113: */
9114: if (src_needs_copy && !src_entry->needs_copy) {
9115: vm_object_pmap_protect(object,
9116: offset,
9117: entry_size,
9118: (src_entry->is_shared ?
9119: PMAP_NULL : map->pmap),
9120: src_entry->vme_start,
9121: src_entry->protection &
9122: ~VM_PROT_WRITE);
9123:
9124: src_entry->needs_copy = TRUE;
9125: }
9126: /*
9127: * Throw away the old object reference of the new entry.
9128: */
9129: vm_object_deallocate(object);
9130:
9131: } else {
9132: new_entry->is_shared = FALSE;
9133:
9134: /*
9135: * The map can be safely unlocked since we
9136: * already hold a reference on the object.
9137: *
9138: * Record the timestamp of the map for later
9139: * verification, and unlock the map.
9140: */
9141: version.main_timestamp = map->timestamp;
9142: vm_map_unlock(map);
9143:
9144: /*
9145: * Perform the copy.
9146: */
9147: if (src_entry->wired_count > 0) {
9148: vm_object_lock(object);
9149: result = vm_object_copy_slowly(
9150: object,
9151: offset,
9152: entry_size,
9153: THREAD_UNINT,
9154: &new_entry->object.vm_object);
9155:
9156: new_entry->offset = 0;
9157: new_entry->needs_copy = FALSE;
9158: } else {
9159: result = vm_object_copy_strategically(
9160: object,
9161: offset,
9162: entry_size,
9163: &new_entry->object.vm_object,
9164: &new_entry->offset,
9165: &new_entry_needs_copy);
9166:
9167: new_entry->needs_copy = new_entry_needs_copy;
9168: }
9169:
9170: /*
9171: * Throw away the old object reference of the new entry.
9172: */
9173: vm_object_deallocate(object);
9174:
9175: if (result != KERN_SUCCESS &&
9176: result != KERN_MEMORY_RESTART_COPY) {
9177: _vm_map_entry_dispose(map_header, new_entry);
9178: break;
9179: }
9180:
9181: /*
9182: * Verify that the map has not substantially
9183: * changed while the copy was being made.
9184: */
9185:
9186: vm_map_lock(map); /* Increments timestamp once! */
9187: if (version.main_timestamp + 1 != map->timestamp) {
9188: /*
9189: * Simple version comparison failed.
9190: *
9191: * Retry the lookup and verify that the
9192: * same object/offset are still present.
9193: */
9194: vm_object_deallocate(new_entry->
9195: object.vm_object);
9196: _vm_map_entry_dispose(map_header, new_entry);
9197: if (result == KERN_MEMORY_RESTART_COPY)
9198: result = KERN_SUCCESS;
9199: continue;
9200: }
9201:
9202: if (result == KERN_MEMORY_RESTART_COPY) {
9203: vm_object_reference(object);
9204: goto RestartCopy;
9205: }
9206: }
9207:
9208: _vm_map_entry_link(map_header,
9209: map_header->links.prev, new_entry);
9210:
9211: *cur_protection &= src_entry->protection;
9212: *max_protection &= src_entry->max_protection;
9213:
9214: map_address += tmp_size;
9215: mapped_size += tmp_size;
9216: src_start += tmp_size;
9217:
9218: } /* end while */
9219:
9220: vm_map_unlock(map);
9221: if (result != KERN_SUCCESS) {
9222: /*
9223: * Free all allocated elements.
9224: */
9225: for (src_entry = map_header->links.next;
9226: src_entry != (struct vm_map_entry *)&map_header->links;
9227: src_entry = new_entry) {
9228: new_entry = src_entry->vme_next;
9229: _vm_map_entry_unlink(map_header, src_entry);
9230: vm_object_deallocate(src_entry->object.vm_object);
9231: _vm_map_entry_dispose(map_header, src_entry);
9232: }
9233: }
9234: return result;
9235: }
9236:
9237: /*
9238: * Routine: vm_remap
9239: *
9240: * Map portion of a task's address space.
9241: * Mapped region must not overlap more than
9242: * one vm memory object. Protections and
9243: * inheritance attributes remain the same
9244: * as in the original task and are out parameters.
9245: * Source and Target task can be identical
9246: * Other attributes are identical as for vm_map()
9247: */
9248: kern_return_t
9249: vm_remap(
9250: vm_map_t target_map,
9251: vm_offset_t *address,
9252: vm_size_t size,
9253: vm_offset_t mask,
9254: boolean_t anywhere,
9255: vm_map_t src_map,
9256: vm_offset_t memory_address,
9257: boolean_t copy,
9258: vm_prot_t *cur_protection,
9259: vm_prot_t *max_protection,
9260: vm_inherit_t inheritance)
9261: {
9262: kern_return_t result;
9263: vm_map_entry_t entry;
9264: vm_map_entry_t insp_entry;
9265: vm_map_entry_t new_entry;
9266: struct vm_map_header map_header;
9267:
9268: if (target_map == VM_MAP_NULL)
9269: return KERN_INVALID_ARGUMENT;
9270:
9271: switch (inheritance) {
9272: case VM_INHERIT_NONE:
9273: case VM_INHERIT_COPY:
9274: case VM_INHERIT_SHARE:
9275: if (size != 0 && src_map != VM_MAP_NULL)
9276: break;
9277: /*FALL THRU*/
9278: default:
9279: return KERN_INVALID_ARGUMENT;
9280: }
9281:
9282: size = round_page(size);
9283:
9284: result = vm_remap_extract(src_map, memory_address,
9285: size, copy, &map_header,
9286: cur_protection,
9287: max_protection,
9288: inheritance,
9289: target_map->hdr.
9290: entries_pageable);
9291: vm_map_deallocate(src_map);
9292:
9293: if (result != KERN_SUCCESS) {
9294: return result;
9295: }
9296:
9297: /*
9298: * Allocate/check a range of free virtual address
9299: * space for the target
9300: */
9301: *address = trunc_page(*address);
9302: vm_map_lock(target_map);
9303: result = vm_remap_range_allocate(target_map, address, size,
9304: mask, anywhere, &insp_entry);
9305:
9306: for (entry = map_header.links.next;
9307: entry != (struct vm_map_entry *)&map_header.links;
9308: entry = new_entry) {
9309: new_entry = entry->vme_next;
9310: _vm_map_entry_unlink(&map_header, entry);
9311: if (result == KERN_SUCCESS) {
9312: entry->vme_start += *address;
9313: entry->vme_end += *address;
9314: vm_map_entry_link(target_map, insp_entry, entry);
9315: insp_entry = entry;
9316: } else {
9317: if (!entry->is_sub_map) {
9318: vm_object_deallocate(entry->object.vm_object);
9319: } else {
9320: vm_map_deallocate(entry->object.sub_map);
9321: }
9322: _vm_map_entry_dispose(&map_header, entry);
9323: }
9324: }
9325:
9326: if (result == KERN_SUCCESS) {
9327: target_map->size += size;
9328: SAVE_HINT(target_map, insp_entry);
9329: }
9330: vm_map_unlock(target_map);
9331:
9332: if (result == KERN_SUCCESS && target_map->wiring_required)
9333: result = vm_map_wire(target_map, *address,
9334: *address + size, *cur_protection, TRUE);
9335: return result;
9336: }
9337:
9338: /*
9339: * Routine: vm_remap_range_allocate
9340: *
9341: * Description:
9342: * Allocate a range in the specified virtual address map.
9343: * returns the address and the map entry just before the allocated
9344: * range
9345: *
9346: * Map must be locked.
9347: */
9348:
9349: kern_return_t
9350: vm_remap_range_allocate(
9351: vm_map_t map,
9352: vm_offset_t *address, /* IN/OUT */
9353: vm_size_t size,
9354: vm_offset_t mask,
9355: boolean_t anywhere,
9356: vm_map_entry_t *map_entry) /* OUT */
9357: {
9358: register vm_map_entry_t entry;
9359: register vm_offset_t start;
9360: register vm_offset_t end;
9361: kern_return_t result = KERN_SUCCESS;
9362:
9363: StartAgain: ;
9364:
9365: start = *address;
9366:
9367: if (anywhere)
9368: {
9369: /*
9370: * Calculate the first possible address.
9371: */
9372:
9373: if (start < map->min_offset)
9374: start = map->min_offset;
9375: if (start > map->max_offset)
9376: return(KERN_NO_SPACE);
9377:
9378: /*
9379: * Look for the first possible address;
9380: * if there's already something at this
9381: * address, we have to start after it.
9382: */
9383:
9384: assert(first_free_is_valid(map));
9385: if (start == map->min_offset) {
9386: if ((entry = map->first_free) != vm_map_to_entry(map))
9387: start = entry->vme_end;
9388: } else {
9389: vm_map_entry_t tmp_entry;
9390: if (vm_map_lookup_entry(map, start, &tmp_entry))
9391: start = tmp_entry->vme_end;
9392: entry = tmp_entry;
9393: }
9394:
9395: /*
9396: * In any case, the "entry" always precedes
9397: * the proposed new region throughout the
9398: * loop:
9399: */
9400:
9401: while (TRUE) {
9402: register vm_map_entry_t next;
9403:
9404: /*
9405: * Find the end of the proposed new region.
9406: * Be sure we didn't go beyond the end, or
9407: * wrap around the address.
9408: */
9409:
9410: end = ((start + mask) & ~mask);
9411: if (end < start)
9412: return(KERN_NO_SPACE);
9413: start = end;
9414: end += size;
9415:
9416: if ((end > map->max_offset) || (end < start)) {
9417: if (map->wait_for_space) {
9418: if (size <= (map->max_offset -
9419: map->min_offset)) {
9420: assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
9421: vm_map_unlock(map);
9422: thread_block((void (*)(void))0);
9423: vm_map_lock(map);
9424: goto StartAgain;
9425: }
9426: }
9427:
9428: return(KERN_NO_SPACE);
9429: }
9430:
9431: /*
9432: * If there are no more entries, we must win.
9433: */
9434:
9435: next = entry->vme_next;
9436: if (next == vm_map_to_entry(map))
9437: break;
9438:
9439: /*
9440: * If there is another entry, it must be
9441: * after the end of the potential new region.
9442: */
9443:
9444: if (next->vme_start >= end)
9445: break;
9446:
9447: /*
9448: * Didn't fit -- move to the next entry.
9449: */
9450:
9451: entry = next;
9452: start = entry->vme_end;
9453: }
9454: *address = start;
9455: } else {
9456: vm_map_entry_t temp_entry;
9457:
9458: /*
9459: * Verify that:
9460: * the address doesn't itself violate
9461: * the mask requirement.
9462: */
9463:
9464: if ((start & mask) != 0)
9465: return(KERN_NO_SPACE);
9466:
9467:
9468: /*
9469: * ... the address is within bounds
9470: */
9471:
9472: end = start + size;
9473:
9474: if ((start < map->min_offset) ||
9475: (end > map->max_offset) ||
9476: (start >= end)) {
9477: return(KERN_INVALID_ADDRESS);
9478: }
9479:
9480: /*
9481: * ... the starting address isn't allocated
9482: */
9483:
9484: if (vm_map_lookup_entry(map, start, &temp_entry))
9485: return(KERN_NO_SPACE);
9486:
9487: entry = temp_entry;
9488:
9489: /*
9490: * ... the next region doesn't overlap the
9491: * end point.
9492: */
9493:
9494: if ((entry->vme_next != vm_map_to_entry(map)) &&
9495: (entry->vme_next->vme_start < end))
9496: return(KERN_NO_SPACE);
9497: }
9498: *map_entry = entry;
9499: return(KERN_SUCCESS);
9500: }
9501:
9502: /*
9503: * vm_map_switch:
9504: *
9505: * Set the address map for the current thr_act to the specified map
9506: */
9507:
9508: vm_map_t
9509: vm_map_switch(
9510: vm_map_t map)
9511: {
9512: int mycpu;
9513: thread_act_t thr_act = current_act();
9514: vm_map_t oldmap = thr_act->map;
9515:
9516: mp_disable_preemption();
9517: mycpu = cpu_number();
9518:
9519: /*
9520: * Deactivate the current map and activate the requested map
9521: */
9522: PMAP_SWITCH_USER(thr_act, map, mycpu);
9523:
9524: mp_enable_preemption();
9525: return(oldmap);
9526: }
9527:
9528:
9529: /*
9530: * Routine: vm_map_write_user
9531: *
9532: * Description:
9533: * Copy out data from a kernel space into space in the
9534: * destination map. The space must already exist in the
9535: * destination map.
9536: * NOTE: This routine should only be called by threads
9537: * which can block on a page fault. i.e. kernel mode user
9538: * threads.
9539: *
9540: */
9541: kern_return_t
9542: vm_map_write_user(
9543: vm_map_t map,
9544: vm_offset_t src_addr,
9545: vm_offset_t dst_addr,
9546: vm_size_t size)
9547: {
9548: thread_act_t thr_act = current_act();
9549: kern_return_t kr = KERN_SUCCESS;
9550:
9551: if(thr_act->map == map) {
9552: if (copyout((char *)src_addr, (char *)dst_addr, size)) {
9553: kr = KERN_INVALID_ADDRESS;
9554: }
9555: } else {
9556: vm_map_t oldmap;
9557:
9558: /* take on the identity of the target map while doing */
9559: /* the transfer */
9560:
9561: vm_map_reference(map);
9562: oldmap = vm_map_switch(map);
9563: if (copyout((char *)src_addr, (char *)dst_addr, size)) {
9564: kr = KERN_INVALID_ADDRESS;
9565: }
9566: vm_map_switch(oldmap);
9567: vm_map_deallocate(map);
9568: }
9569: return kr;
9570: }
9571:
9572: /*
9573: * Routine: vm_map_read_user
9574: *
9575: * Description:
9576: * Copy in data from a user space source map into the
9577: * kernel map. The space must already exist in the
9578: * kernel map.
9579: * NOTE: This routine should only be called by threads
9580: * which can block on a page fault. i.e. kernel mode user
9581: * threads.
9582: *
9583: */
9584: kern_return_t
9585: vm_map_read_user(
9586: vm_map_t map,
9587: vm_offset_t src_addr,
9588: vm_offset_t dst_addr,
9589: vm_size_t size)
9590: {
9591: thread_act_t thr_act = current_act();
9592: kern_return_t kr = KERN_SUCCESS;
9593:
9594: if(thr_act->map == map) {
9595: if (copyin((char *)src_addr, (char *)dst_addr, size)) {
9596: kr = KERN_INVALID_ADDRESS;
9597: }
9598: } else {
9599: vm_map_t oldmap;
9600:
9601: /* take on the identity of the target map while doing */
9602: /* the transfer */
9603:
9604: vm_map_reference(map);
9605: oldmap = vm_map_switch(map);
9606: if (copyin((char *)src_addr, (char *)dst_addr, size)) {
9607: kr = KERN_INVALID_ADDRESS;
9608: }
9609: vm_map_switch(oldmap);
9610: vm_map_deallocate(map);
9611: }
9612: return kr;
9613: }
9614:
9615: /*
9616: * Export routines to other components for the things we access locally through
9617: * macros.
9618: */
9619: #undef current_map
9620: vm_map_t
9621: current_map(void)
9622: {
9623: return (current_map_fast());
9624: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.