![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
37: #include <norma_ipc.h>
38:
39: #include <mach/kern_return.h>
40: #include <mach/port.h>
41: #include <mach/vm_attributes.h>
42: #include <mach/vm_param.h>
43: #include <kern/assert.h>
44: #include <kern/zalloc.h>
45: #include <vm/vm_fault.h>
46: #include <vm/vm_map.h>
47: #include <vm/vm_object.h>
48: #include <vm/vm_page.h>
49: #include <vm/vm_kern.h>
50: #include <ipc/ipc_port.h>
51:
52: /*
53: * Macros to copy a vm_map_entry. We must be careful to correctly
54: * manage the wired page count. vm_map_entry_copy() creates a new
55: * map entry to the same memory - the wired count in the new entry
56: * must be set to zero. vm_map_entry_copy_full() creates a new
57: * entry that is identical to the old entry. This preserves the
58: * wire count; it's used for map splitting and zone changing in
59: * vm_map_copyout.
60: */
61: #define vm_map_entry_copy(NEW,OLD) \
62: MACRO_BEGIN \
63: *(NEW) = *(OLD); \
64: (NEW)->is_shared = FALSE; \
65: (NEW)->needs_wakeup = FALSE; \
66: (NEW)->in_transition = FALSE; \
67: (NEW)->wired_count = 0; \
68: (NEW)->user_wired_count = 0; \
69: MACRO_END
70:
71: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
72:
73: /*
74: * Virtual memory maps provide for the mapping, protection,
75: * and sharing of virtual memory objects. In addition,
76: * this module provides for an efficient virtual copy of
77: * memory from one map to another.
78: *
79: * Synchronization is required prior to most operations.
80: *
81: * Maps consist of an ordered doubly-linked list of simple
82: * entries; a single hint is used to speed up lookups.
83: *
84: * Sharing maps have been deleted from this version of Mach.
85: * All shared objects are now mapped directly into the respective
86: * maps. This requires a change in the copy on write strategy;
87: * the asymmetric (delayed) strategy is used for shared temporary
88: * objects instead of the symmetric (shadow) strategy. This is
89: * selected by the (new) use_shared_copy bit in the object. See
90: * vm_object_copy_temporary in vm_object.c for details. All maps
91: * are now "top level" maps (either task map, kernel map or submap
92: * of the kernel map).
93: *
94: * Since portions of maps are specified by start/end addreses,
95: * which may not align with existing map entries, all
96: * routines merely "clip" entries to these start/end values.
97: * [That is, an entry is split into two, bordering at a
98: * start or end value.] Note that these clippings may not
99: * always be necessary (as the two resulting entries are then
100: * not changed); however, the clipping is done for convenience.
101: * No attempt is currently made to "glue back together" two
102: * abutting entries.
103: *
104: * The symmetric (shadow) copy strategy implements virtual copy
105: * by copying VM object references from one map to
106: * another, and then marking both regions as copy-on-write.
107: * It is important to note that only one writeable reference
108: * to a VM object region exists in any map when this strategy
109: * is used -- this means that shadow object creation can be
110: * delayed until a write operation occurs. The asymmetric (delayed)
111: * strategy allows multiple maps to have writeable references to
112: * the same region of a vm object, and hence cannot delay creating
113: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
114: * Copying of permanent objects is completely different; see
115: * vm_object_copy_strategically() in vm_object.c.
116: */
117:
118: zone_t vm_map_zone; /* zone for vm_map structures */
119: zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */
120: zone_t vm_map_kentry_zone; /* zone for kernel entry structures */
121: zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */
122:
123: boolean_t vm_map_lookup_entry(); /* forward declaration */
124:
125: /*
126: * Placeholder object for submap operations. This object is dropped
127: * into the range by a call to vm_map_find, and removed when
128: * vm_map_submap creates the submap.
129: */
130:
131: vm_object_t vm_submap_object;
132:
133: /*
134: * vm_map_init:
135: *
136: * Initialize the vm_map module. Must be called before
137: * any other vm_map routines.
138: *
139: * Map and entry structures are allocated from zones -- we must
140: * initialize those zones.
141: *
142: * There are three zones of interest:
143: *
144: * vm_map_zone: used to allocate maps.
145: * vm_map_entry_zone: used to allocate map entries.
146: * vm_map_kentry_zone: used to allocate map entries for the kernel.
147: *
148: * The kernel allocates map entries from a special zone that is initially
149: * "crammed" with memory. It would be difficult (perhaps impossible) for
150: * the kernel to allocate more memory to a entry zone when it became
151: * empty since the very act of allocating memory implies the creation
152: * of a new entry.
153: */
154:
155: vm_offset_t kentry_data;
156: vm_size_t kentry_data_size;
157: int kentry_count = 256; /* to init kentry_data_size */
158:
159: void vm_map_init()
160: {
161: vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024,
162: PAGE_SIZE, 0, "maps");
163: vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
164: 1024*1024, PAGE_SIZE*5,
165: 0, "non-kernel map entries");
166: vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
167: kentry_data_size, kentry_data_size,
168: ZONE_FIXED /* XXX */, "kernel map entries");
169:
170: vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy),
171: 16*1024, PAGE_SIZE, 0,
172: "map copies");
173:
174: /*
175: * Cram the kentry zone with initial data.
176: */
177: zcram(vm_map_kentry_zone, kentry_data, kentry_data_size);
178:
179: /*
180: * Submap object is initialized by vm_object_init.
181: */
182: }
183:
184: /*
185: * vm_map_create:
186: *
187: * Creates and returns a new empty VM map with
188: * the given physical map structure, and having
189: * the given lower and upper address bounds.
190: */
191: vm_map_t vm_map_create(pmap, min, max, pageable)
192: pmap_t pmap;
193: vm_offset_t min, max;
194: boolean_t pageable;
195: {
196: register vm_map_t result;
197:
198: result = (vm_map_t) zalloc(vm_map_zone);
199: if (result == VM_MAP_NULL)
200: panic("vm_map_create");
201:
202: vm_map_first_entry(result) = vm_map_to_entry(result);
203: vm_map_last_entry(result) = vm_map_to_entry(result);
204: result->hdr.nentries = 0;
205: result->hdr.entries_pageable = pageable;
206:
207: result->size = 0;
208: result->ref_count = 1;
209: result->pmap = pmap;
210: result->min_offset = min;
211: result->max_offset = max;
212: result->wiring_required = FALSE;
213: result->wait_for_space = FALSE;
214: result->first_free = vm_map_to_entry(result);
215: result->hint = vm_map_to_entry(result);
216: vm_map_lock_init(result);
217: simple_lock_init(&result->ref_lock);
218: simple_lock_init(&result->hint_lock);
219:
220: return(result);
221: }
222:
223: /*
224: * vm_map_entry_create: [ internal use only ]
225: *
226: * Allocates a VM map entry for insertion in the
227: * given map (or map copy). No fields are filled.
228: */
229: #define vm_map_entry_create(map) \
230: _vm_map_entry_create(&(map)->hdr)
231:
232: #define vm_map_copy_entry_create(copy) \
233: _vm_map_entry_create(&(copy)->cpy_hdr)
234:
235: vm_map_entry_t _vm_map_entry_create(map_header)
236: register struct vm_map_header *map_header;
237: {
238: register zone_t zone;
239: register vm_map_entry_t entry;
240:
241: if (map_header->entries_pageable)
242: zone = vm_map_entry_zone;
243: else
244: zone = vm_map_kentry_zone;
245:
246: entry = (vm_map_entry_t) zalloc(zone);
247: if (entry == VM_MAP_ENTRY_NULL)
248: panic("vm_map_entry_create");
249:
250: return(entry);
251: }
252:
253: /*
254: * vm_map_entry_dispose: [ internal use only ]
255: *
256: * Inverse of vm_map_entry_create.
257: */
258: #define vm_map_entry_dispose(map, entry) \
259: _vm_map_entry_dispose(&(map)->hdr, (entry))
260:
261: #define vm_map_copy_entry_dispose(map, entry) \
262: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
263:
264: void _vm_map_entry_dispose(map_header, entry)
265: register struct vm_map_header *map_header;
266: register vm_map_entry_t entry;
267: {
268: register zone_t zone;
269:
270: if (map_header->entries_pageable)
271: zone = vm_map_entry_zone;
272: else
273: zone = vm_map_kentry_zone;
274:
275: zfree(zone, (vm_offset_t) entry);
276: }
277:
278: /*
279: * vm_map_entry_{un,}link:
280: *
281: * Insert/remove entries from maps (or map copies).
282: */
283: #define vm_map_entry_link(map, after_where, entry) \
284: _vm_map_entry_link(&(map)->hdr, after_where, entry)
285:
286: #define vm_map_copy_entry_link(copy, after_where, entry) \
287: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
288:
289: #define _vm_map_entry_link(hdr, after_where, entry) \
290: MACRO_BEGIN \
291: (hdr)->nentries++; \
292: (entry)->vme_prev = (after_where); \
293: (entry)->vme_next = (after_where)->vme_next; \
294: (entry)->vme_prev->vme_next = \
295: (entry)->vme_next->vme_prev = (entry); \
296: MACRO_END
297:
298: #define vm_map_entry_unlink(map, entry) \
299: _vm_map_entry_unlink(&(map)->hdr, entry)
300:
301: #define vm_map_copy_entry_unlink(copy, entry) \
302: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
303:
304: #define _vm_map_entry_unlink(hdr, entry) \
305: MACRO_BEGIN \
306: (hdr)->nentries--; \
307: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
308: (entry)->vme_prev->vme_next = (entry)->vme_next; \
309: MACRO_END
310:
311: /*
312: * vm_map_reference:
313: *
314: * Creates another valid reference to the given map.
315: *
316: */
317: void vm_map_reference(map)
318: register vm_map_t map;
319: {
320: if (map == VM_MAP_NULL)
321: return;
322:
323: simple_lock(&map->ref_lock);
324: map->ref_count++;
325: simple_unlock(&map->ref_lock);
326: }
327:
328: /*
329: * vm_map_deallocate:
330: *
331: * Removes a reference from the specified map,
332: * destroying it if no references remain.
333: * The map should not be locked.
334: */
335: void vm_map_deallocate(map)
336: register vm_map_t map;
337: {
338: register int c;
339:
340: if (map == VM_MAP_NULL)
341: return;
342:
343: simple_lock(&map->ref_lock);
344: c = --map->ref_count;
345: simple_unlock(&map->ref_lock);
346:
347: if (c > 0) {
348: return;
349: }
350:
351: projected_buffer_collect(map);
352: (void) vm_map_delete(map, map->min_offset, map->max_offset);
353:
354: pmap_destroy(map->pmap);
355:
356: zfree(vm_map_zone, (vm_offset_t) map);
357: }
358:
359: /*
360: * SAVE_HINT:
361: *
362: * Saves the specified entry as the hint for
363: * future lookups. Performs necessary interlocks.
364: */
365: #define SAVE_HINT(map,value) \
366: simple_lock(&(map)->hint_lock); \
367: (map)->hint = (value); \
368: simple_unlock(&(map)->hint_lock);
369:
370: /*
371: * vm_map_lookup_entry: [ internal use only ]
372: *
373: * Finds the map entry containing (or
374: * immediately preceding) the specified address
375: * in the given map; the entry is returned
376: * in the "entry" parameter. The boolean
377: * result indicates whether the address is
378: * actually contained in the map.
379: */
380: boolean_t vm_map_lookup_entry(map, address, entry)
381: register vm_map_t map;
382: register vm_offset_t address;
383: vm_map_entry_t *entry; /* OUT */
384: {
385: register vm_map_entry_t cur;
386: register vm_map_entry_t last;
387:
388: /*
389: * Start looking either from the head of the
390: * list, or from the hint.
391: */
392:
393: simple_lock(&map->hint_lock);
394: cur = map->hint;
395: simple_unlock(&map->hint_lock);
396:
397: if (cur == vm_map_to_entry(map))
398: cur = cur->vme_next;
399:
400: if (address >= cur->vme_start) {
401: /*
402: * Go from hint to end of list.
403: *
404: * But first, make a quick check to see if
405: * we are already looking at the entry we
406: * want (which is usually the case).
407: * Note also that we don't need to save the hint
408: * here... it is the same hint (unless we are
409: * at the header, in which case the hint didn't
410: * buy us anything anyway).
411: */
412: last = vm_map_to_entry(map);
413: if ((cur != last) && (cur->vme_end > address)) {
414: *entry = cur;
415: return(TRUE);
416: }
417: }
418: else {
419: /*
420: * Go from start to hint, *inclusively*
421: */
422: last = cur->vme_next;
423: cur = vm_map_first_entry(map);
424: }
425:
426: /*
427: * Search linearly
428: */
429:
430: while (cur != last) {
431: if (cur->vme_end > address) {
432: if (address >= cur->vme_start) {
433: /*
434: * Save this lookup for future
435: * hints, and return
436: */
437:
438: *entry = cur;
439: SAVE_HINT(map, cur);
440: return(TRUE);
441: }
442: break;
443: }
444: cur = cur->vme_next;
445: }
446: *entry = cur->vme_prev;
447: SAVE_HINT(map, *entry);
448: return(FALSE);
449: }
450:
451: /*
452: * Routine: invalid_user_access
453: *
454: * Verifies whether user access is valid.
455: */
456:
457: boolean_t
458: invalid_user_access(map, start, end, prot)
459: vm_map_t map;
460: vm_offset_t start, end;
461: vm_prot_t prot;
462: {
463: vm_map_entry_t entry;
464:
465: return (map == VM_MAP_NULL || map == kernel_map ||
466: !vm_map_lookup_entry(map, start, &entry) ||
467: entry->vme_end < end ||
468: (prot & ~(entry->protection)));
469: }
470:
471:
472: /*
473: * Routine: vm_map_find_entry
474: * Purpose:
475: * Allocate a range in the specified virtual address map,
476: * returning the entry allocated for that range.
477: * Used by kmem_alloc, etc. Returns wired entries.
478: *
479: * The map must be locked.
480: *
481: * If an entry is allocated, the object/offset fields
482: * are initialized to zero. If an object is supplied,
483: * then an existing entry may be extended.
484: */
485: kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry)
486: register vm_map_t map;
487: vm_offset_t *address; /* OUT */
488: vm_size_t size;
489: vm_offset_t mask;
490: vm_object_t object;
491: vm_map_entry_t *o_entry; /* OUT */
492: {
493: register vm_map_entry_t entry, new_entry;
494: register vm_offset_t start;
495: register vm_offset_t end;
496:
497: /*
498: * Look for the first possible address;
499: * if there's already something at this
500: * address, we have to start after it.
501: */
502:
503: if ((entry = map->first_free) == vm_map_to_entry(map))
504: start = map->min_offset;
505: else
506: start = entry->vme_end;
507:
508: /*
509: * In any case, the "entry" always precedes
510: * the proposed new region throughout the loop:
511: */
512:
513: while (TRUE) {
514: register vm_map_entry_t next;
515:
516: /*
517: * Find the end of the proposed new region.
518: * Be sure we didn't go beyond the end, or
519: * wrap around the address.
520: */
521:
522: if (((start + mask) & ~mask) < start)
523: return(KERN_NO_SPACE);
524: start = ((start + mask) & ~mask);
525: end = start + size;
526:
527: if ((end > map->max_offset) || (end < start))
528: return(KERN_NO_SPACE);
529:
530: /*
531: * If there are no more entries, we must win.
532: */
533:
534: next = entry->vme_next;
535: if (next == vm_map_to_entry(map))
536: break;
537:
538: /*
539: * If there is another entry, it must be
540: * after the end of the potential new region.
541: */
542:
543: if (next->vme_start >= end)
544: break;
545:
546: /*
547: * Didn't fit -- move to the next entry.
548: */
549:
550: entry = next;
551: start = entry->vme_end;
552: }
553:
554: /*
555: * At this point,
556: * "start" and "end" should define the endpoints of the
557: * available new range, and
558: * "entry" should refer to the region before the new
559: * range, and
560: *
561: * the map should be locked.
562: */
563:
564: *address = start;
565:
566: /*
567: * See whether we can avoid creating a new entry by
568: * extending one of our neighbors. [So far, we only attempt to
569: * extend from below.]
570: */
571:
572: if ((object != VM_OBJECT_NULL) &&
573: (entry != vm_map_to_entry(map)) &&
574: (entry->vme_end == start) &&
575: (!entry->is_shared) &&
576: (!entry->is_sub_map) &&
577: (entry->object.vm_object == object) &&
578: (entry->needs_copy == FALSE) &&
579: (entry->inheritance == VM_INHERIT_DEFAULT) &&
580: (entry->protection == VM_PROT_DEFAULT) &&
581: (entry->max_protection == VM_PROT_ALL) &&
582: (entry->wired_count == 1) &&
583: (entry->user_wired_count == 0) &&
584: (entry->projected_on == 0)) {
585: /*
586: * Because this is a special case,
587: * we don't need to use vm_object_coalesce.
588: */
589:
590: entry->vme_end = end;
591: new_entry = entry;
592: } else {
593: new_entry = vm_map_entry_create(map);
594:
595: new_entry->vme_start = start;
596: new_entry->vme_end = end;
597:
598: new_entry->is_shared = FALSE;
599: new_entry->is_sub_map = FALSE;
600: new_entry->object.vm_object = VM_OBJECT_NULL;
601: new_entry->offset = (vm_offset_t) 0;
602:
603: new_entry->needs_copy = FALSE;
604:
605: new_entry->inheritance = VM_INHERIT_DEFAULT;
606: new_entry->protection = VM_PROT_DEFAULT;
607: new_entry->max_protection = VM_PROT_ALL;
608: new_entry->wired_count = 1;
609: new_entry->user_wired_count = 0;
610:
611: new_entry->in_transition = FALSE;
612: new_entry->needs_wakeup = FALSE;
613: new_entry->projected_on = 0;
614:
615: /*
616: * Insert the new entry into the list
617: */
618:
619: vm_map_entry_link(map, entry, new_entry);
620: }
621:
622: map->size += size;
623:
624: /*
625: * Update the free space hint and the lookup hint
626: */
627:
628: map->first_free = new_entry;
629: SAVE_HINT(map, new_entry);
630:
631: *o_entry = new_entry;
632: return(KERN_SUCCESS);
633: }
634:
635: int vm_map_pmap_enter_print = FALSE;
636: int vm_map_pmap_enter_enable = FALSE;
637:
638: /*
639: * Routine: vm_map_pmap_enter
640: *
641: * Description:
642: * Force pages from the specified object to be entered into
643: * the pmap at the specified address if they are present.
644: * As soon as a page not found in the object the scan ends.
645: *
646: * Returns:
647: * Nothing.
648: *
649: * In/out conditions:
650: * The source map should not be locked on entry.
651: */
652: void
653: vm_map_pmap_enter(map, addr, end_addr, object, offset, protection)
654: vm_map_t map;
655: register
656: vm_offset_t addr;
657: register
658: vm_offset_t end_addr;
659: register
660: vm_object_t object;
661: vm_offset_t offset;
662: vm_prot_t protection;
663: {
664: while (addr < end_addr) {
665: register vm_page_t m;
666:
667: vm_object_lock(object);
668: vm_object_paging_begin(object);
669:
670: m = vm_page_lookup(object, offset);
671: if (m == VM_PAGE_NULL || m->absent) {
672: vm_object_paging_end(object);
673: vm_object_unlock(object);
674: return;
675: }
676:
677: if (vm_map_pmap_enter_print) {
678: printf("vm_map_pmap_enter:");
679: printf("map: %x, addr: %x, object: %x, offset: %x\n",
680: map, addr, object, offset);
681: }
682:
683: m->busy = TRUE;
684: vm_object_unlock(object);
685:
686: PMAP_ENTER(map->pmap, addr, m,
687: protection, FALSE);
688:
689: vm_object_lock(object);
690: PAGE_WAKEUP_DONE(m);
691: vm_page_lock_queues();
692: if (!m->active && !m->inactive)
693: vm_page_activate(m);
694: vm_page_unlock_queues();
695: vm_object_paging_end(object);
696: vm_object_unlock(object);
697:
698: offset += PAGE_SIZE;
699: addr += PAGE_SIZE;
700: }
701: }
702:
703: /*
704: * Routine: vm_map_enter
705: *
706: * Description:
707: * Allocate a range in the specified virtual address map.
708: * The resulting range will refer to memory defined by
709: * the given memory object and offset into that object.
710: *
711: * Arguments are as defined in the vm_map call.
712: */
713: kern_return_t vm_map_enter(
714: map,
715: address, size, mask, anywhere,
716: object, offset, needs_copy,
717: cur_protection, max_protection, inheritance)
718: register
719: vm_map_t map;
720: vm_offset_t *address; /* IN/OUT */
721: vm_size_t size;
722: vm_offset_t mask;
723: boolean_t anywhere;
724: vm_object_t object;
725: vm_offset_t offset;
726: boolean_t needs_copy;
727: vm_prot_t cur_protection;
728: vm_prot_t max_protection;
729: vm_inherit_t inheritance;
730: {
731: register vm_map_entry_t entry;
732: register vm_offset_t start;
733: register vm_offset_t end;
734: kern_return_t result = KERN_SUCCESS;
735:
736: #define RETURN(value) { result = value; goto BailOut; }
737:
738: StartAgain: ;
739:
740: start = *address;
741:
742: if (anywhere) {
743: vm_map_lock(map);
744:
745: /*
746: * Calculate the first possible address.
747: */
748:
749: if (start < map->min_offset)
750: start = map->min_offset;
751: if (start > map->max_offset)
752: RETURN(KERN_NO_SPACE);
753:
754: /*
755: * Look for the first possible address;
756: * if there's already something at this
757: * address, we have to start after it.
758: */
759:
760: if (start == map->min_offset) {
761: if ((entry = map->first_free) != vm_map_to_entry(map))
762: start = entry->vme_end;
763: } else {
764: vm_map_entry_t tmp_entry;
765: if (vm_map_lookup_entry(map, start, &tmp_entry))
766: start = tmp_entry->vme_end;
767: entry = tmp_entry;
768: }
769:
770: /*
771: * In any case, the "entry" always precedes
772: * the proposed new region throughout the
773: * loop:
774: */
775:
776: while (TRUE) {
777: register vm_map_entry_t next;
778:
779: /*
780: * Find the end of the proposed new region.
781: * Be sure we didn't go beyond the end, or
782: * wrap around the address.
783: */
784:
785: if (((start + mask) & ~mask) < start)
786: return(KERN_NO_SPACE);
787: start = ((start + mask) & ~mask);
788: end = start + size;
789:
790: if ((end > map->max_offset) || (end < start)) {
791: if (map->wait_for_space) {
792: if (size <= (map->max_offset -
793: map->min_offset)) {
794: assert_wait((event_t) map, TRUE);
795: vm_map_unlock(map);
796: thread_block((void (*)()) 0);
797: goto StartAgain;
798: }
799: }
800:
801: RETURN(KERN_NO_SPACE);
802: }
803:
804: /*
805: * If there are no more entries, we must win.
806: */
807:
808: next = entry->vme_next;
809: if (next == vm_map_to_entry(map))
810: break;
811:
812: /*
813: * If there is another entry, it must be
814: * after the end of the potential new region.
815: */
816:
817: if (next->vme_start >= end)
818: break;
819:
820: /*
821: * Didn't fit -- move to the next entry.
822: */
823:
824: entry = next;
825: start = entry->vme_end;
826: }
827: *address = start;
828: } else {
829: vm_map_entry_t temp_entry;
830:
831: /*
832: * Verify that:
833: * the address doesn't itself violate
834: * the mask requirement.
835: */
836:
837: if ((start & mask) != 0)
838: return(KERN_NO_SPACE);
839:
840: vm_map_lock(map);
841:
842: /*
843: * ... the address is within bounds
844: */
845:
846: end = start + size;
847:
848: if ((start < map->min_offset) ||
849: (end > map->max_offset) ||
850: (start >= end)) {
851: RETURN(KERN_INVALID_ADDRESS);
852: }
853:
854: /*
855: * ... the starting address isn't allocated
856: */
857:
858: if (vm_map_lookup_entry(map, start, &temp_entry))
859: RETURN(KERN_NO_SPACE);
860:
861: entry = temp_entry;
862:
863: /*
864: * ... the next region doesn't overlap the
865: * end point.
866: */
867:
868: if ((entry->vme_next != vm_map_to_entry(map)) &&
869: (entry->vme_next->vme_start < end))
870: RETURN(KERN_NO_SPACE);
871: }
872:
873: /*
874: * At this point,
875: * "start" and "end" should define the endpoints of the
876: * available new range, and
877: * "entry" should refer to the region before the new
878: * range, and
879: *
880: * the map should be locked.
881: */
882:
883: /*
884: * See whether we can avoid creating a new entry (and object) by
885: * extending one of our neighbors. [So far, we only attempt to
886: * extend from below.]
887: */
888:
889: if ((object == VM_OBJECT_NULL) &&
890: (entry != vm_map_to_entry(map)) &&
891: (entry->vme_end == start) &&
892: (!entry->is_shared) &&
893: (!entry->is_sub_map) &&
894: (entry->inheritance == inheritance) &&
895: (entry->protection == cur_protection) &&
896: (entry->max_protection == max_protection) &&
897: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
898: (entry->projected_on == 0)) {
899: if (vm_object_coalesce(entry->object.vm_object,
900: VM_OBJECT_NULL,
901: entry->offset,
902: (vm_offset_t) 0,
903: (vm_size_t)(entry->vme_end - entry->vme_start),
904: (vm_size_t)(end - entry->vme_end))) {
905:
906: /*
907: * Coalesced the two objects - can extend
908: * the previous map entry to include the
909: * new range.
910: */
911: map->size += (end - entry->vme_end);
912: entry->vme_end = end;
913: RETURN(KERN_SUCCESS);
914: }
915: }
916:
917: /*
918: * Create a new entry
919: */
920:
921: /**/ {
922: register vm_map_entry_t new_entry;
923:
924: new_entry = vm_map_entry_create(map);
925:
926: new_entry->vme_start = start;
927: new_entry->vme_end = end;
928:
929: new_entry->is_shared = FALSE;
930: new_entry->is_sub_map = FALSE;
931: new_entry->object.vm_object = object;
932: new_entry->offset = offset;
933:
934: new_entry->needs_copy = needs_copy;
935:
936: new_entry->inheritance = inheritance;
937: new_entry->protection = cur_protection;
938: new_entry->max_protection = max_protection;
939: new_entry->wired_count = 0;
940: new_entry->user_wired_count = 0;
941:
942: new_entry->in_transition = FALSE;
943: new_entry->needs_wakeup = FALSE;
944: new_entry->projected_on = 0;
945:
946: /*
947: * Insert the new entry into the list
948: */
949:
950: vm_map_entry_link(map, entry, new_entry);
951: map->size += size;
952:
953: /*
954: * Update the free space hint and the lookup hint
955: */
956:
957: if ((map->first_free == entry) &&
958: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
959: >= new_entry->vme_start))
960: map->first_free = new_entry;
961:
962: SAVE_HINT(map, new_entry);
963:
964: vm_map_unlock(map);
965:
966: if ((object != VM_OBJECT_NULL) &&
967: (vm_map_pmap_enter_enable) &&
968: (!anywhere) &&
969: (!needs_copy) &&
970: (size < (128*1024))) {
971: vm_map_pmap_enter(map, start, end,
972: object, offset, cur_protection);
973: }
974:
975: return(result);
976: /**/ }
977:
978: BailOut: ;
979:
980: vm_map_unlock(map);
981: return(result);
982:
983: #undef RETURN
984: }
985:
986: /*
987: * vm_map_clip_start: [ internal use only ]
988: *
989: * Asserts that the given entry begins at or after
990: * the specified address; if necessary,
991: * it splits the entry into two.
992: */
993: void _vm_map_clip_start();
994: #define vm_map_clip_start(map, entry, startaddr) \
995: MACRO_BEGIN \
996: if ((startaddr) > (entry)->vme_start) \
997: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
998: MACRO_END
999:
1000: void _vm_map_copy_clip_start();
1001: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1002: MACRO_BEGIN \
1003: if ((startaddr) > (entry)->vme_start) \
1004: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1005: MACRO_END
1006:
1007: /*
1008: * This routine is called only when it is known that
1009: * the entry must be split.
1010: */
1011: void _vm_map_clip_start(map_header, entry, start)
1012: register struct vm_map_header *map_header;
1013: register vm_map_entry_t entry;
1014: register vm_offset_t start;
1015: {
1016: register vm_map_entry_t new_entry;
1017:
1018: /*
1019: * Split off the front portion --
1020: * note that we must insert the new
1021: * entry BEFORE this one, so that
1022: * this entry has the specified starting
1023: * address.
1024: */
1025:
1026: new_entry = _vm_map_entry_create(map_header);
1027: vm_map_entry_copy_full(new_entry, entry);
1028:
1029: new_entry->vme_end = start;
1030: entry->offset += (start - entry->vme_start);
1031: entry->vme_start = start;
1032:
1033: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1034:
1035: if (entry->is_sub_map)
1036: vm_map_reference(new_entry->object.sub_map);
1037: else
1038: vm_object_reference(new_entry->object.vm_object);
1039: }
1040:
1041: /*
1042: * vm_map_clip_end: [ internal use only ]
1043: *
1044: * Asserts that the given entry ends at or before
1045: * the specified address; if necessary,
1046: * it splits the entry into two.
1047: */
1048: void _vm_map_clip_end();
1049: #define vm_map_clip_end(map, entry, endaddr) \
1050: MACRO_BEGIN \
1051: if ((endaddr) < (entry)->vme_end) \
1052: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1053: MACRO_END
1054:
1055: void _vm_map_copy_clip_end();
1056: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1057: MACRO_BEGIN \
1058: if ((endaddr) < (entry)->vme_end) \
1059: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1060: MACRO_END
1061:
1062: /*
1063: * This routine is called only when it is known that
1064: * the entry must be split.
1065: */
1066: void _vm_map_clip_end(map_header, entry, end)
1067: register struct vm_map_header *map_header;
1068: register vm_map_entry_t entry;
1069: register vm_offset_t end;
1070: {
1071: register vm_map_entry_t new_entry;
1072:
1073: /*
1074: * Create a new entry and insert it
1075: * AFTER the specified entry
1076: */
1077:
1078: new_entry = _vm_map_entry_create(map_header);
1079: vm_map_entry_copy_full(new_entry, entry);
1080:
1081: new_entry->vme_start = entry->vme_end = end;
1082: new_entry->offset += (end - entry->vme_start);
1083:
1084: _vm_map_entry_link(map_header, entry, new_entry);
1085:
1086: if (entry->is_sub_map)
1087: vm_map_reference(new_entry->object.sub_map);
1088: else
1089: vm_object_reference(new_entry->object.vm_object);
1090: }
1091:
1092: /*
1093: * VM_MAP_RANGE_CHECK: [ internal use only ]
1094: *
1095: * Asserts that the starting and ending region
1096: * addresses fall within the valid range of the map.
1097: */
1098: #define VM_MAP_RANGE_CHECK(map, start, end) \
1099: { \
1100: if (start < vm_map_min(map)) \
1101: start = vm_map_min(map); \
1102: if (end > vm_map_max(map)) \
1103: end = vm_map_max(map); \
1104: if (start > end) \
1105: start = end; \
1106: }
1107:
1108: /*
1109: * vm_map_submap: [ kernel use only ]
1110: *
1111: * Mark the given range as handled by a subordinate map.
1112: *
1113: * This range must have been created with vm_map_find using
1114: * the vm_submap_object, and no other operations may have been
1115: * performed on this range prior to calling vm_map_submap.
1116: *
1117: * Only a limited number of operations can be performed
1118: * within this rage after calling vm_map_submap:
1119: * vm_fault
1120: * [Don't try vm_map_copyin!]
1121: *
1122: * To remove a submapping, one must first remove the
1123: * range from the superior map, and then destroy the
1124: * submap (if desired). [Better yet, don't try it.]
1125: */
1126: kern_return_t vm_map_submap(map, start, end, submap)
1127: register vm_map_t map;
1128: register vm_offset_t start;
1129: register vm_offset_t end;
1130: vm_map_t submap;
1131: {
1132: vm_map_entry_t entry;
1133: register kern_return_t result = KERN_INVALID_ARGUMENT;
1134: register vm_object_t object;
1135:
1136: vm_map_lock(map);
1137:
1138: VM_MAP_RANGE_CHECK(map, start, end);
1139:
1140: if (vm_map_lookup_entry(map, start, &entry)) {
1141: vm_map_clip_start(map, entry, start);
1142: }
1143: else
1144: entry = entry->vme_next;
1145:
1146: vm_map_clip_end(map, entry, end);
1147:
1148: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1149: (!entry->is_sub_map) &&
1150: ((object = entry->object.vm_object) == vm_submap_object) &&
1151: (object->resident_page_count == 0) &&
1152: (object->copy == VM_OBJECT_NULL) &&
1153: (object->shadow == VM_OBJECT_NULL) &&
1154: (!object->pager_created)) {
1155: entry->object.vm_object = VM_OBJECT_NULL;
1156: vm_object_deallocate(object);
1157: entry->is_sub_map = TRUE;
1158: vm_map_reference(entry->object.sub_map = submap);
1159: result = KERN_SUCCESS;
1160: }
1161: vm_map_unlock(map);
1162:
1163: return(result);
1164: }
1165:
1166: /*
1167: * vm_map_protect:
1168: *
1169: * Sets the protection of the specified address
1170: * region in the target map. If "set_max" is
1171: * specified, the maximum protection is to be set;
1172: * otherwise, only the current protection is affected.
1173: */
1174: kern_return_t vm_map_protect(map, start, end, new_prot, set_max)
1175: register vm_map_t map;
1176: register vm_offset_t start;
1177: register vm_offset_t end;
1178: register vm_prot_t new_prot;
1179: register boolean_t set_max;
1180: {
1181: register vm_map_entry_t current;
1182: vm_map_entry_t entry;
1183:
1184: vm_map_lock(map);
1185:
1186: VM_MAP_RANGE_CHECK(map, start, end);
1187:
1188: if (vm_map_lookup_entry(map, start, &entry)) {
1189: vm_map_clip_start(map, entry, start);
1190: }
1191: else
1192: entry = entry->vme_next;
1193:
1194: /*
1195: * Make a first pass to check for protection
1196: * violations.
1197: */
1198:
1199: current = entry;
1200: while ((current != vm_map_to_entry(map)) &&
1201: (current->vme_start < end)) {
1202:
1203: if (current->is_sub_map) {
1204: vm_map_unlock(map);
1205: return(KERN_INVALID_ARGUMENT);
1206: }
1207: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1208: != new_prot) {
1209: vm_map_unlock(map);
1210: return(KERN_PROTECTION_FAILURE);
1211: }
1212:
1213: current = current->vme_next;
1214: }
1215:
1216: /*
1217: * Go back and fix up protections.
1218: * [Note that clipping is not necessary the second time.]
1219: */
1220:
1221: current = entry;
1222:
1223: while ((current != vm_map_to_entry(map)) &&
1224: (current->vme_start < end)) {
1225:
1226: vm_prot_t old_prot;
1227:
1228: vm_map_clip_end(map, current, end);
1229:
1230: old_prot = current->protection;
1231: if (set_max)
1232: current->protection =
1233: (current->max_protection = new_prot) &
1234: old_prot;
1235: else
1236: current->protection = new_prot;
1237:
1238: /*
1239: * Update physical map if necessary.
1240: */
1241:
1242: if (current->protection != old_prot) {
1243: pmap_protect(map->pmap, current->vme_start,
1244: current->vme_end,
1245: current->protection);
1246: }
1247: current = current->vme_next;
1248: }
1249:
1250: vm_map_unlock(map);
1251: return(KERN_SUCCESS);
1252: }
1253:
1254: /*
1255: * vm_map_inherit:
1256: *
1257: * Sets the inheritance of the specified address
1258: * range in the target map. Inheritance
1259: * affects how the map will be shared with
1260: * child maps at the time of vm_map_fork.
1261: */
1262: kern_return_t vm_map_inherit(map, start, end, new_inheritance)
1263: register vm_map_t map;
1264: register vm_offset_t start;
1265: register vm_offset_t end;
1266: register vm_inherit_t new_inheritance;
1267: {
1268: register vm_map_entry_t entry;
1269: vm_map_entry_t temp_entry;
1270:
1271: vm_map_lock(map);
1272:
1273: VM_MAP_RANGE_CHECK(map, start, end);
1274:
1275: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1276: entry = temp_entry;
1277: vm_map_clip_start(map, entry, start);
1278: }
1279: else
1280: entry = temp_entry->vme_next;
1281:
1282: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1283: vm_map_clip_end(map, entry, end);
1284:
1285: entry->inheritance = new_inheritance;
1286:
1287: entry = entry->vme_next;
1288: }
1289:
1290: vm_map_unlock(map);
1291: return(KERN_SUCCESS);
1292: }
1293:
1294: /*
1295: * vm_map_pageable_common:
1296: *
1297: * Sets the pageability of the specified address
1298: * range in the target map. Regions specified
1299: * as not pageable require locked-down physical
1300: * memory and physical page maps. access_type indicates
1301: * types of accesses that must not generate page faults.
1302: * This is checked against protection of memory being locked-down.
1303: * access_type of VM_PROT_NONE makes memory pageable.
1304: *
1305: * The map must not be locked, but a reference
1306: * must remain to the map throughout the call.
1307: *
1308: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1309: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1310: */
1311: kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire)
1312: register vm_map_t map;
1313: register vm_offset_t start;
1314: register vm_offset_t end;
1315: register vm_prot_t access_type;
1316: boolean_t user_wire;
1317: {
1318: register vm_map_entry_t entry;
1319: vm_map_entry_t start_entry;
1320:
1321: vm_map_lock(map);
1322:
1323: VM_MAP_RANGE_CHECK(map, start, end);
1324:
1325: if (vm_map_lookup_entry(map, start, &start_entry)) {
1326: entry = start_entry;
1327: /*
1328: * vm_map_clip_start will be done later.
1329: */
1330: }
1331: else {
1332: /*
1333: * Start address is not in map; this is fatal.
1334: */
1335: vm_map_unlock(map);
1336: return(KERN_FAILURE);
1337: }
1338:
1339: /*
1340: * Actions are rather different for wiring and unwiring,
1341: * so we have two separate cases.
1342: */
1343:
1344: if (access_type == VM_PROT_NONE) {
1345:
1346: vm_map_clip_start(map, entry, start);
1347:
1348: /*
1349: * Unwiring. First ensure that the range to be
1350: * unwired is really wired down.
1351: */
1352: while ((entry != vm_map_to_entry(map)) &&
1353: (entry->vme_start < end)) {
1354:
1355: if ((entry->wired_count == 0) ||
1356: ((entry->vme_end < end) &&
1357: ((entry->vme_next == vm_map_to_entry(map)) ||
1358: (entry->vme_next->vme_start > entry->vme_end))) ||
1359: (user_wire && (entry->user_wired_count == 0))) {
1360: vm_map_unlock(map);
1361: return(KERN_INVALID_ARGUMENT);
1362: }
1363: entry = entry->vme_next;
1364: }
1365:
1366: /*
1367: * Now decrement the wiring count for each region.
1368: * If a region becomes completely unwired,
1369: * unwire its physical pages and mappings.
1370: */
1371: entry = start_entry;
1372: while ((entry != vm_map_to_entry(map)) &&
1373: (entry->vme_start < end)) {
1374: vm_map_clip_end(map, entry, end);
1375:
1376: if (user_wire) {
1377: if (--(entry->user_wired_count) == 0)
1378: entry->wired_count--;
1379: }
1380: else {
1381: entry->wired_count--;
1382: }
1383:
1384: if (entry->wired_count == 0)
1385: vm_fault_unwire(map, entry);
1386:
1387: entry = entry->vme_next;
1388: }
1389: }
1390:
1391: else {
1392: /*
1393: * Wiring. We must do this in two passes:
1394: *
1395: * 1. Holding the write lock, we create any shadow
1396: * or zero-fill objects that need to be created.
1397: * Then we clip each map entry to the region to be
1398: * wired and increment its wiring count. We
1399: * create objects before clipping the map entries
1400: * to avoid object proliferation.
1401: *
1402: * 2. We downgrade to a read lock, and call
1403: * vm_fault_wire to fault in the pages for any
1404: * newly wired area (wired_count is 1).
1405: *
1406: * Downgrading to a read lock for vm_fault_wire avoids
1407: * a possible deadlock with another thread that may have
1408: * faulted on one of the pages to be wired (it would mark
1409: * the page busy, blocking us, then in turn block on the
1410: * map lock that we hold). Because of problems in the
1411: * recursive lock package, we cannot upgrade to a write
1412: * lock in vm_map_lookup. Thus, any actions that require
1413: * the write lock must be done beforehand. Because we
1414: * keep the read lock on the map, the copy-on-write
1415: * status of the entries we modify here cannot change.
1416: */
1417:
1418: /*
1419: * Pass 1.
1420: */
1421: while ((entry != vm_map_to_entry(map)) &&
1422: (entry->vme_start < end)) {
1423: vm_map_clip_end(map, entry, end);
1424:
1425: if (entry->wired_count == 0) {
1426:
1427: /*
1428: * Perform actions of vm_map_lookup that need
1429: * the write lock on the map: create a shadow
1430: * object for a copy-on-write region, or an
1431: * object for a zero-fill region.
1432: */
1433: if (entry->needs_copy &&
1434: ((entry->protection & VM_PROT_WRITE) != 0)) {
1435:
1436: vm_object_shadow(&entry->object.vm_object,
1437: &entry->offset,
1438: (vm_size_t)(entry->vme_end
1439: - entry->vme_start));
1440: entry->needs_copy = FALSE;
1441: }
1442: if (entry->object.vm_object == VM_OBJECT_NULL) {
1443: entry->object.vm_object =
1444: vm_object_allocate(
1445: (vm_size_t)(entry->vme_end
1446: - entry->vme_start));
1447: entry->offset = (vm_offset_t)0;
1448: }
1449: }
1450: vm_map_clip_start(map, entry, start);
1451: vm_map_clip_end(map, entry, end);
1452:
1453: if (user_wire) {
1454: if ((entry->user_wired_count)++ == 0)
1455: entry->wired_count++;
1456: }
1457: else {
1458: entry->wired_count++;
1459: }
1460:
1461: /*
1462: * Check for holes and protection mismatch.
1463: * Holes: Next entry should be contiguous unless
1464: * this is the end of the region.
1465: * Protection: Access requested must be allowed.
1466: */
1467: if (((entry->vme_end < end) &&
1468: ((entry->vme_next == vm_map_to_entry(map)) ||
1469: (entry->vme_next->vme_start > entry->vme_end))) ||
1470: ((entry->protection & access_type) != access_type)) {
1471: /*
1472: * Found a hole or protection problem.
1473: * Object creation actions
1474: * do not need to be undone, but the
1475: * wired counts need to be restored.
1476: */
1477: while ((entry != vm_map_to_entry(map)) &&
1478: (entry->vme_end > start)) {
1479: if (user_wire) {
1480: if (--(entry->user_wired_count) == 0)
1481: entry->wired_count--;
1482: }
1483: else {
1484: entry->wired_count--;
1485: }
1486:
1487: entry = entry->vme_prev;
1488: }
1489:
1490: vm_map_unlock(map);
1491: return(KERN_FAILURE);
1492: }
1493: entry = entry->vme_next;
1494: }
1495:
1496: /*
1497: * Pass 2.
1498: */
1499:
1500: /*
1501: * HACK HACK HACK HACK
1502: *
1503: * If we are wiring in the kernel map or a submap of it,
1504: * unlock the map to avoid deadlocks. We trust that the
1505: * kernel threads are well-behaved, and therefore will
1506: * not do anything destructive to this region of the map
1507: * while we have it unlocked. We cannot trust user threads
1508: * to do the same.
1509: *
1510: * HACK HACK HACK HACK
1511: */
1512: if (vm_map_pmap(map) == kernel_pmap) {
1513: vm_map_unlock(map); /* trust me ... */
1514: }
1515: else {
1516: vm_map_lock_set_recursive(map);
1517: vm_map_lock_write_to_read(map);
1518: }
1519:
1520: entry = start_entry;
1521: while (entry != vm_map_to_entry(map) &&
1522: entry->vme_start < end) {
1523: /*
1524: * Wiring cases:
1525: * Kernel: wired == 1 && user_wired == 0
1526: * User: wired == 1 && user_wired == 1
1527: *
1528: * Don't need to wire if either is > 1. wired = 0 &&
1529: * user_wired == 1 can't happen.
1530: */
1531:
1532: /*
1533: * XXX This assumes that the faults always succeed.
1534: */
1535: if ((entry->wired_count == 1) &&
1536: (entry->user_wired_count <= 1)) {
1537: vm_fault_wire(map, entry);
1538: }
1539: entry = entry->vme_next;
1540: }
1541:
1542: if (vm_map_pmap(map) == kernel_pmap) {
1543: vm_map_lock(map);
1544: }
1545: else {
1546: vm_map_lock_clear_recursive(map);
1547: }
1548: }
1549:
1550: vm_map_unlock(map);
1551:
1552: return(KERN_SUCCESS);
1553: }
1554:
1555: /*
1556: * vm_map_entry_delete: [ internal use only ]
1557: *
1558: * Deallocate the given entry from the target map.
1559: */
1560: void vm_map_entry_delete(map, entry)
1561: register vm_map_t map;
1562: register vm_map_entry_t entry;
1563: {
1564: register vm_offset_t s, e;
1565: register vm_object_t object;
1566: extern vm_object_t kernel_object;
1567:
1568: s = entry->vme_start;
1569: e = entry->vme_end;
1570:
1571: /*Check if projected buffer*/
1572: if (map != kernel_map && entry->projected_on != 0) {
1573: /*Check if projected kernel entry is persistent;
1574: may only manipulate directly if it is*/
1575: if (entry->projected_on->projected_on == 0)
1576: entry->wired_count = 0; /*Avoid unwire fault*/
1577: else
1578: return;
1579: }
1580:
1581: /*
1582: * Get the object. Null objects cannot have pmap entries.
1583: */
1584:
1585: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1586:
1587: /*
1588: * Unwire before removing addresses from the pmap;
1589: * otherwise, unwiring will put the entries back in
1590: * the pmap.
1591: */
1592:
1593: if (entry->wired_count != 0) {
1594: vm_fault_unwire(map, entry);
1595: entry->wired_count = 0;
1596: entry->user_wired_count = 0;
1597: }
1598:
1599: /*
1600: * If the object is shared, we must remove
1601: * *all* references to this data, since we can't
1602: * find all of the physical maps which are sharing
1603: * it.
1604: */
1605:
1606: if (object == kernel_object) {
1607: vm_object_lock(object);
1608: vm_object_page_remove(object, entry->offset,
1609: entry->offset + (e - s));
1610: vm_object_unlock(object);
1611: } else if (entry->is_shared) {
1612: vm_object_pmap_remove(object,
1613: entry->offset,
1614: entry->offset + (e - s));
1615: }
1616: else {
1617: pmap_remove(map->pmap, s, e);
1618: }
1619: }
1620:
1621: /*
1622: * Deallocate the object only after removing all
1623: * pmap entries pointing to its pages.
1624: */
1625:
1626: if (entry->is_sub_map)
1627: vm_map_deallocate(entry->object.sub_map);
1628: else
1629: vm_object_deallocate(entry->object.vm_object);
1630:
1631: vm_map_entry_unlink(map, entry);
1632: map->size -= e - s;
1633:
1634: vm_map_entry_dispose(map, entry);
1635: }
1636:
1637: /*
1638: * vm_map_delete: [ internal use only ]
1639: *
1640: * Deallocates the given address range from the target
1641: * map.
1642: */
1643:
1644: kern_return_t vm_map_delete(map, start, end)
1645: register vm_map_t map;
1646: register vm_offset_t start;
1647: register vm_offset_t end;
1648: {
1649: vm_map_entry_t entry;
1650: vm_map_entry_t first_entry;
1651:
1652: /*
1653: * Find the start of the region, and clip it
1654: */
1655:
1656: if (!vm_map_lookup_entry(map, start, &first_entry))
1657: entry = first_entry->vme_next;
1658: else {
1659: entry = first_entry;
1660: #if NORMA_IPC_xxx
1661: /*
1662: * XXX Had to disable this code because:
1663:
1664: _vm_map_delete(c0804b78,c2198000,c219a000,0,c219a000)+df
1665: [vm/vm_map.c:2007]
1666: _vm_map_remove(c0804b78,c2198000,c219a000,c0817834,
1667: c081786c)+42 [vm/vm_map.c:2094]
1668: _kmem_io_map_deallocate(c0804b78,c2198000,2000,c0817834,
1669: c081786c)+43 [vm/vm_kern.c:818]
1670: _device_write_dealloc(c081786c)+117 [device/ds_routines.c:814]
1671: _ds_write_done(c081786c,0)+2e [device/ds_routines.c:848]
1672: _io_done_thread_continue(c08150c0,c21d4e14,c21d4e30,c08150c0,
1673: c080c114)+14 [device/ds_routines.c:1350]
1674:
1675: */
1676: if (start > entry->vme_start
1677: && end == entry->vme_end
1678: && ! entry->wired_count /* XXX ??? */
1679: && ! entry->is_shared
1680: && ! entry->projected_on
1681: && ! entry->is_sub_map) {
1682: extern vm_object_t kernel_object;
1683: register vm_object_t object = entry->object.vm_object;
1684:
1685: /*
1686: * The region to be deleted lives at the end
1687: * of this entry, and thus all we have to do is
1688: * truncate the entry.
1689: *
1690: * This special case is necessary if we want
1691: * coalescing to do us any good.
1692: *
1693: * XXX Do we have to adjust object size?
1694: */
1695: if (object == kernel_object) {
1696: vm_object_lock(object);
1697: vm_object_page_remove(object,
1698: entry->offset + start,
1699: entry->offset +
1700: (end - start));
1701: vm_object_unlock(object);
1702: } else if (entry->is_shared) {
1703: vm_object_pmap_remove(object,
1704: entry->offset + start,
1705: entry->offset +
1706: (end - start));
1707: } else {
1708: pmap_remove(map->pmap, start, end);
1709: }
1710: object->size -= (end - start); /* XXX */
1711:
1712: entry->vme_end = start;
1713: map->size -= (end - start);
1714:
1715: if (map->wait_for_space) {
1716: thread_wakeup((event_t) map);
1717: }
1718: return KERN_SUCCESS;
1719: }
1720: #endif /* NORMA_IPC */
1721: vm_map_clip_start(map, entry, start);
1722:
1723: /*
1724: * Fix the lookup hint now, rather than each
1725: * time though the loop.
1726: */
1727:
1728: SAVE_HINT(map, entry->vme_prev);
1729: }
1730:
1731: /*
1732: * Save the free space hint
1733: */
1734:
1735: if (map->first_free->vme_start >= start)
1736: map->first_free = entry->vme_prev;
1737:
1738: /*
1739: * Step through all entries in this region
1740: */
1741:
1742: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1743: vm_map_entry_t next;
1744:
1745: vm_map_clip_end(map, entry, end);
1746:
1747: /*
1748: * If the entry is in transition, we must wait
1749: * for it to exit that state. It could be clipped
1750: * while we leave the map unlocked.
1751: */
1752: if(entry->in_transition) {
1753: /*
1754: * Say that we are waiting, and wait for entry.
1755: */
1756: entry->needs_wakeup = TRUE;
1757: vm_map_entry_wait(map, FALSE);
1758: vm_map_lock(map);
1759:
1760: /*
1761: * The entry could have been clipped or it
1762: * may not exist anymore. look it up again.
1763: */
1764: if(!vm_map_lookup_entry(map, start, &entry)) {
1765: entry = entry->vme_next;
1766: }
1767: continue;
1768: }
1769:
1770: next = entry->vme_next;
1771:
1772: vm_map_entry_delete(map, entry);
1773: entry = next;
1774: }
1775:
1776: if (map->wait_for_space)
1777: thread_wakeup((event_t) map);
1778:
1779: return(KERN_SUCCESS);
1780: }
1781:
1782: /*
1783: * vm_map_remove:
1784: *
1785: * Remove the given address range from the target map.
1786: * This is the exported form of vm_map_delete.
1787: */
1788: kern_return_t vm_map_remove(map, start, end)
1789: register vm_map_t map;
1790: register vm_offset_t start;
1791: register vm_offset_t end;
1792: {
1793: register kern_return_t result;
1794:
1795: vm_map_lock(map);
1796: VM_MAP_RANGE_CHECK(map, start, end);
1797: result = vm_map_delete(map, start, end);
1798: vm_map_unlock(map);
1799:
1800: return(result);
1801: }
1802:
1803:
1804: /*
1805: * vm_map_copy_steal_pages:
1806: *
1807: * Steal all the pages from a vm_map_copy page_list by copying ones
1808: * that have not already been stolen.
1809: */
1810: void
1811: vm_map_copy_steal_pages(copy)
1812: vm_map_copy_t copy;
1813: {
1814: register vm_page_t m, new_m;
1815: register int i;
1816: vm_object_t object;
1817:
1818: for (i = 0; i < copy->cpy_npages; i++) {
1819:
1820: /*
1821: * If the page is not tabled, then it's already stolen.
1822: */
1823: m = copy->cpy_page_list[i];
1824: if (!m->tabled)
1825: continue;
1826:
1827: /*
1828: * Page was not stolen, get a new
1829: * one and do the copy now.
1830: */
1831: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) {
1832: VM_PAGE_WAIT((void(*)()) 0);
1833: }
1834:
1835: vm_page_copy(m, new_m);
1836:
1837: object = m->object;
1838: vm_object_lock(object);
1839: vm_page_lock_queues();
1840: if (!m->active && !m->inactive)
1841: vm_page_activate(m);
1842: vm_page_unlock_queues();
1843: PAGE_WAKEUP_DONE(m);
1844: vm_object_paging_end(object);
1845: vm_object_unlock(object);
1846:
1847: copy->cpy_page_list[i] = new_m;
1848: }
1849: }
1850:
1851: /*
1852: * vm_map_copy_page_discard:
1853: *
1854: * Get rid of the pages in a page_list copy. If the pages are
1855: * stolen, they are freed. If the pages are not stolen, they
1856: * are unbusied, and associated state is cleaned up.
1857: */
1858: void vm_map_copy_page_discard(copy)
1859: vm_map_copy_t copy;
1860: {
1861: while (copy->cpy_npages > 0) {
1862: vm_page_t m;
1863:
1864: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1865: VM_PAGE_NULL) {
1866:
1867: /*
1868: * If it's not in the table, then it's
1869: * a stolen page that goes back
1870: * to the free list. Else it belongs
1871: * to some object, and we hold a
1872: * paging reference on that object.
1873: */
1874: if (!m->tabled) {
1875: VM_PAGE_FREE(m);
1876: }
1877: else {
1878: vm_object_t object;
1879:
1880: object = m->object;
1881:
1882: vm_object_lock(object);
1883: vm_page_lock_queues();
1884: if (!m->active && !m->inactive)
1885: vm_page_activate(m);
1886: vm_page_unlock_queues();
1887:
1888: PAGE_WAKEUP_DONE(m);
1889: vm_object_paging_end(object);
1890: vm_object_unlock(object);
1891: }
1892: }
1893: }
1894: }
1895:
1896: /*
1897: * Routine: vm_map_copy_discard
1898: *
1899: * Description:
1900: * Dispose of a map copy object (returned by
1901: * vm_map_copyin).
1902: */
1903: void
1904: vm_map_copy_discard(copy)
1905: vm_map_copy_t copy;
1906: {
1907: free_next_copy:
1908: if (copy == VM_MAP_COPY_NULL)
1909: return;
1910:
1911: switch (copy->type) {
1912: case VM_MAP_COPY_ENTRY_LIST:
1913: while (vm_map_copy_first_entry(copy) !=
1914: vm_map_copy_to_entry(copy)) {
1915: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
1916:
1917: vm_map_copy_entry_unlink(copy, entry);
1918: vm_object_deallocate(entry->object.vm_object);
1919: vm_map_copy_entry_dispose(copy, entry);
1920: }
1921: break;
1922: case VM_MAP_COPY_OBJECT:
1923: vm_object_deallocate(copy->cpy_object);
1924: break;
1925: case VM_MAP_COPY_PAGE_LIST:
1926:
1927: /*
1928: * To clean this up, we have to unbusy all the pages
1929: * and release the paging references in their objects.
1930: */
1931: if (copy->cpy_npages > 0)
1932: vm_map_copy_page_discard(copy);
1933:
1934: /*
1935: * If there's a continuation, abort it. The
1936: * abort routine releases any storage.
1937: */
1938: if (vm_map_copy_has_cont(copy)) {
1939:
1940: /*
1941: * Special case: recognize
1942: * vm_map_copy_discard_cont and optimize
1943: * here to avoid tail recursion.
1944: */
1945: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1946: register vm_map_copy_t new_copy;
1947:
1948: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1949: zfree(vm_map_copy_zone, (vm_offset_t) copy);
1950: copy = new_copy;
1951: goto free_next_copy;
1952: }
1953: else {
1954: vm_map_copy_abort_cont(copy);
1955: }
1956: }
1957:
1958: break;
1959: }
1960: zfree(vm_map_copy_zone, (vm_offset_t) copy);
1961: }
1962:
1963: /*
1964: * Routine: vm_map_copy_copy
1965: *
1966: * Description:
1967: * Move the information in a map copy object to
1968: * a new map copy object, leaving the old one
1969: * empty.
1970: *
1971: * This is used by kernel routines that need
1972: * to look at out-of-line data (in copyin form)
1973: * before deciding whether to return SUCCESS.
1974: * If the routine returns FAILURE, the original
1975: * copy object will be deallocated; therefore,
1976: * these routines must make a copy of the copy
1977: * object and leave the original empty so that
1978: * deallocation will not fail.
1979: */
1980: vm_map_copy_t
1981: vm_map_copy_copy(copy)
1982: vm_map_copy_t copy;
1983: {
1984: vm_map_copy_t new_copy;
1985:
1986: if (copy == VM_MAP_COPY_NULL)
1987: return VM_MAP_COPY_NULL;
1988:
1989: /*
1990: * Allocate a new copy object, and copy the information
1991: * from the old one into it.
1992: */
1993:
1994: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
1995: *new_copy = *copy;
1996:
1997: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
1998: /*
1999: * The links in the entry chain must be
2000: * changed to point to the new copy object.
2001: */
2002: vm_map_copy_first_entry(copy)->vme_prev
2003: = vm_map_copy_to_entry(new_copy);
2004: vm_map_copy_last_entry(copy)->vme_next
2005: = vm_map_copy_to_entry(new_copy);
2006: }
2007:
2008: /*
2009: * Change the old copy object into one that contains
2010: * nothing to be deallocated.
2011: */
2012: copy->type = VM_MAP_COPY_OBJECT;
2013: copy->cpy_object = VM_OBJECT_NULL;
2014:
2015: /*
2016: * Return the new object.
2017: */
2018: return new_copy;
2019: }
2020:
2021: /*
2022: * Routine: vm_map_copy_discard_cont
2023: *
2024: * Description:
2025: * A version of vm_map_copy_discard that can be called
2026: * as a continuation from a vm_map_copy page list.
2027: */
2028: kern_return_t vm_map_copy_discard_cont(cont_args, copy_result)
2029: vm_map_copyin_args_t cont_args;
2030: vm_map_copy_t *copy_result; /* OUT */
2031: {
2032: vm_map_copy_discard((vm_map_copy_t) cont_args);
2033: if (copy_result != (vm_map_copy_t *)0)
2034: *copy_result = VM_MAP_COPY_NULL;
2035: return(KERN_SUCCESS);
2036: }
2037:
2038: /*
2039: * Routine: vm_map_copy_overwrite
2040: *
2041: * Description:
2042: * Copy the memory described by the map copy
2043: * object (copy; returned by vm_map_copyin) onto
2044: * the specified destination region (dst_map, dst_addr).
2045: * The destination must be writeable.
2046: *
2047: * Unlike vm_map_copyout, this routine actually
2048: * writes over previously-mapped memory. If the
2049: * previous mapping was to a permanent (user-supplied)
2050: * memory object, it is preserved.
2051: *
2052: * The attributes (protection and inheritance) of the
2053: * destination region are preserved.
2054: *
2055: * If successful, consumes the copy object.
2056: * Otherwise, the caller is responsible for it.
2057: *
2058: * Implementation notes:
2059: * To overwrite temporary virtual memory, it is
2060: * sufficient to remove the previous mapping and insert
2061: * the new copy. This replacement is done either on
2062: * the whole region (if no permanent virtual memory
2063: * objects are embedded in the destination region) or
2064: * in individual map entries.
2065: *
2066: * To overwrite permanent virtual memory, it is
2067: * necessary to copy each page, as the external
2068: * memory management interface currently does not
2069: * provide any optimizations.
2070: *
2071: * Once a page of permanent memory has been overwritten,
2072: * it is impossible to interrupt this function; otherwise,
2073: * the call would be neither atomic nor location-independent.
2074: * The kernel-state portion of a user thread must be
2075: * interruptible.
2076: *
2077: * It may be expensive to forward all requests that might
2078: * overwrite permanent memory (vm_write, vm_copy) to
2079: * uninterruptible kernel threads. This routine may be
2080: * called by interruptible threads; however, success is
2081: * not guaranteed -- if the request cannot be performed
2082: * atomically and interruptibly, an error indication is
2083: * returned.
2084: */
2085: kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible)
2086: vm_map_t dst_map;
2087: vm_offset_t dst_addr;
2088: vm_map_copy_t copy;
2089: boolean_t interruptible;
2090: {
2091: vm_size_t size;
2092: vm_offset_t start;
2093: vm_map_entry_t tmp_entry;
2094: vm_map_entry_t entry;
2095:
2096: boolean_t contains_permanent_objects = FALSE;
2097:
2098: interruptible = FALSE; /* XXX */
2099:
2100: /*
2101: * Check for null copy object.
2102: */
2103:
2104: if (copy == VM_MAP_COPY_NULL)
2105: return(KERN_SUCCESS);
2106:
2107: /*
2108: * Only works for entry lists at the moment. Will
2109: * support page lists LATER.
2110: */
2111:
2112: #if NORMA_IPC
2113: vm_map_convert_from_page_list(copy);
2114: #else
2115: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2116: #endif
2117:
2118: /*
2119: * Currently this routine only handles page-aligned
2120: * regions. Eventually, it should handle misalignments
2121: * by actually copying pages.
2122: */
2123:
2124: if (!page_aligned(copy->offset) ||
2125: !page_aligned(copy->size) ||
2126: !page_aligned(dst_addr))
2127: return(KERN_INVALID_ARGUMENT);
2128:
2129: size = copy->size;
2130:
2131: if (size == 0) {
2132: vm_map_copy_discard(copy);
2133: return(KERN_SUCCESS);
2134: }
2135:
2136: /*
2137: * Verify that the destination is all writeable
2138: * initially.
2139: */
2140: start_pass_1:
2141: vm_map_lock(dst_map);
2142: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2143: vm_map_unlock(dst_map);
2144: return(KERN_INVALID_ADDRESS);
2145: }
2146: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2147: for (entry = tmp_entry;;) {
2148: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2149: vm_map_entry_t next = entry->vme_next;
2150:
2151: if ( ! (entry->protection & VM_PROT_WRITE)) {
2152: vm_map_unlock(dst_map);
2153: return(KERN_PROTECTION_FAILURE);
2154: }
2155:
2156: /*
2157: * If the entry is in transition, we must wait
2158: * for it to exit that state. Anything could happen
2159: * when we unlock the map, so start over.
2160: */
2161: if (entry->in_transition) {
2162:
2163: /*
2164: * Say that we are waiting, and wait for entry.
2165: */
2166: entry->needs_wakeup = TRUE;
2167: vm_map_entry_wait(dst_map, FALSE);
2168:
2169: goto start_pass_1;
2170: }
2171:
2172: if (size <= sub_size)
2173: break;
2174:
2175: if ((next == vm_map_to_entry(dst_map)) ||
2176: (next->vme_start != entry->vme_end)) {
2177: vm_map_unlock(dst_map);
2178: return(KERN_INVALID_ADDRESS);
2179: }
2180:
2181:
2182: /*
2183: * Check for permanent objects in the destination.
2184: */
2185:
2186: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2187: !entry->object.vm_object->temporary)
2188: contains_permanent_objects = TRUE;
2189:
2190: size -= sub_size;
2191: entry = next;
2192: }
2193:
2194: /*
2195: * If there are permanent objects in the destination, then
2196: * the copy cannot be interrupted.
2197: */
2198:
2199: if (interruptible && contains_permanent_objects)
2200: return(KERN_FAILURE); /* XXX */
2201:
2202: /*
2203: * XXXO If there are no permanent objects in the destination,
2204: * XXXO and the source and destination map entry zones match,
2205: * XXXO and the destination map entry is not shared,
2206: * XXXO then the map entries can be deleted and replaced
2207: * XXXO with those from the copy. The following code is the
2208: * XXXO basic idea of what to do, but there are lots of annoying
2209: * XXXO little details about getting protection and inheritance
2210: * XXXO right. Should add protection, inheritance, and sharing checks
2211: * XXXO to the above pass and make sure that no wiring is involved.
2212: */
2213: /*
2214: * if (!contains_permanent_objects &&
2215: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2216: *
2217: * *
2218: * * Run over copy and adjust entries. Steal code
2219: * * from vm_map_copyout() to do this.
2220: * *
2221: *
2222: * tmp_entry = tmp_entry->vme_prev;
2223: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2224: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2225: *
2226: * vm_map_unlock(dst_map);
2227: * vm_map_copy_discard(copy);
2228: * }
2229: */
2230: /*
2231: *
2232: * Make a second pass, overwriting the data
2233: * At the beginning of each loop iteration,
2234: * the next entry to be overwritten is "tmp_entry"
2235: * (initially, the value returned from the lookup above),
2236: * and the starting address expected in that entry
2237: * is "start".
2238: */
2239:
2240: start = dst_addr;
2241:
2242: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2243: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2244: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2245: vm_object_t object;
2246:
2247: entry = tmp_entry;
2248: size = (entry->vme_end - entry->vme_start);
2249: /*
2250: * Make sure that no holes popped up in the
2251: * address map, and that the protection is
2252: * still valid, in case the map was unlocked
2253: * earlier.
2254: */
2255:
2256: if (entry->vme_start != start) {
2257: vm_map_unlock(dst_map);
2258: return(KERN_INVALID_ADDRESS);
2259: }
2260: assert(entry != vm_map_to_entry(dst_map));
2261:
2262: /*
2263: * Check protection again
2264: */
2265:
2266: if ( ! (entry->protection & VM_PROT_WRITE)) {
2267: vm_map_unlock(dst_map);
2268: return(KERN_PROTECTION_FAILURE);
2269: }
2270:
2271: /*
2272: * Adjust to source size first
2273: */
2274:
2275: if (copy_size < size) {
2276: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2277: size = copy_size;
2278: }
2279:
2280: /*
2281: * Adjust to destination size
2282: */
2283:
2284: if (size < copy_size) {
2285: vm_map_copy_clip_end(copy, copy_entry,
2286: copy_entry->vme_start + size);
2287: copy_size = size;
2288: }
2289:
2290: assert((entry->vme_end - entry->vme_start) == size);
2291: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2292: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2293:
2294: /*
2295: * If the destination contains temporary unshared memory,
2296: * we can perform the copy by throwing it away and
2297: * installing the source data.
2298: */
2299:
2300: object = entry->object.vm_object;
2301: if (!entry->is_shared &&
2302: ((object == VM_OBJECT_NULL) || object->temporary)) {
2303: vm_object_t old_object = entry->object.vm_object;
2304: vm_offset_t old_offset = entry->offset;
2305:
2306: entry->object = copy_entry->object;
2307: entry->offset = copy_entry->offset;
2308: entry->needs_copy = copy_entry->needs_copy;
2309: entry->wired_count = 0;
2310: entry->user_wired_count = 0;
2311:
2312: vm_map_copy_entry_unlink(copy, copy_entry);
2313: vm_map_copy_entry_dispose(copy, copy_entry);
2314:
2315: vm_object_pmap_protect(
2316: old_object,
2317: old_offset,
2318: size,
2319: dst_map->pmap,
2320: tmp_entry->vme_start,
2321: VM_PROT_NONE);
2322:
2323: vm_object_deallocate(old_object);
2324:
2325: /*
2326: * Set up for the next iteration. The map
2327: * has not been unlocked, so the next
2328: * address should be at the end of this
2329: * entry, and the next map entry should be
2330: * the one following it.
2331: */
2332:
2333: start = tmp_entry->vme_end;
2334: tmp_entry = tmp_entry->vme_next;
2335: } else {
2336: vm_map_version_t version;
2337: vm_object_t dst_object = entry->object.vm_object;
2338: vm_offset_t dst_offset = entry->offset;
2339: kern_return_t r;
2340:
2341: /*
2342: * Take an object reference, and record
2343: * the map version information so that the
2344: * map can be safely unlocked.
2345: */
2346:
2347: vm_object_reference(dst_object);
2348:
2349: version.main_timestamp = dst_map->timestamp;
2350:
2351: vm_map_unlock(dst_map);
2352:
2353: /*
2354: * Copy as much as possible in one pass
2355: */
2356:
2357: copy_size = size;
2358: r = vm_fault_copy(
2359: copy_entry->object.vm_object,
2360: copy_entry->offset,
2361: ©_size,
2362: dst_object,
2363: dst_offset,
2364: dst_map,
2365: &version,
2366: FALSE /* XXX interruptible */ );
2367:
2368: /*
2369: * Release the object reference
2370: */
2371:
2372: vm_object_deallocate(dst_object);
2373:
2374: /*
2375: * If a hard error occurred, return it now
2376: */
2377:
2378: if (r != KERN_SUCCESS)
2379: return(r);
2380:
2381: if (copy_size != 0) {
2382: /*
2383: * Dispose of the copied region
2384: */
2385:
2386: vm_map_copy_clip_end(copy, copy_entry,
2387: copy_entry->vme_start + copy_size);
2388: vm_map_copy_entry_unlink(copy, copy_entry);
2389: vm_object_deallocate(copy_entry->object.vm_object);
2390: vm_map_copy_entry_dispose(copy, copy_entry);
2391: }
2392:
2393: /*
2394: * Pick up in the destination map where we left off.
2395: *
2396: * Use the version information to avoid a lookup
2397: * in the normal case.
2398: */
2399:
2400: start += copy_size;
2401: vm_map_lock(dst_map);
2402: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2403: /* We can safely use saved tmp_entry value */
2404:
2405: vm_map_clip_end(dst_map, tmp_entry, start);
2406: tmp_entry = tmp_entry->vme_next;
2407: } else {
2408: /* Must do lookup of tmp_entry */
2409:
2410: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2411: vm_map_unlock(dst_map);
2412: return(KERN_INVALID_ADDRESS);
2413: }
2414: vm_map_clip_start(dst_map, tmp_entry, start);
2415: }
2416: }
2417:
2418: }
2419: vm_map_unlock(dst_map);
2420:
2421: /*
2422: * Throw away the vm_map_copy object
2423: */
2424: vm_map_copy_discard(copy);
2425:
2426: return(KERN_SUCCESS);
2427: }
2428:
2429: /*
2430: * Macro: vm_map_copy_insert
2431: *
2432: * Description:
2433: * Link a copy chain ("copy") into a map at the
2434: * specified location (after "where").
2435: * Side effects:
2436: * The copy chain is destroyed.
2437: * Warning:
2438: * The arguments are evaluated multiple times.
2439: */
2440: #define vm_map_copy_insert(map, where, copy) \
2441: MACRO_BEGIN \
2442: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \
2443: ->vme_next = ((where)->vme_next); \
2444: ((where)->vme_next = vm_map_copy_first_entry(copy)) \
2445: ->vme_prev = (where); \
2446: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \
2447: zfree(vm_map_copy_zone, (vm_offset_t) copy); \
2448: MACRO_END
2449:
2450: /*
2451: * Routine: vm_map_copyout
2452: *
2453: * Description:
2454: * Copy out a copy chain ("copy") into newly-allocated
2455: * space in the destination map.
2456: *
2457: * If successful, consumes the copy object.
2458: * Otherwise, the caller is responsible for it.
2459: */
2460: kern_return_t vm_map_copyout(dst_map, dst_addr, copy)
2461: register
2462: vm_map_t dst_map;
2463: vm_offset_t *dst_addr; /* OUT */
2464: register
2465: vm_map_copy_t copy;
2466: {
2467: vm_size_t size;
2468: vm_size_t adjustment;
2469: vm_offset_t start;
2470: vm_offset_t vm_copy_start;
2471: vm_map_entry_t last;
2472: register
2473: vm_map_entry_t entry;
2474:
2475: /*
2476: * Check for null copy object.
2477: */
2478:
2479: if (copy == VM_MAP_COPY_NULL) {
2480: *dst_addr = 0;
2481: return(KERN_SUCCESS);
2482: }
2483:
2484: /*
2485: * Check for special copy object, created
2486: * by vm_map_copyin_object.
2487: */
2488:
2489: if (copy->type == VM_MAP_COPY_OBJECT) {
2490: vm_object_t object = copy->cpy_object;
2491: vm_size_t offset = copy->offset;
2492: vm_size_t tmp_size = copy->size;
2493: kern_return_t kr;
2494:
2495: *dst_addr = 0;
2496: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2497: (vm_offset_t) 0, TRUE,
2498: object, offset, FALSE,
2499: VM_PROT_DEFAULT, VM_PROT_ALL,
2500: VM_INHERIT_DEFAULT);
2501: if (kr != KERN_SUCCESS)
2502: return(kr);
2503: zfree(vm_map_copy_zone, (vm_offset_t) copy);
2504: return(KERN_SUCCESS);
2505: }
2506:
2507: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2508: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2509:
2510: /*
2511: * Find space for the data
2512: */
2513:
2514: vm_copy_start = trunc_page(copy->offset);
2515: size = round_page(copy->offset + copy->size) - vm_copy_start;
2516:
2517: StartAgain: ;
2518:
2519: vm_map_lock(dst_map);
2520: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
2521: vm_map_min(dst_map) : last->vme_end;
2522:
2523: while (TRUE) {
2524: vm_map_entry_t next = last->vme_next;
2525: vm_offset_t end = start + size;
2526:
2527: if ((end > dst_map->max_offset) || (end < start)) {
2528: if (dst_map->wait_for_space) {
2529: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
2530: assert_wait((event_t) dst_map, TRUE);
2531: vm_map_unlock(dst_map);
2532: thread_block((void (*)()) 0);
2533: goto StartAgain;
2534: }
2535: }
2536: vm_map_unlock(dst_map);
2537: return(KERN_NO_SPACE);
2538: }
2539:
2540: if ((next == vm_map_to_entry(dst_map)) ||
2541: (next->vme_start >= end))
2542: break;
2543:
2544: last = next;
2545: start = last->vme_end;
2546: }
2547:
2548: /*
2549: * Since we're going to just drop the map
2550: * entries from the copy into the destination
2551: * map, they must come from the same pool.
2552: */
2553:
2554: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2555: /*
2556: * Mismatches occur when dealing with the default
2557: * pager.
2558: */
2559: zone_t old_zone;
2560: vm_map_entry_t next, new;
2561:
2562: /*
2563: * Find the zone that the copies were allocated from
2564: */
2565: old_zone = (copy->cpy_hdr.entries_pageable)
2566: ? vm_map_entry_zone
2567: : vm_map_kentry_zone;
2568: entry = vm_map_copy_first_entry(copy);
2569:
2570: /*
2571: * Reinitialize the copy so that vm_map_copy_entry_link
2572: * will work.
2573: */
2574: copy->cpy_hdr.nentries = 0;
2575: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
2576: vm_map_copy_first_entry(copy) =
2577: vm_map_copy_last_entry(copy) =
2578: vm_map_copy_to_entry(copy);
2579:
2580: /*
2581: * Copy each entry.
2582: */
2583: while (entry != vm_map_copy_to_entry(copy)) {
2584: new = vm_map_copy_entry_create(copy);
2585: vm_map_entry_copy_full(new, entry);
2586: vm_map_copy_entry_link(copy,
2587: vm_map_copy_last_entry(copy),
2588: new);
2589: next = entry->vme_next;
2590: zfree(old_zone, (vm_offset_t) entry);
2591: entry = next;
2592: }
2593: }
2594:
2595: /*
2596: * Adjust the addresses in the copy chain, and
2597: * reset the region attributes.
2598: */
2599:
2600: adjustment = start - vm_copy_start;
2601: for (entry = vm_map_copy_first_entry(copy);
2602: entry != vm_map_copy_to_entry(copy);
2603: entry = entry->vme_next) {
2604: entry->vme_start += adjustment;
2605: entry->vme_end += adjustment;
2606:
2607: entry->inheritance = VM_INHERIT_DEFAULT;
2608: entry->protection = VM_PROT_DEFAULT;
2609: entry->max_protection = VM_PROT_ALL;
2610: entry->projected_on = 0;
2611:
2612: /*
2613: * If the entry is now wired,
2614: * map the pages into the destination map.
2615: */
2616: if (entry->wired_count != 0) {
2617: register vm_offset_t va;
2618: vm_offset_t offset;
2619: register vm_object_t object;
2620:
2621: object = entry->object.vm_object;
2622: offset = entry->offset;
2623: va = entry->vme_start;
2624:
2625: pmap_pageable(dst_map->pmap,
2626: entry->vme_start,
2627: entry->vme_end,
2628: TRUE);
2629:
2630: while (va < entry->vme_end) {
2631: register vm_page_t m;
2632:
2633: /*
2634: * Look up the page in the object.
2635: * Assert that the page will be found in the
2636: * top object:
2637: * either
2638: * the object was newly created by
2639: * vm_object_copy_slowly, and has
2640: * copies of all of the pages from
2641: * the source object
2642: * or
2643: * the object was moved from the old
2644: * map entry; because the old map
2645: * entry was wired, all of the pages
2646: * were in the top-level object.
2647: * (XXX not true if we wire pages for
2648: * reading)
2649: */
2650: vm_object_lock(object);
2651: vm_object_paging_begin(object);
2652:
2653: m = vm_page_lookup(object, offset);
2654: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2655: m->absent)
2656: panic("vm_map_copyout: wiring 0x%x", m);
2657:
2658: m->busy = TRUE;
2659: vm_object_unlock(object);
2660:
2661: PMAP_ENTER(dst_map->pmap, va, m,
2662: entry->protection, TRUE);
2663:
2664: vm_object_lock(object);
2665: PAGE_WAKEUP_DONE(m);
2666: /* the page is wired, so we don't have to activate */
2667: vm_object_paging_end(object);
2668: vm_object_unlock(object);
2669:
2670: offset += PAGE_SIZE;
2671: va += PAGE_SIZE;
2672: }
2673: }
2674:
2675:
2676: }
2677:
2678: /*
2679: * Correct the page alignment for the result
2680: */
2681:
2682: *dst_addr = start + (copy->offset - vm_copy_start);
2683:
2684: /*
2685: * Update the hints and the map size
2686: */
2687:
2688: if (dst_map->first_free == last)
2689: dst_map->first_free = vm_map_copy_last_entry(copy);
2690: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2691:
2692: dst_map->size += size;
2693:
2694: /*
2695: * Link in the copy
2696: */
2697:
2698: vm_map_copy_insert(dst_map, last, copy);
2699:
2700: vm_map_unlock(dst_map);
2701:
2702: /*
2703: * XXX If wiring_required, call vm_map_pageable
2704: */
2705:
2706: return(KERN_SUCCESS);
2707: }
2708:
2709: /*
2710: *
2711: * vm_map_copyout_page_list:
2712: *
2713: * Version of vm_map_copyout() for page list vm map copies.
2714: *
2715: */
2716: kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy)
2717: register
2718: vm_map_t dst_map;
2719: vm_offset_t *dst_addr; /* OUT */
2720: register
2721: vm_map_copy_t copy;
2722: {
2723: vm_size_t size;
2724: vm_offset_t start;
2725: vm_offset_t end;
2726: vm_offset_t offset;
2727: vm_map_entry_t last;
2728: register
2729: vm_object_t object;
2730: vm_page_t *page_list, m;
2731: vm_map_entry_t entry;
2732: vm_offset_t old_last_offset;
2733: boolean_t cont_invoked, needs_wakeup = FALSE;
2734: kern_return_t result = KERN_SUCCESS;
2735: vm_map_copy_t orig_copy;
2736: vm_offset_t dst_offset;
2737: boolean_t must_wire;
2738:
2739: /*
2740: * Make sure the pages are stolen, because we are
2741: * going to put them in a new object. Assume that
2742: * all pages are identical to first in this regard.
2743: */
2744:
2745: page_list = ©->cpy_page_list[0];
2746: if ((*page_list)->tabled)
2747: vm_map_copy_steal_pages(copy);
2748:
2749: /*
2750: * Find space for the data
2751: */
2752:
2753: size = round_page(copy->offset + copy->size) -
2754: trunc_page(copy->offset);
2755: StartAgain:
2756: vm_map_lock(dst_map);
2757: must_wire = dst_map->wiring_required;
2758:
2759: last = dst_map->first_free;
2760: if (last == vm_map_to_entry(dst_map)) {
2761: start = vm_map_min(dst_map);
2762: } else {
2763: start = last->vme_end;
2764: }
2765:
2766: while (TRUE) {
2767: vm_map_entry_t next = last->vme_next;
2768: end = start + size;
2769:
2770: if ((end > dst_map->max_offset) || (end < start)) {
2771: if (dst_map->wait_for_space) {
2772: if (size <= (dst_map->max_offset -
2773: dst_map->min_offset)) {
2774: assert_wait((event_t) dst_map, TRUE);
2775: vm_map_unlock(dst_map);
2776: thread_block((void (*)()) 0);
2777: goto StartAgain;
2778: }
2779: }
2780: vm_map_unlock(dst_map);
2781: return(KERN_NO_SPACE);
2782: }
2783:
2784: if ((next == vm_map_to_entry(dst_map)) ||
2785: (next->vme_start >= end)) {
2786: break;
2787: }
2788:
2789: last = next;
2790: start = last->vme_end;
2791: }
2792:
2793: /*
2794: * See whether we can avoid creating a new entry (and object) by
2795: * extending one of our neighbors. [So far, we only attempt to
2796: * extend from below.]
2797: *
2798: * The code path below here is a bit twisted. If any of the
2799: * extension checks fails, we branch to create_object. If
2800: * it all works, we fall out the bottom and goto insert_pages.
2801: */
2802: if (last == vm_map_to_entry(dst_map) ||
2803: last->vme_end != start ||
2804: last->is_shared != FALSE ||
2805: last->is_sub_map != FALSE ||
2806: last->inheritance != VM_INHERIT_DEFAULT ||
2807: last->protection != VM_PROT_DEFAULT ||
2808: last->max_protection != VM_PROT_ALL ||
2809: (must_wire ? (last->wired_count != 1 ||
2810: last->user_wired_count != 1) :
2811: (last->wired_count != 0))) {
2812: goto create_object;
2813: }
2814:
2815: /*
2816: * If this entry needs an object, make one.
2817: */
2818: if (last->object.vm_object == VM_OBJECT_NULL) {
2819: object = vm_object_allocate(
2820: (vm_size_t)(last->vme_end - last->vme_start + size));
2821: last->object.vm_object = object;
2822: last->offset = 0;
2823: vm_object_lock(object);
2824: }
2825: else {
2826: vm_offset_t prev_offset = last->offset;
2827: vm_size_t prev_size = start - last->vme_start;
2828: vm_size_t new_size;
2829:
2830: /*
2831: * This is basically vm_object_coalesce.
2832: */
2833:
2834: object = last->object.vm_object;
2835: vm_object_lock(object);
2836:
2837: /*
2838: * Try to collapse the object first
2839: */
2840: vm_object_collapse(object);
2841:
2842: /*
2843: * Can't coalesce if pages not mapped to
2844: * last may be in use anyway:
2845: * . more than one reference
2846: * . paged out
2847: * . shadows another object
2848: * . has a copy elsewhere
2849: * . paging references (pages might be in page-list)
2850: */
2851:
2852: if ((object->ref_count > 1) ||
2853: object->pager_created ||
2854: (object->shadow != VM_OBJECT_NULL) ||
2855: (object->copy != VM_OBJECT_NULL) ||
2856: (object->paging_in_progress != 0)) {
2857: vm_object_unlock(object);
2858: goto create_object;
2859: }
2860:
2861: /*
2862: * Extend the object if necessary. Don't have to call
2863: * vm_object_page_remove because the pages aren't mapped,
2864: * and vm_page_replace will free up any old ones it encounters.
2865: */
2866: new_size = prev_offset + prev_size + size;
2867: if (new_size > object->size)
2868: object->size = new_size;
2869: }
2870:
2871: /*
2872: * Coalesced the two objects - can extend
2873: * the previous map entry to include the
2874: * new range.
2875: */
2876: dst_map->size += size;
2877: last->vme_end = end;
2878:
2879: SAVE_HINT(dst_map, last);
2880:
2881: goto insert_pages;
2882:
2883: create_object:
2884:
2885: /*
2886: * Create object
2887: */
2888: object = vm_object_allocate(size);
2889:
2890: /*
2891: * Create entry
2892: */
2893:
2894: entry = vm_map_entry_create(dst_map);
2895:
2896: entry->object.vm_object = object;
2897: entry->offset = 0;
2898:
2899: entry->is_shared = FALSE;
2900: entry->is_sub_map = FALSE;
2901: entry->needs_copy = FALSE;
2902:
2903: if (must_wire) {
2904: entry->wired_count = 1;
2905: entry->user_wired_count = 1;
2906: } else {
2907: entry->wired_count = 0;
2908: entry->user_wired_count = 0;
2909: }
2910:
2911: entry->in_transition = TRUE;
2912: entry->needs_wakeup = FALSE;
2913:
2914: entry->vme_start = start;
2915: entry->vme_end = start + size;
2916:
2917: entry->inheritance = VM_INHERIT_DEFAULT;
2918: entry->protection = VM_PROT_DEFAULT;
2919: entry->max_protection = VM_PROT_ALL;
2920: entry->projected_on = 0;
2921:
2922: vm_object_lock(object);
2923:
2924: /*
2925: * Update the hints and the map size
2926: */
2927: if (dst_map->first_free == last) {
2928: dst_map->first_free = entry;
2929: }
2930: SAVE_HINT(dst_map, entry);
2931: dst_map->size += size;
2932:
2933: /*
2934: * Link in the entry
2935: */
2936: vm_map_entry_link(dst_map, last, entry);
2937: last = entry;
2938:
2939: /*
2940: * Transfer pages into new object.
2941: * Scan page list in vm_map_copy.
2942: */
2943: insert_pages:
2944: dst_offset = copy->offset & PAGE_MASK;
2945: cont_invoked = FALSE;
2946: orig_copy = copy;
2947: last->in_transition = TRUE;
2948: old_last_offset = last->offset
2949: + (start - last->vme_start);
2950:
2951: vm_page_lock_queues();
2952:
2953: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2954: m = *page_list;
2955: assert(m && !m->tabled);
2956:
2957: /*
2958: * Must clear busy bit in page before inserting it.
2959: * Ok to skip wakeup logic because nobody else
2960: * can possibly know about this page.
2961: * The page is dirty in its new object.
2962: */
2963:
2964: assert(!m->wanted);
2965:
2966: m->busy = FALSE;
2967: m->dirty = TRUE;
2968: vm_page_replace(m, object, old_last_offset + offset);
2969: if (must_wire) {
2970: vm_page_wire(m);
2971: PMAP_ENTER(dst_map->pmap,
2972: last->vme_start + m->offset - last->offset,
2973: m, last->protection, TRUE);
2974: } else {
2975: vm_page_activate(m);
2976: }
2977:
2978: *page_list++ = VM_PAGE_NULL;
2979: if (--(copy->cpy_npages) == 0 &&
2980: vm_map_copy_has_cont(copy)) {
2981: vm_map_copy_t new_copy;
2982:
2983: /*
2984: * Ok to unlock map because entry is
2985: * marked in_transition.
2986: */
2987: cont_invoked = TRUE;
2988: vm_page_unlock_queues();
2989: vm_object_unlock(object);
2990: vm_map_unlock(dst_map);
2991: vm_map_copy_invoke_cont(copy, &new_copy, &result);
2992:
2993: if (result == KERN_SUCCESS) {
2994:
2995: /*
2996: * If we got back a copy with real pages,
2997: * steal them now. Either all of the
2998: * pages in the list are tabled or none
2999: * of them are; mixtures are not possible.
3000: *
3001: * Save original copy for consume on
3002: * success logic at end of routine.
3003: */
3004: if (copy != orig_copy)
3005: vm_map_copy_discard(copy);
3006:
3007: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
3008: page_list = ©->cpy_page_list[0];
3009: if ((*page_list)->tabled)
3010: vm_map_copy_steal_pages(copy);
3011: }
3012: }
3013: else {
3014: /*
3015: * Continuation failed.
3016: */
3017: vm_map_lock(dst_map);
3018: goto error;
3019: }
3020:
3021: vm_map_lock(dst_map);
3022: vm_object_lock(object);
3023: vm_page_lock_queues();
3024: }
3025: }
3026:
3027: vm_page_unlock_queues();
3028: vm_object_unlock(object);
3029:
3030: *dst_addr = start + dst_offset;
3031:
3032: /*
3033: * Clear the in transition bits. This is easy if we
3034: * didn't have a continuation.
3035: */
3036: error:
3037: if (!cont_invoked) {
3038: /*
3039: * We didn't unlock the map, so nobody could
3040: * be waiting.
3041: */
3042: last->in_transition = FALSE;
3043: assert(!last->needs_wakeup);
3044: needs_wakeup = FALSE;
3045: }
3046: else {
3047: if (!vm_map_lookup_entry(dst_map, start, &entry))
3048: panic("vm_map_copyout_page_list: missing entry");
3049:
3050: /*
3051: * Clear transition bit for all constituent entries that
3052: * were in the original entry. Also check for waiters.
3053: */
3054: while((entry != vm_map_to_entry(dst_map)) &&
3055: (entry->vme_start < end)) {
3056: assert(entry->in_transition);
3057: entry->in_transition = FALSE;
3058: if(entry->needs_wakeup) {
3059: entry->needs_wakeup = FALSE;
3060: needs_wakeup = TRUE;
3061: }
3062: entry = entry->vme_next;
3063: }
3064: }
3065:
3066: if (result != KERN_SUCCESS)
3067: vm_map_delete(dst_map, start, end);
3068:
3069: vm_map_unlock(dst_map);
3070:
3071: if (needs_wakeup)
3072: vm_map_entry_wakeup(dst_map);
3073:
3074: /*
3075: * Consume on success logic.
3076: */
3077: if (copy != orig_copy) {
3078: zfree(vm_map_copy_zone, (vm_offset_t) copy);
3079: }
3080: if (result == KERN_SUCCESS) {
3081: zfree(vm_map_copy_zone, (vm_offset_t) orig_copy);
3082: }
3083:
3084: return(result);
3085: }
3086:
3087: /*
3088: * Routine: vm_map_copyin
3089: *
3090: * Description:
3091: * Copy the specified region (src_addr, len) from the
3092: * source address space (src_map), possibly removing
3093: * the region from the source address space (src_destroy).
3094: *
3095: * Returns:
3096: * A vm_map_copy_t object (copy_result), suitable for
3097: * insertion into another address space (using vm_map_copyout),
3098: * copying over another address space region (using
3099: * vm_map_copy_overwrite). If the copy is unused, it
3100: * should be destroyed (using vm_map_copy_discard).
3101: *
3102: * In/out conditions:
3103: * The source map should not be locked on entry.
3104: */
3105: kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result)
3106: vm_map_t src_map;
3107: vm_offset_t src_addr;
3108: vm_size_t len;
3109: boolean_t src_destroy;
3110: vm_map_copy_t *copy_result; /* OUT */
3111: {
3112: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3113: * in multi-level lookup, this
3114: * entry contains the actual
3115: * vm_object/offset.
3116: */
3117:
3118: vm_offset_t src_start; /* Start of current entry --
3119: * where copy is taking place now
3120: */
3121: vm_offset_t src_end; /* End of entire region to be
3122: * copied */
3123:
3124: register
3125: vm_map_copy_t copy; /* Resulting copy */
3126:
3127: /*
3128: * Check for copies of zero bytes.
3129: */
3130:
3131: if (len == 0) {
3132: *copy_result = VM_MAP_COPY_NULL;
3133: return(KERN_SUCCESS);
3134: }
3135:
3136: /*
3137: * Compute start and end of region
3138: */
3139:
3140: src_start = trunc_page(src_addr);
3141: src_end = round_page(src_addr + len);
3142:
3143: /*
3144: * Check that the end address doesn't overflow
3145: */
3146:
3147: if (src_end <= src_start)
3148: if ((src_end < src_start) || (src_start != 0))
3149: return(KERN_INVALID_ADDRESS);
3150:
3151: /*
3152: * Allocate a header element for the list.
3153: *
3154: * Use the start and end in the header to
3155: * remember the endpoints prior to rounding.
3156: */
3157:
3158: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
3159: vm_map_copy_first_entry(copy) =
3160: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3161: copy->type = VM_MAP_COPY_ENTRY_LIST;
3162: copy->cpy_hdr.nentries = 0;
3163: copy->cpy_hdr.entries_pageable = TRUE;
3164:
3165: copy->offset = src_addr;
3166: copy->size = len;
3167:
3168: #define RETURN(x) \
3169: MACRO_BEGIN \
3170: vm_map_unlock(src_map); \
3171: vm_map_copy_discard(copy); \
3172: MACRO_RETURN(x); \
3173: MACRO_END
3174:
3175: /*
3176: * Find the beginning of the region.
3177: */
3178:
3179: vm_map_lock(src_map);
3180:
3181: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3182: RETURN(KERN_INVALID_ADDRESS);
3183: vm_map_clip_start(src_map, tmp_entry, src_start);
3184:
3185: /*
3186: * Go through entries until we get to the end.
3187: */
3188:
3189: while (TRUE) {
3190: register
3191: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3192: vm_size_t src_size; /* Size of source
3193: * map entry (in both
3194: * maps)
3195: */
3196:
3197: register
3198: vm_object_t src_object; /* Object to copy */
3199: vm_offset_t src_offset;
3200:
3201: boolean_t src_needs_copy; /* Should source map
3202: * be made read-only
3203: * for copy-on-write?
3204: */
3205:
3206: register
3207: vm_map_entry_t new_entry; /* Map entry for copy */
3208: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3209:
3210: boolean_t was_wired; /* Was source wired? */
3211: vm_map_version_t version; /* Version before locks
3212: * dropped to make copy
3213: */
3214:
3215: /*
3216: * Verify that the region can be read.
3217: */
3218:
3219: if (! (src_entry->protection & VM_PROT_READ))
3220: RETURN(KERN_PROTECTION_FAILURE);
3221:
3222: /*
3223: * Clip against the endpoints of the entire region.
3224: */
3225:
3226: vm_map_clip_end(src_map, src_entry, src_end);
3227:
3228: src_size = src_entry->vme_end - src_start;
3229: src_object = src_entry->object.vm_object;
3230: src_offset = src_entry->offset;
3231: was_wired = (src_entry->wired_count != 0);
3232:
3233: /*
3234: * Create a new address map entry to
3235: * hold the result. Fill in the fields from
3236: * the appropriate source entries.
3237: */
3238:
3239: new_entry = vm_map_copy_entry_create(copy);
3240: vm_map_entry_copy(new_entry, src_entry);
3241:
3242: /*
3243: * Attempt non-blocking copy-on-write optimizations.
3244: */
3245:
3246: if (src_destroy &&
3247: (src_object == VM_OBJECT_NULL ||
3248: (src_object->temporary && !src_object->use_shared_copy)))
3249: {
3250: /*
3251: * If we are destroying the source, and the object
3252: * is temporary, and not shared writable,
3253: * we can move the object reference
3254: * from the source to the copy. The copy is
3255: * copy-on-write only if the source is.
3256: * We make another reference to the object, because
3257: * destroying the source entry will deallocate it.
3258: */
3259: vm_object_reference(src_object);
3260:
3261: /*
3262: * Copy is always unwired. vm_map_copy_entry
3263: * set its wired count to zero.
3264: */
3265:
3266: goto CopySuccessful;
3267: }
3268:
3269: if (!was_wired &&
3270: vm_object_copy_temporary(
3271: &new_entry->object.vm_object,
3272: &new_entry->offset,
3273: &src_needs_copy,
3274: &new_entry_needs_copy)) {
3275:
3276: new_entry->needs_copy = new_entry_needs_copy;
3277:
3278: /*
3279: * Handle copy-on-write obligations
3280: */
3281:
3282: if (src_needs_copy && !tmp_entry->needs_copy) {
3283: vm_object_pmap_protect(
3284: src_object,
3285: src_offset,
3286: src_size,
3287: (src_entry->is_shared ? PMAP_NULL
3288: : src_map->pmap),
3289: src_entry->vme_start,
3290: src_entry->protection &
3291: ~VM_PROT_WRITE);
3292:
3293: tmp_entry->needs_copy = TRUE;
3294: }
3295:
3296: /*
3297: * The map has never been unlocked, so it's safe to
3298: * move to the next entry rather than doing another
3299: * lookup.
3300: */
3301:
3302: goto CopySuccessful;
3303: }
3304:
3305: new_entry->needs_copy = FALSE;
3306:
3307: /*
3308: * Take an object reference, so that we may
3309: * release the map lock(s).
3310: */
3311:
3312: assert(src_object != VM_OBJECT_NULL);
3313: vm_object_reference(src_object);
3314:
3315: /*
3316: * Record the timestamp for later verification.
3317: * Unlock the map.
3318: */
3319:
3320: version.main_timestamp = src_map->timestamp;
3321: vm_map_unlock(src_map);
3322:
3323: /*
3324: * Perform the copy
3325: */
3326:
3327: if (was_wired) {
3328: vm_object_lock(src_object);
3329: (void) vm_object_copy_slowly(
3330: src_object,
3331: src_offset,
3332: src_size,
3333: FALSE,
3334: &new_entry->object.vm_object);
3335: new_entry->offset = 0;
3336: new_entry->needs_copy = FALSE;
3337: } else {
3338: kern_return_t result;
3339:
3340: result = vm_object_copy_strategically(src_object,
3341: src_offset,
3342: src_size,
3343: &new_entry->object.vm_object,
3344: &new_entry->offset,
3345: &new_entry_needs_copy);
3346:
3347: new_entry->needs_copy = new_entry_needs_copy;
3348:
3349:
3350: if (result != KERN_SUCCESS) {
3351: vm_map_copy_entry_dispose(copy, new_entry);
3352:
3353: vm_map_lock(src_map);
3354: RETURN(result);
3355: }
3356:
3357: }
3358:
3359: /*
3360: * Throw away the extra reference
3361: */
3362:
3363: vm_object_deallocate(src_object);
3364:
3365: /*
3366: * Verify that the map has not substantially
3367: * changed while the copy was being made.
3368: */
3369:
3370: vm_map_lock(src_map); /* Increments timestamp once! */
3371:
3372: if ((version.main_timestamp + 1) == src_map->timestamp)
3373: goto CopySuccessful;
3374:
3375: /*
3376: * Simple version comparison failed.
3377: *
3378: * Retry the lookup and verify that the
3379: * same object/offset are still present.
3380: *
3381: * [Note: a memory manager that colludes with
3382: * the calling task can detect that we have
3383: * cheated. While the map was unlocked, the
3384: * mapping could have been changed and restored.]
3385: */
3386:
3387: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3388: vm_map_copy_entry_dispose(copy, new_entry);
3389: RETURN(KERN_INVALID_ADDRESS);
3390: }
3391:
3392: src_entry = tmp_entry;
3393: vm_map_clip_start(src_map, src_entry, src_start);
3394:
3395: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3396: goto VerificationFailed;
3397:
3398: if (src_entry->vme_end < new_entry->vme_end)
3399: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3400:
3401: if ((src_entry->object.vm_object != src_object) ||
3402: (src_entry->offset != src_offset) ) {
3403:
3404: /*
3405: * Verification failed.
3406: *
3407: * Start over with this top-level entry.
3408: */
3409:
3410: VerificationFailed: ;
3411:
3412: vm_object_deallocate(new_entry->object.vm_object);
3413: vm_map_copy_entry_dispose(copy, new_entry);
3414: tmp_entry = src_entry;
3415: continue;
3416: }
3417:
3418: /*
3419: * Verification succeeded.
3420: */
3421:
3422: CopySuccessful: ;
3423:
3424: /*
3425: * Link in the new copy entry.
3426: */
3427:
3428: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3429: new_entry);
3430:
3431: /*
3432: * Determine whether the entire region
3433: * has been copied.
3434: */
3435: src_start = new_entry->vme_end;
3436: if ((src_start >= src_end) && (src_end != 0))
3437: break;
3438:
3439: /*
3440: * Verify that there are no gaps in the region
3441: */
3442:
3443: tmp_entry = src_entry->vme_next;
3444: if (tmp_entry->vme_start != src_start)
3445: RETURN(KERN_INVALID_ADDRESS);
3446: }
3447:
3448: /*
3449: * If the source should be destroyed, do it now, since the
3450: * copy was successful.
3451: */
3452: if (src_destroy)
3453: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3454:
3455: vm_map_unlock(src_map);
3456:
3457: *copy_result = copy;
3458: return(KERN_SUCCESS);
3459:
3460: #undef RETURN
3461: }
3462:
3463: /*
3464: * vm_map_copyin_object:
3465: *
3466: * Create a copy object from an object.
3467: * Our caller donates an object reference.
3468: */
3469:
3470: kern_return_t vm_map_copyin_object(object, offset, size, copy_result)
3471: vm_object_t object;
3472: vm_offset_t offset; /* offset of region in object */
3473: vm_size_t size; /* size of region in object */
3474: vm_map_copy_t *copy_result; /* OUT */
3475: {
3476: vm_map_copy_t copy; /* Resulting copy */
3477:
3478: /*
3479: * We drop the object into a special copy object
3480: * that contains the object directly. These copy objects
3481: * are distinguished by entries_pageable == FALSE
3482: * and null links.
3483: */
3484:
3485: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
3486: vm_map_copy_first_entry(copy) =
3487: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3488: copy->type = VM_MAP_COPY_OBJECT;
3489: copy->cpy_object = object;
3490: copy->offset = offset;
3491: copy->size = size;
3492:
3493: *copy_result = copy;
3494: return(KERN_SUCCESS);
3495: }
3496:
3497: /*
3498: * vm_map_copyin_page_list_cont:
3499: *
3500: * Continuation routine for vm_map_copyin_page_list.
3501: *
3502: * If vm_map_copyin_page_list can't fit the entire vm range
3503: * into a single page list object, it creates a continuation.
3504: * When the target of the operation has used the pages in the
3505: * initial page list, it invokes the continuation, which calls
3506: * this routine. If an error happens, the continuation is aborted
3507: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3508: * pages are discarded from the initial page list before invoking
3509: * the continuation.
3510: *
3511: * NOTE: This is not the same sort of continuation used by
3512: * the scheduler.
3513: */
3514:
3515: kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result)
3516: vm_map_copyin_args_t cont_args;
3517: vm_map_copy_t *copy_result; /* OUT */
3518: {
3519: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
3520: register boolean_t do_abort, src_destroy, src_destroy_only;
3521:
3522: /*
3523: * Check for cases that only require memory destruction.
3524: */
3525: do_abort = (copy_result == (vm_map_copy_t *) 0);
3526: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3527: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3528:
3529: if (do_abort || src_destroy_only) {
3530: if (src_destroy)
3531: result = vm_map_remove(cont_args->map,
3532: cont_args->destroy_addr,
3533: cont_args->destroy_addr + cont_args->destroy_len);
3534: if (!do_abort)
3535: *copy_result = VM_MAP_COPY_NULL;
3536: }
3537: else {
3538: result = vm_map_copyin_page_list(cont_args->map,
3539: cont_args->src_addr, cont_args->src_len, src_destroy,
3540: cont_args->steal_pages, copy_result, TRUE);
3541:
3542: if (src_destroy && !cont_args->steal_pages &&
3543: vm_map_copy_has_cont(*copy_result)) {
3544: vm_map_copyin_args_t new_args;
3545: /*
3546: * Transfer old destroy info.
3547: */
3548: new_args = (vm_map_copyin_args_t)
3549: (*copy_result)->cpy_cont_args;
3550: new_args->destroy_addr = cont_args->destroy_addr;
3551: new_args->destroy_len = cont_args->destroy_len;
3552: }
3553: }
3554:
3555: vm_map_deallocate(cont_args->map);
3556: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3557:
3558: return(result);
3559: }
3560:
3561: /*
3562: * vm_map_copyin_page_list:
3563: *
3564: * This is a variant of vm_map_copyin that copies in a list of pages.
3565: * If steal_pages is TRUE, the pages are only in the returned list.
3566: * If steal_pages is FALSE, the pages are busy and still in their
3567: * objects. A continuation may be returned if not all the pages fit:
3568: * the recipient of this copy_result must be prepared to deal with it.
3569: */
3570:
3571: kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy,
3572: steal_pages, copy_result, is_cont)
3573: vm_map_t src_map;
3574: vm_offset_t src_addr;
3575: vm_size_t len;
3576: boolean_t src_destroy;
3577: boolean_t steal_pages;
3578: vm_map_copy_t *copy_result; /* OUT */
3579: boolean_t is_cont;
3580: {
3581: vm_map_entry_t src_entry;
3582: vm_page_t m;
3583: vm_offset_t src_start;
3584: vm_offset_t src_end;
3585: vm_size_t src_size;
3586: register
3587: vm_object_t src_object;
3588: register
3589: vm_offset_t src_offset;
3590: vm_offset_t src_last_offset;
3591: register
3592: vm_map_copy_t copy; /* Resulting copy */
3593: kern_return_t result = KERN_SUCCESS;
3594: boolean_t need_map_lookup;
3595: vm_map_copyin_args_t cont_args;
3596:
3597: /*
3598: * If steal_pages is FALSE, this leaves busy pages in
3599: * the object. A continuation must be used if src_destroy
3600: * is true in this case (!steal_pages && src_destroy).
3601: *
3602: * XXX Still have a more general problem of what happens
3603: * XXX if the same page occurs twice in a list. Deadlock
3604: * XXX can happen if vm_fault_page was called. A
3605: * XXX possible solution is to use a continuation if vm_fault_page
3606: * XXX is called and we cross a map entry boundary.
3607: */
3608:
3609: /*
3610: * Check for copies of zero bytes.
3611: */
3612:
3613: if (len == 0) {
3614: *copy_result = VM_MAP_COPY_NULL;
3615: return(KERN_SUCCESS);
3616: }
3617:
3618: /*
3619: * Compute start and end of region
3620: */
3621:
3622: src_start = trunc_page(src_addr);
3623: src_end = round_page(src_addr + len);
3624:
3625: /*
3626: * Check that the end address doesn't overflow
3627: */
3628:
3629: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3630: return KERN_INVALID_ADDRESS;
3631: }
3632:
3633: /*
3634: * Allocate a header element for the page list.
3635: *
3636: * Record original offset and size, as caller may not
3637: * be page-aligned.
3638: */
3639:
3640: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
3641: copy->type = VM_MAP_COPY_PAGE_LIST;
3642: copy->cpy_npages = 0;
3643: copy->offset = src_addr;
3644: copy->size = len;
3645: copy->cpy_cont = ((kern_return_t (*)()) 0);
3646: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
3647:
3648: /*
3649: * Find the beginning of the region.
3650: */
3651:
3652: do_map_lookup:
3653:
3654: vm_map_lock(src_map);
3655:
3656: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3657: result = KERN_INVALID_ADDRESS;
3658: goto error;
3659: }
3660: need_map_lookup = FALSE;
3661:
3662: /*
3663: * Go through entries until we get to the end.
3664: */
3665:
3666: while (TRUE) {
3667:
3668: if (! (src_entry->protection & VM_PROT_READ)) {
3669: result = KERN_PROTECTION_FAILURE;
3670: goto error;
3671: }
3672:
3673: if (src_end > src_entry->vme_end)
3674: src_size = src_entry->vme_end - src_start;
3675: else
3676: src_size = src_end - src_start;
3677:
3678: src_object = src_entry->object.vm_object;
3679: src_offset = src_entry->offset +
3680: (src_start - src_entry->vme_start);
3681:
3682: /*
3683: * If src_object is NULL, allocate it now;
3684: * we're going to fault on it shortly.
3685: */
3686: if (src_object == VM_OBJECT_NULL) {
3687: src_object = vm_object_allocate((vm_size_t)
3688: src_entry->vme_end -
3689: src_entry->vme_start);
3690: src_entry->object.vm_object = src_object;
3691: }
3692:
3693: /*
3694: * Iterate over pages. Fault in ones that aren't present.
3695: */
3696: src_last_offset = src_offset + src_size;
3697: for (; (src_offset < src_last_offset && !need_map_lookup);
3698: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3699:
3700: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3701: make_continuation:
3702: /*
3703: * At this point we have the max number of
3704: * pages busy for this thread that we're
3705: * willing to allow. Stop here and record
3706: * arguments for the remainder. Note:
3707: * this means that this routine isn't atomic,
3708: * but that's the breaks. Note that only
3709: * the first vm_map_copy_t that comes back
3710: * from this routine has the right offset
3711: * and size; those from continuations are
3712: * page rounded, and short by the amount
3713: * already done.
3714: *
3715: * Reset src_end so the src_destroy
3716: * code at the bottom doesn't do
3717: * something stupid.
3718: */
3719:
3720: cont_args = (vm_map_copyin_args_t)
3721: kalloc(sizeof(vm_map_copyin_args_data_t));
3722: cont_args->map = src_map;
3723: vm_map_reference(src_map);
3724: cont_args->src_addr = src_start;
3725: cont_args->src_len = len - (src_start - src_addr);
3726: if (src_destroy) {
3727: cont_args->destroy_addr = cont_args->src_addr;
3728: cont_args->destroy_len = cont_args->src_len;
3729: }
3730: else {
3731: cont_args->destroy_addr = (vm_offset_t) 0;
3732: cont_args->destroy_len = (vm_offset_t) 0;
3733: }
3734: cont_args->steal_pages = steal_pages;
3735:
3736: copy->cpy_cont_args = (char *) cont_args;
3737: copy->cpy_cont = vm_map_copyin_page_list_cont;
3738:
3739: src_end = src_start;
3740: vm_map_clip_end(src_map, src_entry, src_end);
3741: break;
3742: }
3743:
3744: /*
3745: * Try to find the page of data.
3746: */
3747: vm_object_lock(src_object);
3748: vm_object_paging_begin(src_object);
3749: if (((m = vm_page_lookup(src_object, src_offset)) !=
3750: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3751: !m->absent && !m->error) {
3752:
3753: /*
3754: * This is the page. Mark it busy
3755: * and keep the paging reference on
3756: * the object whilst we do our thing.
3757: */
3758: m->busy = TRUE;
3759:
3760: /*
3761: * Also write-protect the page, so
3762: * that the map`s owner cannot change
3763: * the data. The busy bit will prevent
3764: * faults on the page from succeeding
3765: * until the copy is released; after
3766: * that, the page can be re-entered
3767: * as writable, since we didn`t alter
3768: * the map entry. This scheme is a
3769: * cheap copy-on-write.
3770: *
3771: * Don`t forget the protection and
3772: * the page_lock value!
3773: *
3774: * If the source is being destroyed
3775: * AND not shared writable, we don`t
3776: * have to protect the page, since
3777: * we will destroy the (only)
3778: * writable mapping later.
3779: */
3780: if (!src_destroy ||
3781: src_object->use_shared_copy)
3782: {
3783: pmap_page_protect(m->phys_addr,
3784: src_entry->protection
3785: & ~m->page_lock
3786: & ~VM_PROT_WRITE);
3787: }
3788:
3789: }
3790: else {
3791: vm_prot_t result_prot;
3792: vm_page_t top_page;
3793: kern_return_t kr;
3794:
3795: /*
3796: * Have to fault the page in; must
3797: * unlock the map to do so. While
3798: * the map is unlocked, anything
3799: * can happen, we must lookup the
3800: * map entry before continuing.
3801: */
3802: vm_map_unlock(src_map);
3803: need_map_lookup = TRUE;
3804: retry:
3805: result_prot = VM_PROT_READ;
3806:
3807: kr = vm_fault_page(src_object, src_offset,
3808: VM_PROT_READ, FALSE, FALSE,
3809: &result_prot, &m, &top_page,
3810: FALSE, (void (*)()) 0);
3811: /*
3812: * Cope with what happened.
3813: */
3814: switch (kr) {
3815: case VM_FAULT_SUCCESS:
3816: break;
3817: case VM_FAULT_INTERRUPTED: /* ??? */
3818: case VM_FAULT_RETRY:
3819: vm_object_lock(src_object);
3820: vm_object_paging_begin(src_object);
3821: goto retry;
3822: case VM_FAULT_MEMORY_SHORTAGE:
3823: VM_PAGE_WAIT((void (*)()) 0);
3824: vm_object_lock(src_object);
3825: vm_object_paging_begin(src_object);
3826: goto retry;
3827: case VM_FAULT_FICTITIOUS_SHORTAGE:
3828: vm_page_more_fictitious();
3829: vm_object_lock(src_object);
3830: vm_object_paging_begin(src_object);
3831: goto retry;
3832: case VM_FAULT_MEMORY_ERROR:
3833: /*
3834: * Something broke. If this
3835: * is a continuation, return
3836: * a partial result if possible,
3837: * else fail the whole thing.
3838: * In the continuation case, the
3839: * next continuation call will
3840: * get this error if it persists.
3841: */
3842: vm_map_lock(src_map);
3843: if (is_cont &&
3844: copy->cpy_npages != 0)
3845: goto make_continuation;
3846:
3847: result = KERN_MEMORY_ERROR;
3848: goto error;
3849: }
3850:
3851: if (top_page != VM_PAGE_NULL) {
3852: vm_object_lock(src_object);
3853: VM_PAGE_FREE(top_page);
3854: vm_object_paging_end(src_object);
3855: vm_object_unlock(src_object);
3856: }
3857:
3858: /*
3859: * We do not need to write-protect
3860: * the page, since it cannot have
3861: * been in the pmap (and we did not
3862: * enter it above). The busy bit
3863: * will protect the page from being
3864: * entered as writable until it is
3865: * unlocked.
3866: */
3867:
3868: }
3869:
3870: /*
3871: * The page is busy, its object is locked, and
3872: * we have a paging reference on it. Either
3873: * the map is locked, or need_map_lookup is
3874: * TRUE.
3875: *
3876: * Put the page in the page list.
3877: */
3878: copy->cpy_page_list[copy->cpy_npages++] = m;
3879: vm_object_unlock(m->object);
3880: }
3881:
3882: /*
3883: * DETERMINE whether the entire region
3884: * has been copied.
3885: */
3886: if (src_start >= src_end && src_end != 0) {
3887: if (need_map_lookup)
3888: vm_map_lock(src_map);
3889: break;
3890: }
3891:
3892: /*
3893: * If need_map_lookup is TRUE, have to start over with
3894: * another map lookup. Note that we dropped the map
3895: * lock (to call vm_fault_page) above only in this case.
3896: */
3897: if (need_map_lookup)
3898: goto do_map_lookup;
3899:
3900: /*
3901: * Verify that there are no gaps in the region
3902: */
3903:
3904: src_start = src_entry->vme_end;
3905: src_entry = src_entry->vme_next;
3906: if (src_entry->vme_start != src_start) {
3907: result = KERN_INVALID_ADDRESS;
3908: goto error;
3909: }
3910: }
3911:
3912: /*
3913: * If steal_pages is true, make sure all
3914: * pages in the copy are not in any object
3915: * We try to remove them from the original
3916: * object, but we may have to copy them.
3917: *
3918: * At this point every page in the list is busy
3919: * and holds a paging reference to its object.
3920: * When we're done stealing, every page is busy,
3921: * and in no object (m->tabled == FALSE).
3922: */
3923: src_start = trunc_page(src_addr);
3924: if (steal_pages) {
3925: register int i;
3926: vm_offset_t unwire_end;
3927:
3928: unwire_end = src_start;
3929: for (i = 0; i < copy->cpy_npages; i++) {
3930:
3931: /*
3932: * Remove the page from its object if it
3933: * can be stolen. It can be stolen if:
3934: *
3935: * (1) The source is being destroyed,
3936: * the object is temporary, and
3937: * not shared.
3938: * (2) The page is not precious.
3939: *
3940: * The not shared check consists of two
3941: * parts: (a) there are no objects that
3942: * shadow this object. (b) it is not the
3943: * object in any shared map entries (i.e.,
3944: * use_shared_copy is not set).
3945: *
3946: * The first check (a) means that we can't
3947: * steal pages from objects that are not
3948: * at the top of their shadow chains. This
3949: * should not be a frequent occurrence.
3950: *
3951: * Stealing wired pages requires telling the
3952: * pmap module to let go of them.
3953: *
3954: * NOTE: stealing clean pages from objects
3955: * whose mappings survive requires a call to
3956: * the pmap module. Maybe later.
3957: */
3958: m = copy->cpy_page_list[i];
3959: src_object = m->object;
3960: vm_object_lock(src_object);
3961:
3962: if (src_destroy &&
3963: src_object->temporary &&
3964: (!src_object->shadowed) &&
3965: (!src_object->use_shared_copy) &&
3966: !m->precious) {
3967: vm_offset_t page_vaddr;
3968:
3969: page_vaddr = src_start + (i * PAGE_SIZE);
3970: if (m->wire_count > 0) {
3971:
3972: assert(m->wire_count == 1);
3973: /*
3974: * In order to steal a wired
3975: * page, we have to unwire it
3976: * first. We do this inline
3977: * here because we have the page.
3978: *
3979: * Step 1: Unwire the map entry.
3980: * Also tell the pmap module
3981: * that this piece of the
3982: * pmap is pageable.
3983: */
3984: vm_object_unlock(src_object);
3985: if (page_vaddr >= unwire_end) {
3986: if (!vm_map_lookup_entry(src_map,
3987: page_vaddr, &src_entry))
3988: panic("vm_map_copyin_page_list: missing wired map entry");
3989:
3990: vm_map_clip_start(src_map, src_entry,
3991: page_vaddr);
3992: vm_map_clip_end(src_map, src_entry,
3993: src_start + src_size);
3994:
3995: assert(src_entry->wired_count > 0);
3996: src_entry->wired_count = 0;
3997: src_entry->user_wired_count = 0;
3998: unwire_end = src_entry->vme_end;
3999: pmap_pageable(vm_map_pmap(src_map),
4000: page_vaddr, unwire_end, TRUE);
4001: }
4002:
4003: /*
4004: * Step 2: Unwire the page.
4005: * pmap_remove handles this for us.
4006: */
4007: vm_object_lock(src_object);
4008: }
4009:
4010: /*
4011: * Don't need to remove the mapping;
4012: * vm_map_delete will handle it.
4013: *
4014: * Steal the page. Setting the wire count
4015: * to zero is vm_page_unwire without
4016: * activating the page.
4017: */
4018: vm_page_lock_queues();
4019: vm_page_remove(m);
4020: if (m->wire_count > 0) {
4021: m->wire_count = 0;
4022: vm_page_wire_count--;
4023: } else {
4024: VM_PAGE_QUEUES_REMOVE(m);
4025: }
4026: vm_page_unlock_queues();
4027: }
4028: else {
4029: /*
4030: * Have to copy this page. Have to
4031: * unlock the map while copying,
4032: * hence no further page stealing.
4033: * Hence just copy all the pages.
4034: * Unlock the map while copying;
4035: * This means no further page stealing.
4036: */
4037: vm_object_unlock(src_object);
4038: vm_map_unlock(src_map);
4039:
4040: vm_map_copy_steal_pages(copy);
4041:
4042: vm_map_lock(src_map);
4043: break;
4044: }
4045:
4046: vm_object_paging_end(src_object);
4047: vm_object_unlock(src_object);
4048: }
4049:
4050: /*
4051: * If the source should be destroyed, do it now, since the
4052: * copy was successful.
4053: */
4054:
4055: if (src_destroy) {
4056: (void) vm_map_delete(src_map, src_start, src_end);
4057: }
4058: }
4059: else {
4060: /*
4061: * !steal_pages leaves busy pages in the map.
4062: * This will cause src_destroy to hang. Use
4063: * a continuation to prevent this.
4064: */
4065: if (src_destroy && !vm_map_copy_has_cont(copy)) {
4066: cont_args = (vm_map_copyin_args_t)
4067: kalloc(sizeof(vm_map_copyin_args_data_t));
4068: vm_map_reference(src_map);
4069: cont_args->map = src_map;
4070: cont_args->src_addr = (vm_offset_t) 0;
4071: cont_args->src_len = (vm_size_t) 0;
4072: cont_args->destroy_addr = src_start;
4073: cont_args->destroy_len = src_end - src_start;
4074: cont_args->steal_pages = FALSE;
4075:
4076: copy->cpy_cont_args = (char *) cont_args;
4077: copy->cpy_cont = vm_map_copyin_page_list_cont;
4078: }
4079:
4080: }
4081:
4082: vm_map_unlock(src_map);
4083:
4084: *copy_result = copy;
4085: return(result);
4086:
4087: error:
4088: vm_map_unlock(src_map);
4089: vm_map_copy_discard(copy);
4090: return(result);
4091: }
4092:
4093: /*
4094: * vm_map_fork:
4095: *
4096: * Create and return a new map based on the old
4097: * map, according to the inheritance values on the
4098: * regions in that map.
4099: *
4100: * The source map must not be locked.
4101: */
4102: vm_map_t vm_map_fork(old_map)
4103: vm_map_t old_map;
4104: {
4105: vm_map_t new_map;
4106: register
4107: vm_map_entry_t old_entry;
4108: register
4109: vm_map_entry_t new_entry;
4110: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4111: vm_size_t new_size = 0;
4112: vm_size_t entry_size;
4113: register
4114: vm_object_t object;
4115:
4116: vm_map_lock(old_map);
4117:
4118: new_map = vm_map_create(new_pmap,
4119: old_map->min_offset,
4120: old_map->max_offset,
4121: old_map->hdr.entries_pageable);
4122:
4123: for (
4124: old_entry = vm_map_first_entry(old_map);
4125: old_entry != vm_map_to_entry(old_map);
4126: ) {
4127: if (old_entry->is_sub_map)
4128: panic("vm_map_fork: encountered a submap");
4129:
4130: entry_size = (old_entry->vme_end - old_entry->vme_start);
4131:
4132: switch (old_entry->inheritance) {
4133: case VM_INHERIT_NONE:
4134: break;
4135:
4136: case VM_INHERIT_SHARE:
4137: /*
4138: * New sharing code. New map entry
4139: * references original object. Temporary
4140: * objects use asynchronous copy algorithm for
4141: * future copies. First make sure we have
4142: * the right object. If we need a shadow,
4143: * or someone else already has one, then
4144: * make a new shadow and share it.
4145: */
4146:
4147: object = old_entry->object.vm_object;
4148: if (object == VM_OBJECT_NULL) {
4149: object = vm_object_allocate(
4150: (vm_size_t)(old_entry->vme_end -
4151: old_entry->vme_start));
4152: old_entry->offset = 0;
4153: old_entry->object.vm_object = object;
4154: assert(!old_entry->needs_copy);
4155: }
4156: else if (old_entry->needs_copy || object->shadowed ||
4157: (object->temporary && !old_entry->is_shared &&
4158: object->size > (vm_size_t)(old_entry->vme_end -
4159: old_entry->vme_start))) {
4160:
4161: assert(object->temporary);
4162: assert(!(object->shadowed && old_entry->is_shared));
4163: vm_object_shadow(
4164: &old_entry->object.vm_object,
4165: &old_entry->offset,
4166: (vm_size_t) (old_entry->vme_end -
4167: old_entry->vme_start));
4168:
4169: /*
4170: * If we're making a shadow for other than
4171: * copy on write reasons, then we have
4172: * to remove write permission.
4173: */
4174:
4175: if (!old_entry->needs_copy &&
4176: (old_entry->protection & VM_PROT_WRITE)) {
4177: pmap_protect(vm_map_pmap(old_map),
4178: old_entry->vme_start,
4179: old_entry->vme_end,
4180: old_entry->protection &
4181: ~VM_PROT_WRITE);
4182: }
4183: old_entry->needs_copy = FALSE;
4184: object = old_entry->object.vm_object;
4185: }
4186:
4187: /*
4188: * Set use_shared_copy to indicate that
4189: * object must use shared (delayed) copy-on
4190: * write. This is ignored for permanent objects.
4191: * Bump the reference count for the new entry
4192: */
4193:
4194: vm_object_lock(object);
4195: object->use_shared_copy = TRUE;
4196: object->ref_count++;
4197: vm_object_unlock(object);
4198:
4199: if (old_entry->projected_on != 0) {
4200: /*
4201: * If entry is projected buffer, clone the
4202: * entry exactly.
4203: */
4204:
4205: vm_map_entry_copy_full(new_entry, old_entry);
4206:
4207: } else {
4208: /*
4209: * Clone the entry, using object ref from above.
4210: * Mark both entries as shared.
4211: */
4212:
4213: new_entry = vm_map_entry_create(new_map);
4214: vm_map_entry_copy(new_entry, old_entry);
4215: old_entry->is_shared = TRUE;
4216: new_entry->is_shared = TRUE;
4217: }
4218:
4219: /*
4220: * Insert the entry into the new map -- we
4221: * know we're inserting at the end of the new
4222: * map.
4223: */
4224:
4225: vm_map_entry_link(
4226: new_map,
4227: vm_map_last_entry(new_map),
4228: new_entry);
4229:
4230: /*
4231: * Update the physical map
4232: */
4233:
4234: pmap_copy(new_map->pmap, old_map->pmap,
4235: new_entry->vme_start,
4236: entry_size,
4237: old_entry->vme_start);
4238:
4239: new_size += entry_size;
4240: break;
4241:
4242: case VM_INHERIT_COPY:
4243: if (old_entry->wired_count == 0) {
4244: boolean_t src_needs_copy;
4245: boolean_t new_entry_needs_copy;
4246:
4247: new_entry = vm_map_entry_create(new_map);
4248: vm_map_entry_copy(new_entry, old_entry);
4249:
4250: if (vm_object_copy_temporary(
4251: &new_entry->object.vm_object,
4252: &new_entry->offset,
4253: &src_needs_copy,
4254: &new_entry_needs_copy)) {
4255:
4256: /*
4257: * Handle copy-on-write obligations
4258: */
4259:
4260: if (src_needs_copy && !old_entry->needs_copy) {
4261: vm_object_pmap_protect(
4262: old_entry->object.vm_object,
4263: old_entry->offset,
4264: entry_size,
4265: (old_entry->is_shared ?
4266: PMAP_NULL :
4267: old_map->pmap),
4268: old_entry->vme_start,
4269: old_entry->protection &
4270: ~VM_PROT_WRITE);
4271:
4272: old_entry->needs_copy = TRUE;
4273: }
4274:
4275: new_entry->needs_copy = new_entry_needs_copy;
4276:
4277: /*
4278: * Insert the entry at the end
4279: * of the map.
4280: */
4281:
4282: vm_map_entry_link(new_map,
4283: vm_map_last_entry(new_map),
4284: new_entry);
4285:
4286:
4287: new_size += entry_size;
4288: break;
4289: }
4290:
4291: vm_map_entry_dispose(new_map, new_entry);
4292: }
4293:
4294: /* INNER BLOCK (copy cannot be optimized) */ {
4295:
4296: vm_offset_t start = old_entry->vme_start;
4297: vm_map_copy_t copy;
4298: vm_map_entry_t last = vm_map_last_entry(new_map);
4299:
4300: vm_map_unlock(old_map);
4301: if (vm_map_copyin(old_map,
4302: start,
4303: entry_size,
4304: FALSE,
4305: ©)
4306: != KERN_SUCCESS) {
4307: vm_map_lock(old_map);
4308: if (!vm_map_lookup_entry(old_map, start, &last))
4309: last = last->vme_next;
4310: old_entry = last;
4311: /*
4312: * For some error returns, want to
4313: * skip to the next element.
4314: */
4315:
4316: continue;
4317: }
4318:
4319: /*
4320: * Insert the copy into the new map
4321: */
4322:
4323: vm_map_copy_insert(new_map, last, copy);
4324: new_size += entry_size;
4325:
4326: /*
4327: * Pick up the traversal at the end of
4328: * the copied region.
4329: */
4330:
4331: vm_map_lock(old_map);
4332: start += entry_size;
4333: if (!vm_map_lookup_entry(old_map, start, &last))
4334: last = last->vme_next;
4335: else
4336: vm_map_clip_start(old_map, last, start);
4337: old_entry = last;
4338:
4339: continue;
4340: /* INNER BLOCK (copy cannot be optimized) */ }
4341: }
4342: old_entry = old_entry->vme_next;
4343: }
4344:
4345: new_map->size = new_size;
4346: vm_map_unlock(old_map);
4347:
4348: return(new_map);
4349: }
4350:
4351: /*
4352: * vm_map_lookup:
4353: *
4354: * Finds the VM object, offset, and
4355: * protection for a given virtual address in the
4356: * specified map, assuming a page fault of the
4357: * type specified.
4358: *
4359: * Returns the (object, offset, protection) for
4360: * this address, whether it is wired down, and whether
4361: * this map has the only reference to the data in question.
4362: * In order to later verify this lookup, a "version"
4363: * is returned.
4364: *
4365: * The map should not be locked; it will not be
4366: * locked on exit. In order to guarantee the
4367: * existence of the returned object, it is returned
4368: * locked.
4369: *
4370: * If a lookup is requested with "write protection"
4371: * specified, the map may be changed to perform virtual
4372: * copying operations, although the data referenced will
4373: * remain the same.
4374: */
4375: kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version,
4376: object, offset, out_prot, wired)
4377: vm_map_t *var_map; /* IN/OUT */
4378: register vm_offset_t vaddr;
4379: register vm_prot_t fault_type;
4380:
4381: vm_map_version_t *out_version; /* OUT */
4382: vm_object_t *object; /* OUT */
4383: vm_offset_t *offset; /* OUT */
4384: vm_prot_t *out_prot; /* OUT */
4385: boolean_t *wired; /* OUT */
4386: {
4387: register vm_map_entry_t entry;
4388: register vm_map_t map = *var_map;
4389: register vm_prot_t prot;
4390:
4391: RetryLookup: ;
4392:
4393: /*
4394: * Lookup the faulting address.
4395: */
4396:
4397: vm_map_lock_read(map);
4398:
4399: #define RETURN(why) \
4400: { \
4401: vm_map_unlock_read(map); \
4402: return(why); \
4403: }
4404:
4405: /*
4406: * If the map has an interesting hint, try it before calling
4407: * full blown lookup routine.
4408: */
4409:
4410: simple_lock(&map->hint_lock);
4411: entry = map->hint;
4412: simple_unlock(&map->hint_lock);
4413:
4414: if ((entry == vm_map_to_entry(map)) ||
4415: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4416: vm_map_entry_t tmp_entry;
4417:
4418: /*
4419: * Entry was either not a valid hint, or the vaddr
4420: * was not contained in the entry, so do a full lookup.
4421: */
4422: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4423: RETURN(KERN_INVALID_ADDRESS);
4424:
4425: entry = tmp_entry;
4426: }
4427:
4428: /*
4429: * Handle submaps.
4430: */
4431:
4432: if (entry->is_sub_map) {
4433: vm_map_t old_map = map;
4434:
4435: *var_map = map = entry->object.sub_map;
4436: vm_map_unlock_read(old_map);
4437: goto RetryLookup;
4438: }
4439:
4440: /*
4441: * Check whether this task is allowed to have
4442: * this page.
4443: */
4444:
4445: prot = entry->protection;
4446:
4447: if ((fault_type & (prot)) != fault_type)
4448: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4449: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4450: } else {
4451: RETURN(KERN_PROTECTION_FAILURE);
4452: }
4453:
4454: /*
4455: * If this page is not pageable, we have to get
4456: * it for all possible accesses.
4457: */
4458:
4459: if (*wired = (entry->wired_count != 0))
4460: prot = fault_type = entry->protection;
4461:
4462: /*
4463: * If the entry was copy-on-write, we either ...
4464: */
4465:
4466: if (entry->needs_copy) {
4467: /*
4468: * If we want to write the page, we may as well
4469: * handle that now since we've got the map locked.
4470: *
4471: * If we don't need to write the page, we just
4472: * demote the permissions allowed.
4473: */
4474:
4475: if (fault_type & VM_PROT_WRITE) {
4476: /*
4477: * Make a new object, and place it in the
4478: * object chain. Note that no new references
4479: * have appeared -- one just moved from the
4480: * map to the new object.
4481: */
4482:
4483: if (vm_map_lock_read_to_write(map)) {
4484: goto RetryLookup;
4485: }
4486: map->timestamp++;
4487:
4488: vm_object_shadow(
4489: &entry->object.vm_object,
4490: &entry->offset,
4491: (vm_size_t) (entry->vme_end - entry->vme_start));
4492:
4493: entry->needs_copy = FALSE;
4494:
4495: vm_map_lock_write_to_read(map);
4496: }
4497: else {
4498: /*
4499: * We're attempting to read a copy-on-write
4500: * page -- don't allow writes.
4501: */
4502:
4503: prot &= (~VM_PROT_WRITE);
4504: }
4505: }
4506:
4507: /*
4508: * Create an object if necessary.
4509: */
4510: if (entry->object.vm_object == VM_OBJECT_NULL) {
4511:
4512: if (vm_map_lock_read_to_write(map)) {
4513: goto RetryLookup;
4514: }
4515:
4516: entry->object.vm_object = vm_object_allocate(
4517: (vm_size_t)(entry->vme_end - entry->vme_start));
4518: entry->offset = 0;
4519: vm_map_lock_write_to_read(map);
4520: }
4521:
4522: /*
4523: * Return the object/offset from this entry. If the entry
4524: * was copy-on-write or empty, it has been fixed up. Also
4525: * return the protection.
4526: */
4527:
4528: *offset = (vaddr - entry->vme_start) + entry->offset;
4529: *object = entry->object.vm_object;
4530: *out_prot = prot;
4531:
4532: /*
4533: * Lock the object to prevent it from disappearing
4534: */
4535:
4536: vm_object_lock(*object);
4537:
4538: /*
4539: * Save the version number and unlock the map.
4540: */
4541:
4542: out_version->main_timestamp = map->timestamp;
4543:
4544: RETURN(KERN_SUCCESS);
4545:
4546: #undef RETURN
4547: }
4548:
4549: /*
4550: * vm_map_verify:
4551: *
4552: * Verifies that the map in question has not changed
4553: * since the given version. If successful, the map
4554: * will not change until vm_map_verify_done() is called.
4555: */
4556: boolean_t vm_map_verify(map, version)
4557: register
4558: vm_map_t map;
4559: register
4560: vm_map_version_t *version; /* REF */
4561: {
4562: boolean_t result;
4563:
4564: vm_map_lock_read(map);
4565: result = (map->timestamp == version->main_timestamp);
4566:
4567: if (!result)
4568: vm_map_unlock_read(map);
4569:
4570: return(result);
4571: }
4572:
4573: /*
4574: * vm_map_verify_done:
4575: *
4576: * Releases locks acquired by a vm_map_verify.
4577: *
4578: * This is now a macro in vm/vm_map.h. It does a
4579: * vm_map_unlock_read on the map.
4580: */
4581:
4582: /*
4583: * vm_region:
4584: *
4585: * User call to obtain information about a region in
4586: * a task's address map.
4587: */
4588:
4589: kern_return_t vm_region(map, address, size,
4590: protection, max_protection,
4591: inheritance, is_shared,
4592: object_name, offset_in_object)
4593: vm_map_t map;
4594: vm_offset_t *address; /* IN/OUT */
4595: vm_size_t *size; /* OUT */
4596: vm_prot_t *protection; /* OUT */
4597: vm_prot_t *max_protection; /* OUT */
4598: vm_inherit_t *inheritance; /* OUT */
4599: boolean_t *is_shared; /* OUT */
4600: ipc_port_t *object_name; /* OUT */
4601: vm_offset_t *offset_in_object; /* OUT */
4602: {
4603: vm_map_entry_t tmp_entry;
4604: register
4605: vm_map_entry_t entry;
4606: register
4607: vm_offset_t tmp_offset;
4608: vm_offset_t start;
4609:
4610: if (map == VM_MAP_NULL)
4611: return(KERN_INVALID_ARGUMENT);
4612:
4613: start = *address;
4614:
4615: vm_map_lock_read(map);
4616: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4617: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4618: vm_map_unlock_read(map);
4619: return(KERN_NO_SPACE);
4620: }
4621: } else {
4622: entry = tmp_entry;
4623: }
4624:
4625: start = entry->vme_start;
4626: *protection = entry->protection;
4627: *max_protection = entry->max_protection;
4628: *inheritance = entry->inheritance;
4629: *address = start;
4630: *size = (entry->vme_end - start);
4631:
4632: tmp_offset = entry->offset;
4633:
4634:
4635: if (entry->is_sub_map) {
4636: *is_shared = FALSE;
4637: *object_name = IP_NULL;
4638: *offset_in_object = tmp_offset;
4639: } else {
4640: *is_shared = entry->is_shared;
4641: *object_name = vm_object_name(entry->object.vm_object);
4642: *offset_in_object = tmp_offset;
4643: }
4644:
4645: vm_map_unlock_read(map);
4646:
4647: return(KERN_SUCCESS);
4648: }
4649:
4650: /*
4651: * Routine: vm_map_simplify
4652: *
4653: * Description:
4654: * Attempt to simplify the map representation in
4655: * the vicinity of the given starting address.
4656: * Note:
4657: * This routine is intended primarily to keep the
4658: * kernel maps more compact -- they generally don't
4659: * benefit from the "expand a map entry" technology
4660: * at allocation time because the adjacent entry
4661: * is often wired down.
4662: */
4663: void vm_map_simplify(map, start)
4664: vm_map_t map;
4665: vm_offset_t start;
4666: {
4667: vm_map_entry_t this_entry;
4668: vm_map_entry_t prev_entry;
4669:
4670: vm_map_lock(map);
4671: if (
4672: (vm_map_lookup_entry(map, start, &this_entry)) &&
4673: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4674:
4675: (prev_entry->vme_end == start) &&
4676:
4677: (prev_entry->is_shared == FALSE) &&
4678: (prev_entry->is_sub_map == FALSE) &&
4679:
4680: (this_entry->is_shared == FALSE) &&
4681: (this_entry->is_sub_map == FALSE) &&
4682:
4683: (prev_entry->inheritance == this_entry->inheritance) &&
4684: (prev_entry->protection == this_entry->protection) &&
4685: (prev_entry->max_protection == this_entry->max_protection) &&
4686: (prev_entry->wired_count == this_entry->wired_count) &&
4687: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4688:
4689: (prev_entry->needs_copy == this_entry->needs_copy) &&
4690:
4691: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4692: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4693: == this_entry->offset) &&
4694: (prev_entry->projected_on == 0) &&
4695: (this_entry->projected_on == 0)
4696: ) {
4697: if (map->first_free == this_entry)
4698: map->first_free = prev_entry;
4699:
4700: SAVE_HINT(map, prev_entry);
4701: vm_map_entry_unlink(map, this_entry);
4702: prev_entry->vme_end = this_entry->vme_end;
4703: vm_object_deallocate(this_entry->object.vm_object);
4704: vm_map_entry_dispose(map, this_entry);
4705: }
4706: vm_map_unlock(map);
4707: }
4708:
4709:
4710: /*
4711: * Routine: vm_map_machine_attribute
4712: * Purpose:
4713: * Provide machine-specific attributes to mappings,
4714: * such as cachability etc. for machines that provide
4715: * them. NUMA architectures and machines with big/strange
4716: * caches will use this.
4717: * Note:
4718: * Responsibilities for locking and checking are handled here,
4719: * everything else in the pmap module. If any non-volatile
4720: * information must be kept, the pmap module should handle
4721: * it itself. [This assumes that attributes do not
4722: * need to be inherited, which seems ok to me]
4723: */
4724: kern_return_t vm_map_machine_attribute(map, address, size, attribute, value)
4725: vm_map_t map;
4726: vm_offset_t address;
4727: vm_size_t size;
4728: vm_machine_attribute_t attribute;
4729: vm_machine_attribute_val_t* value; /* IN/OUT */
4730: {
4731: kern_return_t ret;
4732:
4733: if (address < vm_map_min(map) ||
4734: (address + size) > vm_map_max(map))
4735: return KERN_INVALID_ARGUMENT;
4736:
4737: vm_map_lock(map);
4738:
4739: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4740:
4741: vm_map_unlock(map);
4742:
4743: return ret;
4744: }
4745:
4746: #include <mach_kdb.h>
4747:
4748:
4749: #if MACH_KDB
4750:
4751: #define printf kdbprintf
4752:
4753: /*
4754: * vm_map_print: [ debug ]
4755: */
4756: void vm_map_print(map)
4757: register vm_map_t map;
4758: {
4759: register vm_map_entry_t entry;
4760: extern int indent;
4761:
4762: iprintf("Task map 0x%X: pmap=0x%X,",
4763: (vm_offset_t) map, (vm_offset_t) (map->pmap));
4764: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4765: printf("version=%d\n", map->timestamp);
4766: indent += 2;
4767: for (entry = vm_map_first_entry(map);
4768: entry != vm_map_to_entry(map);
4769: entry = entry->vme_next) {
4770: static char *inheritance_name[3] = { "share", "copy", "none"};
4771:
4772: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4773: printf("start=0x%X, end=0x%X, ",
4774: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
4775: printf("prot=%X/%X/%s, ",
4776: entry->protection,
4777: entry->max_protection,
4778: inheritance_name[entry->inheritance]);
4779: if (entry->wired_count != 0) {
4780: printf("wired(");
4781: if (entry->user_wired_count != 0)
4782: printf("u");
4783: if (entry->wired_count >
4784: ((entry->user_wired_count == 0) ? 0 : 1))
4785: printf("k");
4786: printf(") ");
4787: }
4788: if (entry->in_transition) {
4789: printf("in transition");
4790: if (entry->needs_wakeup)
4791: printf("(wake request)");
4792: printf(", ");
4793: }
4794: if (entry->is_sub_map) {
4795: printf("submap=0x%X, offset=0x%X\n",
4796: (vm_offset_t) entry->object.sub_map,
4797: (vm_offset_t) entry->offset);
4798: } else {
4799: printf("object=0x%X, offset=0x%X",
4800: (vm_offset_t) entry->object.vm_object,
4801: (vm_offset_t) entry->offset);
4802: if (entry->is_shared)
4803: printf(", shared");
4804: if (entry->needs_copy)
4805: printf(", copy needed");
4806: printf("\n");
4807:
4808: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4809: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
4810: indent += 2;
4811: vm_object_print(entry->object.vm_object);
4812: indent -= 2;
4813: }
4814: }
4815: }
4816: indent -= 2;
4817: }
4818:
4819: /*
4820: * Routine: vm_map_copy_print
4821: * Purpose:
4822: * Pretty-print a copy object for ddb.
4823: */
4824:
4825: void vm_map_copy_print(copy)
4826: vm_map_copy_t copy;
4827: {
4828: extern int indent;
4829: int i, npages;
4830:
4831: printf("copy object 0x%x\n", copy);
4832:
4833: indent += 2;
4834:
4835: iprintf("type=%d", copy->type);
4836: switch (copy->type) {
4837: case VM_MAP_COPY_ENTRY_LIST:
4838: printf("[entry_list]");
4839: break;
4840:
4841: case VM_MAP_COPY_OBJECT:
4842: printf("[object]");
4843: break;
4844:
4845: case VM_MAP_COPY_PAGE_LIST:
4846: printf("[page_list]");
4847: break;
4848:
4849: default:
4850: printf("[bad type]");
4851: break;
4852: }
4853: printf(", offset=0x%x", copy->offset);
4854: printf(", size=0x%x\n", copy->size);
4855:
4856: switch (copy->type) {
4857: case VM_MAP_COPY_ENTRY_LIST:
4858: /* XXX add stuff here */
4859: break;
4860:
4861: case VM_MAP_COPY_OBJECT:
4862: iprintf("object=0x%x\n", copy->cpy_object);
4863: break;
4864:
4865: case VM_MAP_COPY_PAGE_LIST:
4866: iprintf("npages=%d", copy->cpy_npages);
4867: printf(", cont=%x", copy->cpy_cont);
4868: printf(", cont_args=%x\n", copy->cpy_cont_args);
4869: if (copy->cpy_npages < 0) {
4870: npages = 0;
4871: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4872: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4873: } else {
4874: npages = copy->cpy_npages;
4875: }
4876: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4877: for (i = 0; i < npages - 1; i++) {
4878: printf("0x%x, ", copy->cpy_page_list[i]);
4879: }
4880: if (npages > 0) {
4881: printf("0x%x", copy->cpy_page_list[npages - 1]);
4882: }
4883: printf("}\n");
4884: break;
4885: }
4886:
4887: indent -=2;
4888: }
4889: #endif /* MACH_KDB */
4890:
4891: #if NORMA_IPC
4892: /*
4893: * This should one day be eliminated;
4894: * we should always construct the right flavor of copy object
4895: * the first time. Troublesome areas include vm_read, where vm_map_copyin
4896: * is called without knowing whom the copy object is for.
4897: * There are also situations where we do want a lazy data structure
4898: * even if we are sending to a remote port...
4899: */
4900:
4901: /*
4902: * Convert a copy to a page list. The copy argument is in/out
4903: * because we probably have to allocate a new vm_map_copy structure.
4904: * We take responsibility for discarding the old structure and
4905: * use a continuation to do so. Postponing this discard ensures
4906: * that the objects containing the pages we've marked busy will stick
4907: * around.
4908: */
4909: kern_return_t
4910: vm_map_convert_to_page_list(caller_copy)
4911: vm_map_copy_t *caller_copy;
4912: {
4913: vm_map_entry_t entry, next_entry;
4914: vm_offset_t va;
4915: vm_offset_t offset;
4916: vm_object_t object;
4917: kern_return_t result;
4918: vm_map_copy_t copy, new_copy;
4919: int i, num_pages = 0;
4920:
4921: zone_t entry_zone;
4922:
4923: copy = *caller_copy;
4924:
4925: /*
4926: * We may not have to do anything,
4927: * or may not be able to do anything.
4928: */
4929: if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) {
4930: return KERN_SUCCESS;
4931: }
4932: if (copy->type == VM_MAP_COPY_OBJECT) {
4933: return vm_map_convert_to_page_list_from_object(caller_copy);
4934: }
4935: if (copy->type != VM_MAP_COPY_ENTRY_LIST) {
4936: panic("vm_map_convert_to_page_list: copy type %d!\n",
4937: copy->type);
4938: }
4939:
4940: /*
4941: * Allocate the new copy. Set its continuation to
4942: * discard the old one.
4943: */
4944: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
4945: new_copy->type = VM_MAP_COPY_PAGE_LIST;
4946: new_copy->cpy_npages = 0;
4947: new_copy->offset = copy->offset;
4948: new_copy->size = copy->size;
4949: new_copy->cpy_cont = vm_map_copy_discard_cont;
4950: new_copy->cpy_cont_args = (char *) copy;
4951:
4952: /*
4953: * Iterate over entries.
4954: */
4955: for (entry = vm_map_copy_first_entry(copy);
4956: entry != vm_map_copy_to_entry(copy);
4957: entry = entry->vme_next) {
4958:
4959: object = entry->object.vm_object;
4960: offset = entry->offset;
4961: /*
4962: * Iterate over pages.
4963: */
4964: for (va = entry->vme_start;
4965: va < entry->vme_end;
4966: va += PAGE_SIZE, offset += PAGE_SIZE) {
4967:
4968: vm_page_t m;
4969:
4970: if (new_copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
4971: /*
4972: * What a mess. We need a continuation
4973: * to do the page list, but also one
4974: * to discard the old copy. The right
4975: * thing to do is probably to copy
4976: * out the old copy into the kernel
4977: * map (or some temporary task holding
4978: * map if we're paranoid about large
4979: * copies), and then copyin the page
4980: * list that we really wanted with
4981: * src_destroy. LATER.
4982: */
4983: panic("vm_map_convert_to_page_list: num\n");
4984: }
4985:
4986: /*
4987: * Try to find the page of data.
4988: */
4989: vm_object_lock(object);
4990: vm_object_paging_begin(object);
4991: if (((m = vm_page_lookup(object, offset)) !=
4992: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
4993: !m->absent && !m->error) {
4994:
4995: /*
4996: * This is the page. Mark it busy
4997: * and keep the paging reference on
4998: * the object whilst we do our thing.
4999: */
5000: m->busy = TRUE;
5001:
5002: /*
5003: * Also write-protect the page, so
5004: * that the map`s owner cannot change
5005: * the data. The busy bit will prevent
5006: * faults on the page from succeeding
5007: * until the copy is released; after
5008: * that, the page can be re-entered
5009: * as writable, since we didn`t alter
5010: * the map entry. This scheme is a
5011: * cheap copy-on-write.
5012: *
5013: * Don`t forget the protection and
5014: * the page_lock value!
5015: */
5016:
5017: pmap_page_protect(m->phys_addr,
5018: entry->protection
5019: & ~m->page_lock
5020: & ~VM_PROT_WRITE);
5021:
5022: }
5023: else {
5024: vm_prot_t result_prot;
5025: vm_page_t top_page;
5026: kern_return_t kr;
5027:
5028: retry:
5029: result_prot = VM_PROT_READ;
5030:
5031: kr = vm_fault_page(object, offset,
5032: VM_PROT_READ, FALSE, FALSE,
5033: &result_prot, &m, &top_page,
5034: FALSE, (void (*)()) 0);
5035: if (kr == VM_FAULT_MEMORY_SHORTAGE) {
5036: VM_PAGE_WAIT((void (*)()) 0);
5037: vm_object_lock(object);
5038: vm_object_paging_begin(object);
5039: goto retry;
5040: }
5041: if (kr != VM_FAULT_SUCCESS) {
5042: /* XXX what about data_error? */
5043: vm_object_lock(object);
5044: vm_object_paging_begin(object);
5045: goto retry;
5046: }
5047: if (top_page != VM_PAGE_NULL) {
5048: vm_object_lock(object);
5049: VM_PAGE_FREE(top_page);
5050: vm_object_paging_end(object);
5051: vm_object_unlock(object);
5052: }
5053: }
5054: assert(m);
5055: m->busy = TRUE;
5056: new_copy->cpy_page_list[new_copy->cpy_npages++] = m;
5057: vm_object_unlock(object);
5058: }
5059: }
5060:
5061: *caller_copy = new_copy;
5062: return KERN_SUCCESS;
5063: }
5064:
5065: kern_return_t
5066: vm_map_convert_to_page_list_from_object(caller_copy)
5067: vm_map_copy_t *caller_copy;
5068: {
5069: vm_object_t object;
5070: vm_offset_t offset;
5071: vm_map_copy_t copy, new_copy;
5072:
5073: copy = *caller_copy;
5074: assert(copy->type == VM_MAP_COPY_OBJECT);
5075: object = copy->cpy_object;
5076: assert(object->size == round_page(object->size));
5077:
5078: /*
5079: * Allocate the new copy. Set its continuation to
5080: * discard the old one.
5081: */
5082: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
5083: new_copy->type = VM_MAP_COPY_PAGE_LIST;
5084: new_copy->cpy_npages = 0;
5085: new_copy->offset = copy->offset;
5086: new_copy->size = copy->size;
5087: new_copy->cpy_cont = vm_map_copy_discard_cont;
5088: new_copy->cpy_cont_args = (char *) copy;
5089:
5090: /*
5091: * XXX memory_object_lock_request can probably bust this
5092: * XXX See continuation comment in previous routine for solution.
5093: */
5094: assert(object->size <= VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE);
5095:
5096: for (offset = 0; offset < object->size; offset += PAGE_SIZE) {
5097: vm_page_t m;
5098:
5099: /*
5100: * Try to find the page of data.
5101: */
5102: vm_object_lock(object);
5103: vm_object_paging_begin(object);
5104: m = vm_page_lookup(object, offset);
5105: if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious &&
5106: !m->absent && !m->error) {
5107:
5108: /*
5109: * This is the page. Mark it busy
5110: * and keep the paging reference on
5111: * the object whilst we do our thing.
5112: */
5113: m->busy = TRUE;
5114: }
5115: else {
5116: vm_prot_t result_prot;
5117: vm_page_t top_page;
5118: kern_return_t kr;
5119:
5120: retry:
5121: result_prot = VM_PROT_READ;
5122:
5123: kr = vm_fault_page(object, offset,
5124: VM_PROT_READ, FALSE, FALSE,
5125: &result_prot, &m, &top_page,
5126: FALSE, (void (*)()) 0);
5127: if (kr == VM_FAULT_MEMORY_SHORTAGE) {
5128: VM_PAGE_WAIT((void (*)()) 0);
5129: vm_object_lock(object);
5130: vm_object_paging_begin(object);
5131: goto retry;
5132: }
5133: if (kr != VM_FAULT_SUCCESS) {
5134: /* XXX what about data_error? */
5135: vm_object_lock(object);
5136: vm_object_paging_begin(object);
5137: goto retry;
5138: }
5139:
5140: if (top_page != VM_PAGE_NULL) {
5141: vm_object_lock(object);
5142: VM_PAGE_FREE(top_page);
5143: vm_object_paging_end(object);
5144: vm_object_unlock(object);
5145: }
5146: }
5147: assert(m);
5148: m->busy = TRUE;
5149: new_copy->cpy_page_list[new_copy->cpy_npages++] = m;
5150: vm_object_unlock(object);
5151: }
5152:
5153: *caller_copy = new_copy;
5154: return (KERN_SUCCESS);
5155: }
5156:
5157: kern_return_t
5158: vm_map_convert_from_page_list(copy)
5159: vm_map_copy_t copy;
5160: {
5161: vm_object_t object;
5162: int i;
5163: vm_map_entry_t new_entry;
5164: vm_page_t *page_list;
5165:
5166: /*
5167: * Check type of copy object.
5168: */
5169: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
5170: return KERN_SUCCESS;
5171: }
5172: if (copy->type == VM_MAP_COPY_OBJECT) {
5173: printf("vm_map_convert_from_page_list: COPY_OBJECT?");
5174: return KERN_SUCCESS;
5175: }
5176: if (copy->type != VM_MAP_COPY_PAGE_LIST) {
5177: panic("vm_map_convert_from_page_list 0x%x %d",
5178: copy,
5179: copy->type);
5180: }
5181:
5182: /*
5183: * Make sure the pages are loose. This may be
5184: * a "Can't Happen", but just to be safe ...
5185: */
5186: page_list = ©->cpy_page_list[0];
5187: if ((*page_list)->tabled)
5188: vm_map_copy_steal_pages(copy);
5189:
5190: /*
5191: * Create object, and stuff pages into it.
5192: */
5193: object = vm_object_allocate(copy->cpy_npages);
5194: for (i = 0; i < copy->cpy_npages; i++) {
5195: register vm_page_t m = *page_list++;
5196: vm_page_insert(m, object, i * PAGE_SIZE);
5197: m->busy = FALSE;
5198: m->dirty = TRUE;
5199: vm_page_activate(m);
5200: }
5201:
5202: /*
5203: * XXX If this page list contained a continuation, then
5204: * XXX we're screwed. The right thing to do is probably do
5205: * XXX the copyout, and then copyin the entry list we really
5206: * XXX wanted.
5207: */
5208: if (vm_map_copy_has_cont(copy))
5209: panic("convert_from_page_list: continuation");
5210:
5211: /*
5212: * Change type of copy object
5213: */
5214: vm_map_copy_first_entry(copy) =
5215: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
5216: copy->type = VM_MAP_COPY_ENTRY_LIST;
5217: copy->cpy_hdr.nentries = 0;
5218: copy->cpy_hdr.entries_pageable = TRUE;
5219:
5220: /*
5221: * Allocate and initialize an entry for object
5222: */
5223: new_entry = vm_map_copy_entry_create(copy);
5224: new_entry->vme_start = trunc_page(copy->offset);
5225: new_entry->vme_end = round_page(copy->offset + copy->size);
5226: new_entry->object.vm_object = object;
5227: new_entry->offset = 0;
5228: new_entry->is_shared = FALSE;
5229: new_entry->is_sub_map = FALSE;
5230: new_entry->needs_copy = FALSE;
5231: new_entry->protection = VM_PROT_DEFAULT;
5232: new_entry->max_protection = VM_PROT_ALL;
5233: new_entry->inheritance = VM_INHERIT_DEFAULT;
5234: new_entry->wired_count = 0;
5235: new_entry->user_wired_count = 0;
5236: new_entry->projected_on = 0;
5237:
5238: /*
5239: * Insert entry into copy object, and return.
5240: */
5241: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry);
5242: return(KERN_SUCCESS);
5243: }
5244: #endif /* NORMA_IPC */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.