--- Gnu-Mach/vm/vm_map.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_map.c 2020/09/02 04:45:24 1.1.1.3 @@ -34,28 +34,51 @@ * Virtual memory mapping module. */ -#include - +#include #include #include #include #include #include -#include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include +#if MACH_KDB +#include +#include +#endif /* MACH_KDB */ + + +/* Forward declarations */ +kern_return_t vm_map_delete( + vm_map_t map, + vm_offset_t start, + vm_offset_t end); + +kern_return_t vm_map_copyout_page_list( + vm_map_t dst_map, + vm_offset_t *dst_addr, /* OUT */ + vm_map_copy_t copy); + +void vm_map_copy_page_discard (vm_map_copy_t copy); + /* * Macros to copy a vm_map_entry. We must be careful to correctly * manage the wired page count. vm_map_entry_copy() creates a new * map entry to the same memory - the wired count in the new entry * must be set to zero. vm_map_entry_copy_full() creates a new * entry that is identical to the old entry. This preserves the - * wire count; it's used for map splitting and zone changing in + * wire count; it's used for map splitting and cache changing in * vm_map_copyout. */ #define vm_map_entry_copy(NEW,OLD) \ @@ -79,7 +102,7 @@ MACRO_END * Synchronization is required prior to most operations. * * Maps consist of an ordered doubly-linked list of simple - * entries; a single hint is used to speed up lookups. + * entries; a hint and a red-black tree are used to speed up lookups. * * Sharing maps have been deleted from this version of Mach. * All shared objects are now mapped directly into the respective @@ -89,7 +112,7 @@ MACRO_END * selected by the (new) use_shared_copy bit in the object. See * vm_object_copy_temporary in vm_object.c for details. All maps * are now "top level" maps (either task map, kernel map or submap - * of the kernel map). + * of the kernel map). * * Since portions of maps are specified by start/end addreses, * which may not align with existing map entries, all @@ -115,10 +138,10 @@ MACRO_END * vm_object_copy_strategically() in vm_object.c. */ -zone_t vm_map_zone; /* zone for vm_map structures */ -zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ -zone_t vm_map_kentry_zone; /* zone for kernel entry structures */ -zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ +struct kmem_cache vm_map_cache; /* cache for vm_map structures */ +struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */ +struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */ +struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */ boolean_t vm_map_lookup_entry(); /* forward declaration */ @@ -128,7 +151,8 @@ boolean_t vm_map_lookup_entry(); /* forw * vm_map_submap creates the submap. */ -vm_object_t vm_submap_object; +static struct vm_object vm_submap_object_store; +vm_object_t vm_submap_object = &vm_submap_object_store; /* * vm_map_init: @@ -136,51 +160,82 @@ vm_object_t vm_submap_object; * Initialize the vm_map module. Must be called before * any other vm_map routines. * - * Map and entry structures are allocated from zones -- we must - * initialize those zones. + * Map and entry structures are allocated from caches -- we must + * initialize those caches. * - * There are three zones of interest: + * There are three caches of interest: * - * vm_map_zone: used to allocate maps. - * vm_map_entry_zone: used to allocate map entries. - * vm_map_kentry_zone: used to allocate map entries for the kernel. - * - * The kernel allocates map entries from a special zone that is initially - * "crammed" with memory. It would be difficult (perhaps impossible) for - * the kernel to allocate more memory to a entry zone when it became - * empty since the very act of allocating memory implies the creation - * of a new entry. + * vm_map_cache: used to allocate maps. + * vm_map_entry_cache: used to allocate map entries. + * vm_map_kentry_cache: used to allocate map entries for the kernel. + * + * Kernel map entries are allocated from a special cache, using a custom + * page allocation function to avoid recursion. It would be difficult + * (perhaps impossible) for the kernel to allocate more memory to an entry + * cache when it became empty since the very act of allocating memory + * implies the creation of a new entry. */ vm_offset_t kentry_data; -vm_size_t kentry_data_size; -int kentry_count = 256; /* to init kentry_data_size */ +vm_size_t kentry_data_size = KENTRY_DATA_SIZE; -void vm_map_init() +static vm_offset_t kentry_pagealloc(vm_size_t size) { - vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024, - PAGE_SIZE, 0, "maps"); - vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - 1024*1024, PAGE_SIZE*5, - 0, "non-kernel map entries"); - vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - kentry_data_size, kentry_data_size, - ZONE_FIXED /* XXX */, "kernel map entries"); + vm_offset_t result; - vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy), - 16*1024, PAGE_SIZE, 0, - "map copies"); + if (size > kentry_data_size) + panic("vm_map: kentry memory exhausted"); - /* - * Cram the kentry zone with initial data. - */ - zcram(vm_map_kentry_zone, kentry_data, kentry_data_size); + result = kentry_data; + kentry_data += size; + kentry_data_size -= size; + return result; +} + +void vm_map_init(void) +{ + kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0, + NULL, NULL, NULL, 0); + kmem_cache_init(&vm_map_entry_cache, "vm_map_entry", + sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0); + kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry", + sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc, + NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB + | KMEM_CACHE_NORECLAIM); + kmem_cache_init(&vm_map_copy_cache, "vm_map_copy", + sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0); /* * Submap object is initialized by vm_object_init. */ } +void vm_map_setup(map, pmap, min, max, pageable) + vm_map_t map; + pmap_t pmap; + vm_offset_t min, max; + boolean_t pageable; +{ + vm_map_first_entry(map) = vm_map_to_entry(map); + vm_map_last_entry(map) = vm_map_to_entry(map); + map->hdr.nentries = 0; + map->hdr.entries_pageable = pageable; + rbtree_init(&map->hdr.tree); + + map->size = 0; + map->ref_count = 1; + map->pmap = pmap; + map->min_offset = min; + map->max_offset = max; + map->wiring_required = FALSE; + map->wait_for_space = FALSE; + map->first_free = vm_map_to_entry(map); + map->hint = vm_map_to_entry(map); + vm_map_lock_init(map); + simple_lock_init(&map->ref_lock); + simple_lock_init(&map->hint_lock); +} + /* * vm_map_create: * @@ -195,27 +250,11 @@ vm_map_t vm_map_create(pmap, min, max, p { register vm_map_t result; - result = (vm_map_t) zalloc(vm_map_zone); + result = (vm_map_t) kmem_cache_alloc(&vm_map_cache); if (result == VM_MAP_NULL) panic("vm_map_create"); - vm_map_first_entry(result) = vm_map_to_entry(result); - vm_map_last_entry(result) = vm_map_to_entry(result); - result->hdr.nentries = 0; - result->hdr.entries_pageable = pageable; - - result->size = 0; - result->ref_count = 1; - result->pmap = pmap; - result->min_offset = min; - result->max_offset = max; - result->wiring_required = FALSE; - result->wait_for_space = FALSE; - result->first_free = vm_map_to_entry(result); - result->hint = vm_map_to_entry(result); - vm_map_lock_init(result); - simple_lock_init(&result->ref_lock); - simple_lock_init(&result->hint_lock); + vm_map_setup(result, pmap, min, max, pageable); return(result); } @@ -235,15 +274,15 @@ vm_map_t vm_map_create(pmap, min, max, p vm_map_entry_t _vm_map_entry_create(map_header) register struct vm_map_header *map_header; { - register zone_t zone; + register kmem_cache_t cache; register vm_map_entry_t entry; if (map_header->entries_pageable) - zone = vm_map_entry_zone; + cache = &vm_map_entry_cache; else - zone = vm_map_kentry_zone; + cache = &vm_map_kentry_cache; - entry = (vm_map_entry_t) zalloc(zone); + entry = (vm_map_entry_t) kmem_cache_alloc(cache); if (entry == VM_MAP_ENTRY_NULL) panic("vm_map_entry_create"); @@ -265,20 +304,50 @@ void _vm_map_entry_dispose(map_header, e register struct vm_map_header *map_header; register vm_map_entry_t entry; { - register zone_t zone; + register kmem_cache_t cache; if (map_header->entries_pageable) - zone = vm_map_entry_zone; + cache = &vm_map_entry_cache; else - zone = vm_map_kentry_zone; + cache = &vm_map_kentry_cache; - zfree(zone, (vm_offset_t) entry); + kmem_cache_free(cache, (vm_offset_t) entry); +} + +/* + * Red-black tree lookup/insert comparison functions + */ +static inline int vm_map_entry_cmp_lookup(vm_offset_t addr, + const struct rbtree_node *node) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(node, struct vm_map_entry, tree_node); + + if (addr < entry->vme_start) + return -1; + else if (addr < entry->vme_end) + return 0; + else + return 1; +} + +static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a, + const struct rbtree_node *b) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(a, struct vm_map_entry, tree_node); + return vm_map_entry_cmp_lookup(entry->vme_start, b); } /* * vm_map_entry_{un,}link: * * Insert/remove entries from maps (or map copies). + * + * The start and end addresses of the entries must be properly set + * before using these macros. */ #define vm_map_entry_link(map, after_where, entry) \ _vm_map_entry_link(&(map)->hdr, after_where, entry) @@ -293,6 +362,8 @@ void _vm_map_entry_dispose(map_header, e (entry)->vme_next = (after_where)->vme_next; \ (entry)->vme_prev->vme_next = \ (entry)->vme_next->vme_prev = (entry); \ + rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \ + vm_map_entry_cmp_insert); \ MACRO_END #define vm_map_entry_unlink(map, entry) \ @@ -306,6 +377,7 @@ void _vm_map_entry_dispose(map_header, e (hdr)->nentries--; \ (entry)->vme_next->vme_prev = (entry)->vme_prev; \ (entry)->vme_prev->vme_next = (entry)->vme_next; \ + rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \ MACRO_END /* @@ -353,7 +425,7 @@ void vm_map_deallocate(map) pmap_destroy(map->pmap); - zfree(vm_map_zone, (vm_offset_t) map); + kmem_cache_free(&vm_map_cache, (vm_offset_t) map); } /* @@ -382,70 +454,49 @@ boolean_t vm_map_lookup_entry(map, addre register vm_offset_t address; vm_map_entry_t *entry; /* OUT */ { - register vm_map_entry_t cur; - register vm_map_entry_t last; + register struct rbtree_node *node; + register vm_map_entry_t hint; /* - * Start looking either from the head of the - * list, or from the hint. + * First, make a quick check to see if we are already + * looking at the entry we want (which is often the case). */ simple_lock(&map->hint_lock); - cur = map->hint; + hint = map->hint; simple_unlock(&map->hint_lock); - if (cur == vm_map_to_entry(map)) - cur = cur->vme_next; - - if (address >= cur->vme_start) { - /* - * Go from hint to end of list. - * - * But first, make a quick check to see if - * we are already looking at the entry we - * want (which is usually the case). - * Note also that we don't need to save the hint - * here... it is the same hint (unless we are - * at the header, in which case the hint didn't - * buy us anything anyway). - */ - last = vm_map_to_entry(map); - if ((cur != last) && (cur->vme_end > address)) { - *entry = cur; + if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) { + if (address < hint->vme_end) { + *entry = hint; return(TRUE); + } else { + vm_map_entry_t next = hint->vme_next; + + if ((next == vm_map_to_entry(map)) + || (address < next->vme_start)) { + *entry = hint; + return(FALSE); + } } } - else { - /* - * Go from start to hint, *inclusively* - */ - last = cur->vme_next; - cur = vm_map_first_entry(map); - } /* - * Search linearly + * If the hint didn't help, use the red-black tree. */ - while (cur != last) { - if (cur->vme_end > address) { - if (address >= cur->vme_start) { - /* - * Save this lookup for future - * hints, and return - */ + node = rbtree_lookup_nearest(&map->hdr.tree, address, + vm_map_entry_cmp_lookup, RBTREE_LEFT); - *entry = cur; - SAVE_HINT(map, cur); - return(TRUE); - } - break; - } - cur = cur->vme_next; + if (node == NULL) { + *entry = vm_map_to_entry(map); + SAVE_HINT(map, *entry); + return(FALSE); + } else { + *entry = rbtree_entry(node, struct vm_map_entry, tree_node); + SAVE_HINT(map, *entry); + return((address < (*entry)->vme_end) ? TRUE : FALSE); } - *entry = cur->vme_prev; - SAVE_HINT(map, *entry); - return(FALSE); } /* @@ -519,13 +570,17 @@ kern_return_t vm_map_find_entry(map, add * wrap around the address. */ - if (((start + mask) & ~mask) < start) + if (((start + mask) & ~mask) < start) { + printf_once("no more room for vm_map_find_entry in %p\n", map); return(KERN_NO_SPACE); + } start = ((start + mask) & ~mask); end = start + size; - if ((end > map->max_offset) || (end < start)) + if ((end > map->max_offset) || (end < start)) { + printf_once("no more room for vm_map_find_entry in %p\n", map); return(KERN_NO_SPACE); + } /* * If there are no more entries, we must win. @@ -644,7 +699,7 @@ int vm_map_pmap_enter_enable = FALSE; * As soon as a page not found in the object the scan ends. * * Returns: - * Nothing. + * Nothing. * * In/out conditions: * The source map should not be locked on entry. @@ -676,7 +731,7 @@ vm_map_pmap_enter(map, addr, end_addr, o if (vm_map_pmap_enter_print) { printf("vm_map_pmap_enter:"); - printf("map: %x, addr: %x, object: %x, offset: %x\n", + printf("map: %p, addr: %lx, object: %p, offset: %lx\n", map, addr, object, offset); } @@ -735,6 +790,9 @@ kern_return_t vm_map_enter( #define RETURN(value) { result = value; goto BailOut; } + if (size == 0) + return KERN_INVALID_ARGUMENT; + StartAgain: ; start = *address; @@ -782,8 +840,10 @@ kern_return_t vm_map_enter( * wrap around the address. */ - if (((start + mask) & ~mask) < start) - return(KERN_NO_SPACE); + if (((start + mask) & ~mask) < start) { + printf_once("no more room for vm_map_enter in %p\n", map); + RETURN(KERN_NO_SPACE); + } start = ((start + mask) & ~mask); end = start + size; @@ -797,7 +857,8 @@ kern_return_t vm_map_enter( goto StartAgain; } } - + + printf_once("no more room for vm_map_enter in %p\n", map); RETURN(KERN_NO_SPACE); } @@ -895,7 +956,7 @@ kern_return_t vm_map_enter( (entry->protection == cur_protection) && (entry->max_protection == max_protection) && (entry->wired_count == 0) && /* implies user_wired_count == 0 */ - (entry->projected_on == 0)) { + (entry->projected_on == 0)) { if (vm_object_coalesce(entry->object.vm_object, VM_OBJECT_NULL, entry->offset, @@ -966,9 +1027,9 @@ kern_return_t vm_map_enter( if ((object != VM_OBJECT_NULL) && (vm_map_pmap_enter_enable) && (!anywhere) && - (!needs_copy) && + (!needs_copy) && (size < (128*1024))) { - vm_map_pmap_enter(map, start, end, + vm_map_pmap_enter(map, start, end, object, offset, cur_protection); } @@ -1353,7 +1414,7 @@ kern_return_t vm_map_pageable_common(map (entry->vme_start < end)) { if ((entry->wired_count == 0) || - ((entry->vme_end < end) && + ((entry->vme_end < end) && ((entry->vme_next == vm_map_to_entry(map)) || (entry->vme_next->vme_start > entry->vme_end))) || (user_wire && (entry->user_wired_count == 0))) { @@ -1380,7 +1441,7 @@ kern_return_t vm_map_pageable_common(map else { entry->wired_count--; } - + if (entry->wired_count == 0) vm_fault_unwire(map, entry); @@ -1442,7 +1503,7 @@ kern_return_t vm_map_pageable_common(map if (entry->object.vm_object == VM_OBJECT_NULL) { entry->object.vm_object = vm_object_allocate( - (vm_size_t)(entry->vme_end + (vm_size_t)(entry->vme_end - entry->vme_start)); entry->offset = (vm_offset_t)0; } @@ -1464,7 +1525,7 @@ kern_return_t vm_map_pageable_common(map * this is the end of the region. * Protection: Access requested must be allowed. */ - if (((entry->vme_end < end) && + if (((entry->vme_end < end) && ((entry->vme_next == vm_map_to_entry(map)) || (entry->vme_next->vme_start > entry->vme_end))) || ((entry->protection & access_type) != access_type)) { @@ -1556,7 +1617,7 @@ kern_return_t vm_map_pageable_common(map * vm_map_entry_delete: [ internal use only ] * * Deallocate the given entry from the target map. - */ + */ void vm_map_entry_delete(map, entry) register vm_map_t map; register vm_map_entry_t entry; @@ -1657,67 +1718,6 @@ kern_return_t vm_map_delete(map, start, entry = first_entry->vme_next; else { entry = first_entry; -#if NORMA_IPC_xxx - /* - * XXX Had to disable this code because: - - _vm_map_delete(c0804b78,c2198000,c219a000,0,c219a000)+df - [vm/vm_map.c:2007] - _vm_map_remove(c0804b78,c2198000,c219a000,c0817834, - c081786c)+42 [vm/vm_map.c:2094] - _kmem_io_map_deallocate(c0804b78,c2198000,2000,c0817834, - c081786c)+43 [vm/vm_kern.c:818] - _device_write_dealloc(c081786c)+117 [device/ds_routines.c:814] - _ds_write_done(c081786c,0)+2e [device/ds_routines.c:848] - _io_done_thread_continue(c08150c0,c21d4e14,c21d4e30,c08150c0, - c080c114)+14 [device/ds_routines.c:1350] - - */ - if (start > entry->vme_start - && end == entry->vme_end - && ! entry->wired_count /* XXX ??? */ - && ! entry->is_shared - && ! entry->projected_on - && ! entry->is_sub_map) { - extern vm_object_t kernel_object; - register vm_object_t object = entry->object.vm_object; - - /* - * The region to be deleted lives at the end - * of this entry, and thus all we have to do is - * truncate the entry. - * - * This special case is necessary if we want - * coalescing to do us any good. - * - * XXX Do we have to adjust object size? - */ - if (object == kernel_object) { - vm_object_lock(object); - vm_object_page_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - vm_object_unlock(object); - } else if (entry->is_shared) { - vm_object_pmap_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - } else { - pmap_remove(map->pmap, start, end); - } - object->size -= (end - start); /* XXX */ - - entry->vme_end = start; - map->size -= (end - start); - - if (map->wait_for_space) { - thread_wakeup((event_t) map); - } - return KERN_SUCCESS; - } -#endif NORMA_IPC vm_map_clip_start(map, entry, start); /* @@ -1828,7 +1828,7 @@ vm_map_copy_t copy; * Page was not stolen, get a new * one and do the copy now. */ - while ((new_m = vm_page_grab()) == VM_PAGE_NULL) { + while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) { VM_PAGE_WAIT((void(*)()) 0); } @@ -1946,7 +1946,7 @@ free_next_copy: register vm_map_copy_t new_copy; new_copy = (vm_map_copy_t) copy->cpy_cont_args; - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); copy = new_copy; goto free_next_copy; } @@ -1957,7 +1957,7 @@ free_next_copy: break; } - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } /* @@ -1991,7 +1991,7 @@ vm_map_copy_copy(copy) * from the old one into it. */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); *new_copy = *copy; if (copy->type == VM_MAP_COPY_ENTRY_LIST) { @@ -2109,11 +2109,7 @@ kern_return_t vm_map_copy_overwrite(dst_ * support page lists LATER. */ -#if NORMA_IPC - vm_map_convert_from_page_list(copy); -#else assert(copy->type == VM_MAP_COPY_ENTRY_LIST); -#endif /* * Currently this routine only handles page-aligned @@ -2196,13 +2192,15 @@ start_pass_1: * the copy cannot be interrupted. */ - if (interruptible && contains_permanent_objects) + if (interruptible && contains_permanent_objects) { + vm_map_unlock(dst_map); return(KERN_FAILURE); /* XXX */ + } /* * XXXO If there are no permanent objects in the destination, - * XXXO and the source and destination map entry zones match, - * XXXO and the destination map entry is not shared, + * XXXO and the source and destination map entry caches match, + * XXXO and the destination map entry is not shared, * XXXO then the map entries can be deleted and replaced * XXXO with those from the copy. The following code is the * XXXO basic idea of what to do, but there are lots of annoying @@ -2243,7 +2241,7 @@ start_pass_1: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy); vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start); vm_object_t object; - + entry = tmp_entry; size = (entry->vme_end - entry->vme_start); /* @@ -2428,7 +2426,7 @@ start_pass_1: /* * Macro: vm_map_copy_insert - * + * * Description: * Link a copy chain ("copy") into a map at the * specified location (after "where"). @@ -2439,12 +2437,16 @@ start_pass_1: */ #define vm_map_copy_insert(map, where, copy) \ MACRO_BEGIN \ + struct rbtree_node *node, *tmp; \ + rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \ + rbtree_insert(&(map)->hdr.tree, node, \ + vm_map_entry_cmp_insert); \ (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \ ->vme_next = ((where)->vme_next); \ ((where)->vme_next = vm_map_copy_first_entry(copy)) \ ->vme_prev = (where); \ (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \ - zfree(vm_map_copy_zone, (vm_offset_t) copy); \ + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \ MACRO_END /* @@ -2500,7 +2502,7 @@ kern_return_t vm_map_copyout(dst_map, ds VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return(kr); - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); return(KERN_SUCCESS); } @@ -2534,6 +2536,7 @@ kern_return_t vm_map_copyout(dst_map, ds } } vm_map_unlock(dst_map); + printf_once("no more room for vm_map_copyout in %p\n", dst_map); return(KERN_NO_SPACE); } @@ -2556,15 +2559,15 @@ kern_return_t vm_map_copyout(dst_map, ds * Mismatches occur when dealing with the default * pager. */ - zone_t old_zone; + kmem_cache_t old_cache; vm_map_entry_t next, new; /* - * Find the zone that the copies were allocated from + * Find the cache that the copies were allocated from */ - old_zone = (copy->cpy_hdr.entries_pageable) - ? vm_map_entry_zone - : vm_map_kentry_zone; + old_cache = (copy->cpy_hdr.entries_pageable) + ? &vm_map_entry_cache + : &vm_map_kentry_cache; entry = vm_map_copy_first_entry(copy); /* @@ -2587,7 +2590,7 @@ kern_return_t vm_map_copyout(dst_map, ds vm_map_copy_last_entry(copy), new); next = entry->vme_next; - zfree(old_zone, (vm_offset_t) entry); + kmem_cache_free(old_cache, (vm_offset_t) entry); entry = next; } } @@ -2778,6 +2781,7 @@ StartAgain: } } vm_map_unlock(dst_map); + printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map); return(KERN_NO_SPACE); } @@ -2811,7 +2815,7 @@ StartAgain: (last->wired_count != 0))) { goto create_object; } - + /* * If this entry needs an object, make one. */ @@ -2913,7 +2917,7 @@ create_object: entry->vme_start = start; entry->vme_end = start + size; - + entry->inheritance = VM_INHERIT_DEFAULT; entry->protection = VM_PROT_DEFAULT; entry->max_protection = VM_PROT_ALL; @@ -2937,7 +2941,7 @@ create_object: last = entry; /* - * Transfer pages into new object. + * Transfer pages into new object. * Scan page list in vm_map_copy. */ insert_pages: @@ -3028,7 +3032,7 @@ insert_pages: vm_object_unlock(object); *dst_addr = start + dst_offset; - + /* * Clear the in transition bits. This is easy if we * didn't have a continuation. @@ -3062,7 +3066,7 @@ error: entry = entry->vme_next; } } - + if (result != KERN_SUCCESS) vm_map_delete(dst_map, start, end); @@ -3075,12 +3079,12 @@ error: * Consume on success logic. */ if (copy != orig_copy) { - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } if (result == KERN_SUCCESS) { - zfree(vm_map_copy_zone, (vm_offset_t) orig_copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy); } - + return(result); } @@ -3151,20 +3155,21 @@ kern_return_t vm_map_copyin(src_map, src /* * Allocate a header element for the list. * - * Use the start and end in the header to + * Use the start and end in the header to * remember the endpoints prior to rounding. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); copy->type = VM_MAP_COPY_ENTRY_LIST; copy->cpy_hdr.nentries = 0; copy->cpy_hdr.entries_pageable = TRUE; + rbtree_init(©->cpy_hdr.tree); copy->offset = src_addr; copy->size = len; - + #define RETURN(x) \ MACRO_BEGIN \ vm_map_unlock(src_map); \ @@ -3345,7 +3350,7 @@ kern_return_t vm_map_copyin(src_map, src &new_entry_needs_copy); new_entry->needs_copy = new_entry_needs_copy; - + if (result != KERN_SUCCESS) { vm_map_copy_entry_dispose(copy, new_entry); @@ -3427,7 +3432,7 @@ kern_return_t vm_map_copyin(src_map, src vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - + /* * Determine whether the entire region * has been copied. @@ -3447,7 +3452,7 @@ kern_return_t vm_map_copyin(src_map, src /* * If the source should be destroyed, do it now, since the - * copy was successful. + * copy was successful. */ if (src_destroy) (void) vm_map_delete(src_map, trunc_page(src_addr), src_end); @@ -3482,7 +3487,7 @@ kern_return_t vm_map_copyin_object(objec * and null links. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL; copy->type = VM_MAP_COPY_OBJECT; @@ -3498,7 +3503,7 @@ kern_return_t vm_map_copyin_object(objec * vm_map_copyin_page_list_cont: * * Continuation routine for vm_map_copyin_page_list. - * + * * If vm_map_copyin_page_list can't fit the entire vm range * into a single page list object, it creates a continuation. * When the target of the operation has used the pages in the @@ -3527,7 +3532,7 @@ vm_map_copy_t *copy_result; /* OUT */ src_destroy_only = (cont_args->src_len == (vm_size_t) 0); if (do_abort || src_destroy_only) { - if (src_destroy) + if (src_destroy) result = vm_map_remove(cont_args->map, cont_args->destroy_addr, cont_args->destroy_addr + cont_args->destroy_len); @@ -3551,7 +3556,7 @@ vm_map_copy_t *copy_result; /* OUT */ new_args->destroy_len = cont_args->destroy_len; } } - + vm_map_deallocate(cont_args->map); kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t)); @@ -3637,14 +3642,14 @@ kern_return_t vm_map_copyin_page_list(sr * be page-aligned. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); copy->type = VM_MAP_COPY_PAGE_LIST; copy->cpy_npages = 0; copy->offset = src_addr; copy->size = len; copy->cpy_cont = ((kern_return_t (*)()) 0); copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL; - + /* * Find the beginning of the region. */ @@ -3717,7 +3722,7 @@ make_continuation: * something stupid. */ - cont_args = (vm_map_copyin_args_t) + cont_args = (vm_map_copyin_args_t) kalloc(sizeof(vm_map_copyin_args_data_t)); cont_args->map = src_map; vm_map_reference(src_map); @@ -3791,7 +3796,7 @@ make_continuation: vm_prot_t result_prot; vm_page_t top_page; kern_return_t kr; - + /* * Have to fault the page in; must * unlock the map to do so. While @@ -3803,7 +3808,7 @@ make_continuation: need_map_lookup = TRUE; retry: result_prot = VM_PROT_READ; - + kr = vm_fault_page(src_object, src_offset, VM_PROT_READ, FALSE, FALSE, &result_prot, &m, &top_page, @@ -3847,7 +3852,7 @@ retry: result = KERN_MEMORY_ERROR; goto error; } - + if (top_page != VM_PAGE_NULL) { vm_object_lock(src_object); VM_PAGE_FREE(top_page); @@ -3872,13 +3877,13 @@ retry: * we have a paging reference on it. Either * the map is locked, or need_map_lookup is * TRUE. - * + * * Put the page in the page list. */ copy->cpy_page_list[copy->cpy_npages++] = m; vm_object_unlock(m->object); } - + /* * DETERMINE whether the entire region * has been copied. @@ -3932,7 +3937,7 @@ retry: * Remove the page from its object if it * can be stolen. It can be stolen if: * - * (1) The source is being destroyed, + * (1) The source is being destroyed, * the object is temporary, and * not shared. * (2) The page is not precious. @@ -3950,7 +3955,7 @@ retry: * * Stealing wired pages requires telling the * pmap module to let go of them. - * + * * NOTE: stealing clean pages from objects * whose mappings survive requires a call to * the pmap module. Maybe later. @@ -3965,7 +3970,7 @@ retry: (!src_object->use_shared_copy) && !m->precious) { vm_offset_t page_vaddr; - + page_vaddr = src_start + (i * PAGE_SIZE); if (m->wire_count > 0) { @@ -4063,7 +4068,7 @@ retry: * a continuation to prevent this. */ if (src_destroy && !vm_map_copy_has_cont(copy)) { - cont_args = (vm_map_copyin_args_t) + cont_args = (vm_map_copyin_args_t) kalloc(sizeof(vm_map_copyin_args_data_t)); vm_map_reference(src_map); cont_args->map = src_map; @@ -4076,7 +4081,7 @@ retry: copy->cpy_cont_args = (char *) cont_args; copy->cpy_cont = vm_map_copyin_page_list_cont; } - + } vm_map_unlock(src_map); @@ -4165,7 +4170,7 @@ vm_map_t vm_map_fork(old_map) &old_entry->offset, (vm_size_t) (old_entry->vme_end - old_entry->vme_start)); - + /* * If we're making a shadow for other than * copy on write reasons, then we have @@ -4196,6 +4201,8 @@ vm_map_t vm_map_fork(old_map) object->ref_count++; vm_object_unlock(object); + new_entry = vm_map_entry_create(new_map); + if (old_entry->projected_on != 0) { /* * If entry is projected buffer, clone the @@ -4210,7 +4217,6 @@ vm_map_t vm_map_fork(old_map) * Mark both entries as shared. */ - new_entry = vm_map_entry_create(new_map); vm_map_entry_copy(new_entry, old_entry); old_entry->is_shared = TRUE; new_entry->is_shared = TRUE; @@ -4302,7 +4308,7 @@ vm_map_t vm_map_fork(old_map) start, entry_size, FALSE, - ©) + ©) != KERN_SUCCESS) { vm_map_lock(old_map); if (!vm_map_lookup_entry(old_map, start, &last)) @@ -4436,7 +4442,7 @@ kern_return_t vm_map_lookup(var_map, vad vm_map_unlock_read(old_map); goto RetryLookup; } - + /* * Check whether this task is allowed to have * this page. @@ -4444,19 +4450,20 @@ kern_return_t vm_map_lookup(var_map, vad prot = entry->protection; - if ((fault_type & (prot)) != fault_type) + if ((fault_type & (prot)) != fault_type) { if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) { RETURN(KERN_WRITE_PROTECTION_FAILURE); } else { RETURN(KERN_PROTECTION_FAILURE); } + } /* * If this page is not pageable, we have to get * it for all possible accesses. */ - if (*wired = (entry->wired_count != 0)) + if ((*wired = (entry->wired_count != 0))) prot = fault_type = entry->protection; /* @@ -4489,9 +4496,9 @@ kern_return_t vm_map_lookup(var_map, vad &entry->object.vm_object, &entry->offset, (vm_size_t) (entry->vme_end - entry->vme_start)); - + entry->needs_copy = FALSE; - + vm_map_lock_write_to_read(map); } else { @@ -4542,7 +4549,7 @@ kern_return_t vm_map_lookup(var_map, vad out_version->main_timestamp = map->timestamp; RETURN(KERN_SUCCESS); - + #undef RETURN } @@ -4692,7 +4699,7 @@ void vm_map_simplify(map, start) ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start)) == this_entry->offset) && (prev_entry->projected_on == 0) && - (this_entry->projected_on == 0) + (this_entry->projected_on == 0) ) { if (map->first_free == this_entry) map->first_free = prev_entry; @@ -4743,8 +4750,6 @@ kern_return_t vm_map_machine_attribute(m return ret; } -#include - #if MACH_KDB @@ -4757,7 +4762,6 @@ void vm_map_print(map) register vm_map_t map; { register vm_map_entry_t entry; - extern int indent; iprintf("Task map 0x%X: pmap=0x%X,", (vm_offset_t) map, (vm_offset_t) (map->pmap)); @@ -4825,7 +4829,6 @@ void vm_map_print(map) void vm_map_copy_print(copy) vm_map_copy_t copy; { - extern int indent; int i, npages; printf("copy object 0x%x\n", copy); @@ -4837,11 +4840,11 @@ void vm_map_copy_print(copy) case VM_MAP_COPY_ENTRY_LIST: printf("[entry_list]"); break; - + case VM_MAP_COPY_OBJECT: printf("[object]"); break; - + case VM_MAP_COPY_PAGE_LIST: printf("[page_list]"); break; @@ -4886,359 +4889,4 @@ void vm_map_copy_print(copy) indent -=2; } -#endif MACH_KDB - -#if NORMA_IPC -/* - * This should one day be eliminated; - * we should always construct the right flavor of copy object - * the first time. Troublesome areas include vm_read, where vm_map_copyin - * is called without knowing whom the copy object is for. - * There are also situations where we do want a lazy data structure - * even if we are sending to a remote port... - */ - -/* - * Convert a copy to a page list. The copy argument is in/out - * because we probably have to allocate a new vm_map_copy structure. - * We take responsibility for discarding the old structure and - * use a continuation to do so. Postponing this discard ensures - * that the objects containing the pages we've marked busy will stick - * around. - */ -kern_return_t -vm_map_convert_to_page_list(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_map_entry_t entry, next_entry; - vm_offset_t va; - vm_offset_t offset; - vm_object_t object; - kern_return_t result; - vm_map_copy_t copy, new_copy; - int i, num_pages = 0; - - zone_t entry_zone; - - copy = *caller_copy; - - /* - * We may not have to do anything, - * or may not be able to do anything. - */ - if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - return vm_map_convert_to_page_list_from_object(caller_copy); - } - if (copy->type != VM_MAP_COPY_ENTRY_LIST) { - panic("vm_map_convert_to_page_list: copy type %d!\n", - copy->type); - } - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * Iterate over entries. - */ - for (entry = vm_map_copy_first_entry(copy); - entry != vm_map_copy_to_entry(copy); - entry = entry->vme_next) { - - object = entry->object.vm_object; - offset = entry->offset; - /* - * Iterate over pages. - */ - for (va = entry->vme_start; - va < entry->vme_end; - va += PAGE_SIZE, offset += PAGE_SIZE) { - - vm_page_t m; - - if (new_copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) { - /* - * What a mess. We need a continuation - * to do the page list, but also one - * to discard the old copy. The right - * thing to do is probably to copy - * out the old copy into the kernel - * map (or some temporary task holding - * map if we're paranoid about large - * copies), and then copyin the page - * list that we really wanted with - * src_destroy. LATER. - */ - panic("vm_map_convert_to_page_list: num\n"); - } - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - if (((m = vm_page_lookup(object, offset)) != - VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - - /* - * Also write-protect the page, so - * that the map`s owner cannot change - * the data. The busy bit will prevent - * faults on the page from succeeding - * until the copy is released; after - * that, the page can be re-entered - * as writable, since we didn`t alter - * the map entry. This scheme is a - * cheap copy-on-write. - * - * Don`t forget the protection and - * the page_lock value! - */ - - pmap_page_protect(m->phys_addr, - entry->protection - & ~m->page_lock - & ~VM_PROT_WRITE); - - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - } - - *caller_copy = new_copy; - return KERN_SUCCESS; -} - -kern_return_t -vm_map_convert_to_page_list_from_object(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_object_t object; - vm_offset_t offset; - vm_map_copy_t copy, new_copy; - - copy = *caller_copy; - assert(copy->type == VM_MAP_COPY_OBJECT); - object = copy->cpy_object; - assert(object->size == round_page(object->size)); - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * XXX memory_object_lock_request can probably bust this - * XXX See continuation comment in previous routine for solution. - */ - assert(object->size <= VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE); - - for (offset = 0; offset < object->size; offset += PAGE_SIZE) { - vm_page_t m; - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - m = vm_page_lookup(object, offset); - if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - - *caller_copy = new_copy; - return (KERN_SUCCESS); -} - -kern_return_t -vm_map_convert_from_page_list(copy) - vm_map_copy_t copy; -{ - vm_object_t object; - int i; - vm_map_entry_t new_entry; - vm_page_t *page_list; - - /* - * Check type of copy object. - */ - if (copy->type == VM_MAP_COPY_ENTRY_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - printf("vm_map_convert_from_page_list: COPY_OBJECT?"); - return KERN_SUCCESS; - } - if (copy->type != VM_MAP_COPY_PAGE_LIST) { - panic("vm_map_convert_from_page_list 0x%x %d", - copy, - copy->type); - } - - /* - * Make sure the pages are loose. This may be - * a "Can't Happen", but just to be safe ... - */ - page_list = ©->cpy_page_list[0]; - if ((*page_list)->tabled) - vm_map_copy_steal_pages(copy); - - /* - * Create object, and stuff pages into it. - */ - object = vm_object_allocate(copy->cpy_npages); - for (i = 0; i < copy->cpy_npages; i++) { - register vm_page_t m = *page_list++; - vm_page_insert(m, object, i * PAGE_SIZE); - m->busy = FALSE; - m->dirty = TRUE; - vm_page_activate(m); - } - - /* - * XXX If this page list contained a continuation, then - * XXX we're screwed. The right thing to do is probably do - * XXX the copyout, and then copyin the entry list we really - * XXX wanted. - */ - if (vm_map_copy_has_cont(copy)) - panic("convert_from_page_list: continuation"); - - /* - * Change type of copy object - */ - vm_map_copy_first_entry(copy) = - vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); - copy->type = VM_MAP_COPY_ENTRY_LIST; - copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = TRUE; - - /* - * Allocate and initialize an entry for object - */ - new_entry = vm_map_copy_entry_create(copy); - new_entry->vme_start = trunc_page(copy->offset); - new_entry->vme_end = round_page(copy->offset + copy->size); - new_entry->object.vm_object = object; - new_entry->offset = 0; - new_entry->is_shared = FALSE; - new_entry->is_sub_map = FALSE; - new_entry->needs_copy = FALSE; - new_entry->protection = VM_PROT_DEFAULT; - new_entry->max_protection = VM_PROT_ALL; - new_entry->inheritance = VM_INHERIT_DEFAULT; - new_entry->wired_count = 0; - new_entry->user_wired_count = 0; - new_entry->projected_on = 0; - - /* - * Insert entry into copy object, and return. - */ - vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - return(KERN_SUCCESS); -} -#endif NORMA_IPC +#endif /* MACH_KDB */