--- Gnu-Mach/vm/vm_map.c 2020/09/02 04:47:45 1.1.1.4 +++ Gnu-Mach/vm/vm_map.c 2020/09/02 04:54:11 1.1.1.7 @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -126,7 +127,6 @@ MACRO_END struct kmem_cache vm_map_cache; /* cache for vm_map structures */ struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */ -struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */ struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */ /* @@ -147,47 +147,24 @@ vm_object_t vm_submap_object = &vm_subm * Map and entry structures are allocated from caches -- we must * initialize those caches. * - * There are three caches of interest: + * There are two caches of interest: * * vm_map_cache: used to allocate maps. * vm_map_entry_cache: used to allocate map entries. - * vm_map_kentry_cache: used to allocate map entries for the kernel. * - * Kernel map entries are allocated from a special cache, using a custom - * page allocation function to avoid recursion. It would be difficult - * (perhaps impossible) for the kernel to allocate more memory to an entry - * cache when it became empty since the very act of allocating memory - * implies the creation of a new entry. + * We make sure the map entry cache allocates memory directly from the + * physical allocator to avoid recursion with this module. */ -vm_offset_t kentry_data; -vm_size_t kentry_data_size = KENTRY_DATA_SIZE; - -static vm_offset_t kentry_pagealloc(vm_size_t size) -{ - vm_offset_t result; - - if (size > kentry_data_size) - panic("vm_map: kentry memory exhausted"); - - result = kentry_data; - kentry_data += size; - kentry_data_size -= size; - return result; -} - void vm_map_init(void) { kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0, - NULL, NULL, NULL, 0); + NULL, 0); kmem_cache_init(&vm_map_entry_cache, "vm_map_entry", - sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0); - kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry", - sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc, - NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB - | KMEM_CACHE_NORECLAIM); + sizeof(struct vm_map_entry), 0, NULL, + KMEM_CACHE_NOOFFSLAB | KMEM_CACHE_PHYSMEM); kmem_cache_init(&vm_map_copy_cache, "vm_map_copy", - sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0); + sizeof(struct vm_map_copy), 0, NULL, 0); /* * Submap object is initialized by vm_object_init. @@ -198,16 +175,16 @@ void vm_map_setup( vm_map_t map, pmap_t pmap, vm_offset_t min, - vm_offset_t max, - boolean_t pageable) + vm_offset_t max) { vm_map_first_entry(map) = vm_map_to_entry(map); vm_map_last_entry(map) = vm_map_to_entry(map); map->hdr.nentries = 0; - map->hdr.entries_pageable = pageable; rbtree_init(&map->hdr.tree); + rbtree_init(&map->hdr.gap_tree); map->size = 0; + map->user_wired = 0; map->ref_count = 1; map->pmap = pmap; map->min_offset = min; @@ -216,6 +193,7 @@ void vm_map_setup( map->wait_for_space = FALSE; map->first_free = vm_map_to_entry(map); map->hint = vm_map_to_entry(map); + map->name = NULL; vm_map_lock_init(map); simple_lock_init(&map->ref_lock); simple_lock_init(&map->hint_lock); @@ -231,8 +209,7 @@ void vm_map_setup( vm_map_t vm_map_create( pmap_t pmap, vm_offset_t min, - vm_offset_t max, - boolean_t pageable) + vm_offset_t max) { vm_map_t result; @@ -240,11 +217,53 @@ vm_map_t vm_map_create( if (result == VM_MAP_NULL) panic("vm_map_create"); - vm_map_setup(result, pmap, min, max, pageable); + vm_map_setup(result, pmap, min, max); return(result); } +void vm_map_lock(struct vm_map *map) +{ + lock_write(&map->lock); + + /* + * XXX Memory allocation may occur while a map is locked, + * for example when clipping entries. If the system is running + * low on memory, allocating may block until pages are + * available. But if a map used by the default pager is + * kept locked, a deadlock occurs. + * + * This workaround temporarily elevates the current thread + * VM privileges to avoid that particular deadlock, and does + * so regardless of the map for convenience, and because it's + * currently impossible to predict which map the default pager + * may depend on. + * + * This workaround isn't reliable, and only makes exhaustion + * less likely. In particular pageout may cause lots of data + * to be passed between the kernel and the pagers, often + * in the form of large copy maps. Making the minimum + * number of pages depend on the total number of pages + * should make exhaustion even less likely. + */ + + if (current_thread()) { + current_thread()->vm_privilege++; + assert(current_thread()->vm_privilege != 0); + } + + map->timestamp++; +} + +void vm_map_unlock(struct vm_map *map) +{ + if (current_thread()) { + current_thread()->vm_privilege--; + } + + lock_write_done(&map->lock); +} + /* * vm_map_entry_create: [ internal use only ] * @@ -260,15 +279,9 @@ vm_map_t vm_map_create( vm_map_entry_t _vm_map_entry_create(map_header) const struct vm_map_header *map_header; { - kmem_cache_t cache; vm_map_entry_t entry; - if (map_header->entries_pageable) - cache = &vm_map_entry_cache; - else - cache = &vm_map_kentry_cache; - - entry = (vm_map_entry_t) kmem_cache_alloc(cache); + entry = (vm_map_entry_t) kmem_cache_alloc(&vm_map_entry_cache); if (entry == VM_MAP_ENTRY_NULL) panic("vm_map_entry_create"); @@ -290,14 +303,9 @@ void _vm_map_entry_dispose(map_header, e const struct vm_map_header *map_header; vm_map_entry_t entry; { - kmem_cache_t cache; - - if (map_header->entries_pageable) - cache = &vm_map_entry_cache; - else - cache = &vm_map_kentry_cache; + (void)map_header; - kmem_cache_free(cache, (vm_offset_t) entry); + kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry); } /* @@ -328,6 +336,141 @@ static inline int vm_map_entry_cmp_inser } /* + * Gap management functions + */ +static inline int vm_map_entry_gap_cmp_lookup(vm_size_t gap_size, + const struct rbtree_node *node) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(node, struct vm_map_entry, gap_node); + + if (gap_size < entry->gap_size) + return -1; + else if (gap_size == entry->gap_size) + return 0; + else + return 1; +} + +static inline int vm_map_entry_gap_cmp_insert(const struct rbtree_node *a, + const struct rbtree_node *b) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(a, struct vm_map_entry, gap_node); + return vm_map_entry_gap_cmp_lookup(entry->gap_size, b); +} + +static int +vm_map_gap_valid(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + return entry != (struct vm_map_entry *)&hdr->links; +} + +static void +vm_map_gap_compute(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *next; + + next = entry->vme_next; + + if (vm_map_gap_valid(hdr, next)) { + entry->gap_size = next->vme_start - entry->vme_end; + } else { + entry->gap_size = hdr->vme_end - entry->vme_end; + } +} + +static void +vm_map_gap_insert_single(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *tmp; + struct rbtree_node *node; + unsigned long slot; + + if (!vm_map_gap_valid(hdr, entry)) { + return; + } + + vm_map_gap_compute(hdr, entry); + + if (entry->gap_size == 0) { + return; + } + + node = rbtree_lookup_slot(&hdr->gap_tree, entry->gap_size, + vm_map_entry_gap_cmp_lookup, slot); + + if (node == NULL) { + rbtree_insert_slot(&hdr->gap_tree, slot, &entry->gap_node); + list_init(&entry->gap_list); + entry->in_gap_tree = 1; + } else { + tmp = rbtree_entry(node, struct vm_map_entry, gap_node); + list_insert_tail(&tmp->gap_list, &entry->gap_list); + entry->in_gap_tree = 0; + } +} + +static void +vm_map_gap_remove_single(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *tmp; + + if (!vm_map_gap_valid(hdr, entry)) { + return; + } + + if (entry->gap_size == 0) { + return; + } + + if (!entry->in_gap_tree) { + list_remove(&entry->gap_list); + return; + } + + rbtree_remove(&hdr->gap_tree, &entry->gap_node); + + if (list_empty(&entry->gap_list)) { + return; + } + + tmp = list_first_entry(&entry->gap_list, struct vm_map_entry, gap_list); + assert(tmp->gap_size == entry->gap_size); + list_remove(&tmp->gap_list); + list_set_head(&tmp->gap_list, &entry->gap_list); + assert(!tmp->in_gap_tree); + rbtree_insert(&hdr->gap_tree, &tmp->gap_node, + vm_map_entry_gap_cmp_insert); + tmp->in_gap_tree = 1; +} + +static void +vm_map_gap_update(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry); + vm_map_gap_insert_single(hdr, entry); +} + +static void +vm_map_gap_insert(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry); +} + +static void +vm_map_gap_remove(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry); + vm_map_gap_remove_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry->vme_prev); +} + +/* * vm_map_entry_{un,}link: * * Insert/remove entries from maps (or map copies). @@ -350,6 +493,7 @@ static inline int vm_map_entry_cmp_inser (entry)->vme_next->vme_prev = (entry); \ rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \ vm_map_entry_cmp_insert); \ + vm_map_gap_insert((hdr), (entry)); \ MACRO_END #define vm_map_entry_unlink(map, entry) \ @@ -364,6 +508,7 @@ static inline int vm_map_entry_cmp_inser (entry)->vme_next->vme_prev = (entry)->vme_prev; \ (entry)->vme_prev->vme_next = (entry)->vme_next; \ rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \ + vm_map_gap_remove((hdr), (entry)); \ MACRO_END /* @@ -504,6 +649,104 @@ invalid_user_access( (prot & ~(entry->protection))); } +/* + * Find a range of available space from the specified map. + * + * If successful, this function returns the map entry immediately preceding + * the range, and writes the range address in startp. If the map contains + * no entry, the entry returned points to the map header. + * Otherwise, NULL is returned. + * + * If map_locked is true, this function will not wait for more space in case + * of failure. Otherwise, the map is locked. + */ +static struct vm_map_entry * +vm_map_find_entry_anywhere(struct vm_map *map, + vm_size_t size, + vm_offset_t mask, + boolean_t map_locked, + vm_offset_t *startp) +{ + struct vm_map_entry *entry; + struct rbtree_node *node; + vm_size_t max_size; + vm_offset_t start, end; + + assert(size != 0); + + if (!map_locked) { + vm_map_lock(map); + } + +restart: + if (map->hdr.nentries == 0) { + entry = vm_map_to_entry(map); + start = (map->min_offset + mask) & ~mask; + end = start + size; + + if ((end <= start) || (end > map->max_offset)) { + goto error; + } + + *startp = start; + return entry; + } + + entry = map->first_free; + + if (entry != vm_map_to_entry(map)) { + start = (entry->vme_end + mask) & ~mask; + end = start + size; + + if ((end > start) + && (end <= map->max_offset) + && (end <= (entry->vme_end + entry->gap_size))) { + *startp = start; + return entry; + } + } + + max_size = size + mask; + + if (max_size < size) { + goto error; + } + + node = rbtree_lookup_nearest(&map->hdr.gap_tree, max_size, + vm_map_entry_gap_cmp_lookup, RBTREE_RIGHT); + + if (node == NULL) { + if (map_locked || !map->wait_for_space) { + goto error; + } + + assert_wait((event_t)map, TRUE); + vm_map_unlock(map); + thread_block(NULL); + vm_map_lock(map); + goto restart; + } + + entry = rbtree_entry(node, struct vm_map_entry, gap_node); + assert(entry->in_gap_tree); + + if (!list_empty(&entry->gap_list)) { + entry = list_last_entry(&entry->gap_list, + struct vm_map_entry, gap_list); + } + + assert(entry->gap_size >= max_size); + start = (entry->vme_end + mask) & ~mask; + end = start + size; + assert(end > start); + assert(end <= (entry->vme_end + entry->gap_size)); + *startp = start; + return entry; + +error: + printf("no more room in %p (%s)\n", map, map->name); + return NULL; +} /* * Routine: vm_map_find_entry @@ -530,67 +773,14 @@ kern_return_t vm_map_find_entry( vm_offset_t start; vm_offset_t end; - /* - * Look for the first possible address; - * if there's already something at this - * address, we have to start after it. - */ - - if ((entry = map->first_free) == vm_map_to_entry(map)) - start = map->min_offset; - else - start = entry->vme_end; - - /* - * In any case, the "entry" always precedes - * the proposed new region throughout the loop: - */ - - while (TRUE) { - vm_map_entry_t next; - - /* - * Find the end of the proposed new region. - * Be sure we didn't go beyond the end, or - * wrap around the address. - */ - - if (((start + mask) & ~mask) < start) { - printf_once("no more room for vm_map_find_entry in %p\n", map); - return(KERN_NO_SPACE); - } - start = ((start + mask) & ~mask); - end = start + size; - - if ((end > map->max_offset) || (end < start)) { - printf_once("no more room for vm_map_find_entry in %p\n", map); - return(KERN_NO_SPACE); - } - - /* - * If there are no more entries, we must win. - */ - - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ - - if (next->vme_start >= end) - break; - - /* - * Didn't fit -- move to the next entry. - */ + entry = vm_map_find_entry_anywhere(map, size, mask, TRUE, &start); - entry = next; - start = entry->vme_end; + if (entry == NULL) { + return KERN_NO_SPACE; } + end = start + size; + /* * At this point, * "start" and "end" should define the endpoints of the @@ -628,6 +818,7 @@ kern_return_t vm_map_find_entry( */ entry->vme_end = end; + vm_map_gap_update(&map->hdr, entry); new_entry = entry; } else { new_entry = vm_map_entry_create(map); @@ -770,98 +961,16 @@ kern_return_t vm_map_enter( if (size == 0) return KERN_INVALID_ARGUMENT; - StartAgain: ; - start = *address; if (anywhere) { - vm_map_lock(map); + entry = vm_map_find_entry_anywhere(map, size, mask, FALSE, &start); - /* - * Calculate the first possible address. - */ - - if (start < map->min_offset) - start = map->min_offset; - if (start > map->max_offset) + if (entry == NULL) { RETURN(KERN_NO_SPACE); - - /* - * Look for the first possible address; - * if there's already something at this - * address, we have to start after it. - */ - - if (start == map->min_offset) { - if ((entry = map->first_free) != vm_map_to_entry(map)) - start = entry->vme_end; - } else { - vm_map_entry_t tmp_entry; - if (vm_map_lookup_entry(map, start, &tmp_entry)) - start = tmp_entry->vme_end; - entry = tmp_entry; } - /* - * In any case, the "entry" always precedes - * the proposed new region throughout the - * loop: - */ - - while (TRUE) { - vm_map_entry_t next; - - /* - * Find the end of the proposed new region. - * Be sure we didn't go beyond the end, or - * wrap around the address. - */ - - if (((start + mask) & ~mask) < start) { - printf_once("no more room for vm_map_enter in %p\n", map); - RETURN(KERN_NO_SPACE); - } - start = ((start + mask) & ~mask); - end = start + size; - - if ((end > map->max_offset) || (end < start)) { - if (map->wait_for_space) { - if (size <= (map->max_offset - - map->min_offset)) { - assert_wait((event_t) map, TRUE); - vm_map_unlock(map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - - printf_once("no more room for vm_map_enter in %p\n", map); - RETURN(KERN_NO_SPACE); - } - - /* - * If there are no more entries, we must win. - */ - - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ - - if (next->vme_start >= end) - break; - - /* - * Didn't fit -- move to the next entry. - */ - - entry = next; - start = entry->vme_end; - } + end = start + size; *address = start; } else { vm_map_entry_t temp_entry; @@ -948,6 +1057,7 @@ kern_return_t vm_map_enter( */ map->size += (end - entry->vme_end); entry->vme_end = end; + vm_map_gap_update(&map->hdr, entry); RETURN(KERN_SUCCESS); } } @@ -1367,7 +1477,7 @@ kern_return_t vm_map_pageable_common( * Start address is not in map; this is fatal. */ vm_map_unlock(map); - return(KERN_FAILURE); + return(KERN_NO_SPACE); } /* @@ -1380,19 +1490,17 @@ kern_return_t vm_map_pageable_common( vm_map_clip_start(map, entry, start); /* - * Unwiring. First ensure that the range to be - * unwired is really wired down. + * Unwiring. First ensure that there are no holes + * in the specified range. */ while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { - if ((entry->wired_count == 0) || - ((entry->vme_end < end) && - ((entry->vme_next == vm_map_to_entry(map)) || - (entry->vme_next->vme_start > entry->vme_end))) || - (user_wire && (entry->user_wired_count == 0))) { + if ((entry->vme_end < end) && + ((entry->vme_next == vm_map_to_entry(map)) || + (entry->vme_next->vme_start > entry->vme_end))) { vm_map_unlock(map); - return(KERN_INVALID_ARGUMENT); + return(KERN_NO_SPACE); } entry = entry->vme_next; } @@ -1407,9 +1515,19 @@ kern_return_t vm_map_pageable_common( (entry->vme_start < end)) { vm_map_clip_end(map, entry, end); + /* Skip unwired entries */ + if (entry->wired_count == 0) { + assert(entry->user_wired_count == 0); + entry = entry->vme_next; + continue; + } + if (user_wire) { if (--(entry->user_wired_count) == 0) + { + map->user_wired -= entry->vme_end - entry->vme_start; entry->wired_count--; + } } else { entry->wired_count--; @@ -1486,7 +1604,10 @@ kern_return_t vm_map_pageable_common( if (user_wire) { if ((entry->user_wired_count)++ == 0) + { + map->user_wired += entry->vme_end - entry->vme_start; entry->wired_count++; + } } else { entry->wired_count++; @@ -1512,7 +1633,10 @@ kern_return_t vm_map_pageable_common( (entry->vme_end > start)) { if (user_wire) { if (--(entry->user_wired_count) == 0) + { + map->user_wired -= entry->vme_end - entry->vme_start; entry->wired_count--; + } } else { entry->wired_count--; @@ -1522,7 +1646,7 @@ kern_return_t vm_map_pageable_common( } vm_map_unlock(map); - return(KERN_FAILURE); + return(KERN_NO_SPACE); } entry = entry->vme_next; } @@ -1627,6 +1751,8 @@ void vm_map_entry_delete( if (entry->wired_count != 0) { vm_fault_unwire(map, entry); entry->wired_count = 0; + if (entry->user_wired_count) + map->user_wired -= entry->vme_end - entry->vme_start; entry->user_wired_count = 0; } @@ -1800,7 +1926,7 @@ vm_map_copy_steal_pages(vm_map_copy_t co * Page was not stolen, get a new * one and do the copy now. */ - while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) { + while ((new_m = vm_page_grab()) == VM_PAGE_NULL) { VM_PAGE_WAIT((void(*)()) 0); } @@ -2168,7 +2294,6 @@ start_pass_1: /* * XXXO If there are no permanent objects in the destination, - * XXXO and the source and destination map entry caches match, * XXXO and the destination map entry is not shared, * XXXO then the map entries can be deleted and replaced * XXXO with those from the copy. The following code is the @@ -2178,8 +2303,7 @@ start_pass_1: * XXXO to the above pass and make sure that no wiring is involved. */ /* - * if (!contains_permanent_objects && - * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) { + * if (!contains_permanent_objects) { * * * * * Run over copy and adjust entries. Steal code @@ -2274,6 +2398,8 @@ start_pass_1: entry->offset = copy_entry->offset; entry->needs_copy = copy_entry->needs_copy; entry->wired_count = 0; + if (entry->user_wired_count) + dst_map->user_wired -= entry->vme_end - entry->vme_start; entry->user_wired_count = 0; vm_map_copy_entry_unlink(copy, copy_entry); @@ -2394,29 +2520,40 @@ start_pass_1: } /* - * Macro: vm_map_copy_insert + * Routine: vm_map_copy_insert * * Description: * Link a copy chain ("copy") into a map at the * specified location (after "where"). * Side effects: * The copy chain is destroyed. - * Warning: - * The arguments are evaluated multiple times. */ -#define vm_map_copy_insert(map, where, copy) \ - MACRO_BEGIN \ - struct rbtree_node *node, *tmp; \ - rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \ - rbtree_insert(&(map)->hdr.tree, node, \ - vm_map_entry_cmp_insert); \ - (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \ - ->vme_next = ((where)->vme_next); \ - ((where)->vme_next = vm_map_copy_first_entry(copy)) \ - ->vme_prev = (where); \ - (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \ - kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \ - MACRO_END +static void +vm_map_copy_insert(struct vm_map *map, struct vm_map_entry *where, + struct vm_map_copy *copy) +{ + struct vm_map_entry *entry; + + assert(copy->type == VM_MAP_COPY_ENTRY_LIST); + + for (;;) { + entry = vm_map_copy_first_entry(copy); + + if (entry == vm_map_copy_to_entry(copy)) { + break; + } + + /* + * TODO Turn copy maps into their own type so they don't + * use any of the tree operations. + */ + vm_map_copy_entry_unlink(copy, entry); + vm_map_entry_link(map, where, entry); + where = entry; + } + + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t)copy); +} /* * Routine: vm_map_copyout @@ -2481,84 +2618,11 @@ kern_return_t vm_map_copyout( vm_copy_start = trunc_page(copy->offset); size = round_page(copy->offset + copy->size) - vm_copy_start; + last = vm_map_find_entry_anywhere(dst_map, size, 0, FALSE, &start); - StartAgain: ; - - vm_map_lock(dst_map); - start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ? - vm_map_min(dst_map) : last->vme_end; - - while (TRUE) { - vm_map_entry_t next = last->vme_next; - vm_offset_t end = start + size; - - if ((end > dst_map->max_offset) || (end < start)) { - if (dst_map->wait_for_space) { - if (size <= (dst_map->max_offset - dst_map->min_offset)) { - assert_wait((event_t) dst_map, TRUE); - vm_map_unlock(dst_map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - vm_map_unlock(dst_map); - printf_once("no more room for vm_map_copyout in %p\n", dst_map); - return(KERN_NO_SPACE); - } - - if ((next == vm_map_to_entry(dst_map)) || - (next->vme_start >= end)) - break; - - last = next; - start = last->vme_end; - } - - /* - * Since we're going to just drop the map - * entries from the copy into the destination - * map, they must come from the same pool. - */ - - if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) { - /* - * Mismatches occur when dealing with the default - * pager. - */ - kmem_cache_t old_cache; - vm_map_entry_t next, new; - - /* - * Find the cache that the copies were allocated from - */ - old_cache = (copy->cpy_hdr.entries_pageable) - ? &vm_map_entry_cache - : &vm_map_kentry_cache; - entry = vm_map_copy_first_entry(copy); - - /* - * Reinitialize the copy so that vm_map_copy_entry_link - * will work. - */ - copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable; - vm_map_copy_first_entry(copy) = - vm_map_copy_last_entry(copy) = - vm_map_copy_to_entry(copy); - - /* - * Copy each entry. - */ - while (entry != vm_map_copy_to_entry(copy)) { - new = vm_map_copy_entry_create(copy); - vm_map_entry_copy_full(new, entry); - vm_map_copy_entry_link(copy, - vm_map_copy_last_entry(copy), - new); - next = entry->vme_next; - kmem_cache_free(old_cache, (vm_offset_t) entry); - entry = next; - } + if (last == NULL) { + vm_map_unlock(dst_map); + return KERN_NO_SPACE; } /* @@ -2573,6 +2637,11 @@ kern_return_t vm_map_copyout( entry->vme_start += adjustment; entry->vme_end += adjustment; + /* + * XXX There is no need to update the gap tree here. + * See vm_map_copy_insert. + */ + entry->inheritance = VM_INHERIT_DEFAULT; entry->protection = VM_PROT_DEFAULT; entry->max_protection = VM_PROT_ALL; @@ -2718,44 +2787,19 @@ kern_return_t vm_map_copyout_page_list( size = round_page(copy->offset + copy->size) - trunc_page(copy->offset); -StartAgain: - vm_map_lock(dst_map); - must_wire = dst_map->wiring_required; - last = dst_map->first_free; - if (last == vm_map_to_entry(dst_map)) { - start = vm_map_min(dst_map); - } else { - start = last->vme_end; - } + vm_map_lock(dst_map); - while (TRUE) { - vm_map_entry_t next = last->vme_next; - end = start + size; + last = vm_map_find_entry_anywhere(dst_map, size, 0, TRUE, &start); - if ((end > dst_map->max_offset) || (end < start)) { - if (dst_map->wait_for_space) { - if (size <= (dst_map->max_offset - - dst_map->min_offset)) { - assert_wait((event_t) dst_map, TRUE); - vm_map_unlock(dst_map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - vm_map_unlock(dst_map); - printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map); - return(KERN_NO_SPACE); - } + if (last == NULL) { + vm_map_unlock(dst_map); + return KERN_NO_SPACE; + } - if ((next == vm_map_to_entry(dst_map)) || - (next->vme_start >= end)) { - break; - } + end = start + size; - last = next; - start = last->vme_end; - } + must_wire = dst_map->wiring_required; /* * See whether we can avoid creating a new entry (and object) by @@ -2842,6 +2886,7 @@ StartAgain: */ dst_map->size += size; last->vme_end = end; + vm_map_gap_update(&dst_map->hdr, last); SAVE_HINT(dst_map, last); @@ -2869,6 +2914,7 @@ create_object: if (must_wire) { entry->wired_count = 1; + dst_map->user_wired += entry->vme_end - entry->vme_start; entry->user_wired_count = 1; } else { entry->wired_count = 0; @@ -3100,6 +3146,14 @@ kern_return_t vm_map_copyin( } /* + * Check that the end address doesn't overflow + */ + + if ((src_addr + len) <= src_addr) { + return KERN_INVALID_ADDRESS; + } + + /* * Compute start and end of region */ @@ -3107,12 +3161,12 @@ kern_return_t vm_map_copyin( src_end = round_page(src_addr + len); /* - * Check that the end address doesn't overflow + * XXX VM maps shouldn't end at maximum address */ - if (src_end <= src_start) - if ((src_end < src_start) || (src_start != 0)) - return(KERN_INVALID_ADDRESS); + if (src_end == 0) { + return KERN_INVALID_ADDRESS; + } /* * Allocate a header element for the list. @@ -3126,8 +3180,8 @@ kern_return_t vm_map_copyin( vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); copy->type = VM_MAP_COPY_ENTRY_LIST; copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = TRUE; rbtree_init(©->cpy_hdr.tree); + rbtree_init(©->cpy_hdr.gap_tree); copy->offset = src_addr; copy->size = len; @@ -3442,8 +3496,7 @@ kern_return_t vm_map_copyin_object( /* * We drop the object into a special copy object * that contains the object directly. These copy objects - * are distinguished by entries_pageable == FALSE - * and null links. + * are distinguished by links. */ copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); @@ -3576,6 +3629,14 @@ kern_return_t vm_map_copyin_page_list( } /* + * Check that the end address doesn't overflow + */ + + if ((src_addr + len) <= src_addr) { + return KERN_INVALID_ADDRESS; + } + + /* * Compute start and end of region */ @@ -3583,10 +3644,10 @@ kern_return_t vm_map_copyin_page_list( src_end = round_page(src_addr + len); /* - * Check that the end address doesn't overflow + * XXX VM maps shouldn't end at maximum address */ - if (src_end <= src_start && (src_end < src_start || src_start != 0)) { + if (src_end == 0) { return KERN_INVALID_ADDRESS; } @@ -3954,6 +4015,8 @@ retry: assert(src_entry->wired_count > 0); src_entry->wired_count = 0; + if (src_entry->user_wired_count) + src_map->user_wired -= src_entry->vme_end - src_entry->vme_start; src_entry->user_wired_count = 0; unwire_end = src_entry->vme_end; pmap_pageable(vm_map_pmap(src_map), @@ -4069,12 +4132,14 @@ vm_map_t vm_map_fork(vm_map_t old_map) vm_size_t entry_size; vm_object_t object; + if (new_pmap == PMAP_NULL) + return VM_MAP_NULL; + vm_map_lock(old_map); new_map = vm_map_create(new_pmap, old_map->min_offset, - old_map->max_offset, - old_map->hdr.entries_pageable); + old_map->max_offset); for ( old_entry = vm_map_first_entry(old_map); @@ -4648,8 +4713,8 @@ void vm_map_simplify( map->first_free = prev_entry; SAVE_HINT(map, prev_entry); - vm_map_entry_unlink(map, this_entry); prev_entry->vme_end = this_entry->vme_end; + vm_map_entry_unlink(map, this_entry); vm_object_deallocate(this_entry->object.vm_object); vm_map_entry_dispose(map, this_entry); } @@ -4701,24 +4766,30 @@ kern_return_t vm_map_machine_attribute( /* * vm_map_print: [ debug ] */ -void vm_map_print(vm_map_t map) +void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const char *modif) { + vm_map_t map; vm_map_entry_t entry; - iprintf("Task map 0x%X: pmap=0x%X,", - (vm_offset_t) map, (vm_offset_t) (map->pmap)); + if (!have_addr) + map = current_thread()->task->map; + else + map = (vm_map_t)addr; + + iprintf("Map 0x%X: name=\"%s\", pmap=0x%X,", + (vm_offset_t) map, map->name, (vm_offset_t) (map->pmap)); printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries); printf("version=%d\n", map->timestamp); - indent += 2; + indent += 1; for (entry = vm_map_first_entry(map); entry != vm_map_to_entry(map); entry = entry->vme_next) { static char *inheritance_name[3] = { "share", "copy", "none"}; iprintf("map entry 0x%X: ", (vm_offset_t) entry); - printf("start=0x%X, end=0x%X, ", + printf("start=0x%X, end=0x%X\n", (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end); - printf("prot=%X/%X/%s, ", + iprintf("prot=%X/%X/%s, ", entry->protection, entry->max_protection, inheritance_name[entry->inheritance]); @@ -4753,13 +4824,13 @@ void vm_map_print(vm_map_t map) if ((entry->vme_prev == vm_map_to_entry(map)) || (entry->vme_prev->object.vm_object != entry->object.vm_object)) { - indent += 2; + indent += 1; vm_object_print(entry->object.vm_object); - indent -= 2; + indent -= 1; } } } - indent -= 2; + indent -= 1; } /* @@ -4775,7 +4846,7 @@ void vm_map_copy_print(copy) printf("copy object 0x%x\n", copy); - indent += 2; + indent += 1; iprintf("type=%d", copy->type); switch (copy->type) { @@ -4829,6 +4900,6 @@ void vm_map_copy_print(copy) break; } - indent -= 2; + indent -= 1; } #endif /* MACH_KDB */