--- Gnu-Mach/vm/vm_map.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_map.c 2020/09/02 04:54:11 1.1.1.7 @@ -34,28 +34,38 @@ * Virtual memory mapping module. */ -#include - +#include #include #include #include #include #include -#include +#include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include +#if MACH_KDB +#include +#include +#endif /* MACH_KDB */ + /* * Macros to copy a vm_map_entry. We must be careful to correctly * manage the wired page count. vm_map_entry_copy() creates a new * map entry to the same memory - the wired count in the new entry * must be set to zero. vm_map_entry_copy_full() creates a new * entry that is identical to the old entry. This preserves the - * wire count; it's used for map splitting and zone changing in + * wire count; it's used for map splitting and cache changing in * vm_map_copyout. */ #define vm_map_entry_copy(NEW,OLD) \ @@ -79,7 +89,7 @@ MACRO_END * Synchronization is required prior to most operations. * * Maps consist of an ordered doubly-linked list of simple - * entries; a single hint is used to speed up lookups. + * entries; a hint and a red-black tree are used to speed up lookups. * * Sharing maps have been deleted from this version of Mach. * All shared objects are now mapped directly into the respective @@ -89,7 +99,7 @@ MACRO_END * selected by the (new) use_shared_copy bit in the object. See * vm_object_copy_temporary in vm_object.c for details. All maps * are now "top level" maps (either task map, kernel map or submap - * of the kernel map). + * of the kernel map). * * Since portions of maps are specified by start/end addreses, * which may not align with existing map entries, all @@ -115,12 +125,9 @@ MACRO_END * vm_object_copy_strategically() in vm_object.c. */ -zone_t vm_map_zone; /* zone for vm_map structures */ -zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ -zone_t vm_map_kentry_zone; /* zone for kernel entry structures */ -zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ - -boolean_t vm_map_lookup_entry(); /* forward declaration */ +struct kmem_cache vm_map_cache; /* cache for vm_map structures */ +struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */ +struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */ /* * Placeholder object for submap operations. This object is dropped @@ -128,7 +135,8 @@ boolean_t vm_map_lookup_entry(); /* forw * vm_map_submap creates the submap. */ -vm_object_t vm_submap_object; +static struct vm_object vm_submap_object_store; +vm_object_t vm_submap_object = &vm_submap_object_store; /* * vm_map_init: @@ -136,51 +144,61 @@ vm_object_t vm_submap_object; * Initialize the vm_map module. Must be called before * any other vm_map routines. * - * Map and entry structures are allocated from zones -- we must - * initialize those zones. + * Map and entry structures are allocated from caches -- we must + * initialize those caches. * - * There are three zones of interest: + * There are two caches of interest: * - * vm_map_zone: used to allocate maps. - * vm_map_entry_zone: used to allocate map entries. - * vm_map_kentry_zone: used to allocate map entries for the kernel. + * vm_map_cache: used to allocate maps. + * vm_map_entry_cache: used to allocate map entries. * - * The kernel allocates map entries from a special zone that is initially - * "crammed" with memory. It would be difficult (perhaps impossible) for - * the kernel to allocate more memory to a entry zone when it became - * empty since the very act of allocating memory implies the creation - * of a new entry. + * We make sure the map entry cache allocates memory directly from the + * physical allocator to avoid recursion with this module. */ -vm_offset_t kentry_data; -vm_size_t kentry_data_size; -int kentry_count = 256; /* to init kentry_data_size */ - -void vm_map_init() +void vm_map_init(void) { - vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024, - PAGE_SIZE, 0, "maps"); - vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - 1024*1024, PAGE_SIZE*5, - 0, "non-kernel map entries"); - vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - kentry_data_size, kentry_data_size, - ZONE_FIXED /* XXX */, "kernel map entries"); - - vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy), - 16*1024, PAGE_SIZE, 0, - "map copies"); - - /* - * Cram the kentry zone with initial data. - */ - zcram(vm_map_kentry_zone, kentry_data, kentry_data_size); + kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0, + NULL, 0); + kmem_cache_init(&vm_map_entry_cache, "vm_map_entry", + sizeof(struct vm_map_entry), 0, NULL, + KMEM_CACHE_NOOFFSLAB | KMEM_CACHE_PHYSMEM); + kmem_cache_init(&vm_map_copy_cache, "vm_map_copy", + sizeof(struct vm_map_copy), 0, NULL, 0); /* * Submap object is initialized by vm_object_init. */ } +void vm_map_setup( + vm_map_t map, + pmap_t pmap, + vm_offset_t min, + vm_offset_t max) +{ + vm_map_first_entry(map) = vm_map_to_entry(map); + vm_map_last_entry(map) = vm_map_to_entry(map); + map->hdr.nentries = 0; + rbtree_init(&map->hdr.tree); + rbtree_init(&map->hdr.gap_tree); + + map->size = 0; + map->user_wired = 0; + map->ref_count = 1; + map->pmap = pmap; + map->min_offset = min; + map->max_offset = max; + map->wiring_required = FALSE; + map->wait_for_space = FALSE; + map->first_free = vm_map_to_entry(map); + map->hint = vm_map_to_entry(map); + map->name = NULL; + vm_map_lock_init(map); + simple_lock_init(&map->ref_lock); + simple_lock_init(&map->hint_lock); +} + /* * vm_map_create: * @@ -188,38 +206,64 @@ void vm_map_init() * the given physical map structure, and having * the given lower and upper address bounds. */ -vm_map_t vm_map_create(pmap, min, max, pageable) - pmap_t pmap; - vm_offset_t min, max; - boolean_t pageable; +vm_map_t vm_map_create( + pmap_t pmap, + vm_offset_t min, + vm_offset_t max) { - register vm_map_t result; + vm_map_t result; - result = (vm_map_t) zalloc(vm_map_zone); + result = (vm_map_t) kmem_cache_alloc(&vm_map_cache); if (result == VM_MAP_NULL) panic("vm_map_create"); - vm_map_first_entry(result) = vm_map_to_entry(result); - vm_map_last_entry(result) = vm_map_to_entry(result); - result->hdr.nentries = 0; - result->hdr.entries_pageable = pageable; - - result->size = 0; - result->ref_count = 1; - result->pmap = pmap; - result->min_offset = min; - result->max_offset = max; - result->wiring_required = FALSE; - result->wait_for_space = FALSE; - result->first_free = vm_map_to_entry(result); - result->hint = vm_map_to_entry(result); - vm_map_lock_init(result); - simple_lock_init(&result->ref_lock); - simple_lock_init(&result->hint_lock); + vm_map_setup(result, pmap, min, max); return(result); } +void vm_map_lock(struct vm_map *map) +{ + lock_write(&map->lock); + + /* + * XXX Memory allocation may occur while a map is locked, + * for example when clipping entries. If the system is running + * low on memory, allocating may block until pages are + * available. But if a map used by the default pager is + * kept locked, a deadlock occurs. + * + * This workaround temporarily elevates the current thread + * VM privileges to avoid that particular deadlock, and does + * so regardless of the map for convenience, and because it's + * currently impossible to predict which map the default pager + * may depend on. + * + * This workaround isn't reliable, and only makes exhaustion + * less likely. In particular pageout may cause lots of data + * to be passed between the kernel and the pagers, often + * in the form of large copy maps. Making the minimum + * number of pages depend on the total number of pages + * should make exhaustion even less likely. + */ + + if (current_thread()) { + current_thread()->vm_privilege++; + assert(current_thread()->vm_privilege != 0); + } + + map->timestamp++; +} + +void vm_map_unlock(struct vm_map *map) +{ + if (current_thread()) { + current_thread()->vm_privilege--; + } + + lock_write_done(&map->lock); +} + /* * vm_map_entry_create: [ internal use only ] * @@ -233,17 +277,11 @@ vm_map_t vm_map_create(pmap, min, max, p _vm_map_entry_create(&(copy)->cpy_hdr) vm_map_entry_t _vm_map_entry_create(map_header) - register struct vm_map_header *map_header; + const struct vm_map_header *map_header; { - register zone_t zone; - register vm_map_entry_t entry; - - if (map_header->entries_pageable) - zone = vm_map_entry_zone; - else - zone = vm_map_kentry_zone; + vm_map_entry_t entry; - entry = (vm_map_entry_t) zalloc(zone); + entry = (vm_map_entry_t) kmem_cache_alloc(&vm_map_entry_cache); if (entry == VM_MAP_ENTRY_NULL) panic("vm_map_entry_create"); @@ -262,23 +300,183 @@ vm_map_entry_t _vm_map_entry_create(map_ _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry)) void _vm_map_entry_dispose(map_header, entry) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; + const struct vm_map_header *map_header; + vm_map_entry_t entry; +{ + (void)map_header; + + kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry); +} + +/* + * Red-black tree lookup/insert comparison functions + */ +static inline int vm_map_entry_cmp_lookup(vm_offset_t addr, + const struct rbtree_node *node) { - register zone_t zone; + struct vm_map_entry *entry; - if (map_header->entries_pageable) - zone = vm_map_entry_zone; + entry = rbtree_entry(node, struct vm_map_entry, tree_node); + + if (addr < entry->vme_start) + return -1; + else if (addr < entry->vme_end) + return 0; else - zone = vm_map_kentry_zone; + return 1; +} + +static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a, + const struct rbtree_node *b) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(a, struct vm_map_entry, tree_node); + return vm_map_entry_cmp_lookup(entry->vme_start, b); +} + +/* + * Gap management functions + */ +static inline int vm_map_entry_gap_cmp_lookup(vm_size_t gap_size, + const struct rbtree_node *node) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(node, struct vm_map_entry, gap_node); + + if (gap_size < entry->gap_size) + return -1; + else if (gap_size == entry->gap_size) + return 0; + else + return 1; +} + +static inline int vm_map_entry_gap_cmp_insert(const struct rbtree_node *a, + const struct rbtree_node *b) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(a, struct vm_map_entry, gap_node); + return vm_map_entry_gap_cmp_lookup(entry->gap_size, b); +} + +static int +vm_map_gap_valid(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + return entry != (struct vm_map_entry *)&hdr->links; +} + +static void +vm_map_gap_compute(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *next; + + next = entry->vme_next; + + if (vm_map_gap_valid(hdr, next)) { + entry->gap_size = next->vme_start - entry->vme_end; + } else { + entry->gap_size = hdr->vme_end - entry->vme_end; + } +} + +static void +vm_map_gap_insert_single(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *tmp; + struct rbtree_node *node; + unsigned long slot; + + if (!vm_map_gap_valid(hdr, entry)) { + return; + } + + vm_map_gap_compute(hdr, entry); + + if (entry->gap_size == 0) { + return; + } - zfree(zone, (vm_offset_t) entry); + node = rbtree_lookup_slot(&hdr->gap_tree, entry->gap_size, + vm_map_entry_gap_cmp_lookup, slot); + + if (node == NULL) { + rbtree_insert_slot(&hdr->gap_tree, slot, &entry->gap_node); + list_init(&entry->gap_list); + entry->in_gap_tree = 1; + } else { + tmp = rbtree_entry(node, struct vm_map_entry, gap_node); + list_insert_tail(&tmp->gap_list, &entry->gap_list); + entry->in_gap_tree = 0; + } +} + +static void +vm_map_gap_remove_single(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + struct vm_map_entry *tmp; + + if (!vm_map_gap_valid(hdr, entry)) { + return; + } + + if (entry->gap_size == 0) { + return; + } + + if (!entry->in_gap_tree) { + list_remove(&entry->gap_list); + return; + } + + rbtree_remove(&hdr->gap_tree, &entry->gap_node); + + if (list_empty(&entry->gap_list)) { + return; + } + + tmp = list_first_entry(&entry->gap_list, struct vm_map_entry, gap_list); + assert(tmp->gap_size == entry->gap_size); + list_remove(&tmp->gap_list); + list_set_head(&tmp->gap_list, &entry->gap_list); + assert(!tmp->in_gap_tree); + rbtree_insert(&hdr->gap_tree, &tmp->gap_node, + vm_map_entry_gap_cmp_insert); + tmp->in_gap_tree = 1; +} + +static void +vm_map_gap_update(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry); + vm_map_gap_insert_single(hdr, entry); +} + +static void +vm_map_gap_insert(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry); +} + +static void +vm_map_gap_remove(struct vm_map_header *hdr, struct vm_map_entry *entry) +{ + vm_map_gap_remove_single(hdr, entry); + vm_map_gap_remove_single(hdr, entry->vme_prev); + vm_map_gap_insert_single(hdr, entry->vme_prev); } /* * vm_map_entry_{un,}link: * * Insert/remove entries from maps (or map copies). + * + * The start and end addresses of the entries must be properly set + * before using these macros. */ #define vm_map_entry_link(map, after_where, entry) \ _vm_map_entry_link(&(map)->hdr, after_where, entry) @@ -293,6 +491,9 @@ void _vm_map_entry_dispose(map_header, e (entry)->vme_next = (after_where)->vme_next; \ (entry)->vme_prev->vme_next = \ (entry)->vme_next->vme_prev = (entry); \ + rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \ + vm_map_entry_cmp_insert); \ + vm_map_gap_insert((hdr), (entry)); \ MACRO_END #define vm_map_entry_unlink(map, entry) \ @@ -306,6 +507,8 @@ void _vm_map_entry_dispose(map_header, e (hdr)->nentries--; \ (entry)->vme_next->vme_prev = (entry)->vme_prev; \ (entry)->vme_prev->vme_next = (entry)->vme_next; \ + rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \ + vm_map_gap_remove((hdr), (entry)); \ MACRO_END /* @@ -314,8 +517,7 @@ void _vm_map_entry_dispose(map_header, e * Creates another valid reference to the given map. * */ -void vm_map_reference(map) - register vm_map_t map; +void vm_map_reference(vm_map_t map) { if (map == VM_MAP_NULL) return; @@ -332,10 +534,9 @@ void vm_map_reference(map) * destroying it if no references remain. * The map should not be locked. */ -void vm_map_deallocate(map) - register vm_map_t map; +void vm_map_deallocate(vm_map_t map) { - register int c; + int c; if (map == VM_MAP_NULL) return; @@ -353,7 +554,7 @@ void vm_map_deallocate(map) pmap_destroy(map->pmap); - zfree(vm_map_zone, (vm_offset_t) map); + kmem_cache_free(&vm_map_cache, (vm_offset_t) map); } /* @@ -377,75 +578,54 @@ void vm_map_deallocate(map) * result indicates whether the address is * actually contained in the map. */ -boolean_t vm_map_lookup_entry(map, address, entry) - register vm_map_t map; - register vm_offset_t address; - vm_map_entry_t *entry; /* OUT */ +boolean_t vm_map_lookup_entry( + vm_map_t map, + vm_offset_t address, + vm_map_entry_t *entry) /* OUT */ { - register vm_map_entry_t cur; - register vm_map_entry_t last; + struct rbtree_node *node; + vm_map_entry_t hint; /* - * Start looking either from the head of the - * list, or from the hint. + * First, make a quick check to see if we are already + * looking at the entry we want (which is often the case). */ simple_lock(&map->hint_lock); - cur = map->hint; + hint = map->hint; simple_unlock(&map->hint_lock); - if (cur == vm_map_to_entry(map)) - cur = cur->vme_next; - - if (address >= cur->vme_start) { - /* - * Go from hint to end of list. - * - * But first, make a quick check to see if - * we are already looking at the entry we - * want (which is usually the case). - * Note also that we don't need to save the hint - * here... it is the same hint (unless we are - * at the header, in which case the hint didn't - * buy us anything anyway). - */ - last = vm_map_to_entry(map); - if ((cur != last) && (cur->vme_end > address)) { - *entry = cur; + if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) { + if (address < hint->vme_end) { + *entry = hint; return(TRUE); + } else { + vm_map_entry_t next = hint->vme_next; + + if ((next == vm_map_to_entry(map)) + || (address < next->vme_start)) { + *entry = hint; + return(FALSE); + } } } - else { - /* - * Go from start to hint, *inclusively* - */ - last = cur->vme_next; - cur = vm_map_first_entry(map); - } /* - * Search linearly + * If the hint didn't help, use the red-black tree. */ - while (cur != last) { - if (cur->vme_end > address) { - if (address >= cur->vme_start) { - /* - * Save this lookup for future - * hints, and return - */ + node = rbtree_lookup_nearest(&map->hdr.tree, address, + vm_map_entry_cmp_lookup, RBTREE_LEFT); - *entry = cur; - SAVE_HINT(map, cur); - return(TRUE); - } - break; - } - cur = cur->vme_next; + if (node == NULL) { + *entry = vm_map_to_entry(map); + SAVE_HINT(map, *entry); + return(FALSE); + } else { + *entry = rbtree_entry(node, struct vm_map_entry, tree_node); + SAVE_HINT(map, *entry); + return((address < (*entry)->vme_end) ? TRUE : FALSE); } - *entry = cur->vme_prev; - SAVE_HINT(map, *entry); - return(FALSE); } /* @@ -455,10 +635,11 @@ boolean_t vm_map_lookup_entry(map, addre */ boolean_t -invalid_user_access(map, start, end, prot) - vm_map_t map; - vm_offset_t start, end; - vm_prot_t prot; +invalid_user_access( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot) { vm_map_entry_t entry; @@ -468,6 +649,104 @@ invalid_user_access(map, start, end, pro (prot & ~(entry->protection))); } +/* + * Find a range of available space from the specified map. + * + * If successful, this function returns the map entry immediately preceding + * the range, and writes the range address in startp. If the map contains + * no entry, the entry returned points to the map header. + * Otherwise, NULL is returned. + * + * If map_locked is true, this function will not wait for more space in case + * of failure. Otherwise, the map is locked. + */ +static struct vm_map_entry * +vm_map_find_entry_anywhere(struct vm_map *map, + vm_size_t size, + vm_offset_t mask, + boolean_t map_locked, + vm_offset_t *startp) +{ + struct vm_map_entry *entry; + struct rbtree_node *node; + vm_size_t max_size; + vm_offset_t start, end; + + assert(size != 0); + + if (!map_locked) { + vm_map_lock(map); + } + +restart: + if (map->hdr.nentries == 0) { + entry = vm_map_to_entry(map); + start = (map->min_offset + mask) & ~mask; + end = start + size; + + if ((end <= start) || (end > map->max_offset)) { + goto error; + } + + *startp = start; + return entry; + } + + entry = map->first_free; + + if (entry != vm_map_to_entry(map)) { + start = (entry->vme_end + mask) & ~mask; + end = start + size; + + if ((end > start) + && (end <= map->max_offset) + && (end <= (entry->vme_end + entry->gap_size))) { + *startp = start; + return entry; + } + } + + max_size = size + mask; + + if (max_size < size) { + goto error; + } + + node = rbtree_lookup_nearest(&map->hdr.gap_tree, max_size, + vm_map_entry_gap_cmp_lookup, RBTREE_RIGHT); + + if (node == NULL) { + if (map_locked || !map->wait_for_space) { + goto error; + } + + assert_wait((event_t)map, TRUE); + vm_map_unlock(map); + thread_block(NULL); + vm_map_lock(map); + goto restart; + } + + entry = rbtree_entry(node, struct vm_map_entry, gap_node); + assert(entry->in_gap_tree); + + if (!list_empty(&entry->gap_list)) { + entry = list_last_entry(&entry->gap_list, + struct vm_map_entry, gap_list); + } + + assert(entry->gap_size >= max_size); + start = (entry->vme_end + mask) & ~mask; + end = start + size; + assert(end > start); + assert(end <= (entry->vme_end + entry->gap_size)); + *startp = start; + return entry; + +error: + printf("no more room in %p (%s)\n", map, map->name); + return NULL; +} /* * Routine: vm_map_find_entry @@ -482,75 +761,26 @@ invalid_user_access(map, start, end, pro * are initialized to zero. If an object is supplied, * then an existing entry may be extended. */ -kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry) - register vm_map_t map; - vm_offset_t *address; /* OUT */ - vm_size_t size; - vm_offset_t mask; - vm_object_t object; - vm_map_entry_t *o_entry; /* OUT */ +kern_return_t vm_map_find_entry( + vm_map_t map, + vm_offset_t *address, /* OUT */ + vm_size_t size, + vm_offset_t mask, + vm_object_t object, + vm_map_entry_t *o_entry) /* OUT */ { - register vm_map_entry_t entry, new_entry; - register vm_offset_t start; - register vm_offset_t end; - - /* - * Look for the first possible address; - * if there's already something at this - * address, we have to start after it. - */ - - if ((entry = map->first_free) == vm_map_to_entry(map)) - start = map->min_offset; - else - start = entry->vme_end; - - /* - * In any case, the "entry" always precedes - * the proposed new region throughout the loop: - */ - - while (TRUE) { - register vm_map_entry_t next; - - /* - * Find the end of the proposed new region. - * Be sure we didn't go beyond the end, or - * wrap around the address. - */ - - if (((start + mask) & ~mask) < start) - return(KERN_NO_SPACE); - start = ((start + mask) & ~mask); - end = start + size; - - if ((end > map->max_offset) || (end < start)) - return(KERN_NO_SPACE); - - /* - * If there are no more entries, we must win. - */ - - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ - - if (next->vme_start >= end) - break; + vm_map_entry_t entry, new_entry; + vm_offset_t start; + vm_offset_t end; - /* - * Didn't fit -- move to the next entry. - */ + entry = vm_map_find_entry_anywhere(map, size, mask, TRUE, &start); - entry = next; - start = entry->vme_end; + if (entry == NULL) { + return KERN_NO_SPACE; } + end = start + size; + /* * At this point, * "start" and "end" should define the endpoints of the @@ -588,6 +818,7 @@ kern_return_t vm_map_find_entry(map, add */ entry->vme_end = end; + vm_map_gap_update(&map->hdr, entry); new_entry = entry; } else { new_entry = vm_map_entry_create(map); @@ -632,8 +863,8 @@ kern_return_t vm_map_find_entry(map, add return(KERN_SUCCESS); } -int vm_map_pmap_enter_print = FALSE; -int vm_map_pmap_enter_enable = FALSE; +boolean_t vm_map_pmap_enter_print = FALSE; +boolean_t vm_map_pmap_enter_enable = FALSE; /* * Routine: vm_map_pmap_enter @@ -644,25 +875,22 @@ int vm_map_pmap_enter_enable = FALSE; * As soon as a page not found in the object the scan ends. * * Returns: - * Nothing. + * Nothing. * * In/out conditions: * The source map should not be locked on entry. */ void -vm_map_pmap_enter(map, addr, end_addr, object, offset, protection) - vm_map_t map; - register - vm_offset_t addr; - register - vm_offset_t end_addr; - register - vm_object_t object; - vm_offset_t offset; - vm_prot_t protection; +vm_map_pmap_enter( + vm_map_t map, + vm_offset_t addr, + vm_offset_t end_addr, + vm_object_t object, + vm_offset_t offset, + vm_prot_t protection) { while (addr < end_addr) { - register vm_page_t m; + vm_page_t m; vm_object_lock(object); vm_object_paging_begin(object); @@ -676,7 +904,7 @@ vm_map_pmap_enter(map, addr, end_addr, o if (vm_map_pmap_enter_print) { printf("vm_map_pmap_enter:"); - printf("map: %x, addr: %x, object: %x, offset: %x\n", + printf("map: %p, addr: %lx, object: %p, offset: %lx\n", map, addr, object, offset); } @@ -711,119 +939,38 @@ vm_map_pmap_enter(map, addr, end_addr, o * Arguments are as defined in the vm_map call. */ kern_return_t vm_map_enter( - map, - address, size, mask, anywhere, - object, offset, needs_copy, - cur_protection, max_protection, inheritance) - register - vm_map_t map; - vm_offset_t *address; /* IN/OUT */ - vm_size_t size; - vm_offset_t mask; - boolean_t anywhere; - vm_object_t object; - vm_offset_t offset; - boolean_t needs_copy; - vm_prot_t cur_protection; - vm_prot_t max_protection; - vm_inherit_t inheritance; -{ - register vm_map_entry_t entry; - register vm_offset_t start; - register vm_offset_t end; - kern_return_t result = KERN_SUCCESS; + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t size, + vm_offset_t mask, + boolean_t anywhere, + vm_object_t object, + vm_offset_t offset, + boolean_t needs_copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) +{ + vm_map_entry_t entry; + vm_offset_t start; + vm_offset_t end; + kern_return_t result = KERN_SUCCESS; #define RETURN(value) { result = value; goto BailOut; } - StartAgain: ; + if (size == 0) + return KERN_INVALID_ARGUMENT; start = *address; if (anywhere) { - vm_map_lock(map); + entry = vm_map_find_entry_anywhere(map, size, mask, FALSE, &start); - /* - * Calculate the first possible address. - */ - - if (start < map->min_offset) - start = map->min_offset; - if (start > map->max_offset) + if (entry == NULL) { RETURN(KERN_NO_SPACE); - - /* - * Look for the first possible address; - * if there's already something at this - * address, we have to start after it. - */ - - if (start == map->min_offset) { - if ((entry = map->first_free) != vm_map_to_entry(map)) - start = entry->vme_end; - } else { - vm_map_entry_t tmp_entry; - if (vm_map_lookup_entry(map, start, &tmp_entry)) - start = tmp_entry->vme_end; - entry = tmp_entry; } - /* - * In any case, the "entry" always precedes - * the proposed new region throughout the - * loop: - */ - - while (TRUE) { - register vm_map_entry_t next; - - /* - * Find the end of the proposed new region. - * Be sure we didn't go beyond the end, or - * wrap around the address. - */ - - if (((start + mask) & ~mask) < start) - return(KERN_NO_SPACE); - start = ((start + mask) & ~mask); - end = start + size; - - if ((end > map->max_offset) || (end < start)) { - if (map->wait_for_space) { - if (size <= (map->max_offset - - map->min_offset)) { - assert_wait((event_t) map, TRUE); - vm_map_unlock(map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - - RETURN(KERN_NO_SPACE); - } - - /* - * If there are no more entries, we must win. - */ - - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ - - if (next->vme_start >= end) - break; - - /* - * Didn't fit -- move to the next entry. - */ - - entry = next; - start = entry->vme_end; - } + end = start + size; *address = start; } else { vm_map_entry_t temp_entry; @@ -895,7 +1042,7 @@ kern_return_t vm_map_enter( (entry->protection == cur_protection) && (entry->max_protection == max_protection) && (entry->wired_count == 0) && /* implies user_wired_count == 0 */ - (entry->projected_on == 0)) { + (entry->projected_on == 0)) { if (vm_object_coalesce(entry->object.vm_object, VM_OBJECT_NULL, entry->offset, @@ -910,6 +1057,7 @@ kern_return_t vm_map_enter( */ map->size += (end - entry->vme_end); entry->vme_end = end; + vm_map_gap_update(&map->hdr, entry); RETURN(KERN_SUCCESS); } } @@ -919,7 +1067,7 @@ kern_return_t vm_map_enter( */ /**/ { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; new_entry = vm_map_entry_create(map); @@ -966,9 +1114,9 @@ kern_return_t vm_map_enter( if ((object != VM_OBJECT_NULL) && (vm_map_pmap_enter_enable) && (!anywhere) && - (!needs_copy) && + (!needs_copy) && (size < (128*1024))) { - vm_map_pmap_enter(map, start, end, + vm_map_pmap_enter(map, start, end, object, offset, cur_protection); } @@ -990,14 +1138,12 @@ kern_return_t vm_map_enter( * the specified address; if necessary, * it splits the entry into two. */ -void _vm_map_clip_start(); #define vm_map_clip_start(map, entry, startaddr) \ MACRO_BEGIN \ if ((startaddr) > (entry)->vme_start) \ _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \ MACRO_END -void _vm_map_copy_clip_start(); #define vm_map_copy_clip_start(copy, entry, startaddr) \ MACRO_BEGIN \ if ((startaddr) > (entry)->vme_start) \ @@ -1008,12 +1154,12 @@ void _vm_map_copy_clip_start(); * This routine is called only when it is known that * the entry must be split. */ -void _vm_map_clip_start(map_header, entry, start) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; - register vm_offset_t start; +void _vm_map_clip_start( + struct vm_map_header *map_header, + vm_map_entry_t entry, + vm_offset_t start) { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; /* * Split off the front portion -- @@ -1045,14 +1191,12 @@ void _vm_map_clip_start(map_header, entr * the specified address; if necessary, * it splits the entry into two. */ -void _vm_map_clip_end(); #define vm_map_clip_end(map, entry, endaddr) \ MACRO_BEGIN \ if ((endaddr) < (entry)->vme_end) \ _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \ MACRO_END -void _vm_map_copy_clip_end(); #define vm_map_copy_clip_end(copy, entry, endaddr) \ MACRO_BEGIN \ if ((endaddr) < (entry)->vme_end) \ @@ -1063,12 +1207,12 @@ void _vm_map_copy_clip_end(); * This routine is called only when it is known that * the entry must be split. */ -void _vm_map_clip_end(map_header, entry, end) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; - register vm_offset_t end; +void _vm_map_clip_end( + struct vm_map_header *map_header, + vm_map_entry_t entry, + vm_offset_t end) { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; /* * Create a new entry and insert it @@ -1123,15 +1267,15 @@ void _vm_map_clip_end(map_header, entry, * range from the superior map, and then destroy the * submap (if desired). [Better yet, don't try it.] */ -kern_return_t vm_map_submap(map, start, end, submap) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - vm_map_t submap; +kern_return_t vm_map_submap( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_map_t submap) { vm_map_entry_t entry; - register kern_return_t result = KERN_INVALID_ARGUMENT; - register vm_object_t object; + kern_return_t result = KERN_INVALID_ARGUMENT; + vm_object_t object; vm_map_lock(map); @@ -1171,15 +1315,15 @@ kern_return_t vm_map_submap(map, start, * specified, the maximum protection is to be set; * otherwise, only the current protection is affected. */ -kern_return_t vm_map_protect(map, start, end, new_prot, set_max) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_prot_t new_prot; - register boolean_t set_max; +kern_return_t vm_map_protect( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t new_prot, + boolean_t set_max) { - register vm_map_entry_t current; - vm_map_entry_t entry; + vm_map_entry_t current; + vm_map_entry_t entry; vm_map_lock(map); @@ -1259,13 +1403,13 @@ kern_return_t vm_map_protect(map, start, * affects how the map will be shared with * child maps at the time of vm_map_fork. */ -kern_return_t vm_map_inherit(map, start, end, new_inheritance) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_inherit_t new_inheritance; +kern_return_t vm_map_inherit( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_inherit_t new_inheritance) { - register vm_map_entry_t entry; + vm_map_entry_t entry; vm_map_entry_t temp_entry; vm_map_lock(map); @@ -1308,14 +1452,14 @@ kern_return_t vm_map_inherit(map, start, * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable, * or vm_map_pageable_user); don't call vm_map_pageable directly. */ -kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_prot_t access_type; - boolean_t user_wire; +kern_return_t vm_map_pageable_common( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t access_type, + boolean_t user_wire) { - register vm_map_entry_t entry; + vm_map_entry_t entry; vm_map_entry_t start_entry; vm_map_lock(map); @@ -1333,7 +1477,7 @@ kern_return_t vm_map_pageable_common(map * Start address is not in map; this is fatal. */ vm_map_unlock(map); - return(KERN_FAILURE); + return(KERN_NO_SPACE); } /* @@ -1346,19 +1490,17 @@ kern_return_t vm_map_pageable_common(map vm_map_clip_start(map, entry, start); /* - * Unwiring. First ensure that the range to be - * unwired is really wired down. + * Unwiring. First ensure that there are no holes + * in the specified range. */ while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { - if ((entry->wired_count == 0) || - ((entry->vme_end < end) && - ((entry->vme_next == vm_map_to_entry(map)) || - (entry->vme_next->vme_start > entry->vme_end))) || - (user_wire && (entry->user_wired_count == 0))) { + if ((entry->vme_end < end) && + ((entry->vme_next == vm_map_to_entry(map)) || + (entry->vme_next->vme_start > entry->vme_end))) { vm_map_unlock(map); - return(KERN_INVALID_ARGUMENT); + return(KERN_NO_SPACE); } entry = entry->vme_next; } @@ -1373,14 +1515,24 @@ kern_return_t vm_map_pageable_common(map (entry->vme_start < end)) { vm_map_clip_end(map, entry, end); + /* Skip unwired entries */ + if (entry->wired_count == 0) { + assert(entry->user_wired_count == 0); + entry = entry->vme_next; + continue; + } + if (user_wire) { if (--(entry->user_wired_count) == 0) + { + map->user_wired -= entry->vme_end - entry->vme_start; entry->wired_count--; + } } else { entry->wired_count--; } - + if (entry->wired_count == 0) vm_fault_unwire(map, entry); @@ -1442,7 +1594,7 @@ kern_return_t vm_map_pageable_common(map if (entry->object.vm_object == VM_OBJECT_NULL) { entry->object.vm_object = vm_object_allocate( - (vm_size_t)(entry->vme_end + (vm_size_t)(entry->vme_end - entry->vme_start)); entry->offset = (vm_offset_t)0; } @@ -1452,7 +1604,10 @@ kern_return_t vm_map_pageable_common(map if (user_wire) { if ((entry->user_wired_count)++ == 0) + { + map->user_wired += entry->vme_end - entry->vme_start; entry->wired_count++; + } } else { entry->wired_count++; @@ -1464,7 +1619,7 @@ kern_return_t vm_map_pageable_common(map * this is the end of the region. * Protection: Access requested must be allowed. */ - if (((entry->vme_end < end) && + if (((entry->vme_end < end) && ((entry->vme_next == vm_map_to_entry(map)) || (entry->vme_next->vme_start > entry->vme_end))) || ((entry->protection & access_type) != access_type)) { @@ -1478,7 +1633,10 @@ kern_return_t vm_map_pageable_common(map (entry->vme_end > start)) { if (user_wire) { if (--(entry->user_wired_count) == 0) + { + map->user_wired -= entry->vme_end - entry->vme_start; entry->wired_count--; + } } else { entry->wired_count--; @@ -1488,7 +1646,7 @@ kern_return_t vm_map_pageable_common(map } vm_map_unlock(map); - return(KERN_FAILURE); + return(KERN_NO_SPACE); } entry = entry->vme_next; } @@ -1556,13 +1714,13 @@ kern_return_t vm_map_pageable_common(map * vm_map_entry_delete: [ internal use only ] * * Deallocate the given entry from the target map. - */ -void vm_map_entry_delete(map, entry) - register vm_map_t map; - register vm_map_entry_t entry; + */ +void vm_map_entry_delete( + vm_map_t map, + vm_map_entry_t entry) { - register vm_offset_t s, e; - register vm_object_t object; + vm_offset_t s, e; + vm_object_t object; extern vm_object_t kernel_object; s = entry->vme_start; @@ -1593,6 +1751,8 @@ void vm_map_entry_delete(map, entry) if (entry->wired_count != 0) { vm_fault_unwire(map, entry); entry->wired_count = 0; + if (entry->user_wired_count) + map->user_wired -= entry->vme_end - entry->vme_start; entry->user_wired_count = 0; } @@ -1641,10 +1801,10 @@ void vm_map_entry_delete(map, entry) * map. */ -kern_return_t vm_map_delete(map, start, end) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; +kern_return_t vm_map_delete( + vm_map_t map, + vm_offset_t start, + vm_offset_t end) { vm_map_entry_t entry; vm_map_entry_t first_entry; @@ -1657,67 +1817,6 @@ kern_return_t vm_map_delete(map, start, entry = first_entry->vme_next; else { entry = first_entry; -#if NORMA_IPC_xxx - /* - * XXX Had to disable this code because: - - _vm_map_delete(c0804b78,c2198000,c219a000,0,c219a000)+df - [vm/vm_map.c:2007] - _vm_map_remove(c0804b78,c2198000,c219a000,c0817834, - c081786c)+42 [vm/vm_map.c:2094] - _kmem_io_map_deallocate(c0804b78,c2198000,2000,c0817834, - c081786c)+43 [vm/vm_kern.c:818] - _device_write_dealloc(c081786c)+117 [device/ds_routines.c:814] - _ds_write_done(c081786c,0)+2e [device/ds_routines.c:848] - _io_done_thread_continue(c08150c0,c21d4e14,c21d4e30,c08150c0, - c080c114)+14 [device/ds_routines.c:1350] - - */ - if (start > entry->vme_start - && end == entry->vme_end - && ! entry->wired_count /* XXX ??? */ - && ! entry->is_shared - && ! entry->projected_on - && ! entry->is_sub_map) { - extern vm_object_t kernel_object; - register vm_object_t object = entry->object.vm_object; - - /* - * The region to be deleted lives at the end - * of this entry, and thus all we have to do is - * truncate the entry. - * - * This special case is necessary if we want - * coalescing to do us any good. - * - * XXX Do we have to adjust object size? - */ - if (object == kernel_object) { - vm_object_lock(object); - vm_object_page_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - vm_object_unlock(object); - } else if (entry->is_shared) { - vm_object_pmap_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - } else { - pmap_remove(map->pmap, start, end); - } - object->size -= (end - start); /* XXX */ - - entry->vme_end = start; - map->size -= (end - start); - - if (map->wait_for_space) { - thread_wakeup((event_t) map); - } - return KERN_SUCCESS; - } -#endif NORMA_IPC vm_map_clip_start(map, entry, start); /* @@ -1785,12 +1884,12 @@ kern_return_t vm_map_delete(map, start, * Remove the given address range from the target map. * This is the exported form of vm_map_delete. */ -kern_return_t vm_map_remove(map, start, end) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; +kern_return_t vm_map_remove( + vm_map_t map, + vm_offset_t start, + vm_offset_t end) { - register kern_return_t result; + kern_return_t result; vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); @@ -1808,12 +1907,11 @@ kern_return_t vm_map_remove(map, start, * that have not already been stolen. */ void -vm_map_copy_steal_pages(copy) -vm_map_copy_t copy; +vm_map_copy_steal_pages(vm_map_copy_t copy) { - register vm_page_t m, new_m; - register int i; - vm_object_t object; + vm_page_t m, new_m; + int i; + vm_object_t object; for (i = 0; i < copy->cpy_npages; i++) { @@ -1855,8 +1953,7 @@ vm_map_copy_t copy; * stolen, they are freed. If the pages are not stolen, they * are unbusied, and associated state is cleaned up. */ -void vm_map_copy_page_discard(copy) -vm_map_copy_t copy; +void vm_map_copy_page_discard(vm_map_copy_t copy) { while (copy->cpy_npages > 0) { vm_page_t m; @@ -1901,8 +1998,7 @@ vm_map_copy_t copy; * vm_map_copyin). */ void -vm_map_copy_discard(copy) - vm_map_copy_t copy; +vm_map_copy_discard(vm_map_copy_t copy) { free_next_copy: if (copy == VM_MAP_COPY_NULL) @@ -1943,10 +2039,10 @@ free_next_copy: * here to avoid tail recursion. */ if (copy->cpy_cont == vm_map_copy_discard_cont) { - register vm_map_copy_t new_copy; + vm_map_copy_t new_copy; new_copy = (vm_map_copy_t) copy->cpy_cont_args; - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); copy = new_copy; goto free_next_copy; } @@ -1957,7 +2053,7 @@ free_next_copy: break; } - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } /* @@ -1978,8 +2074,7 @@ free_next_copy: * deallocation will not fail. */ vm_map_copy_t -vm_map_copy_copy(copy) - vm_map_copy_t copy; +vm_map_copy_copy(vm_map_copy_t copy) { vm_map_copy_t new_copy; @@ -1991,7 +2086,7 @@ vm_map_copy_copy(copy) * from the old one into it. */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); *new_copy = *copy; if (copy->type == VM_MAP_COPY_ENTRY_LIST) { @@ -2025,9 +2120,9 @@ vm_map_copy_copy(copy) * A version of vm_map_copy_discard that can be called * as a continuation from a vm_map_copy page list. */ -kern_return_t vm_map_copy_discard_cont(cont_args, copy_result) -vm_map_copyin_args_t cont_args; -vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copy_discard_cont( +vm_map_copyin_args_t cont_args, +vm_map_copy_t *copy_result) /* OUT */ { vm_map_copy_discard((vm_map_copy_t) cont_args); if (copy_result != (vm_map_copy_t *)0) @@ -2082,11 +2177,11 @@ vm_map_copy_t *copy_result; /* OUT */ * atomically and interruptibly, an error indication is * returned. */ -kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible) - vm_map_t dst_map; - vm_offset_t dst_addr; - vm_map_copy_t copy; - boolean_t interruptible; +kern_return_t vm_map_copy_overwrite( + vm_map_t dst_map, + vm_offset_t dst_addr, + vm_map_copy_t copy, + boolean_t interruptible) { vm_size_t size; vm_offset_t start; @@ -2109,11 +2204,7 @@ kern_return_t vm_map_copy_overwrite(dst_ * support page lists LATER. */ -#if NORMA_IPC - vm_map_convert_from_page_list(copy); -#else assert(copy->type == VM_MAP_COPY_ENTRY_LIST); -#endif /* * Currently this routine only handles page-aligned @@ -2196,13 +2287,14 @@ start_pass_1: * the copy cannot be interrupted. */ - if (interruptible && contains_permanent_objects) + if (interruptible && contains_permanent_objects) { + vm_map_unlock(dst_map); return(KERN_FAILURE); /* XXX */ + } /* * XXXO If there are no permanent objects in the destination, - * XXXO and the source and destination map entry zones match, - * XXXO and the destination map entry is not shared, + * XXXO and the destination map entry is not shared, * XXXO then the map entries can be deleted and replaced * XXXO with those from the copy. The following code is the * XXXO basic idea of what to do, but there are lots of annoying @@ -2211,8 +2303,7 @@ start_pass_1: * XXXO to the above pass and make sure that no wiring is involved. */ /* - * if (!contains_permanent_objects && - * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) { + * if (!contains_permanent_objects) { * * * * * Run over copy and adjust entries. Steal code @@ -2243,7 +2334,7 @@ start_pass_1: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy); vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start); vm_object_t object; - + entry = tmp_entry; size = (entry->vme_end - entry->vme_start); /* @@ -2307,6 +2398,8 @@ start_pass_1: entry->offset = copy_entry->offset; entry->needs_copy = copy_entry->needs_copy; entry->wired_count = 0; + if (entry->user_wired_count) + dst_map->user_wired -= entry->vme_end - entry->vme_start; entry->user_wired_count = 0; vm_map_copy_entry_unlink(copy, copy_entry); @@ -2427,25 +2520,40 @@ start_pass_1: } /* - * Macro: vm_map_copy_insert - * + * Routine: vm_map_copy_insert + * * Description: * Link a copy chain ("copy") into a map at the * specified location (after "where"). * Side effects: * The copy chain is destroyed. - * Warning: - * The arguments are evaluated multiple times. */ -#define vm_map_copy_insert(map, where, copy) \ - MACRO_BEGIN \ - (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \ - ->vme_next = ((where)->vme_next); \ - ((where)->vme_next = vm_map_copy_first_entry(copy)) \ - ->vme_prev = (where); \ - (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \ - zfree(vm_map_copy_zone, (vm_offset_t) copy); \ - MACRO_END +static void +vm_map_copy_insert(struct vm_map *map, struct vm_map_entry *where, + struct vm_map_copy *copy) +{ + struct vm_map_entry *entry; + + assert(copy->type == VM_MAP_COPY_ENTRY_LIST); + + for (;;) { + entry = vm_map_copy_first_entry(copy); + + if (entry == vm_map_copy_to_entry(copy)) { + break; + } + + /* + * TODO Turn copy maps into their own type so they don't + * use any of the tree operations. + */ + vm_map_copy_entry_unlink(copy, entry); + vm_map_entry_link(map, where, entry); + where = entry; + } + + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t)copy); +} /* * Routine: vm_map_copyout @@ -2457,19 +2565,16 @@ start_pass_1: * If successful, consumes the copy object. * Otherwise, the caller is responsible for it. */ -kern_return_t vm_map_copyout(dst_map, dst_addr, copy) - register - vm_map_t dst_map; - vm_offset_t *dst_addr; /* OUT */ - register - vm_map_copy_t copy; +kern_return_t vm_map_copyout( + vm_map_t dst_map, + vm_offset_t *dst_addr, /* OUT */ + vm_map_copy_t copy) { vm_size_t size; vm_size_t adjustment; vm_offset_t start; vm_offset_t vm_copy_start; vm_map_entry_t last; - register vm_map_entry_t entry; /* @@ -2500,7 +2605,7 @@ kern_return_t vm_map_copyout(dst_map, ds VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return(kr); - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); return(KERN_SUCCESS); } @@ -2513,83 +2618,11 @@ kern_return_t vm_map_copyout(dst_map, ds vm_copy_start = trunc_page(copy->offset); size = round_page(copy->offset + copy->size) - vm_copy_start; + last = vm_map_find_entry_anywhere(dst_map, size, 0, FALSE, &start); - StartAgain: ; - - vm_map_lock(dst_map); - start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ? - vm_map_min(dst_map) : last->vme_end; - - while (TRUE) { - vm_map_entry_t next = last->vme_next; - vm_offset_t end = start + size; - - if ((end > dst_map->max_offset) || (end < start)) { - if (dst_map->wait_for_space) { - if (size <= (dst_map->max_offset - dst_map->min_offset)) { - assert_wait((event_t) dst_map, TRUE); - vm_map_unlock(dst_map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - vm_map_unlock(dst_map); - return(KERN_NO_SPACE); - } - - if ((next == vm_map_to_entry(dst_map)) || - (next->vme_start >= end)) - break; - - last = next; - start = last->vme_end; - } - - /* - * Since we're going to just drop the map - * entries from the copy into the destination - * map, they must come from the same pool. - */ - - if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) { - /* - * Mismatches occur when dealing with the default - * pager. - */ - zone_t old_zone; - vm_map_entry_t next, new; - - /* - * Find the zone that the copies were allocated from - */ - old_zone = (copy->cpy_hdr.entries_pageable) - ? vm_map_entry_zone - : vm_map_kentry_zone; - entry = vm_map_copy_first_entry(copy); - - /* - * Reinitialize the copy so that vm_map_copy_entry_link - * will work. - */ - copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable; - vm_map_copy_first_entry(copy) = - vm_map_copy_last_entry(copy) = - vm_map_copy_to_entry(copy); - - /* - * Copy each entry. - */ - while (entry != vm_map_copy_to_entry(copy)) { - new = vm_map_copy_entry_create(copy); - vm_map_entry_copy_full(new, entry); - vm_map_copy_entry_link(copy, - vm_map_copy_last_entry(copy), - new); - next = entry->vme_next; - zfree(old_zone, (vm_offset_t) entry); - entry = next; - } + if (last == NULL) { + vm_map_unlock(dst_map); + return KERN_NO_SPACE; } /* @@ -2604,6 +2637,11 @@ kern_return_t vm_map_copyout(dst_map, ds entry->vme_start += adjustment; entry->vme_end += adjustment; + /* + * XXX There is no need to update the gap tree here. + * See vm_map_copy_insert. + */ + entry->inheritance = VM_INHERIT_DEFAULT; entry->protection = VM_PROT_DEFAULT; entry->max_protection = VM_PROT_ALL; @@ -2614,9 +2652,9 @@ kern_return_t vm_map_copyout(dst_map, ds * map the pages into the destination map. */ if (entry->wired_count != 0) { - register vm_offset_t va; - vm_offset_t offset; - register vm_object_t object; + vm_offset_t va; + vm_offset_t offset; + vm_object_t object; object = entry->object.vm_object; offset = entry->offset; @@ -2628,7 +2666,7 @@ kern_return_t vm_map_copyout(dst_map, ds TRUE); while (va < entry->vme_end) { - register vm_page_t m; + vm_page_t m; /* * Look up the page in the object. @@ -2713,19 +2751,16 @@ kern_return_t vm_map_copyout(dst_map, ds * Version of vm_map_copyout() for page list vm map copies. * */ -kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy) - register - vm_map_t dst_map; - vm_offset_t *dst_addr; /* OUT */ - register - vm_map_copy_t copy; +kern_return_t vm_map_copyout_page_list( + vm_map_t dst_map, + vm_offset_t *dst_addr, /* OUT */ + vm_map_copy_t copy) { vm_size_t size; vm_offset_t start; vm_offset_t end; vm_offset_t offset; vm_map_entry_t last; - register vm_object_t object; vm_page_t *page_list, m; vm_map_entry_t entry; @@ -2752,43 +2787,19 @@ kern_return_t vm_map_copyout_page_list(d size = round_page(copy->offset + copy->size) - trunc_page(copy->offset); -StartAgain: - vm_map_lock(dst_map); - must_wire = dst_map->wiring_required; - last = dst_map->first_free; - if (last == vm_map_to_entry(dst_map)) { - start = vm_map_min(dst_map); - } else { - start = last->vme_end; - } + vm_map_lock(dst_map); - while (TRUE) { - vm_map_entry_t next = last->vme_next; - end = start + size; + last = vm_map_find_entry_anywhere(dst_map, size, 0, TRUE, &start); - if ((end > dst_map->max_offset) || (end < start)) { - if (dst_map->wait_for_space) { - if (size <= (dst_map->max_offset - - dst_map->min_offset)) { - assert_wait((event_t) dst_map, TRUE); - vm_map_unlock(dst_map); - thread_block((void (*)()) 0); - goto StartAgain; - } - } - vm_map_unlock(dst_map); - return(KERN_NO_SPACE); - } + if (last == NULL) { + vm_map_unlock(dst_map); + return KERN_NO_SPACE; + } - if ((next == vm_map_to_entry(dst_map)) || - (next->vme_start >= end)) { - break; - } + end = start + size; - last = next; - start = last->vme_end; - } + must_wire = dst_map->wiring_required; /* * See whether we can avoid creating a new entry (and object) by @@ -2811,7 +2822,7 @@ StartAgain: (last->wired_count != 0))) { goto create_object; } - + /* * If this entry needs an object, make one. */ @@ -2875,6 +2886,7 @@ StartAgain: */ dst_map->size += size; last->vme_end = end; + vm_map_gap_update(&dst_map->hdr, last); SAVE_HINT(dst_map, last); @@ -2902,6 +2914,7 @@ create_object: if (must_wire) { entry->wired_count = 1; + dst_map->user_wired += entry->vme_end - entry->vme_start; entry->user_wired_count = 1; } else { entry->wired_count = 0; @@ -2913,7 +2926,7 @@ create_object: entry->vme_start = start; entry->vme_end = start + size; - + entry->inheritance = VM_INHERIT_DEFAULT; entry->protection = VM_PROT_DEFAULT; entry->max_protection = VM_PROT_ALL; @@ -2937,7 +2950,7 @@ create_object: last = entry; /* - * Transfer pages into new object. + * Transfer pages into new object. * Scan page list in vm_map_copy. */ insert_pages: @@ -3028,7 +3041,7 @@ insert_pages: vm_object_unlock(object); *dst_addr = start + dst_offset; - + /* * Clear the in transition bits. This is easy if we * didn't have a continuation. @@ -3062,7 +3075,7 @@ error: entry = entry->vme_next; } } - + if (result != KERN_SUCCESS) vm_map_delete(dst_map, start, end); @@ -3075,12 +3088,12 @@ error: * Consume on success logic. */ if (copy != orig_copy) { - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } if (result == KERN_SUCCESS) { - zfree(vm_map_copy_zone, (vm_offset_t) orig_copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy); } - + return(result); } @@ -3102,12 +3115,12 @@ error: * In/out conditions: * The source map should not be locked on entry. */ -kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result) - vm_map_t src_map; - vm_offset_t src_addr; - vm_size_t len; - boolean_t src_destroy; - vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin( + vm_map_t src_map, + vm_offset_t src_addr, + vm_size_t len, + boolean_t src_destroy, + vm_map_copy_t *copy_result) /* OUT */ { vm_map_entry_t tmp_entry; /* Result of last map lookup -- * in multi-level lookup, this @@ -3121,7 +3134,6 @@ kern_return_t vm_map_copyin(src_map, src vm_offset_t src_end; /* End of entire region to be * copied */ - register vm_map_copy_t copy; /* Resulting copy */ /* @@ -3134,6 +3146,14 @@ kern_return_t vm_map_copyin(src_map, src } /* + * Check that the end address doesn't overflow + */ + + if ((src_addr + len) <= src_addr) { + return KERN_INVALID_ADDRESS; + } + + /* * Compute start and end of region */ @@ -3141,30 +3161,31 @@ kern_return_t vm_map_copyin(src_map, src src_end = round_page(src_addr + len); /* - * Check that the end address doesn't overflow + * XXX VM maps shouldn't end at maximum address */ - if (src_end <= src_start) - if ((src_end < src_start) || (src_start != 0)) - return(KERN_INVALID_ADDRESS); + if (src_end == 0) { + return KERN_INVALID_ADDRESS; + } /* * Allocate a header element for the list. * - * Use the start and end in the header to + * Use the start and end in the header to * remember the endpoints prior to rounding. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); copy->type = VM_MAP_COPY_ENTRY_LIST; copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = TRUE; + rbtree_init(©->cpy_hdr.tree); + rbtree_init(©->cpy_hdr.gap_tree); copy->offset = src_addr; copy->size = len; - + #define RETURN(x) \ MACRO_BEGIN \ vm_map_unlock(src_map); \ @@ -3187,14 +3208,12 @@ kern_return_t vm_map_copyin(src_map, src */ while (TRUE) { - register vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */ vm_size_t src_size; /* Size of source * map entry (in both * maps) */ - register vm_object_t src_object; /* Object to copy */ vm_offset_t src_offset; @@ -3203,7 +3222,6 @@ kern_return_t vm_map_copyin(src_map, src * for copy-on-write? */ - register vm_map_entry_t new_entry; /* Map entry for copy */ boolean_t new_entry_needs_copy; /* Will new entry be COW? */ @@ -3345,7 +3363,7 @@ kern_return_t vm_map_copyin(src_map, src &new_entry_needs_copy); new_entry->needs_copy = new_entry_needs_copy; - + if (result != KERN_SUCCESS) { vm_map_copy_entry_dispose(copy, new_entry); @@ -3427,7 +3445,7 @@ kern_return_t vm_map_copyin(src_map, src vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - + /* * Determine whether the entire region * has been copied. @@ -3447,7 +3465,7 @@ kern_return_t vm_map_copyin(src_map, src /* * If the source should be destroyed, do it now, since the - * copy was successful. + * copy was successful. */ if (src_destroy) (void) vm_map_delete(src_map, trunc_page(src_addr), src_end); @@ -3467,22 +3485,21 @@ kern_return_t vm_map_copyin(src_map, src * Our caller donates an object reference. */ -kern_return_t vm_map_copyin_object(object, offset, size, copy_result) - vm_object_t object; - vm_offset_t offset; /* offset of region in object */ - vm_size_t size; /* size of region in object */ - vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin_object( + vm_object_t object, + vm_offset_t offset, /* offset of region in object */ + vm_size_t size, /* size of region in object */ + vm_map_copy_t *copy_result) /* OUT */ { vm_map_copy_t copy; /* Resulting copy */ /* * We drop the object into a special copy object * that contains the object directly. These copy objects - * are distinguished by entries_pageable == FALSE - * and null links. + * are distinguished by links. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL; copy->type = VM_MAP_COPY_OBJECT; @@ -3498,7 +3515,7 @@ kern_return_t vm_map_copyin_object(objec * vm_map_copyin_page_list_cont: * * Continuation routine for vm_map_copyin_page_list. - * + * * If vm_map_copyin_page_list can't fit the entire vm range * into a single page list object, it creates a continuation. * When the target of the operation has used the pages in the @@ -3512,12 +3529,12 @@ kern_return_t vm_map_copyin_object(objec * the scheduler. */ -kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result) -vm_map_copyin_args_t cont_args; -vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin_page_list_cont( + vm_map_copyin_args_t cont_args, + vm_map_copy_t *copy_result) /* OUT */ { kern_return_t result = 0; /* '=0' to quiet gcc warnings */ - register boolean_t do_abort, src_destroy, src_destroy_only; + boolean_t do_abort, src_destroy, src_destroy_only; /* * Check for cases that only require memory destruction. @@ -3527,7 +3544,7 @@ vm_map_copy_t *copy_result; /* OUT */ src_destroy_only = (cont_args->src_len == (vm_size_t) 0); if (do_abort || src_destroy_only) { - if (src_destroy) + if (src_destroy) result = vm_map_remove(cont_args->map, cont_args->destroy_addr, cont_args->destroy_addr + cont_args->destroy_len); @@ -3551,7 +3568,7 @@ vm_map_copy_t *copy_result; /* OUT */ new_args->destroy_len = cont_args->destroy_len; } } - + vm_map_deallocate(cont_args->map); kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t)); @@ -3568,27 +3585,23 @@ vm_map_copy_t *copy_result; /* OUT */ * the recipient of this copy_result must be prepared to deal with it. */ -kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy, - steal_pages, copy_result, is_cont) - vm_map_t src_map; - vm_offset_t src_addr; - vm_size_t len; - boolean_t src_destroy; - boolean_t steal_pages; - vm_map_copy_t *copy_result; /* OUT */ - boolean_t is_cont; +kern_return_t vm_map_copyin_page_list( + vm_map_t src_map, + vm_offset_t src_addr, + vm_size_t len, + boolean_t src_destroy, + boolean_t steal_pages, + vm_map_copy_t *copy_result, /* OUT */ + boolean_t is_cont) { vm_map_entry_t src_entry; vm_page_t m; vm_offset_t src_start; vm_offset_t src_end; vm_size_t src_size; - register vm_object_t src_object; - register vm_offset_t src_offset; vm_offset_t src_last_offset; - register vm_map_copy_t copy; /* Resulting copy */ kern_return_t result = KERN_SUCCESS; boolean_t need_map_lookup; @@ -3616,6 +3629,14 @@ kern_return_t vm_map_copyin_page_list(sr } /* + * Check that the end address doesn't overflow + */ + + if ((src_addr + len) <= src_addr) { + return KERN_INVALID_ADDRESS; + } + + /* * Compute start and end of region */ @@ -3623,10 +3644,10 @@ kern_return_t vm_map_copyin_page_list(sr src_end = round_page(src_addr + len); /* - * Check that the end address doesn't overflow + * XXX VM maps shouldn't end at maximum address */ - if (src_end <= src_start && (src_end < src_start || src_start != 0)) { + if (src_end == 0) { return KERN_INVALID_ADDRESS; } @@ -3637,14 +3658,14 @@ kern_return_t vm_map_copyin_page_list(sr * be page-aligned. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); copy->type = VM_MAP_COPY_PAGE_LIST; copy->cpy_npages = 0; copy->offset = src_addr; copy->size = len; copy->cpy_cont = ((kern_return_t (*)()) 0); copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL; - + /* * Find the beginning of the region. */ @@ -3717,7 +3738,7 @@ make_continuation: * something stupid. */ - cont_args = (vm_map_copyin_args_t) + cont_args = (vm_map_copyin_args_t) kalloc(sizeof(vm_map_copyin_args_data_t)); cont_args->map = src_map; vm_map_reference(src_map); @@ -3791,7 +3812,7 @@ make_continuation: vm_prot_t result_prot; vm_page_t top_page; kern_return_t kr; - + /* * Have to fault the page in; must * unlock the map to do so. While @@ -3803,7 +3824,7 @@ make_continuation: need_map_lookup = TRUE; retry: result_prot = VM_PROT_READ; - + kr = vm_fault_page(src_object, src_offset, VM_PROT_READ, FALSE, FALSE, &result_prot, &m, &top_page, @@ -3847,7 +3868,7 @@ retry: result = KERN_MEMORY_ERROR; goto error; } - + if (top_page != VM_PAGE_NULL) { vm_object_lock(src_object); VM_PAGE_FREE(top_page); @@ -3872,13 +3893,13 @@ retry: * we have a paging reference on it. Either * the map is locked, or need_map_lookup is * TRUE. - * + * * Put the page in the page list. */ copy->cpy_page_list[copy->cpy_npages++] = m; vm_object_unlock(m->object); } - + /* * DETERMINE whether the entire region * has been copied. @@ -3922,7 +3943,7 @@ retry: */ src_start = trunc_page(src_addr); if (steal_pages) { - register int i; + int i; vm_offset_t unwire_end; unwire_end = src_start; @@ -3932,7 +3953,7 @@ retry: * Remove the page from its object if it * can be stolen. It can be stolen if: * - * (1) The source is being destroyed, + * (1) The source is being destroyed, * the object is temporary, and * not shared. * (2) The page is not precious. @@ -3950,7 +3971,7 @@ retry: * * Stealing wired pages requires telling the * pmap module to let go of them. - * + * * NOTE: stealing clean pages from objects * whose mappings survive requires a call to * the pmap module. Maybe later. @@ -3965,7 +3986,7 @@ retry: (!src_object->use_shared_copy) && !m->precious) { vm_offset_t page_vaddr; - + page_vaddr = src_start + (i * PAGE_SIZE); if (m->wire_count > 0) { @@ -3994,6 +4015,8 @@ retry: assert(src_entry->wired_count > 0); src_entry->wired_count = 0; + if (src_entry->user_wired_count) + src_map->user_wired -= src_entry->vme_end - src_entry->vme_start; src_entry->user_wired_count = 0; unwire_end = src_entry->vme_end; pmap_pageable(vm_map_pmap(src_map), @@ -4063,7 +4086,7 @@ retry: * a continuation to prevent this. */ if (src_destroy && !vm_map_copy_has_cont(copy)) { - cont_args = (vm_map_copyin_args_t) + cont_args = (vm_map_copyin_args_t) kalloc(sizeof(vm_map_copyin_args_data_t)); vm_map_reference(src_map); cont_args->map = src_map; @@ -4076,7 +4099,7 @@ retry: copy->cpy_cont_args = (char *) cont_args; copy->cpy_cont = vm_map_copyin_page_list_cont; } - + } vm_map_unlock(src_map); @@ -4099,26 +4122,24 @@ error: * * The source map must not be locked. */ -vm_map_t vm_map_fork(old_map) - vm_map_t old_map; +vm_map_t vm_map_fork(vm_map_t old_map) { vm_map_t new_map; - register vm_map_entry_t old_entry; - register vm_map_entry_t new_entry; pmap_t new_pmap = pmap_create((vm_size_t) 0); vm_size_t new_size = 0; vm_size_t entry_size; - register vm_object_t object; + if (new_pmap == PMAP_NULL) + return VM_MAP_NULL; + vm_map_lock(old_map); new_map = vm_map_create(new_pmap, old_map->min_offset, - old_map->max_offset, - old_map->hdr.entries_pageable); + old_map->max_offset); for ( old_entry = vm_map_first_entry(old_map); @@ -4165,7 +4186,7 @@ vm_map_t vm_map_fork(old_map) &old_entry->offset, (vm_size_t) (old_entry->vme_end - old_entry->vme_start)); - + /* * If we're making a shadow for other than * copy on write reasons, then we have @@ -4196,6 +4217,8 @@ vm_map_t vm_map_fork(old_map) object->ref_count++; vm_object_unlock(object); + new_entry = vm_map_entry_create(new_map); + if (old_entry->projected_on != 0) { /* * If entry is projected buffer, clone the @@ -4210,7 +4233,6 @@ vm_map_t vm_map_fork(old_map) * Mark both entries as shared. */ - new_entry = vm_map_entry_create(new_map); vm_map_entry_copy(new_entry, old_entry); old_entry->is_shared = TRUE; new_entry->is_shared = TRUE; @@ -4302,7 +4324,7 @@ vm_map_t vm_map_fork(old_map) start, entry_size, FALSE, - ©) + ©) != KERN_SUCCESS) { vm_map_lock(old_map); if (!vm_map_lookup_entry(old_map, start, &last)) @@ -4372,21 +4394,20 @@ vm_map_t vm_map_fork(old_map) * copying operations, although the data referenced will * remain the same. */ -kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version, - object, offset, out_prot, wired) - vm_map_t *var_map; /* IN/OUT */ - register vm_offset_t vaddr; - register vm_prot_t fault_type; - - vm_map_version_t *out_version; /* OUT */ - vm_object_t *object; /* OUT */ - vm_offset_t *offset; /* OUT */ - vm_prot_t *out_prot; /* OUT */ - boolean_t *wired; /* OUT */ -{ - register vm_map_entry_t entry; - register vm_map_t map = *var_map; - register vm_prot_t prot; +kern_return_t vm_map_lookup( + vm_map_t *var_map, /* IN/OUT */ + vm_offset_t vaddr, + vm_prot_t fault_type, + + vm_map_version_t *out_version, /* OUT */ + vm_object_t *object, /* OUT */ + vm_offset_t *offset, /* OUT */ + vm_prot_t *out_prot, /* OUT */ + boolean_t *wired) /* OUT */ +{ + vm_map_entry_t entry; + vm_map_t map = *var_map; + vm_prot_t prot; RetryLookup: ; @@ -4436,7 +4457,7 @@ kern_return_t vm_map_lookup(var_map, vad vm_map_unlock_read(old_map); goto RetryLookup; } - + /* * Check whether this task is allowed to have * this page. @@ -4444,19 +4465,20 @@ kern_return_t vm_map_lookup(var_map, vad prot = entry->protection; - if ((fault_type & (prot)) != fault_type) + if ((fault_type & (prot)) != fault_type) { if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) { RETURN(KERN_WRITE_PROTECTION_FAILURE); } else { RETURN(KERN_PROTECTION_FAILURE); } + } /* * If this page is not pageable, we have to get * it for all possible accesses. */ - if (*wired = (entry->wired_count != 0)) + if ((*wired = (entry->wired_count != 0))) prot = fault_type = entry->protection; /* @@ -4489,9 +4511,9 @@ kern_return_t vm_map_lookup(var_map, vad &entry->object.vm_object, &entry->offset, (vm_size_t) (entry->vme_end - entry->vme_start)); - + entry->needs_copy = FALSE; - + vm_map_lock_write_to_read(map); } else { @@ -4542,7 +4564,7 @@ kern_return_t vm_map_lookup(var_map, vad out_version->main_timestamp = map->timestamp; RETURN(KERN_SUCCESS); - + #undef RETURN } @@ -4553,11 +4575,9 @@ kern_return_t vm_map_lookup(var_map, vad * since the given version. If successful, the map * will not change until vm_map_verify_done() is called. */ -boolean_t vm_map_verify(map, version) - register - vm_map_t map; - register - vm_map_version_t *version; /* REF */ +boolean_t vm_map_verify( + vm_map_t map, + vm_map_version_t *version) /* REF */ { boolean_t result; @@ -4586,24 +4606,19 @@ boolean_t vm_map_verify(map, version) * a task's address map. */ -kern_return_t vm_region(map, address, size, - protection, max_protection, - inheritance, is_shared, - object_name, offset_in_object) - vm_map_t map; - vm_offset_t *address; /* IN/OUT */ - vm_size_t *size; /* OUT */ - vm_prot_t *protection; /* OUT */ - vm_prot_t *max_protection; /* OUT */ - vm_inherit_t *inheritance; /* OUT */ - boolean_t *is_shared; /* OUT */ - ipc_port_t *object_name; /* OUT */ - vm_offset_t *offset_in_object; /* OUT */ +kern_return_t vm_region( + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t *size, /* OUT */ + vm_prot_t *protection, /* OUT */ + vm_prot_t *max_protection, /* OUT */ + vm_inherit_t *inheritance, /* OUT */ + boolean_t *is_shared, /* OUT */ + ipc_port_t *object_name, /* OUT */ + vm_offset_t *offset_in_object) /* OUT */ { vm_map_entry_t tmp_entry; - register vm_map_entry_t entry; - register vm_offset_t tmp_offset; vm_offset_t start; @@ -4660,9 +4675,9 @@ kern_return_t vm_region(map, address, si * at allocation time because the adjacent entry * is often wired down. */ -void vm_map_simplify(map, start) - vm_map_t map; - vm_offset_t start; +void vm_map_simplify( + vm_map_t map, + vm_offset_t start) { vm_map_entry_t this_entry; vm_map_entry_t prev_entry; @@ -4692,14 +4707,14 @@ void vm_map_simplify(map, start) ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start)) == this_entry->offset) && (prev_entry->projected_on == 0) && - (this_entry->projected_on == 0) + (this_entry->projected_on == 0) ) { if (map->first_free == this_entry) map->first_free = prev_entry; SAVE_HINT(map, prev_entry); - vm_map_entry_unlink(map, this_entry); prev_entry->vme_end = this_entry->vme_end; + vm_map_entry_unlink(map, this_entry); vm_object_deallocate(this_entry->object.vm_object); vm_map_entry_dispose(map, this_entry); } @@ -4721,12 +4736,12 @@ void vm_map_simplify(map, start) * it itself. [This assumes that attributes do not * need to be inherited, which seems ok to me] */ -kern_return_t vm_map_machine_attribute(map, address, size, attribute, value) - vm_map_t map; - vm_offset_t address; - vm_size_t size; - vm_machine_attribute_t attribute; - vm_machine_attribute_val_t* value; /* IN/OUT */ +kern_return_t vm_map_machine_attribute( + vm_map_t map, + vm_offset_t address, + vm_size_t size, + vm_machine_attribute_t attribute, + vm_machine_attribute_val_t* value) /* IN/OUT */ { kern_return_t ret; @@ -4743,8 +4758,6 @@ kern_return_t vm_map_machine_attribute(m return ret; } -#include - #if MACH_KDB @@ -4753,26 +4766,30 @@ kern_return_t vm_map_machine_attribute(m /* * vm_map_print: [ debug ] */ -void vm_map_print(map) - register vm_map_t map; +void vm_map_print(db_expr_t addr, boolean_t have_addr, db_expr_t count, const char *modif) { - register vm_map_entry_t entry; - extern int indent; + vm_map_t map; + vm_map_entry_t entry; + + if (!have_addr) + map = current_thread()->task->map; + else + map = (vm_map_t)addr; - iprintf("Task map 0x%X: pmap=0x%X,", - (vm_offset_t) map, (vm_offset_t) (map->pmap)); + iprintf("Map 0x%X: name=\"%s\", pmap=0x%X,", + (vm_offset_t) map, map->name, (vm_offset_t) (map->pmap)); printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries); printf("version=%d\n", map->timestamp); - indent += 2; + indent += 1; for (entry = vm_map_first_entry(map); entry != vm_map_to_entry(map); entry = entry->vme_next) { static char *inheritance_name[3] = { "share", "copy", "none"}; iprintf("map entry 0x%X: ", (vm_offset_t) entry); - printf("start=0x%X, end=0x%X, ", + printf("start=0x%X, end=0x%X\n", (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end); - printf("prot=%X/%X/%s, ", + iprintf("prot=%X/%X/%s, ", entry->protection, entry->max_protection, inheritance_name[entry->inheritance]); @@ -4807,13 +4824,13 @@ void vm_map_print(map) if ((entry->vme_prev == vm_map_to_entry(map)) || (entry->vme_prev->object.vm_object != entry->object.vm_object)) { - indent += 2; + indent += 1; vm_object_print(entry->object.vm_object); - indent -= 2; + indent -= 1; } } } - indent -= 2; + indent -= 1; } /* @@ -4823,25 +4840,24 @@ void vm_map_print(map) */ void vm_map_copy_print(copy) - vm_map_copy_t copy; + const vm_map_copy_t copy; { - extern int indent; int i, npages; printf("copy object 0x%x\n", copy); - indent += 2; + indent += 1; iprintf("type=%d", copy->type); switch (copy->type) { case VM_MAP_COPY_ENTRY_LIST: printf("[entry_list]"); break; - + case VM_MAP_COPY_OBJECT: printf("[object]"); break; - + case VM_MAP_COPY_PAGE_LIST: printf("[page_list]"); break; @@ -4884,361 +4900,6 @@ void vm_map_copy_print(copy) break; } - indent -=2; -} -#endif MACH_KDB - -#if NORMA_IPC -/* - * This should one day be eliminated; - * we should always construct the right flavor of copy object - * the first time. Troublesome areas include vm_read, where vm_map_copyin - * is called without knowing whom the copy object is for. - * There are also situations where we do want a lazy data structure - * even if we are sending to a remote port... - */ - -/* - * Convert a copy to a page list. The copy argument is in/out - * because we probably have to allocate a new vm_map_copy structure. - * We take responsibility for discarding the old structure and - * use a continuation to do so. Postponing this discard ensures - * that the objects containing the pages we've marked busy will stick - * around. - */ -kern_return_t -vm_map_convert_to_page_list(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_map_entry_t entry, next_entry; - vm_offset_t va; - vm_offset_t offset; - vm_object_t object; - kern_return_t result; - vm_map_copy_t copy, new_copy; - int i, num_pages = 0; - - zone_t entry_zone; - - copy = *caller_copy; - - /* - * We may not have to do anything, - * or may not be able to do anything. - */ - if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - return vm_map_convert_to_page_list_from_object(caller_copy); - } - if (copy->type != VM_MAP_COPY_ENTRY_LIST) { - panic("vm_map_convert_to_page_list: copy type %d!\n", - copy->type); - } - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * Iterate over entries. - */ - for (entry = vm_map_copy_first_entry(copy); - entry != vm_map_copy_to_entry(copy); - entry = entry->vme_next) { - - object = entry->object.vm_object; - offset = entry->offset; - /* - * Iterate over pages. - */ - for (va = entry->vme_start; - va < entry->vme_end; - va += PAGE_SIZE, offset += PAGE_SIZE) { - - vm_page_t m; - - if (new_copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) { - /* - * What a mess. We need a continuation - * to do the page list, but also one - * to discard the old copy. The right - * thing to do is probably to copy - * out the old copy into the kernel - * map (or some temporary task holding - * map if we're paranoid about large - * copies), and then copyin the page - * list that we really wanted with - * src_destroy. LATER. - */ - panic("vm_map_convert_to_page_list: num\n"); - } - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - if (((m = vm_page_lookup(object, offset)) != - VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - - /* - * Also write-protect the page, so - * that the map`s owner cannot change - * the data. The busy bit will prevent - * faults on the page from succeeding - * until the copy is released; after - * that, the page can be re-entered - * as writable, since we didn`t alter - * the map entry. This scheme is a - * cheap copy-on-write. - * - * Don`t forget the protection and - * the page_lock value! - */ - - pmap_page_protect(m->phys_addr, - entry->protection - & ~m->page_lock - & ~VM_PROT_WRITE); - - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - } - - *caller_copy = new_copy; - return KERN_SUCCESS; -} - -kern_return_t -vm_map_convert_to_page_list_from_object(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_object_t object; - vm_offset_t offset; - vm_map_copy_t copy, new_copy; - - copy = *caller_copy; - assert(copy->type == VM_MAP_COPY_OBJECT); - object = copy->cpy_object; - assert(object->size == round_page(object->size)); - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * XXX memory_object_lock_request can probably bust this - * XXX See continuation comment in previous routine for solution. - */ - assert(object->size <= VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE); - - for (offset = 0; offset < object->size; offset += PAGE_SIZE) { - vm_page_t m; - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - m = vm_page_lookup(object, offset); - if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - - *caller_copy = new_copy; - return (KERN_SUCCESS); -} - -kern_return_t -vm_map_convert_from_page_list(copy) - vm_map_copy_t copy; -{ - vm_object_t object; - int i; - vm_map_entry_t new_entry; - vm_page_t *page_list; - - /* - * Check type of copy object. - */ - if (copy->type == VM_MAP_COPY_ENTRY_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - printf("vm_map_convert_from_page_list: COPY_OBJECT?"); - return KERN_SUCCESS; - } - if (copy->type != VM_MAP_COPY_PAGE_LIST) { - panic("vm_map_convert_from_page_list 0x%x %d", - copy, - copy->type); - } - - /* - * Make sure the pages are loose. This may be - * a "Can't Happen", but just to be safe ... - */ - page_list = ©->cpy_page_list[0]; - if ((*page_list)->tabled) - vm_map_copy_steal_pages(copy); - - /* - * Create object, and stuff pages into it. - */ - object = vm_object_allocate(copy->cpy_npages); - for (i = 0; i < copy->cpy_npages; i++) { - register vm_page_t m = *page_list++; - vm_page_insert(m, object, i * PAGE_SIZE); - m->busy = FALSE; - m->dirty = TRUE; - vm_page_activate(m); - } - - /* - * XXX If this page list contained a continuation, then - * XXX we're screwed. The right thing to do is probably do - * XXX the copyout, and then copyin the entry list we really - * XXX wanted. - */ - if (vm_map_copy_has_cont(copy)) - panic("convert_from_page_list: continuation"); - - /* - * Change type of copy object - */ - vm_map_copy_first_entry(copy) = - vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); - copy->type = VM_MAP_COPY_ENTRY_LIST; - copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = TRUE; - - /* - * Allocate and initialize an entry for object - */ - new_entry = vm_map_copy_entry_create(copy); - new_entry->vme_start = trunc_page(copy->offset); - new_entry->vme_end = round_page(copy->offset + copy->size); - new_entry->object.vm_object = object; - new_entry->offset = 0; - new_entry->is_shared = FALSE; - new_entry->is_sub_map = FALSE; - new_entry->needs_copy = FALSE; - new_entry->protection = VM_PROT_DEFAULT; - new_entry->max_protection = VM_PROT_ALL; - new_entry->inheritance = VM_INHERIT_DEFAULT; - new_entry->wired_count = 0; - new_entry->user_wired_count = 0; - new_entry->projected_on = 0; - - /* - * Insert entry into copy object, and return. - */ - vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - return(KERN_SUCCESS); + indent -= 1; } -#endif NORMA_IPC +#endif /* MACH_KDB */