--- Gnu-Mach/vm/vm_map.c 2020/09/02 04:42:52 1.1.1.2 +++ Gnu-Mach/vm/vm_map.c 2020/09/02 04:47:45 1.1.1.4 @@ -34,28 +34,37 @@ * Virtual memory mapping module. */ -#include - +#include #include #include #include #include #include -#include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include +#if MACH_KDB +#include +#include +#endif /* MACH_KDB */ + /* * Macros to copy a vm_map_entry. We must be careful to correctly * manage the wired page count. vm_map_entry_copy() creates a new * map entry to the same memory - the wired count in the new entry * must be set to zero. vm_map_entry_copy_full() creates a new * entry that is identical to the old entry. This preserves the - * wire count; it's used for map splitting and zone changing in + * wire count; it's used for map splitting and cache changing in * vm_map_copyout. */ #define vm_map_entry_copy(NEW,OLD) \ @@ -79,7 +88,7 @@ MACRO_END * Synchronization is required prior to most operations. * * Maps consist of an ordered doubly-linked list of simple - * entries; a single hint is used to speed up lookups. + * entries; a hint and a red-black tree are used to speed up lookups. * * Sharing maps have been deleted from this version of Mach. * All shared objects are now mapped directly into the respective @@ -115,12 +124,10 @@ MACRO_END * vm_object_copy_strategically() in vm_object.c. */ -zone_t vm_map_zone; /* zone for vm_map structures */ -zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ -zone_t vm_map_kentry_zone; /* zone for kernel entry structures */ -zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ - -boolean_t vm_map_lookup_entry(); /* forward declaration */ +struct kmem_cache vm_map_cache; /* cache for vm_map structures */ +struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */ +struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */ +struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */ /* * Placeholder object for submap operations. This object is dropped @@ -128,7 +135,8 @@ boolean_t vm_map_lookup_entry(); /* forw * vm_map_submap creates the submap. */ -vm_object_t vm_submap_object; +static struct vm_object vm_submap_object_store; +vm_object_t vm_submap_object = &vm_submap_object_store; /* * vm_map_init: @@ -136,51 +144,83 @@ vm_object_t vm_submap_object; * Initialize the vm_map module. Must be called before * any other vm_map routines. * - * Map and entry structures are allocated from zones -- we must - * initialize those zones. + * Map and entry structures are allocated from caches -- we must + * initialize those caches. * - * There are three zones of interest: + * There are three caches of interest: * - * vm_map_zone: used to allocate maps. - * vm_map_entry_zone: used to allocate map entries. - * vm_map_kentry_zone: used to allocate map entries for the kernel. - * - * The kernel allocates map entries from a special zone that is initially - * "crammed" with memory. It would be difficult (perhaps impossible) for - * the kernel to allocate more memory to a entry zone when it became - * empty since the very act of allocating memory implies the creation - * of a new entry. + * vm_map_cache: used to allocate maps. + * vm_map_entry_cache: used to allocate map entries. + * vm_map_kentry_cache: used to allocate map entries for the kernel. + * + * Kernel map entries are allocated from a special cache, using a custom + * page allocation function to avoid recursion. It would be difficult + * (perhaps impossible) for the kernel to allocate more memory to an entry + * cache when it became empty since the very act of allocating memory + * implies the creation of a new entry. */ vm_offset_t kentry_data; -vm_size_t kentry_data_size; -int kentry_count = 256; /* to init kentry_data_size */ +vm_size_t kentry_data_size = KENTRY_DATA_SIZE; -void vm_map_init() +static vm_offset_t kentry_pagealloc(vm_size_t size) { - vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024, - PAGE_SIZE, 0, "maps"); - vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - 1024*1024, PAGE_SIZE*5, - 0, "non-kernel map entries"); - vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), - kentry_data_size, kentry_data_size, - ZONE_FIXED /* XXX */, "kernel map entries"); + vm_offset_t result; - vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy), - 16*1024, PAGE_SIZE, 0, - "map copies"); + if (size > kentry_data_size) + panic("vm_map: kentry memory exhausted"); - /* - * Cram the kentry zone with initial data. - */ - zcram(vm_map_kentry_zone, kentry_data, kentry_data_size); + result = kentry_data; + kentry_data += size; + kentry_data_size -= size; + return result; +} + +void vm_map_init(void) +{ + kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0, + NULL, NULL, NULL, 0); + kmem_cache_init(&vm_map_entry_cache, "vm_map_entry", + sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0); + kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry", + sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc, + NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB + | KMEM_CACHE_NORECLAIM); + kmem_cache_init(&vm_map_copy_cache, "vm_map_copy", + sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0); /* * Submap object is initialized by vm_object_init. */ } +void vm_map_setup( + vm_map_t map, + pmap_t pmap, + vm_offset_t min, + vm_offset_t max, + boolean_t pageable) +{ + vm_map_first_entry(map) = vm_map_to_entry(map); + vm_map_last_entry(map) = vm_map_to_entry(map); + map->hdr.nentries = 0; + map->hdr.entries_pageable = pageable; + rbtree_init(&map->hdr.tree); + + map->size = 0; + map->ref_count = 1; + map->pmap = pmap; + map->min_offset = min; + map->max_offset = max; + map->wiring_required = FALSE; + map->wait_for_space = FALSE; + map->first_free = vm_map_to_entry(map); + map->hint = vm_map_to_entry(map); + vm_map_lock_init(map); + simple_lock_init(&map->ref_lock); + simple_lock_init(&map->hint_lock); +} + /* * vm_map_create: * @@ -188,34 +228,19 @@ void vm_map_init() * the given physical map structure, and having * the given lower and upper address bounds. */ -vm_map_t vm_map_create(pmap, min, max, pageable) - pmap_t pmap; - vm_offset_t min, max; - boolean_t pageable; +vm_map_t vm_map_create( + pmap_t pmap, + vm_offset_t min, + vm_offset_t max, + boolean_t pageable) { - register vm_map_t result; + vm_map_t result; - result = (vm_map_t) zalloc(vm_map_zone); + result = (vm_map_t) kmem_cache_alloc(&vm_map_cache); if (result == VM_MAP_NULL) panic("vm_map_create"); - vm_map_first_entry(result) = vm_map_to_entry(result); - vm_map_last_entry(result) = vm_map_to_entry(result); - result->hdr.nentries = 0; - result->hdr.entries_pageable = pageable; - - result->size = 0; - result->ref_count = 1; - result->pmap = pmap; - result->min_offset = min; - result->max_offset = max; - result->wiring_required = FALSE; - result->wait_for_space = FALSE; - result->first_free = vm_map_to_entry(result); - result->hint = vm_map_to_entry(result); - vm_map_lock_init(result); - simple_lock_init(&result->ref_lock); - simple_lock_init(&result->hint_lock); + vm_map_setup(result, pmap, min, max, pageable); return(result); } @@ -233,17 +258,17 @@ vm_map_t vm_map_create(pmap, min, max, p _vm_map_entry_create(&(copy)->cpy_hdr) vm_map_entry_t _vm_map_entry_create(map_header) - register struct vm_map_header *map_header; + const struct vm_map_header *map_header; { - register zone_t zone; - register vm_map_entry_t entry; + kmem_cache_t cache; + vm_map_entry_t entry; if (map_header->entries_pageable) - zone = vm_map_entry_zone; + cache = &vm_map_entry_cache; else - zone = vm_map_kentry_zone; + cache = &vm_map_kentry_cache; - entry = (vm_map_entry_t) zalloc(zone); + entry = (vm_map_entry_t) kmem_cache_alloc(cache); if (entry == VM_MAP_ENTRY_NULL) panic("vm_map_entry_create"); @@ -262,23 +287,53 @@ vm_map_entry_t _vm_map_entry_create(map_ _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry)) void _vm_map_entry_dispose(map_header, entry) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; + const struct vm_map_header *map_header; + vm_map_entry_t entry; { - register zone_t zone; + kmem_cache_t cache; if (map_header->entries_pageable) - zone = vm_map_entry_zone; + cache = &vm_map_entry_cache; else - zone = vm_map_kentry_zone; + cache = &vm_map_kentry_cache; - zfree(zone, (vm_offset_t) entry); + kmem_cache_free(cache, (vm_offset_t) entry); +} + +/* + * Red-black tree lookup/insert comparison functions + */ +static inline int vm_map_entry_cmp_lookup(vm_offset_t addr, + const struct rbtree_node *node) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(node, struct vm_map_entry, tree_node); + + if (addr < entry->vme_start) + return -1; + else if (addr < entry->vme_end) + return 0; + else + return 1; +} + +static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a, + const struct rbtree_node *b) +{ + struct vm_map_entry *entry; + + entry = rbtree_entry(a, struct vm_map_entry, tree_node); + return vm_map_entry_cmp_lookup(entry->vme_start, b); } /* * vm_map_entry_{un,}link: * * Insert/remove entries from maps (or map copies). + * + * The start and end addresses of the entries must be properly set + * before using these macros. */ #define vm_map_entry_link(map, after_where, entry) \ _vm_map_entry_link(&(map)->hdr, after_where, entry) @@ -293,6 +348,8 @@ void _vm_map_entry_dispose(map_header, e (entry)->vme_next = (after_where)->vme_next; \ (entry)->vme_prev->vme_next = \ (entry)->vme_next->vme_prev = (entry); \ + rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \ + vm_map_entry_cmp_insert); \ MACRO_END #define vm_map_entry_unlink(map, entry) \ @@ -306,6 +363,7 @@ void _vm_map_entry_dispose(map_header, e (hdr)->nentries--; \ (entry)->vme_next->vme_prev = (entry)->vme_prev; \ (entry)->vme_prev->vme_next = (entry)->vme_next; \ + rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \ MACRO_END /* @@ -314,8 +372,7 @@ void _vm_map_entry_dispose(map_header, e * Creates another valid reference to the given map. * */ -void vm_map_reference(map) - register vm_map_t map; +void vm_map_reference(vm_map_t map) { if (map == VM_MAP_NULL) return; @@ -332,10 +389,9 @@ void vm_map_reference(map) * destroying it if no references remain. * The map should not be locked. */ -void vm_map_deallocate(map) - register vm_map_t map; +void vm_map_deallocate(vm_map_t map) { - register int c; + int c; if (map == VM_MAP_NULL) return; @@ -353,7 +409,7 @@ void vm_map_deallocate(map) pmap_destroy(map->pmap); - zfree(vm_map_zone, (vm_offset_t) map); + kmem_cache_free(&vm_map_cache, (vm_offset_t) map); } /* @@ -377,75 +433,54 @@ void vm_map_deallocate(map) * result indicates whether the address is * actually contained in the map. */ -boolean_t vm_map_lookup_entry(map, address, entry) - register vm_map_t map; - register vm_offset_t address; - vm_map_entry_t *entry; /* OUT */ +boolean_t vm_map_lookup_entry( + vm_map_t map, + vm_offset_t address, + vm_map_entry_t *entry) /* OUT */ { - register vm_map_entry_t cur; - register vm_map_entry_t last; + struct rbtree_node *node; + vm_map_entry_t hint; /* - * Start looking either from the head of the - * list, or from the hint. + * First, make a quick check to see if we are already + * looking at the entry we want (which is often the case). */ simple_lock(&map->hint_lock); - cur = map->hint; + hint = map->hint; simple_unlock(&map->hint_lock); - if (cur == vm_map_to_entry(map)) - cur = cur->vme_next; - - if (address >= cur->vme_start) { - /* - * Go from hint to end of list. - * - * But first, make a quick check to see if - * we are already looking at the entry we - * want (which is usually the case). - * Note also that we don't need to save the hint - * here... it is the same hint (unless we are - * at the header, in which case the hint didn't - * buy us anything anyway). - */ - last = vm_map_to_entry(map); - if ((cur != last) && (cur->vme_end > address)) { - *entry = cur; + if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) { + if (address < hint->vme_end) { + *entry = hint; return(TRUE); + } else { + vm_map_entry_t next = hint->vme_next; + + if ((next == vm_map_to_entry(map)) + || (address < next->vme_start)) { + *entry = hint; + return(FALSE); + } } } - else { - /* - * Go from start to hint, *inclusively* - */ - last = cur->vme_next; - cur = vm_map_first_entry(map); - } /* - * Search linearly + * If the hint didn't help, use the red-black tree. */ - while (cur != last) { - if (cur->vme_end > address) { - if (address >= cur->vme_start) { - /* - * Save this lookup for future - * hints, and return - */ + node = rbtree_lookup_nearest(&map->hdr.tree, address, + vm_map_entry_cmp_lookup, RBTREE_LEFT); - *entry = cur; - SAVE_HINT(map, cur); - return(TRUE); - } - break; - } - cur = cur->vme_next; + if (node == NULL) { + *entry = vm_map_to_entry(map); + SAVE_HINT(map, *entry); + return(FALSE); + } else { + *entry = rbtree_entry(node, struct vm_map_entry, tree_node); + SAVE_HINT(map, *entry); + return((address < (*entry)->vme_end) ? TRUE : FALSE); } - *entry = cur->vme_prev; - SAVE_HINT(map, *entry); - return(FALSE); } /* @@ -455,10 +490,11 @@ boolean_t vm_map_lookup_entry(map, addre */ boolean_t -invalid_user_access(map, start, end, prot) - vm_map_t map; - vm_offset_t start, end; - vm_prot_t prot; +invalid_user_access( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot) { vm_map_entry_t entry; @@ -482,17 +518,17 @@ invalid_user_access(map, start, end, pro * are initialized to zero. If an object is supplied, * then an existing entry may be extended. */ -kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry) - register vm_map_t map; - vm_offset_t *address; /* OUT */ - vm_size_t size; - vm_offset_t mask; - vm_object_t object; - vm_map_entry_t *o_entry; /* OUT */ +kern_return_t vm_map_find_entry( + vm_map_t map, + vm_offset_t *address, /* OUT */ + vm_size_t size, + vm_offset_t mask, + vm_object_t object, + vm_map_entry_t *o_entry) /* OUT */ { - register vm_map_entry_t entry, new_entry; - register vm_offset_t start; - register vm_offset_t end; + vm_map_entry_t entry, new_entry; + vm_offset_t start; + vm_offset_t end; /* * Look for the first possible address; @@ -511,7 +547,7 @@ kern_return_t vm_map_find_entry(map, add */ while (TRUE) { - register vm_map_entry_t next; + vm_map_entry_t next; /* * Find the end of the proposed new region. @@ -519,13 +555,17 @@ kern_return_t vm_map_find_entry(map, add * wrap around the address. */ - if (((start + mask) & ~mask) < start) + if (((start + mask) & ~mask) < start) { + printf_once("no more room for vm_map_find_entry in %p\n", map); return(KERN_NO_SPACE); + } start = ((start + mask) & ~mask); end = start + size; - if ((end > map->max_offset) || (end < start)) + if ((end > map->max_offset) || (end < start)) { + printf_once("no more room for vm_map_find_entry in %p\n", map); return(KERN_NO_SPACE); + } /* * If there are no more entries, we must win. @@ -632,8 +672,8 @@ kern_return_t vm_map_find_entry(map, add return(KERN_SUCCESS); } -int vm_map_pmap_enter_print = FALSE; -int vm_map_pmap_enter_enable = FALSE; +boolean_t vm_map_pmap_enter_print = FALSE; +boolean_t vm_map_pmap_enter_enable = FALSE; /* * Routine: vm_map_pmap_enter @@ -650,19 +690,16 @@ int vm_map_pmap_enter_enable = FALSE; * The source map should not be locked on entry. */ void -vm_map_pmap_enter(map, addr, end_addr, object, offset, protection) - vm_map_t map; - register - vm_offset_t addr; - register - vm_offset_t end_addr; - register - vm_object_t object; - vm_offset_t offset; - vm_prot_t protection; +vm_map_pmap_enter( + vm_map_t map, + vm_offset_t addr, + vm_offset_t end_addr, + vm_object_t object, + vm_offset_t offset, + vm_prot_t protection) { while (addr < end_addr) { - register vm_page_t m; + vm_page_t m; vm_object_lock(object); vm_object_paging_begin(object); @@ -676,7 +713,7 @@ vm_map_pmap_enter(map, addr, end_addr, o if (vm_map_pmap_enter_print) { printf("vm_map_pmap_enter:"); - printf("map: %x, addr: %x, object: %x, offset: %x\n", + printf("map: %p, addr: %lx, object: %p, offset: %lx\n", map, addr, object, offset); } @@ -711,30 +748,28 @@ vm_map_pmap_enter(map, addr, end_addr, o * Arguments are as defined in the vm_map call. */ kern_return_t vm_map_enter( - map, - address, size, mask, anywhere, - object, offset, needs_copy, - cur_protection, max_protection, inheritance) - register - vm_map_t map; - vm_offset_t *address; /* IN/OUT */ - vm_size_t size; - vm_offset_t mask; - boolean_t anywhere; - vm_object_t object; - vm_offset_t offset; - boolean_t needs_copy; - vm_prot_t cur_protection; - vm_prot_t max_protection; - vm_inherit_t inheritance; -{ - register vm_map_entry_t entry; - register vm_offset_t start; - register vm_offset_t end; - kern_return_t result = KERN_SUCCESS; + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t size, + vm_offset_t mask, + boolean_t anywhere, + vm_object_t object, + vm_offset_t offset, + boolean_t needs_copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) +{ + vm_map_entry_t entry; + vm_offset_t start; + vm_offset_t end; + kern_return_t result = KERN_SUCCESS; #define RETURN(value) { result = value; goto BailOut; } + if (size == 0) + return KERN_INVALID_ARGUMENT; + StartAgain: ; start = *address; @@ -774,7 +809,7 @@ kern_return_t vm_map_enter( */ while (TRUE) { - register vm_map_entry_t next; + vm_map_entry_t next; /* * Find the end of the proposed new region. @@ -782,8 +817,10 @@ kern_return_t vm_map_enter( * wrap around the address. */ - if (((start + mask) & ~mask) < start) - return(KERN_NO_SPACE); + if (((start + mask) & ~mask) < start) { + printf_once("no more room for vm_map_enter in %p\n", map); + RETURN(KERN_NO_SPACE); + } start = ((start + mask) & ~mask); end = start + size; @@ -798,6 +835,7 @@ kern_return_t vm_map_enter( } } + printf_once("no more room for vm_map_enter in %p\n", map); RETURN(KERN_NO_SPACE); } @@ -919,7 +957,7 @@ kern_return_t vm_map_enter( */ /**/ { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; new_entry = vm_map_entry_create(map); @@ -990,14 +1028,12 @@ kern_return_t vm_map_enter( * the specified address; if necessary, * it splits the entry into two. */ -void _vm_map_clip_start(); #define vm_map_clip_start(map, entry, startaddr) \ MACRO_BEGIN \ if ((startaddr) > (entry)->vme_start) \ _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \ MACRO_END -void _vm_map_copy_clip_start(); #define vm_map_copy_clip_start(copy, entry, startaddr) \ MACRO_BEGIN \ if ((startaddr) > (entry)->vme_start) \ @@ -1008,12 +1044,12 @@ void _vm_map_copy_clip_start(); * This routine is called only when it is known that * the entry must be split. */ -void _vm_map_clip_start(map_header, entry, start) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; - register vm_offset_t start; +void _vm_map_clip_start( + struct vm_map_header *map_header, + vm_map_entry_t entry, + vm_offset_t start) { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; /* * Split off the front portion -- @@ -1045,14 +1081,12 @@ void _vm_map_clip_start(map_header, entr * the specified address; if necessary, * it splits the entry into two. */ -void _vm_map_clip_end(); #define vm_map_clip_end(map, entry, endaddr) \ MACRO_BEGIN \ if ((endaddr) < (entry)->vme_end) \ _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \ MACRO_END -void _vm_map_copy_clip_end(); #define vm_map_copy_clip_end(copy, entry, endaddr) \ MACRO_BEGIN \ if ((endaddr) < (entry)->vme_end) \ @@ -1063,12 +1097,12 @@ void _vm_map_copy_clip_end(); * This routine is called only when it is known that * the entry must be split. */ -void _vm_map_clip_end(map_header, entry, end) - register struct vm_map_header *map_header; - register vm_map_entry_t entry; - register vm_offset_t end; +void _vm_map_clip_end( + struct vm_map_header *map_header, + vm_map_entry_t entry, + vm_offset_t end) { - register vm_map_entry_t new_entry; + vm_map_entry_t new_entry; /* * Create a new entry and insert it @@ -1123,15 +1157,15 @@ void _vm_map_clip_end(map_header, entry, * range from the superior map, and then destroy the * submap (if desired). [Better yet, don't try it.] */ -kern_return_t vm_map_submap(map, start, end, submap) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - vm_map_t submap; +kern_return_t vm_map_submap( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_map_t submap) { vm_map_entry_t entry; - register kern_return_t result = KERN_INVALID_ARGUMENT; - register vm_object_t object; + kern_return_t result = KERN_INVALID_ARGUMENT; + vm_object_t object; vm_map_lock(map); @@ -1171,15 +1205,15 @@ kern_return_t vm_map_submap(map, start, * specified, the maximum protection is to be set; * otherwise, only the current protection is affected. */ -kern_return_t vm_map_protect(map, start, end, new_prot, set_max) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_prot_t new_prot; - register boolean_t set_max; +kern_return_t vm_map_protect( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t new_prot, + boolean_t set_max) { - register vm_map_entry_t current; - vm_map_entry_t entry; + vm_map_entry_t current; + vm_map_entry_t entry; vm_map_lock(map); @@ -1259,13 +1293,13 @@ kern_return_t vm_map_protect(map, start, * affects how the map will be shared with * child maps at the time of vm_map_fork. */ -kern_return_t vm_map_inherit(map, start, end, new_inheritance) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_inherit_t new_inheritance; +kern_return_t vm_map_inherit( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_inherit_t new_inheritance) { - register vm_map_entry_t entry; + vm_map_entry_t entry; vm_map_entry_t temp_entry; vm_map_lock(map); @@ -1308,14 +1342,14 @@ kern_return_t vm_map_inherit(map, start, * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable, * or vm_map_pageable_user); don't call vm_map_pageable directly. */ -kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; - register vm_prot_t access_type; - boolean_t user_wire; +kern_return_t vm_map_pageable_common( + vm_map_t map, + vm_offset_t start, + vm_offset_t end, + vm_prot_t access_type, + boolean_t user_wire) { - register vm_map_entry_t entry; + vm_map_entry_t entry; vm_map_entry_t start_entry; vm_map_lock(map); @@ -1557,12 +1591,12 @@ kern_return_t vm_map_pageable_common(map * * Deallocate the given entry from the target map. */ -void vm_map_entry_delete(map, entry) - register vm_map_t map; - register vm_map_entry_t entry; +void vm_map_entry_delete( + vm_map_t map, + vm_map_entry_t entry) { - register vm_offset_t s, e; - register vm_object_t object; + vm_offset_t s, e; + vm_object_t object; extern vm_object_t kernel_object; s = entry->vme_start; @@ -1641,10 +1675,10 @@ void vm_map_entry_delete(map, entry) * map. */ -kern_return_t vm_map_delete(map, start, end) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; +kern_return_t vm_map_delete( + vm_map_t map, + vm_offset_t start, + vm_offset_t end) { vm_map_entry_t entry; vm_map_entry_t first_entry; @@ -1657,67 +1691,6 @@ kern_return_t vm_map_delete(map, start, entry = first_entry->vme_next; else { entry = first_entry; -#if NORMA_IPC_xxx - /* - * XXX Had to disable this code because: - - _vm_map_delete(c0804b78,c2198000,c219a000,0,c219a000)+df - [vm/vm_map.c:2007] - _vm_map_remove(c0804b78,c2198000,c219a000,c0817834, - c081786c)+42 [vm/vm_map.c:2094] - _kmem_io_map_deallocate(c0804b78,c2198000,2000,c0817834, - c081786c)+43 [vm/vm_kern.c:818] - _device_write_dealloc(c081786c)+117 [device/ds_routines.c:814] - _ds_write_done(c081786c,0)+2e [device/ds_routines.c:848] - _io_done_thread_continue(c08150c0,c21d4e14,c21d4e30,c08150c0, - c080c114)+14 [device/ds_routines.c:1350] - - */ - if (start > entry->vme_start - && end == entry->vme_end - && ! entry->wired_count /* XXX ??? */ - && ! entry->is_shared - && ! entry->projected_on - && ! entry->is_sub_map) { - extern vm_object_t kernel_object; - register vm_object_t object = entry->object.vm_object; - - /* - * The region to be deleted lives at the end - * of this entry, and thus all we have to do is - * truncate the entry. - * - * This special case is necessary if we want - * coalescing to do us any good. - * - * XXX Do we have to adjust object size? - */ - if (object == kernel_object) { - vm_object_lock(object); - vm_object_page_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - vm_object_unlock(object); - } else if (entry->is_shared) { - vm_object_pmap_remove(object, - entry->offset + start, - entry->offset + - (end - start)); - } else { - pmap_remove(map->pmap, start, end); - } - object->size -= (end - start); /* XXX */ - - entry->vme_end = start; - map->size -= (end - start); - - if (map->wait_for_space) { - thread_wakeup((event_t) map); - } - return KERN_SUCCESS; - } -#endif /* NORMA_IPC */ vm_map_clip_start(map, entry, start); /* @@ -1785,12 +1758,12 @@ kern_return_t vm_map_delete(map, start, * Remove the given address range from the target map. * This is the exported form of vm_map_delete. */ -kern_return_t vm_map_remove(map, start, end) - register vm_map_t map; - register vm_offset_t start; - register vm_offset_t end; +kern_return_t vm_map_remove( + vm_map_t map, + vm_offset_t start, + vm_offset_t end) { - register kern_return_t result; + kern_return_t result; vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); @@ -1808,12 +1781,11 @@ kern_return_t vm_map_remove(map, start, * that have not already been stolen. */ void -vm_map_copy_steal_pages(copy) -vm_map_copy_t copy; +vm_map_copy_steal_pages(vm_map_copy_t copy) { - register vm_page_t m, new_m; - register int i; - vm_object_t object; + vm_page_t m, new_m; + int i; + vm_object_t object; for (i = 0; i < copy->cpy_npages; i++) { @@ -1855,8 +1827,7 @@ vm_map_copy_t copy; * stolen, they are freed. If the pages are not stolen, they * are unbusied, and associated state is cleaned up. */ -void vm_map_copy_page_discard(copy) -vm_map_copy_t copy; +void vm_map_copy_page_discard(vm_map_copy_t copy) { while (copy->cpy_npages > 0) { vm_page_t m; @@ -1901,8 +1872,7 @@ vm_map_copy_t copy; * vm_map_copyin). */ void -vm_map_copy_discard(copy) - vm_map_copy_t copy; +vm_map_copy_discard(vm_map_copy_t copy) { free_next_copy: if (copy == VM_MAP_COPY_NULL) @@ -1943,10 +1913,10 @@ free_next_copy: * here to avoid tail recursion. */ if (copy->cpy_cont == vm_map_copy_discard_cont) { - register vm_map_copy_t new_copy; + vm_map_copy_t new_copy; new_copy = (vm_map_copy_t) copy->cpy_cont_args; - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); copy = new_copy; goto free_next_copy; } @@ -1957,7 +1927,7 @@ free_next_copy: break; } - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } /* @@ -1978,8 +1948,7 @@ free_next_copy: * deallocation will not fail. */ vm_map_copy_t -vm_map_copy_copy(copy) - vm_map_copy_t copy; +vm_map_copy_copy(vm_map_copy_t copy) { vm_map_copy_t new_copy; @@ -1991,7 +1960,7 @@ vm_map_copy_copy(copy) * from the old one into it. */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); *new_copy = *copy; if (copy->type == VM_MAP_COPY_ENTRY_LIST) { @@ -2025,9 +1994,9 @@ vm_map_copy_copy(copy) * A version of vm_map_copy_discard that can be called * as a continuation from a vm_map_copy page list. */ -kern_return_t vm_map_copy_discard_cont(cont_args, copy_result) -vm_map_copyin_args_t cont_args; -vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copy_discard_cont( +vm_map_copyin_args_t cont_args, +vm_map_copy_t *copy_result) /* OUT */ { vm_map_copy_discard((vm_map_copy_t) cont_args); if (copy_result != (vm_map_copy_t *)0) @@ -2082,11 +2051,11 @@ vm_map_copy_t *copy_result; /* OUT */ * atomically and interruptibly, an error indication is * returned. */ -kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible) - vm_map_t dst_map; - vm_offset_t dst_addr; - vm_map_copy_t copy; - boolean_t interruptible; +kern_return_t vm_map_copy_overwrite( + vm_map_t dst_map, + vm_offset_t dst_addr, + vm_map_copy_t copy, + boolean_t interruptible) { vm_size_t size; vm_offset_t start; @@ -2109,11 +2078,7 @@ kern_return_t vm_map_copy_overwrite(dst_ * support page lists LATER. */ -#if NORMA_IPC - vm_map_convert_from_page_list(copy); -#else assert(copy->type == VM_MAP_COPY_ENTRY_LIST); -#endif /* * Currently this routine only handles page-aligned @@ -2196,12 +2161,14 @@ start_pass_1: * the copy cannot be interrupted. */ - if (interruptible && contains_permanent_objects) + if (interruptible && contains_permanent_objects) { + vm_map_unlock(dst_map); return(KERN_FAILURE); /* XXX */ + } /* * XXXO If there are no permanent objects in the destination, - * XXXO and the source and destination map entry zones match, + * XXXO and the source and destination map entry caches match, * XXXO and the destination map entry is not shared, * XXXO then the map entries can be deleted and replaced * XXXO with those from the copy. The following code is the @@ -2439,12 +2406,16 @@ start_pass_1: */ #define vm_map_copy_insert(map, where, copy) \ MACRO_BEGIN \ + struct rbtree_node *node, *tmp; \ + rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \ + rbtree_insert(&(map)->hdr.tree, node, \ + vm_map_entry_cmp_insert); \ (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \ ->vme_next = ((where)->vme_next); \ ((where)->vme_next = vm_map_copy_first_entry(copy)) \ ->vme_prev = (where); \ (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \ - zfree(vm_map_copy_zone, (vm_offset_t) copy); \ + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \ MACRO_END /* @@ -2457,19 +2428,16 @@ start_pass_1: * If successful, consumes the copy object. * Otherwise, the caller is responsible for it. */ -kern_return_t vm_map_copyout(dst_map, dst_addr, copy) - register - vm_map_t dst_map; - vm_offset_t *dst_addr; /* OUT */ - register - vm_map_copy_t copy; +kern_return_t vm_map_copyout( + vm_map_t dst_map, + vm_offset_t *dst_addr, /* OUT */ + vm_map_copy_t copy) { vm_size_t size; vm_size_t adjustment; vm_offset_t start; vm_offset_t vm_copy_start; vm_map_entry_t last; - register vm_map_entry_t entry; /* @@ -2500,7 +2468,7 @@ kern_return_t vm_map_copyout(dst_map, ds VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return(kr); - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); return(KERN_SUCCESS); } @@ -2534,6 +2502,7 @@ kern_return_t vm_map_copyout(dst_map, ds } } vm_map_unlock(dst_map); + printf_once("no more room for vm_map_copyout in %p\n", dst_map); return(KERN_NO_SPACE); } @@ -2556,15 +2525,15 @@ kern_return_t vm_map_copyout(dst_map, ds * Mismatches occur when dealing with the default * pager. */ - zone_t old_zone; + kmem_cache_t old_cache; vm_map_entry_t next, new; /* - * Find the zone that the copies were allocated from + * Find the cache that the copies were allocated from */ - old_zone = (copy->cpy_hdr.entries_pageable) - ? vm_map_entry_zone - : vm_map_kentry_zone; + old_cache = (copy->cpy_hdr.entries_pageable) + ? &vm_map_entry_cache + : &vm_map_kentry_cache; entry = vm_map_copy_first_entry(copy); /* @@ -2587,7 +2556,7 @@ kern_return_t vm_map_copyout(dst_map, ds vm_map_copy_last_entry(copy), new); next = entry->vme_next; - zfree(old_zone, (vm_offset_t) entry); + kmem_cache_free(old_cache, (vm_offset_t) entry); entry = next; } } @@ -2614,9 +2583,9 @@ kern_return_t vm_map_copyout(dst_map, ds * map the pages into the destination map. */ if (entry->wired_count != 0) { - register vm_offset_t va; - vm_offset_t offset; - register vm_object_t object; + vm_offset_t va; + vm_offset_t offset; + vm_object_t object; object = entry->object.vm_object; offset = entry->offset; @@ -2628,7 +2597,7 @@ kern_return_t vm_map_copyout(dst_map, ds TRUE); while (va < entry->vme_end) { - register vm_page_t m; + vm_page_t m; /* * Look up the page in the object. @@ -2713,19 +2682,16 @@ kern_return_t vm_map_copyout(dst_map, ds * Version of vm_map_copyout() for page list vm map copies. * */ -kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy) - register - vm_map_t dst_map; - vm_offset_t *dst_addr; /* OUT */ - register - vm_map_copy_t copy; +kern_return_t vm_map_copyout_page_list( + vm_map_t dst_map, + vm_offset_t *dst_addr, /* OUT */ + vm_map_copy_t copy) { vm_size_t size; vm_offset_t start; vm_offset_t end; vm_offset_t offset; vm_map_entry_t last; - register vm_object_t object; vm_page_t *page_list, m; vm_map_entry_t entry; @@ -2778,6 +2744,7 @@ StartAgain: } } vm_map_unlock(dst_map); + printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map); return(KERN_NO_SPACE); } @@ -3075,10 +3042,10 @@ error: * Consume on success logic. */ if (copy != orig_copy) { - zfree(vm_map_copy_zone, (vm_offset_t) copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); } if (result == KERN_SUCCESS) { - zfree(vm_map_copy_zone, (vm_offset_t) orig_copy); + kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy); } return(result); @@ -3102,12 +3069,12 @@ error: * In/out conditions: * The source map should not be locked on entry. */ -kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result) - vm_map_t src_map; - vm_offset_t src_addr; - vm_size_t len; - boolean_t src_destroy; - vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin( + vm_map_t src_map, + vm_offset_t src_addr, + vm_size_t len, + boolean_t src_destroy, + vm_map_copy_t *copy_result) /* OUT */ { vm_map_entry_t tmp_entry; /* Result of last map lookup -- * in multi-level lookup, this @@ -3121,7 +3088,6 @@ kern_return_t vm_map_copyin(src_map, src vm_offset_t src_end; /* End of entire region to be * copied */ - register vm_map_copy_t copy; /* Resulting copy */ /* @@ -3155,12 +3121,13 @@ kern_return_t vm_map_copyin(src_map, src * remember the endpoints prior to rounding. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); copy->type = VM_MAP_COPY_ENTRY_LIST; copy->cpy_hdr.nentries = 0; copy->cpy_hdr.entries_pageable = TRUE; + rbtree_init(©->cpy_hdr.tree); copy->offset = src_addr; copy->size = len; @@ -3187,14 +3154,12 @@ kern_return_t vm_map_copyin(src_map, src */ while (TRUE) { - register vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */ vm_size_t src_size; /* Size of source * map entry (in both * maps) */ - register vm_object_t src_object; /* Object to copy */ vm_offset_t src_offset; @@ -3203,7 +3168,6 @@ kern_return_t vm_map_copyin(src_map, src * for copy-on-write? */ - register vm_map_entry_t new_entry; /* Map entry for copy */ boolean_t new_entry_needs_copy; /* Will new entry be COW? */ @@ -3467,11 +3431,11 @@ kern_return_t vm_map_copyin(src_map, src * Our caller donates an object reference. */ -kern_return_t vm_map_copyin_object(object, offset, size, copy_result) - vm_object_t object; - vm_offset_t offset; /* offset of region in object */ - vm_size_t size; /* size of region in object */ - vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin_object( + vm_object_t object, + vm_offset_t offset, /* offset of region in object */ + vm_size_t size, /* size of region in object */ + vm_map_copy_t *copy_result) /* OUT */ { vm_map_copy_t copy; /* Resulting copy */ @@ -3482,7 +3446,7 @@ kern_return_t vm_map_copyin_object(objec * and null links. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); vm_map_copy_first_entry(copy) = vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL; copy->type = VM_MAP_COPY_OBJECT; @@ -3512,12 +3476,12 @@ kern_return_t vm_map_copyin_object(objec * the scheduler. */ -kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result) -vm_map_copyin_args_t cont_args; -vm_map_copy_t *copy_result; /* OUT */ +kern_return_t vm_map_copyin_page_list_cont( + vm_map_copyin_args_t cont_args, + vm_map_copy_t *copy_result) /* OUT */ { kern_return_t result = 0; /* '=0' to quiet gcc warnings */ - register boolean_t do_abort, src_destroy, src_destroy_only; + boolean_t do_abort, src_destroy, src_destroy_only; /* * Check for cases that only require memory destruction. @@ -3568,27 +3532,23 @@ vm_map_copy_t *copy_result; /* OUT */ * the recipient of this copy_result must be prepared to deal with it. */ -kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy, - steal_pages, copy_result, is_cont) - vm_map_t src_map; - vm_offset_t src_addr; - vm_size_t len; - boolean_t src_destroy; - boolean_t steal_pages; - vm_map_copy_t *copy_result; /* OUT */ - boolean_t is_cont; +kern_return_t vm_map_copyin_page_list( + vm_map_t src_map, + vm_offset_t src_addr, + vm_size_t len, + boolean_t src_destroy, + boolean_t steal_pages, + vm_map_copy_t *copy_result, /* OUT */ + boolean_t is_cont) { vm_map_entry_t src_entry; vm_page_t m; vm_offset_t src_start; vm_offset_t src_end; vm_size_t src_size; - register vm_object_t src_object; - register vm_offset_t src_offset; vm_offset_t src_last_offset; - register vm_map_copy_t copy; /* Resulting copy */ kern_return_t result = KERN_SUCCESS; boolean_t need_map_lookup; @@ -3637,7 +3597,7 @@ kern_return_t vm_map_copyin_page_list(sr * be page-aligned. */ - copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); + copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache); copy->type = VM_MAP_COPY_PAGE_LIST; copy->cpy_npages = 0; copy->offset = src_addr; @@ -3922,7 +3882,7 @@ retry: */ src_start = trunc_page(src_addr); if (steal_pages) { - register int i; + int i; vm_offset_t unwire_end; unwire_end = src_start; @@ -4099,18 +4059,14 @@ error: * * The source map must not be locked. */ -vm_map_t vm_map_fork(old_map) - vm_map_t old_map; +vm_map_t vm_map_fork(vm_map_t old_map) { vm_map_t new_map; - register vm_map_entry_t old_entry; - register vm_map_entry_t new_entry; pmap_t new_pmap = pmap_create((vm_size_t) 0); vm_size_t new_size = 0; vm_size_t entry_size; - register vm_object_t object; vm_map_lock(old_map); @@ -4196,6 +4152,8 @@ vm_map_t vm_map_fork(old_map) object->ref_count++; vm_object_unlock(object); + new_entry = vm_map_entry_create(new_map); + if (old_entry->projected_on != 0) { /* * If entry is projected buffer, clone the @@ -4210,7 +4168,6 @@ vm_map_t vm_map_fork(old_map) * Mark both entries as shared. */ - new_entry = vm_map_entry_create(new_map); vm_map_entry_copy(new_entry, old_entry); old_entry->is_shared = TRUE; new_entry->is_shared = TRUE; @@ -4372,21 +4329,20 @@ vm_map_t vm_map_fork(old_map) * copying operations, although the data referenced will * remain the same. */ -kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version, - object, offset, out_prot, wired) - vm_map_t *var_map; /* IN/OUT */ - register vm_offset_t vaddr; - register vm_prot_t fault_type; - - vm_map_version_t *out_version; /* OUT */ - vm_object_t *object; /* OUT */ - vm_offset_t *offset; /* OUT */ - vm_prot_t *out_prot; /* OUT */ - boolean_t *wired; /* OUT */ -{ - register vm_map_entry_t entry; - register vm_map_t map = *var_map; - register vm_prot_t prot; +kern_return_t vm_map_lookup( + vm_map_t *var_map, /* IN/OUT */ + vm_offset_t vaddr, + vm_prot_t fault_type, + + vm_map_version_t *out_version, /* OUT */ + vm_object_t *object, /* OUT */ + vm_offset_t *offset, /* OUT */ + vm_prot_t *out_prot, /* OUT */ + boolean_t *wired) /* OUT */ +{ + vm_map_entry_t entry; + vm_map_t map = *var_map; + vm_prot_t prot; RetryLookup: ; @@ -4444,19 +4400,20 @@ kern_return_t vm_map_lookup(var_map, vad prot = entry->protection; - if ((fault_type & (prot)) != fault_type) + if ((fault_type & (prot)) != fault_type) { if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) { RETURN(KERN_WRITE_PROTECTION_FAILURE); } else { RETURN(KERN_PROTECTION_FAILURE); } + } /* * If this page is not pageable, we have to get * it for all possible accesses. */ - if (*wired = (entry->wired_count != 0)) + if ((*wired = (entry->wired_count != 0))) prot = fault_type = entry->protection; /* @@ -4553,11 +4510,9 @@ kern_return_t vm_map_lookup(var_map, vad * since the given version. If successful, the map * will not change until vm_map_verify_done() is called. */ -boolean_t vm_map_verify(map, version) - register - vm_map_t map; - register - vm_map_version_t *version; /* REF */ +boolean_t vm_map_verify( + vm_map_t map, + vm_map_version_t *version) /* REF */ { boolean_t result; @@ -4586,24 +4541,19 @@ boolean_t vm_map_verify(map, version) * a task's address map. */ -kern_return_t vm_region(map, address, size, - protection, max_protection, - inheritance, is_shared, - object_name, offset_in_object) - vm_map_t map; - vm_offset_t *address; /* IN/OUT */ - vm_size_t *size; /* OUT */ - vm_prot_t *protection; /* OUT */ - vm_prot_t *max_protection; /* OUT */ - vm_inherit_t *inheritance; /* OUT */ - boolean_t *is_shared; /* OUT */ - ipc_port_t *object_name; /* OUT */ - vm_offset_t *offset_in_object; /* OUT */ +kern_return_t vm_region( + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t *size, /* OUT */ + vm_prot_t *protection, /* OUT */ + vm_prot_t *max_protection, /* OUT */ + vm_inherit_t *inheritance, /* OUT */ + boolean_t *is_shared, /* OUT */ + ipc_port_t *object_name, /* OUT */ + vm_offset_t *offset_in_object) /* OUT */ { vm_map_entry_t tmp_entry; - register vm_map_entry_t entry; - register vm_offset_t tmp_offset; vm_offset_t start; @@ -4660,9 +4610,9 @@ kern_return_t vm_region(map, address, si * at allocation time because the adjacent entry * is often wired down. */ -void vm_map_simplify(map, start) - vm_map_t map; - vm_offset_t start; +void vm_map_simplify( + vm_map_t map, + vm_offset_t start) { vm_map_entry_t this_entry; vm_map_entry_t prev_entry; @@ -4721,12 +4671,12 @@ void vm_map_simplify(map, start) * it itself. [This assumes that attributes do not * need to be inherited, which seems ok to me] */ -kern_return_t vm_map_machine_attribute(map, address, size, attribute, value) - vm_map_t map; - vm_offset_t address; - vm_size_t size; - vm_machine_attribute_t attribute; - vm_machine_attribute_val_t* value; /* IN/OUT */ +kern_return_t vm_map_machine_attribute( + vm_map_t map, + vm_offset_t address, + vm_size_t size, + vm_machine_attribute_t attribute, + vm_machine_attribute_val_t* value) /* IN/OUT */ { kern_return_t ret; @@ -4743,8 +4693,6 @@ kern_return_t vm_map_machine_attribute(m return ret; } -#include - #if MACH_KDB @@ -4753,11 +4701,9 @@ kern_return_t vm_map_machine_attribute(m /* * vm_map_print: [ debug ] */ -void vm_map_print(map) - register vm_map_t map; +void vm_map_print(vm_map_t map) { - register vm_map_entry_t entry; - extern int indent; + vm_map_entry_t entry; iprintf("Task map 0x%X: pmap=0x%X,", (vm_offset_t) map, (vm_offset_t) (map->pmap)); @@ -4823,9 +4769,8 @@ void vm_map_print(map) */ void vm_map_copy_print(copy) - vm_map_copy_t copy; + const vm_map_copy_t copy; { - extern int indent; int i, npages; printf("copy object 0x%x\n", copy); @@ -4884,361 +4829,6 @@ void vm_map_copy_print(copy) break; } - indent -=2; + indent -= 2; } #endif /* MACH_KDB */ - -#if NORMA_IPC -/* - * This should one day be eliminated; - * we should always construct the right flavor of copy object - * the first time. Troublesome areas include vm_read, where vm_map_copyin - * is called without knowing whom the copy object is for. - * There are also situations where we do want a lazy data structure - * even if we are sending to a remote port... - */ - -/* - * Convert a copy to a page list. The copy argument is in/out - * because we probably have to allocate a new vm_map_copy structure. - * We take responsibility for discarding the old structure and - * use a continuation to do so. Postponing this discard ensures - * that the objects containing the pages we've marked busy will stick - * around. - */ -kern_return_t -vm_map_convert_to_page_list(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_map_entry_t entry, next_entry; - vm_offset_t va; - vm_offset_t offset; - vm_object_t object; - kern_return_t result; - vm_map_copy_t copy, new_copy; - int i, num_pages = 0; - - zone_t entry_zone; - - copy = *caller_copy; - - /* - * We may not have to do anything, - * or may not be able to do anything. - */ - if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - return vm_map_convert_to_page_list_from_object(caller_copy); - } - if (copy->type != VM_MAP_COPY_ENTRY_LIST) { - panic("vm_map_convert_to_page_list: copy type %d!\n", - copy->type); - } - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * Iterate over entries. - */ - for (entry = vm_map_copy_first_entry(copy); - entry != vm_map_copy_to_entry(copy); - entry = entry->vme_next) { - - object = entry->object.vm_object; - offset = entry->offset; - /* - * Iterate over pages. - */ - for (va = entry->vme_start; - va < entry->vme_end; - va += PAGE_SIZE, offset += PAGE_SIZE) { - - vm_page_t m; - - if (new_copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) { - /* - * What a mess. We need a continuation - * to do the page list, but also one - * to discard the old copy. The right - * thing to do is probably to copy - * out the old copy into the kernel - * map (or some temporary task holding - * map if we're paranoid about large - * copies), and then copyin the page - * list that we really wanted with - * src_destroy. LATER. - */ - panic("vm_map_convert_to_page_list: num\n"); - } - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - if (((m = vm_page_lookup(object, offset)) != - VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - - /* - * Also write-protect the page, so - * that the map`s owner cannot change - * the data. The busy bit will prevent - * faults on the page from succeeding - * until the copy is released; after - * that, the page can be re-entered - * as writable, since we didn`t alter - * the map entry. This scheme is a - * cheap copy-on-write. - * - * Don`t forget the protection and - * the page_lock value! - */ - - pmap_page_protect(m->phys_addr, - entry->protection - & ~m->page_lock - & ~VM_PROT_WRITE); - - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - } - - *caller_copy = new_copy; - return KERN_SUCCESS; -} - -kern_return_t -vm_map_convert_to_page_list_from_object(caller_copy) - vm_map_copy_t *caller_copy; -{ - vm_object_t object; - vm_offset_t offset; - vm_map_copy_t copy, new_copy; - - copy = *caller_copy; - assert(copy->type == VM_MAP_COPY_OBJECT); - object = copy->cpy_object; - assert(object->size == round_page(object->size)); - - /* - * Allocate the new copy. Set its continuation to - * discard the old one. - */ - new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); - new_copy->type = VM_MAP_COPY_PAGE_LIST; - new_copy->cpy_npages = 0; - new_copy->offset = copy->offset; - new_copy->size = copy->size; - new_copy->cpy_cont = vm_map_copy_discard_cont; - new_copy->cpy_cont_args = (char *) copy; - - /* - * XXX memory_object_lock_request can probably bust this - * XXX See continuation comment in previous routine for solution. - */ - assert(object->size <= VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE); - - for (offset = 0; offset < object->size; offset += PAGE_SIZE) { - vm_page_t m; - - /* - * Try to find the page of data. - */ - vm_object_lock(object); - vm_object_paging_begin(object); - m = vm_page_lookup(object, offset); - if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious && - !m->absent && !m->error) { - - /* - * This is the page. Mark it busy - * and keep the paging reference on - * the object whilst we do our thing. - */ - m->busy = TRUE; - } - else { - vm_prot_t result_prot; - vm_page_t top_page; - kern_return_t kr; - -retry: - result_prot = VM_PROT_READ; - - kr = vm_fault_page(object, offset, - VM_PROT_READ, FALSE, FALSE, - &result_prot, &m, &top_page, - FALSE, (void (*)()) 0); - if (kr == VM_FAULT_MEMORY_SHORTAGE) { - VM_PAGE_WAIT((void (*)()) 0); - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - if (kr != VM_FAULT_SUCCESS) { - /* XXX what about data_error? */ - vm_object_lock(object); - vm_object_paging_begin(object); - goto retry; - } - - if (top_page != VM_PAGE_NULL) { - vm_object_lock(object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(object); - vm_object_unlock(object); - } - } - assert(m); - m->busy = TRUE; - new_copy->cpy_page_list[new_copy->cpy_npages++] = m; - vm_object_unlock(object); - } - - *caller_copy = new_copy; - return (KERN_SUCCESS); -} - -kern_return_t -vm_map_convert_from_page_list(copy) - vm_map_copy_t copy; -{ - vm_object_t object; - int i; - vm_map_entry_t new_entry; - vm_page_t *page_list; - - /* - * Check type of copy object. - */ - if (copy->type == VM_MAP_COPY_ENTRY_LIST) { - return KERN_SUCCESS; - } - if (copy->type == VM_MAP_COPY_OBJECT) { - printf("vm_map_convert_from_page_list: COPY_OBJECT?"); - return KERN_SUCCESS; - } - if (copy->type != VM_MAP_COPY_PAGE_LIST) { - panic("vm_map_convert_from_page_list 0x%x %d", - copy, - copy->type); - } - - /* - * Make sure the pages are loose. This may be - * a "Can't Happen", but just to be safe ... - */ - page_list = ©->cpy_page_list[0]; - if ((*page_list)->tabled) - vm_map_copy_steal_pages(copy); - - /* - * Create object, and stuff pages into it. - */ - object = vm_object_allocate(copy->cpy_npages); - for (i = 0; i < copy->cpy_npages; i++) { - register vm_page_t m = *page_list++; - vm_page_insert(m, object, i * PAGE_SIZE); - m->busy = FALSE; - m->dirty = TRUE; - vm_page_activate(m); - } - - /* - * XXX If this page list contained a continuation, then - * XXX we're screwed. The right thing to do is probably do - * XXX the copyout, and then copyin the entry list we really - * XXX wanted. - */ - if (vm_map_copy_has_cont(copy)) - panic("convert_from_page_list: continuation"); - - /* - * Change type of copy object - */ - vm_map_copy_first_entry(copy) = - vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); - copy->type = VM_MAP_COPY_ENTRY_LIST; - copy->cpy_hdr.nentries = 0; - copy->cpy_hdr.entries_pageable = TRUE; - - /* - * Allocate and initialize an entry for object - */ - new_entry = vm_map_copy_entry_create(copy); - new_entry->vme_start = trunc_page(copy->offset); - new_entry->vme_end = round_page(copy->offset + copy->size); - new_entry->object.vm_object = object; - new_entry->offset = 0; - new_entry->is_shared = FALSE; - new_entry->is_sub_map = FALSE; - new_entry->needs_copy = FALSE; - new_entry->protection = VM_PROT_DEFAULT; - new_entry->max_protection = VM_PROT_ALL; - new_entry->inheritance = VM_INHERIT_DEFAULT; - new_entry->wired_count = 0; - new_entry->user_wired_count = 0; - new_entry->projected_on = 0; - - /* - * Insert entry into copy object, and return. - */ - vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - return(KERN_SUCCESS); -} -#endif /* NORMA_IPC */