--- Gnu-Mach/vm/vm_object.c 2020/09/02 04:36:57 1.1.1.1 +++ Gnu-Mach/vm/vm_object.c 2020/09/02 04:54:12 1.1.1.6 @@ -33,24 +33,21 @@ * Virtual memory object module. */ -#include -#include - -#if NORMA_VM -#include -#endif /* NORMA_VM */ +#include +#include #include -#include "memory_object_default.h" -#include "memory_object_user.h" -#include "vm_param.h" +#include +#include +#include #include #include #include +#include #include #include #include -#include +#include #include #include #include @@ -58,14 +55,15 @@ #include #include +#if MACH_KDB +#include +#endif /* MACH_KDB */ void memory_object_release( ipc_port_t pager, pager_request_t pager_request, ipc_port_t pager_name); /* forward */ -void vm_object_deactivate_pages(vm_object_t); - /* * Virtual memory objects maintain the actual data * associated with allocated virtual memory. A given @@ -144,13 +142,14 @@ void vm_object_deactivate_pages(vm_objec * ZZZ Continue this comment. */ -zone_t vm_object_zone; /* vm backing store zone */ +struct kmem_cache vm_object_cache; /* vm backing store cache */ /* * All wired-down kernel memory belongs to a single virtual * memory object (kernel_object) to avoid wasting data structures. */ -vm_object_t kernel_object; +static struct vm_object kernel_object_store; +vm_object_t kernel_object = &kernel_object_store; /* * Virtual memory objects that are not referenced by @@ -165,8 +164,9 @@ vm_object_t kernel_object; * * The kernel may choose to terminate objects from this * queue in order to reclaim storage. The current policy - * is to permit a fixed maximum number of unreferenced - * objects (vm_object_cached_max). + * is to let memory pressure dynamically adjust the number + * of unreferenced objects. The pageout daemon attempts to + * collect objects after removing pages from them. * * A simple lock (accessed by routines * vm_object_cache_{lock,lock_try,unlock}) governs the @@ -181,8 +181,6 @@ vm_object_t kernel_object; * not be held to make simple references. */ queue_head_t vm_object_cached_list; -int vm_object_cached_count; -int vm_object_cached_max = 200; /* may be patched*/ decl_simple_lock_data(,vm_object_cached_lock_data) @@ -194,6 +192,15 @@ decl_simple_lock_data(,vm_object_cached_ simple_unlock(&vm_object_cached_lock_data) /* + * Number of physical pages referenced by cached objects. + * This counter is protected by its own lock to work around + * lock ordering issues. + */ +int vm_object_cached_pages; + +decl_simple_lock_data(,vm_object_cached_pages_lock_data) + +/* * Virtual memory objects are initialized from * a template (see vm_object_allocate). * @@ -201,7 +208,7 @@ decl_simple_lock_data(,vm_object_cached_ * object structure, be sure to add initialization * (see vm_object_init). */ -vm_object_t vm_object_template; +struct vm_object vm_object_template; /* * vm_object_allocate: @@ -209,17 +216,26 @@ vm_object_t vm_object_template; * Returns a new object with the given size. */ +static void _vm_object_setup( + vm_object_t object, + vm_size_t size) +{ + *object = vm_object_template; + queue_init(&object->memq); + vm_object_lock_init(object); + object->size = size; +} + vm_object_t _vm_object_allocate( vm_size_t size) { - register vm_object_t object; + vm_object_t object; - object = (vm_object_t) zalloc(vm_object_zone); + object = (vm_object_t) kmem_cache_alloc(&vm_object_cache); + if (!object) + return 0; - *object = *vm_object_template; - queue_init(&object->memq); - vm_object_lock_init(object); - object->size = size; + _vm_object_setup(object, size); return object; } @@ -227,17 +243,17 @@ vm_object_t _vm_object_allocate( vm_object_t vm_object_allocate( vm_size_t size) { - register vm_object_t object; - register ipc_port_t port; + vm_object_t object; + ipc_port_t port; object = _vm_object_allocate(size); -#if !NORMA_VM + if (object == 0) + panic("vm_object_allocate"); port = ipc_port_alloc_kernel(); if (port == IP_NULL) panic("vm_object_allocate"); object->pager_name = port; ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME); -#endif /* !NORMA_VM */ return object; } @@ -249,10 +265,8 @@ vm_object_t vm_object_allocate( */ void vm_object_bootstrap(void) { - vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object), - round_page(512*1024), - round_page(12*1024), - 0, "objects"); + kmem_cache_init(&vm_object_cache, "vm_object", + sizeof(struct vm_object), 0, NULL, 0); queue_init(&vm_object_cached_list); simple_lock_init(&vm_object_cached_lock_data); @@ -261,53 +275,51 @@ void vm_object_bootstrap(void) * Fill in a template object, for quick initialization */ - vm_object_template = (vm_object_t) zalloc(vm_object_zone); - bzero((char *) vm_object_template, sizeof *vm_object_template); + vm_object_template.ref_count = 1; + vm_object_template.size = 0; + vm_object_template.resident_page_count = 0; + vm_object_template.copy = VM_OBJECT_NULL; + vm_object_template.shadow = VM_OBJECT_NULL; + vm_object_template.shadow_offset = (vm_offset_t) 0; + + vm_object_template.pager = IP_NULL; + vm_object_template.paging_offset = 0; + vm_object_template.pager_request = PAGER_REQUEST_NULL; + vm_object_template.pager_name = IP_NULL; + + vm_object_template.pager_created = FALSE; + vm_object_template.pager_initialized = FALSE; + vm_object_template.pager_ready = FALSE; - vm_object_template->ref_count = 1; - vm_object_template->size = 0; - vm_object_template->resident_page_count = 0; - vm_object_template->copy = VM_OBJECT_NULL; - vm_object_template->shadow = VM_OBJECT_NULL; - vm_object_template->shadow_offset = (vm_offset_t) 0; - - vm_object_template->pager = IP_NULL; - vm_object_template->paging_offset = 0; - vm_object_template->pager_request = PAGER_REQUEST_NULL; - vm_object_template->pager_name = IP_NULL; - - vm_object_template->pager_created = FALSE; - vm_object_template->pager_initialized = FALSE; - vm_object_template->pager_ready = FALSE; - - vm_object_template->copy_strategy = MEMORY_OBJECT_COPY_NONE; + vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE; /* ignored if temporary, will be reset before * permanent object becomes ready */ - vm_object_template->use_shared_copy = FALSE; - vm_object_template->shadowed = FALSE; + vm_object_template.use_shared_copy = FALSE; + vm_object_template.shadowed = FALSE; - vm_object_template->absent_count = 0; - vm_object_template->all_wanted = 0; /* all bits FALSE */ + vm_object_template.absent_count = 0; + vm_object_template.all_wanted = 0; /* all bits FALSE */ - vm_object_template->paging_in_progress = 0; - vm_object_template->can_persist = FALSE; - vm_object_template->internal = TRUE; - vm_object_template->temporary = TRUE; - vm_object_template->alive = TRUE; - vm_object_template->lock_in_progress = FALSE; - vm_object_template->lock_restart = FALSE; - vm_object_template->use_old_pageout = TRUE; /* XXX change later */ - vm_object_template->last_alloc = (vm_offset_t) 0; + vm_object_template.paging_in_progress = 0; + vm_object_template.used_for_pageout = FALSE; + vm_object_template.can_persist = FALSE; + vm_object_template.cached = FALSE; + vm_object_template.internal = TRUE; + vm_object_template.temporary = TRUE; + vm_object_template.alive = TRUE; + vm_object_template.lock_in_progress = FALSE; + vm_object_template.lock_restart = FALSE; + vm_object_template.last_alloc = (vm_offset_t) 0; #if MACH_PAGEMAP - vm_object_template->existence_info = VM_EXTERNAL_NULL; + vm_object_template.existence_info = VM_EXTERNAL_NULL; #endif /* MACH_PAGEMAP */ /* * Initialize the "kernel object" */ - kernel_object = _vm_object_allocate( + _vm_object_setup(kernel_object, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS); /* @@ -315,7 +327,7 @@ void vm_object_bootstrap(void) * kernel object so that no limit is imposed on submap sizes. */ - vm_submap_object = _vm_object_allocate( + _vm_object_setup(vm_submap_object, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS); #if MACH_PAGEMAP @@ -325,7 +337,6 @@ void vm_object_bootstrap(void) void vm_object_init(void) { -#if !NORMA_VM /* * Finish initializing the kernel object. * The submap object doesn't need a name port. @@ -335,7 +346,56 @@ void vm_object_init(void) ipc_kobject_set(kernel_object->pager_name, (ipc_kobject_t) kernel_object, IKOT_PAGING_NAME); -#endif /* !NORMA_VM */ +} + +/* + * Object cache management functions. + * + * Both the cache and the object must be locked + * before calling these functions. + */ + +static void vm_object_cache_add( + vm_object_t object) +{ + assert(!object->cached); + queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list); + object->cached = TRUE; +} + +static void vm_object_cache_remove( + vm_object_t object) +{ + assert(object->cached); + queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); + object->cached = FALSE; +} + +void vm_object_collect( + register vm_object_t object) +{ + vm_object_unlock(object); + + /* + * The cache lock must be acquired in the proper order. + */ + + vm_object_cache_lock(); + vm_object_lock(object); + + /* + * If the object was referenced while the lock was + * dropped, cancel the termination. + */ + + if (!vm_object_collectable(object)) { + vm_object_unlock(object); + vm_object_cache_unlock(); + return; + } + + vm_object_cache_remove(object); + vm_object_terminate(object); } /* @@ -344,7 +404,7 @@ void vm_object_init(void) * Gets another reference to the given object. */ void vm_object_reference( - register vm_object_t object) + vm_object_t object) { if (object == VM_OBJECT_NULL) return; @@ -367,7 +427,7 @@ void vm_object_reference( * No object may be locked. */ void vm_object_deallocate( - register vm_object_t object) + vm_object_t object) { vm_object_t temp; @@ -398,102 +458,31 @@ void vm_object_deallocate( /* * See whether this object can persist. If so, enter - * it in the cache, then deactivate all of its - * pages. + * it in the cache. */ - if (object->can_persist) { - boolean_t overflow; - - /* - * Enter the object onto the queue - * of "cached" objects. Remember whether - * we've caused the queue to overflow, - * as a hint. - */ - - queue_enter(&vm_object_cached_list, object, - vm_object_t, cached_list); - overflow = (++vm_object_cached_count > vm_object_cached_max); + if (object->can_persist && (object->resident_page_count > 0)) { + vm_object_cache_add(object); vm_object_cache_unlock(); - - vm_object_deactivate_pages(object); vm_object_unlock(object); + return; + } - /* - * If we didn't overflow, or if the queue has - * been reduced back to below the specified - * minimum, then quit. - */ - if (!overflow) - return; - - while (TRUE) { - vm_object_cache_lock(); - if (vm_object_cached_count <= - vm_object_cached_max) { - vm_object_cache_unlock(); - return; - } - - /* - * If we must trim down the queue, take - * the first object, and proceed to - * terminate it instead of the original - * object. Have to wait for pager init. - * if it's in progress. - */ - object= (vm_object_t) - queue_first(&vm_object_cached_list); - vm_object_lock(object); - - if (!(object->pager_created && - !object->pager_initialized)) { - - /* - * Ok to terminate, hang on to lock. - */ - break; - } - - vm_object_assert_wait(object, - VM_OBJECT_EVENT_INITIALIZED, FALSE); - vm_object_unlock(object); - vm_object_cache_unlock(); - thread_block((void (*)()) 0); - - /* - * Continue loop to check if cache still - * needs to be trimmed. - */ - } + if (object->pager_created && + !object->pager_initialized) { /* - * Actually remove object from cache. + * Have to wait for initialization. + * Put reference back and retry + * when it's initialized. */ - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - - assert(object->ref_count == 0); - } - else { - if (object->pager_created && - !object->pager_initialized) { - - /* - * Have to wait for initialization. - * Put reference back and retry - * when it's initialized. - */ - object->ref_count++; - vm_object_assert_wait(object, - VM_OBJECT_EVENT_INITIALIZED, FALSE); - vm_object_unlock(object); - vm_object_cache_unlock(); - thread_block((void (*)()) 0); - continue; - } + object->ref_count++; + vm_object_assert_wait(object, + VM_OBJECT_EVENT_INITIALIZED, FALSE); + vm_object_unlock(object); + vm_object_cache_unlock(); + thread_block((void (*)()) 0); + continue; } /* @@ -520,8 +509,6 @@ void vm_object_deallocate( } } -boolean_t vm_object_terminate_remove_all = FALSE; - /* * Routine: vm_object_terminate * Purpose: @@ -536,10 +523,10 @@ boolean_t vm_object_terminate_remove_all * object will cease to exist. */ void vm_object_terminate( - register vm_object_t object) + vm_object_t object) { - register vm_page_t p; - vm_object_t shadow_object; + vm_page_t p; + vm_object_t shadow_object; /* * Make sure the object isn't already being terminated @@ -588,10 +575,6 @@ void vm_object_terminate( VM_PAGE_CHECK(p); - if (p->busy && !p->absent) - panic("vm_object_terminate.2 0x%x 0x%x", - object, p); - VM_PAGE_FREE(p); } } else while (!queue_empty(&object->memq)) { @@ -599,9 +582,6 @@ void vm_object_terminate( VM_PAGE_CHECK(p); - if (p->busy && !p->absent) - panic("vm_object_terminate.3 0x%x 0x%x", object, p); - vm_page_lock_queues(); VM_PAGE_QUEUES_REMOVE(p); vm_page_unlock_queues(); @@ -619,9 +599,6 @@ void vm_object_terminate( goto free_page; } - if (p->fictitious) - panic("vm_object_terminate.4 0x%x 0x%x", object, p); - if (!p->dirty) p->dirty = pmap_is_modified(p->phys_addr); @@ -636,6 +613,15 @@ void vm_object_terminate( assert(object->ref_count == 0); assert(object->paging_in_progress == 0); + assert(!object->cached); + + if (!object->internal) { + assert(object->resident_page_count == 0); + + vm_page_lock_queues(); + vm_object_external_count--; + vm_page_unlock_queues(); + } /* * Throw away port rights... note that they may @@ -656,11 +642,7 @@ void vm_object_terminate( object->pager_name); } else if (object->pager_name != IP_NULL) { /* consumes our right for pager_name */ -#if NORMA_VM - ipc_port_release_send(object->pager_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(object->pager_name); -#endif /* NORMA_VM */ } #if MACH_PAGEMAP @@ -671,7 +653,7 @@ void vm_object_terminate( * Free the space for the object. */ - zfree(vm_object_zone, (vm_offset_t) object); + kmem_cache_free(&vm_object_cache, (vm_offset_t) object); } /* @@ -746,7 +728,6 @@ void memory_object_release( void vm_object_abort_activity( vm_object_t object) { - register vm_page_t p; vm_page_t next; @@ -800,17 +781,12 @@ void vm_object_abort_activity( * or from port destruction handling (via vm_object_destroy). */ kern_return_t memory_object_destroy( - register vm_object_t object, kern_return_t reason) { ipc_port_t old_object, old_name; pager_request_t old_control; -#ifdef lint - reason++; -#endif /* lint */ - if (object == VM_OBJECT_NULL) return KERN_SUCCESS; @@ -865,11 +841,7 @@ kern_return_t memory_object_destroy( old_name); } else if (old_name != IP_NULL) { /* consumes our right for name */ -#if NORMA_VM - ipc_port_release_send(object->pager_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(object->pager_name); -#endif /* NORMA_VM */ } /* @@ -882,28 +854,6 @@ kern_return_t memory_object_destroy( } /* - * vm_object_deactivate_pages - * - * Deactivate all pages in the specified object. (Keep its pages - * in memory even though it is no longer referenced.) - * - * The object must be locked. - */ -void vm_object_deactivate_pages( - register vm_object_t object) -{ - register vm_page_t p; - - queue_iterate(&object->memq, p, vm_page_t, listq) { - vm_page_lock_queues(); - if (!p->busy) - vm_page_deactivate(p); - vm_page_unlock_queues(); - } -} - - -/* * Routine: vm_object_pmap_protect * * Purpose: @@ -929,9 +879,9 @@ void vm_object_deactivate_pages( boolean_t vm_object_pmap_protect_by_page = FALSE; void vm_object_pmap_protect( - register vm_object_t object, - register vm_offset_t offset, - vm_offset_t size, + vm_object_t object, + vm_offset_t offset, + vm_size_t size, pmap_t pmap, vm_offset_t pmap_start, vm_prot_t prot) @@ -952,8 +902,8 @@ void vm_object_pmap_protect( } { - register vm_page_t p; - register vm_offset_t end; + vm_page_t p; + vm_offset_t end; end = offset + size; @@ -984,7 +934,7 @@ void vm_object_pmap_protect( * Must follow shadow chain to remove access * to pages in shadowed objects. */ - register vm_object_t next_object; + vm_object_t next_object; next_object = object->shadow; if (next_object != VM_OBJECT_NULL) { @@ -1021,11 +971,11 @@ void vm_object_pmap_protect( * The object must *not* be locked. */ void vm_object_pmap_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p; + vm_page_t p; if (object == VM_OBJECT_NULL) return; @@ -1071,7 +1021,6 @@ void vm_object_pmap_remove( * VM_OBJECT_NULL. */ kern_return_t vm_object_copy_slowly( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1125,7 +1074,6 @@ kern_return_t vm_object_copy_slowly( vm_prot_t prot = VM_PROT_READ; vm_page_t _result_page; vm_page_t top_page; - register vm_page_t result_page; vm_object_lock(src_object); @@ -1245,8 +1193,6 @@ kern_return_t vm_object_copy_slowly( * The object should be unlocked on entry and exit. */ -vm_object_t vm_object_copy_delayed(); /* forward declaration */ - boolean_t vm_object_copy_temporary( vm_object_t *_object, /* INOUT */ vm_offset_t *_offset, /* INOUT */ @@ -1255,10 +1201,6 @@ boolean_t vm_object_copy_temporary( { vm_object_t object = *_object; -#ifdef lint - ++*_offset; -#endif /* lint */ - if (object == VM_OBJECT_NULL) { *_src_needs_copy = FALSE; *_dst_needs_copy = FALSE; @@ -1358,16 +1300,6 @@ kern_return_t vm_object_copy_call( vm_page_t p; /* - * Set the backing object for the new - * temporary object. - */ - - assert(src_object->ref_count > 0); - src_object->ref_count++; - vm_object_paging_begin(src_object); - vm_object_unlock(src_object); - - /* * Create a memory object port to be associated * with this new vm_object. * @@ -1380,10 +1312,18 @@ kern_return_t vm_object_copy_call( */ new_memory_object = ipc_port_alloc_kernel(); - if (new_memory_object == IP_NULL) { - panic("vm_object_copy_call: allocate memory object port"); - /* XXX Shouldn't panic here. */ - } + if (new_memory_object == IP_NULL) + return KERN_RESOURCE_SHORTAGE; + + /* + * Set the backing object for the new + * temporary object. + */ + + assert(src_object->ref_count > 0); + src_object->ref_count++; + vm_object_paging_begin(src_object); + vm_object_unlock(src_object); /* we hold a naked receive right for new_memory_object */ (void) ipc_port_make_send(new_memory_object); @@ -1488,7 +1428,7 @@ vm_object_t vm_object_copy_delayed( * synchronization required in the "push" * operation described above. * - * The copy-on-write is said to be assymetric because + * The copy-on-write is said to be asymmetric because * the original object is *not* marked copy-on-write. * A copied page is pushed to the copy object, regardless * which party attempted to modify the page. @@ -1621,7 +1561,6 @@ vm_object_t vm_object_copy_delayed( * and may be interrupted. */ kern_return_t vm_object_copy_strategically( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1734,8 +1673,8 @@ void vm_object_shadow( vm_offset_t *offset, /* IN/OUT */ vm_size_t length) { - register vm_object_t source; - register vm_object_t result; + vm_object_t source; + vm_object_t result; source = *object; @@ -1861,22 +1800,15 @@ vm_object_t vm_object_lookup( if (IP_VALID(port)) { ip_lock(port); if (ip_active(port) && -#if NORMA_VM - (ip_kotype(port) == IKOT_PAGER)) { -#else /* NORMA_VM */ (ip_kotype(port) == IKOT_PAGING_REQUEST)) { -#endif /* NORMA_VM */ vm_object_cache_lock(); object = (vm_object_t) port->ip_kobject; vm_object_lock(object); assert(object->alive); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -1903,11 +1835,8 @@ vm_object_t vm_object_lookup_name( assert(object->alive); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -1939,11 +1868,8 @@ void vm_object_destroy( object = (vm_object_t) pager->ip_kobject; vm_object_lock(object); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; object->can_persist = FALSE; @@ -1976,16 +1902,10 @@ void vm_object_destroy( */ ipc_port_release_send(pager); -#if !NORMA_VM if (old_request != IP_NULL) ipc_port_dealloc_kernel(old_request); -#endif /* !NORMA_VM */ if (old_name != IP_NULL) -#if NORMA_VM - ipc_port_release_send(old_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(old_name); -#endif /* NORMA_VM */ /* * Restart pending page requests @@ -2014,7 +1934,6 @@ vm_object_t vm_object_enter( vm_size_t size, boolean_t internal) { - register vm_object_t object; vm_object_t new_object; boolean_t must_init; @@ -2098,11 +2017,8 @@ restart: if ((object != VM_OBJECT_NULL) && !must_init) { vm_object_lock(object); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -2138,27 +2054,6 @@ restart: object->pager_created = TRUE; object->pager = pager; -#if NORMA_VM - - /* - * Let the xmm system know that we want to use the pager. - * - * Name port will be provided by the xmm system - * when set_attributes_common is called. - */ - - object->internal = internal; - object->pager_ready = internal; - if (internal) { - assert(object->temporary); - } else { - object->temporary = FALSE; - } - object->pager_name = IP_NULL; - - (void) xmm_memory_object_init(object); -#else /* NORMA_VM */ - /* * Allocate request port. */ @@ -2198,6 +2093,9 @@ restart: object->internal = FALSE; object->temporary = FALSE; + assert(object->resident_page_count == 0); + vm_object_external_count++; + /* user pager objects are not ready until marked so */ object->pager_ready = FALSE; @@ -2207,7 +2105,6 @@ restart: PAGE_SIZE); } -#endif /* NORMA_VM */ vm_object_lock(object); object->pager_initialized = TRUE; @@ -2253,7 +2150,6 @@ restart: * daemon will be using this routine. */ void vm_object_pager_create( - register vm_object_t object) { ipc_port_t pager; @@ -2359,7 +2255,6 @@ void vm_object_remove( else if (ip_kotype(port) != IKOT_NONE) panic("vm_object_remove: bad object port"); } -#if !NORMA_VM if ((port = object->pager_request) != IP_NULL) { if (ip_kotype(port) == IKOT_PAGING_REQUEST) ipc_kobject_set(port, IKO_NULL, IKOT_NONE); @@ -2372,7 +2267,6 @@ void vm_object_remove( else if (ip_kotype(port) != IKOT_NONE) panic("vm_object_remove: bad name port"); } -#endif /* !NORMA_VM */ } /* @@ -2400,14 +2294,14 @@ boolean_t vm_object_collapse_bypass_allo * so the caller should hold a reference for the object. */ void vm_object_collapse( - register vm_object_t object) + vm_object_t object) { - register vm_object_t backing_object; - register vm_offset_t backing_offset; - register vm_size_t size; - register vm_offset_t new_offset; - register vm_page_t p, pp; - ipc_port_t old_name_port; + vm_object_t backing_object; + vm_offset_t backing_offset; + vm_size_t size; + vm_offset_t new_offset; + vm_page_t p, pp; + ipc_port_t old_name_port; if (!vm_object_collapse_allowed) return; @@ -2520,9 +2414,7 @@ void vm_object_collapse( if (p->offset < backing_offset || new_offset >= size) { - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } else { pp = vm_page_lookup(object, new_offset); if (pp != VM_PAGE_NULL && !pp->absent) { @@ -2531,41 +2423,12 @@ void vm_object_collapse( * Throw away the backing object's * page. */ - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } else { - if (pp != VM_PAGE_NULL) { - /* - * Parent has an absent page... - * it's not being paged in, so - * it must really be missing from - * the parent. - * - * Throw out the absent page... - * any faults looking for that - * page will restart with the new - * one. - */ - - /* - * This should never happen -- the - * parent cannot have ever had an - * external memory object, and thus - * cannot have absent pages. - */ - panic("vm_object_collapse: bad case"); - - vm_page_lock_queues(); - vm_page_free(pp); - vm_page_unlock_queues(); - - /* - * Fall through to move the backing - * object's page up. - */ - } + assert(pp == VM_PAGE_NULL || ! + "vm_object_collapse: bad case"); + /* * Parent now has no page. * Move the backing object's page up. @@ -2594,11 +2457,11 @@ void vm_object_collapse( /* Fall through to... */ default: - printf("vm_object_collapse: %#x (pager %#x, request %#x) up to %#x\n", + printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n", backing_object, backing_object->pager, backing_object->pager_request, object); if (vm_object_collapse_debug > 2) - Debugger("vm_object_collapse"); + SoftDebugger("vm_object_collapse"); } object->pager = backing_object->pager; @@ -2611,10 +2474,6 @@ void vm_object_collapse( object->pager_created = backing_object->pager_created; object->pager_request = backing_object->pager_request; -#if NORMA_VM - old_name_port = object->pager_name; - object->pager_name = backing_object->pager_name; -#else /* NORMA_VM */ if (object->pager_request != IP_NULL) ipc_kobject_set(object->pager_request, (ipc_kobject_t) object, @@ -2628,7 +2487,6 @@ void vm_object_collapse( ipc_kobject_set(object->pager_name, (ipc_kobject_t) object, IKOT_PAGING_NAME); -#endif /* NORMA_VM */ vm_object_cache_unlock(); @@ -2672,17 +2530,14 @@ void vm_object_collapse( ); assert(backing_object->alive); + assert(!backing_object->cached); backing_object->alive = FALSE; vm_object_unlock(backing_object); vm_object_unlock(object); if (old_name_port != IP_NULL) -#if NORMA_VM - ipc_port_release_send(old_name_port); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(old_name_port); -#endif /* NORMA_VM */ - zfree(vm_object_zone, (vm_offset_t) backing_object); + kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object); vm_object_lock(object); object_collapses++; @@ -2792,11 +2647,11 @@ unsigned int vm_object_page_remove_looku unsigned int vm_object_page_remove_iterate = 0; void vm_object_page_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p, next; + vm_page_t p, next; /* * One and two page removals are most popular. @@ -2804,7 +2659,7 @@ void vm_object_page_remove( * It balances vm_object_lookup vs iteration. */ - if (atop(end - start) < (unsigned)object->resident_page_count/16) { + if (atop(end - start) < object->resident_page_count/16) { vm_object_page_remove_lookup++; for (; start < end; start += PAGE_SIZE) { @@ -2813,9 +2668,7 @@ void vm_object_page_remove( if (!p->fictitious) pmap_page_protect(p->phys_addr, VM_PROT_NONE); - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } } } else { @@ -2828,9 +2681,7 @@ void vm_object_page_remove( if (!p->fictitious) pmap_page_protect(p->phys_addr, VM_PROT_NONE); - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } p = next; } @@ -2861,7 +2712,7 @@ void vm_object_page_remove( */ boolean_t vm_object_coalesce( - register vm_object_t prev_object, + vm_object_t prev_object, vm_object_t next_object, vm_offset_t prev_offset, vm_offset_t next_offset, @@ -2870,10 +2721,6 @@ boolean_t vm_object_coalesce( { vm_size_t newsize; -#ifdef lint - next_offset++; -#endif /* lint */ - if (next_object != VM_OBJECT_NULL) { return FALSE; } @@ -2901,6 +2748,7 @@ boolean_t vm_object_coalesce( if ((prev_object->ref_count > 1) || prev_object->pager_created || + prev_object->used_for_pageout || (prev_object->shadow != VM_OBJECT_NULL) || (prev_object->copy != VM_OBJECT_NULL) || (prev_object->paging_in_progress != 0)) { @@ -2959,11 +2807,7 @@ ipc_port_t vm_object_name( p = object->pager_name; if (p != IP_NULL) -#if NORMA_VM - p = ipc_port_copy_send(p); -#else /* NORMA_VM */ p = ipc_port_make_send(p); -#endif /* NORMA_VM */ vm_object_unlock(object); return p; @@ -3003,12 +2847,11 @@ vm_object_page_map( if ((old_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { - vm_page_lock_queues(); - vm_page_free(old_page); - vm_page_unlock_queues(); + VM_PAGE_FREE(old_page); } - vm_page_init(m, addr); + vm_page_init(m); + m->phys_addr = addr; m->private = TRUE; /* don`t free page */ m->wire_count = 1; vm_page_lock_queues(); @@ -3020,10 +2863,9 @@ vm_object_page_map( } } -#include - #if MACH_KDB +#include #define printf kdbprintf boolean_t vm_object_print_pages = FALSE; @@ -3034,21 +2876,21 @@ boolean_t vm_object_print_pages = FALSE; void vm_object_print( vm_object_t object) { - register vm_page_t p; - extern indent; + vm_page_t p; - register int count; + int count; if (object == VM_OBJECT_NULL) return; - iprintf("Object 0x%X: size=0x%X", - (vm_offset_t) object, (vm_offset_t) object->size); - printf(", %d references, %d resident pages,", object->ref_count, - object->resident_page_count); + iprintf("Object 0x%X: size=0x%X, %d references", + (vm_offset_t) object, (vm_offset_t) object->size, + object->ref_count); + printf("\n"); + iprintf("%lu resident pages,", object->resident_page_count); printf(" %d absent pages,", object->absent_count); printf(" %d paging ops\n", object->paging_in_progress); - indent += 2; + indent += 1; iprintf("memory object=0x%X (offset=0x%X),", (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset); printf("control=0x%X, name=0x%X\n", @@ -3067,7 +2909,7 @@ void vm_object_print( (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset); printf("copy=0x%X\n", (vm_offset_t) object->copy); - indent += 2; + indent += 1; if (vm_object_print_pages) { count = 0; @@ -3084,7 +2926,7 @@ void vm_object_print( if (count != 0) printf("\n"); } - indent -= 4; + indent -= 2; } #endif /* MACH_KDB */