--- Gnu-Mach/vm/vm_object.c 2020/09/02 04:45:25 1.1.1.2 +++ Gnu-Mach/vm/vm_object.c 2020/09/02 04:52:07 1.1.1.5 @@ -59,14 +59,11 @@ #include #endif /* MACH_KDB */ - void memory_object_release( ipc_port_t pager, pager_request_t pager_request, ipc_port_t pager_name); /* forward */ -void vm_object_deactivate_pages(vm_object_t); - /* * Virtual memory objects maintain the actual data * associated with allocated virtual memory. A given @@ -167,8 +164,9 @@ vm_object_t kernel_object = &kernel_obj * * The kernel may choose to terminate objects from this * queue in order to reclaim storage. The current policy - * is to permit a fixed maximum number of unreferenced - * objects (vm_object_cached_max). + * is to let memory pressure dynamically adjust the number + * of unreferenced objects. The pageout daemon attempts to + * collect objects after removing pages from them. * * A simple lock (accessed by routines * vm_object_cache_{lock,lock_try,unlock}) governs the @@ -184,7 +182,6 @@ vm_object_t kernel_object = &kernel_obj */ queue_head_t vm_object_cached_list; int vm_object_cached_count; -int vm_object_cached_max = 4000; /* may be patched*/ decl_simple_lock_data(,vm_object_cached_lock_data) @@ -233,9 +230,11 @@ static void _vm_object_setup( vm_object_t _vm_object_allocate( vm_size_t size) { - register vm_object_t object; + vm_object_t object; object = (vm_object_t) kmem_cache_alloc(&vm_object_cache); + if (!object) + return 0; _vm_object_setup(object, size); @@ -245,10 +244,12 @@ vm_object_t _vm_object_allocate( vm_object_t vm_object_allocate( vm_size_t size) { - register vm_object_t object; - register ipc_port_t port; + vm_object_t object; + ipc_port_t port; object = _vm_object_allocate(size); + if (object == 0) + panic("vm_object_allocate"); port = ipc_port_alloc_kernel(); if (port == IP_NULL) panic("vm_object_allocate"); @@ -266,7 +267,7 @@ vm_object_t vm_object_allocate( void vm_object_bootstrap(void) { kmem_cache_init(&vm_object_cache, "vm_object", - sizeof(struct vm_object), 0, NULL, NULL, NULL, 0); + sizeof(struct vm_object), 0, NULL, 0); queue_init(&vm_object_cached_list); simple_lock_init(&vm_object_cached_lock_data); @@ -301,7 +302,9 @@ void vm_object_bootstrap(void) vm_object_template.all_wanted = 0; /* all bits FALSE */ vm_object_template.paging_in_progress = 0; + vm_object_template.used_for_pageout = FALSE; vm_object_template.can_persist = FALSE; + vm_object_template.cached = FALSE; vm_object_template.internal = TRUE; vm_object_template.temporary = TRUE; vm_object_template.alive = TRUE; @@ -348,12 +351,66 @@ void vm_object_init(void) } /* + * Object cache management functions. + * + * Both the cache and the object must be locked + * before calling these functions. + */ + +static void vm_object_cache_add( + vm_object_t object) +{ + assert(!object->cached); + queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list); + vm_object_cached_count++; + vm_object_cached_pages_update(object->resident_page_count); + object->cached = TRUE; +} + +static void vm_object_cache_remove( + vm_object_t object) +{ + assert(object->cached); + queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); + vm_object_cached_count--; + vm_object_cached_pages_update(-object->resident_page_count); + object->cached = FALSE; +} + +void vm_object_collect( + register vm_object_t object) +{ + vm_object_unlock(object); + + /* + * The cache lock must be acquired in the proper order. + */ + + vm_object_cache_lock(); + vm_object_lock(object); + + /* + * If the object was referenced while the lock was + * dropped, cancel the termination. + */ + + if (!vm_object_collectable(object)) { + vm_object_unlock(object); + vm_object_cache_unlock(); + return; + } + + vm_object_cache_remove(object); + vm_object_terminate(object); +} + +/* * vm_object_reference: * * Gets another reference to the given object. */ void vm_object_reference( - register vm_object_t object) + vm_object_t object) { if (object == VM_OBJECT_NULL) return; @@ -376,7 +433,7 @@ void vm_object_reference( * No object may be locked. */ void vm_object_deallocate( - register vm_object_t object) + vm_object_t object) { vm_object_t temp; @@ -407,103 +464,31 @@ void vm_object_deallocate( /* * See whether this object can persist. If so, enter - * it in the cache, then deactivate all of its - * pages. + * it in the cache. */ - if (object->can_persist) { - boolean_t overflow; - - /* - * Enter the object onto the queue - * of "cached" objects. Remember whether - * we've caused the queue to overflow, - * as a hint. - */ - - queue_enter(&vm_object_cached_list, object, - vm_object_t, cached_list); - overflow = (++vm_object_cached_count > vm_object_cached_max); - vm_object_cached_pages_update(object->resident_page_count); + if (object->can_persist && (object->resident_page_count > 0)) { + vm_object_cache_add(object); vm_object_cache_unlock(); - - vm_object_deactivate_pages(object); vm_object_unlock(object); + return; + } - /* - * If we didn't overflow, or if the queue has - * been reduced back to below the specified - * minimum, then quit. - */ - if (!overflow) - return; - - while (TRUE) { - vm_object_cache_lock(); - if (vm_object_cached_count <= - vm_object_cached_max) { - vm_object_cache_unlock(); - return; - } - - /* - * If we must trim down the queue, take - * the first object, and proceed to - * terminate it instead of the original - * object. Have to wait for pager init. - * if it's in progress. - */ - object= (vm_object_t) - queue_first(&vm_object_cached_list); - vm_object_lock(object); - - if (!(object->pager_created && - !object->pager_initialized)) { - - /* - * Ok to terminate, hang on to lock. - */ - break; - } - - vm_object_assert_wait(object, - VM_OBJECT_EVENT_INITIALIZED, FALSE); - vm_object_unlock(object); - vm_object_cache_unlock(); - thread_block((void (*)()) 0); - - /* - * Continue loop to check if cache still - * needs to be trimmed. - */ - } + if (object->pager_created && + !object->pager_initialized) { /* - * Actually remove object from cache. + * Have to wait for initialization. + * Put reference back and retry + * when it's initialized. */ - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - - assert(object->ref_count == 0); - } - else { - if (object->pager_created && - !object->pager_initialized) { - - /* - * Have to wait for initialization. - * Put reference back and retry - * when it's initialized. - */ - object->ref_count++; - vm_object_assert_wait(object, - VM_OBJECT_EVENT_INITIALIZED, FALSE); - vm_object_unlock(object); - vm_object_cache_unlock(); - thread_block((void (*)()) 0); - continue; - } + object->ref_count++; + vm_object_assert_wait(object, + VM_OBJECT_EVENT_INITIALIZED, FALSE); + vm_object_unlock(object); + vm_object_cache_unlock(); + thread_block((void (*)()) 0); + continue; } /* @@ -530,8 +515,6 @@ void vm_object_deallocate( } } -boolean_t vm_object_terminate_remove_all = FALSE; - /* * Routine: vm_object_terminate * Purpose: @@ -546,10 +529,10 @@ boolean_t vm_object_terminate_remove_all * object will cease to exist. */ void vm_object_terminate( - register vm_object_t object) + vm_object_t object) { - register vm_page_t p; - vm_object_t shadow_object; + vm_page_t p; + vm_object_t shadow_object; /* * Make sure the object isn't already being terminated @@ -598,10 +581,6 @@ void vm_object_terminate( VM_PAGE_CHECK(p); - if (p->busy && !p->absent) - panic("vm_object_terminate.2 0x%x 0x%x", - object, p); - VM_PAGE_FREE(p); } } else while (!queue_empty(&object->memq)) { @@ -609,9 +588,6 @@ void vm_object_terminate( VM_PAGE_CHECK(p); - if (p->busy && !p->absent) - panic("vm_object_terminate.3 0x%x 0x%x", object, p); - vm_page_lock_queues(); VM_PAGE_QUEUES_REMOVE(p); vm_page_unlock_queues(); @@ -629,9 +605,6 @@ void vm_object_terminate( goto free_page; } - if (p->fictitious) - panic("vm_object_terminate.4 0x%x 0x%x", object, p); - if (!p->dirty) p->dirty = pmap_is_modified(p->phys_addr); @@ -646,6 +619,7 @@ void vm_object_terminate( assert(object->ref_count == 0); assert(object->paging_in_progress == 0); + assert(!object->cached); /* * Throw away port rights... note that they may @@ -752,7 +726,6 @@ void memory_object_release( void vm_object_abort_activity( vm_object_t object) { - register vm_page_t p; vm_page_t next; @@ -806,17 +779,12 @@ void vm_object_abort_activity( * or from port destruction handling (via vm_object_destroy). */ kern_return_t memory_object_destroy( - register vm_object_t object, kern_return_t reason) { ipc_port_t old_object, old_name; pager_request_t old_control; -#ifdef lint - reason++; -#endif /* lint */ - if (object == VM_OBJECT_NULL) return KERN_SUCCESS; @@ -884,28 +852,6 @@ kern_return_t memory_object_destroy( } /* - * vm_object_deactivate_pages - * - * Deactivate all pages in the specified object. (Keep its pages - * in memory even though it is no longer referenced.) - * - * The object must be locked. - */ -void vm_object_deactivate_pages( - register vm_object_t object) -{ - register vm_page_t p; - - queue_iterate(&object->memq, p, vm_page_t, listq) { - vm_page_lock_queues(); - if (!p->busy) - vm_page_deactivate(p); - vm_page_unlock_queues(); - } -} - - -/* * Routine: vm_object_pmap_protect * * Purpose: @@ -931,8 +877,8 @@ void vm_object_deactivate_pages( boolean_t vm_object_pmap_protect_by_page = FALSE; void vm_object_pmap_protect( - register vm_object_t object, - register vm_offset_t offset, + vm_object_t object, + vm_offset_t offset, vm_size_t size, pmap_t pmap, vm_offset_t pmap_start, @@ -954,8 +900,8 @@ void vm_object_pmap_protect( } { - register vm_page_t p; - register vm_offset_t end; + vm_page_t p; + vm_offset_t end; end = offset + size; @@ -986,7 +932,7 @@ void vm_object_pmap_protect( * Must follow shadow chain to remove access * to pages in shadowed objects. */ - register vm_object_t next_object; + vm_object_t next_object; next_object = object->shadow; if (next_object != VM_OBJECT_NULL) { @@ -1023,11 +969,11 @@ void vm_object_pmap_protect( * The object must *not* be locked. */ void vm_object_pmap_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p; + vm_page_t p; if (object == VM_OBJECT_NULL) return; @@ -1073,7 +1019,6 @@ void vm_object_pmap_remove( * VM_OBJECT_NULL. */ kern_return_t vm_object_copy_slowly( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1127,7 +1072,6 @@ kern_return_t vm_object_copy_slowly( vm_prot_t prot = VM_PROT_READ; vm_page_t _result_page; vm_page_t top_page; - register vm_page_t result_page; vm_object_lock(src_object); @@ -1247,8 +1191,6 @@ kern_return_t vm_object_copy_slowly( * The object should be unlocked on entry and exit. */ -vm_object_t vm_object_copy_delayed(); /* forward declaration */ - boolean_t vm_object_copy_temporary( vm_object_t *_object, /* INOUT */ vm_offset_t *_offset, /* INOUT */ @@ -1257,10 +1199,6 @@ boolean_t vm_object_copy_temporary( { vm_object_t object = *_object; -#ifdef lint - ++*_offset; -#endif /* lint */ - if (object == VM_OBJECT_NULL) { *_src_needs_copy = FALSE; *_dst_needs_copy = FALSE; @@ -1360,16 +1298,6 @@ kern_return_t vm_object_copy_call( vm_page_t p; /* - * Set the backing object for the new - * temporary object. - */ - - assert(src_object->ref_count > 0); - src_object->ref_count++; - vm_object_paging_begin(src_object); - vm_object_unlock(src_object); - - /* * Create a memory object port to be associated * with this new vm_object. * @@ -1382,10 +1310,18 @@ kern_return_t vm_object_copy_call( */ new_memory_object = ipc_port_alloc_kernel(); - if (new_memory_object == IP_NULL) { - panic("vm_object_copy_call: allocate memory object port"); - /* XXX Shouldn't panic here. */ - } + if (new_memory_object == IP_NULL) + return KERN_RESOURCE_SHORTAGE; + + /* + * Set the backing object for the new + * temporary object. + */ + + assert(src_object->ref_count > 0); + src_object->ref_count++; + vm_object_paging_begin(src_object); + vm_object_unlock(src_object); /* we hold a naked receive right for new_memory_object */ (void) ipc_port_make_send(new_memory_object); @@ -1490,7 +1426,7 @@ vm_object_t vm_object_copy_delayed( * synchronization required in the "push" * operation described above. * - * The copy-on-write is said to be assymetric because + * The copy-on-write is said to be asymmetric because * the original object is *not* marked copy-on-write. * A copied page is pushed to the copy object, regardless * which party attempted to modify the page. @@ -1623,7 +1559,6 @@ vm_object_t vm_object_copy_delayed( * and may be interrupted. */ kern_return_t vm_object_copy_strategically( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1736,8 +1671,8 @@ void vm_object_shadow( vm_offset_t *offset, /* IN/OUT */ vm_size_t length) { - register vm_object_t source; - register vm_object_t result; + vm_object_t source; + vm_object_t result; source = *object; @@ -1870,12 +1805,8 @@ vm_object_t vm_object_lookup( assert(object->alive); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - vm_object_cached_pages_update(-object->resident_page_count); - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -1902,12 +1833,8 @@ vm_object_t vm_object_lookup_name( assert(object->alive); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - vm_object_cached_pages_update(-object->resident_page_count); - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -1939,12 +1866,8 @@ void vm_object_destroy( object = (vm_object_t) pager->ip_kobject; vm_object_lock(object); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - vm_object_cached_pages_update(-object->resident_page_count); - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; object->can_persist = FALSE; @@ -2009,7 +1932,6 @@ vm_object_t vm_object_enter( vm_size_t size, boolean_t internal) { - register vm_object_t object; vm_object_t new_object; boolean_t must_init; @@ -2093,12 +2015,8 @@ restart: if ((object != VM_OBJECT_NULL) && !must_init) { vm_object_lock(object); - if (object->ref_count == 0) { - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - vm_object_cached_pages_update(-object->resident_page_count); - } + if (object->ref_count == 0) + vm_object_cache_remove(object); object->ref_count++; vm_object_unlock(object); @@ -2227,7 +2145,6 @@ restart: * daemon will be using this routine. */ void vm_object_pager_create( - register vm_object_t object) { ipc_port_t pager; @@ -2372,14 +2289,14 @@ boolean_t vm_object_collapse_bypass_allo * so the caller should hold a reference for the object. */ void vm_object_collapse( - register vm_object_t object) + vm_object_t object) { - register vm_object_t backing_object; - register vm_offset_t backing_offset; - register vm_size_t size; - register vm_offset_t new_offset; - register vm_page_t p, pp; - ipc_port_t old_name_port; + vm_object_t backing_object; + vm_offset_t backing_offset; + vm_size_t size; + vm_offset_t new_offset; + vm_page_t p, pp; + ipc_port_t old_name_port; if (!vm_object_collapse_allowed) return; @@ -2504,34 +2421,9 @@ void vm_object_collapse( VM_PAGE_FREE(p); } else { - if (pp != VM_PAGE_NULL) { - /* - * Parent has an absent page... - * it's not being paged in, so - * it must really be missing from - * the parent. - * - * Throw out the absent page... - * any faults looking for that - * page will restart with the new - * one. - */ - - /* - * This should never happen -- the - * parent cannot have ever had an - * external memory object, and thus - * cannot have absent pages. - */ - panic("vm_object_collapse: bad case"); - - VM_PAGE_FREE(pp); - - /* - * Fall through to move the backing - * object's page up. - */ - } + assert(pp == VM_PAGE_NULL || ! + "vm_object_collapse: bad case"); + /* * Parent now has no page. * Move the backing object's page up. @@ -2633,6 +2525,7 @@ void vm_object_collapse( ); assert(backing_object->alive); + assert(!backing_object->cached); backing_object->alive = FALSE; vm_object_unlock(backing_object); @@ -2749,11 +2642,11 @@ unsigned int vm_object_page_remove_looku unsigned int vm_object_page_remove_iterate = 0; void vm_object_page_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p, next; + vm_page_t p, next; /* * One and two page removals are most popular. @@ -2761,7 +2654,7 @@ void vm_object_page_remove( * It balances vm_object_lookup vs iteration. */ - if (atop(end - start) < (unsigned)object->resident_page_count/16) { + if (atop(end - start) < object->resident_page_count/16) { vm_object_page_remove_lookup++; for (; start < end; start += PAGE_SIZE) { @@ -2814,7 +2707,7 @@ void vm_object_page_remove( */ boolean_t vm_object_coalesce( - register vm_object_t prev_object, + vm_object_t prev_object, vm_object_t next_object, vm_offset_t prev_offset, vm_offset_t next_offset, @@ -2823,10 +2716,6 @@ boolean_t vm_object_coalesce( { vm_size_t newsize; -#ifdef lint - next_offset++; -#endif /* lint */ - if (next_object != VM_OBJECT_NULL) { return FALSE; } @@ -2854,6 +2743,7 @@ boolean_t vm_object_coalesce( if ((prev_object->ref_count > 1) || prev_object->pager_created || + prev_object->used_for_pageout || (prev_object->shadow != VM_OBJECT_NULL) || (prev_object->copy != VM_OBJECT_NULL) || (prev_object->paging_in_progress != 0)) { @@ -2955,7 +2845,8 @@ vm_object_page_map( VM_PAGE_FREE(old_page); } - vm_page_init(m, addr); + vm_page_init(m); + m->phys_addr = addr; m->private = TRUE; /* don`t free page */ m->wire_count = 1; vm_page_lock_queues(); @@ -2980,20 +2871,21 @@ boolean_t vm_object_print_pages = FALSE; void vm_object_print( vm_object_t object) { - register vm_page_t p; + vm_page_t p; - register int count; + int count; if (object == VM_OBJECT_NULL) return; - iprintf("Object 0x%X: size=0x%X", - (vm_offset_t) object, (vm_offset_t) object->size); - printf(", %d references, %d resident pages,", object->ref_count, - object->resident_page_count); + iprintf("Object 0x%X: size=0x%X, %d references", + (vm_offset_t) object, (vm_offset_t) object->size, + object->ref_count); + printf("\n"); + iprintf("%lu resident pages,", object->resident_page_count); printf(" %d absent pages,", object->absent_count); printf(" %d paging ops\n", object->paging_in_progress); - indent += 2; + indent += 1; iprintf("memory object=0x%X (offset=0x%X),", (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset); printf("control=0x%X, name=0x%X\n", @@ -3012,7 +2904,7 @@ void vm_object_print( (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset); printf("copy=0x%X\n", (vm_offset_t) object->copy); - indent += 2; + indent += 1; if (vm_object_print_pages) { count = 0; @@ -3029,7 +2921,7 @@ void vm_object_print( if (count != 0) printf("\n"); } - indent -= 4; + indent -= 2; } #endif /* MACH_KDB */