--- Gnu-Mach/vm/vm_object.c 2020/09/02 04:36:57 1.1.1.1 +++ Gnu-Mach/vm/vm_object.c 2020/09/02 04:47:46 1.1.1.3 @@ -33,24 +33,21 @@ * Virtual memory object module. */ -#include -#include - -#if NORMA_VM -#include -#endif /* NORMA_VM */ +#include +#include #include -#include "memory_object_default.h" -#include "memory_object_user.h" -#include "vm_param.h" +#include +#include +#include #include #include #include +#include #include #include #include -#include +#include #include #include #include @@ -58,13 +55,9 @@ #include #include - -void memory_object_release( - ipc_port_t pager, - pager_request_t pager_request, - ipc_port_t pager_name); /* forward */ - -void vm_object_deactivate_pages(vm_object_t); +#if MACH_KDB +#include +#endif /* MACH_KDB */ /* * Virtual memory objects maintain the actual data @@ -144,13 +137,14 @@ void vm_object_deactivate_pages(vm_objec * ZZZ Continue this comment. */ -zone_t vm_object_zone; /* vm backing store zone */ +struct kmem_cache vm_object_cache; /* vm backing store cache */ /* * All wired-down kernel memory belongs to a single virtual * memory object (kernel_object) to avoid wasting data structures. */ -vm_object_t kernel_object; +static struct vm_object kernel_object_store; +vm_object_t kernel_object = &kernel_object_store; /* * Virtual memory objects that are not referenced by @@ -182,7 +176,7 @@ vm_object_t kernel_object; */ queue_head_t vm_object_cached_list; int vm_object_cached_count; -int vm_object_cached_max = 200; /* may be patched*/ +int vm_object_cached_max = 4000; /* may be patched*/ decl_simple_lock_data(,vm_object_cached_lock_data) @@ -194,6 +188,15 @@ decl_simple_lock_data(,vm_object_cached_ simple_unlock(&vm_object_cached_lock_data) /* + * Number of physical pages referenced by cached objects. + * This counter is protected by its own lock to work around + * lock ordering issues. + */ +int vm_object_cached_pages; + +decl_simple_lock_data(,vm_object_cached_pages_lock_data) + +/* * Virtual memory objects are initialized from * a template (see vm_object_allocate). * @@ -201,7 +204,7 @@ decl_simple_lock_data(,vm_object_cached_ * object structure, be sure to add initialization * (see vm_object_init). */ -vm_object_t vm_object_template; +struct vm_object vm_object_template; /* * vm_object_allocate: @@ -209,17 +212,24 @@ vm_object_t vm_object_template; * Returns a new object with the given size. */ +static void _vm_object_setup( + vm_object_t object, + vm_size_t size) +{ + *object = vm_object_template; + queue_init(&object->memq); + vm_object_lock_init(object); + object->size = size; +} + vm_object_t _vm_object_allocate( vm_size_t size) { - register vm_object_t object; + vm_object_t object; - object = (vm_object_t) zalloc(vm_object_zone); + object = (vm_object_t) kmem_cache_alloc(&vm_object_cache); - *object = *vm_object_template; - queue_init(&object->memq); - vm_object_lock_init(object); - object->size = size; + _vm_object_setup(object, size); return object; } @@ -227,17 +237,15 @@ vm_object_t _vm_object_allocate( vm_object_t vm_object_allocate( vm_size_t size) { - register vm_object_t object; - register ipc_port_t port; + vm_object_t object; + ipc_port_t port; object = _vm_object_allocate(size); -#if !NORMA_VM port = ipc_port_alloc_kernel(); if (port == IP_NULL) panic("vm_object_allocate"); object->pager_name = port; ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME); -#endif /* !NORMA_VM */ return object; } @@ -249,10 +257,8 @@ vm_object_t vm_object_allocate( */ void vm_object_bootstrap(void) { - vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object), - round_page(512*1024), - round_page(12*1024), - 0, "objects"); + kmem_cache_init(&vm_object_cache, "vm_object", + sizeof(struct vm_object), 0, NULL, NULL, NULL, 0); queue_init(&vm_object_cached_list); simple_lock_init(&vm_object_cached_lock_data); @@ -261,53 +267,50 @@ void vm_object_bootstrap(void) * Fill in a template object, for quick initialization */ - vm_object_template = (vm_object_t) zalloc(vm_object_zone); - bzero((char *) vm_object_template, sizeof *vm_object_template); - - vm_object_template->ref_count = 1; - vm_object_template->size = 0; - vm_object_template->resident_page_count = 0; - vm_object_template->copy = VM_OBJECT_NULL; - vm_object_template->shadow = VM_OBJECT_NULL; - vm_object_template->shadow_offset = (vm_offset_t) 0; - - vm_object_template->pager = IP_NULL; - vm_object_template->paging_offset = 0; - vm_object_template->pager_request = PAGER_REQUEST_NULL; - vm_object_template->pager_name = IP_NULL; - - vm_object_template->pager_created = FALSE; - vm_object_template->pager_initialized = FALSE; - vm_object_template->pager_ready = FALSE; + vm_object_template.ref_count = 1; + vm_object_template.size = 0; + vm_object_template.resident_page_count = 0; + vm_object_template.copy = VM_OBJECT_NULL; + vm_object_template.shadow = VM_OBJECT_NULL; + vm_object_template.shadow_offset = (vm_offset_t) 0; + + vm_object_template.pager = IP_NULL; + vm_object_template.paging_offset = 0; + vm_object_template.pager_request = PAGER_REQUEST_NULL; + vm_object_template.pager_name = IP_NULL; + + vm_object_template.pager_created = FALSE; + vm_object_template.pager_initialized = FALSE; + vm_object_template.pager_ready = FALSE; - vm_object_template->copy_strategy = MEMORY_OBJECT_COPY_NONE; + vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE; /* ignored if temporary, will be reset before * permanent object becomes ready */ - vm_object_template->use_shared_copy = FALSE; - vm_object_template->shadowed = FALSE; + vm_object_template.use_shared_copy = FALSE; + vm_object_template.shadowed = FALSE; - vm_object_template->absent_count = 0; - vm_object_template->all_wanted = 0; /* all bits FALSE */ + vm_object_template.absent_count = 0; + vm_object_template.all_wanted = 0; /* all bits FALSE */ - vm_object_template->paging_in_progress = 0; - vm_object_template->can_persist = FALSE; - vm_object_template->internal = TRUE; - vm_object_template->temporary = TRUE; - vm_object_template->alive = TRUE; - vm_object_template->lock_in_progress = FALSE; - vm_object_template->lock_restart = FALSE; - vm_object_template->use_old_pageout = TRUE; /* XXX change later */ - vm_object_template->last_alloc = (vm_offset_t) 0; + vm_object_template.paging_in_progress = 0; + vm_object_template.can_persist = FALSE; + vm_object_template.internal = TRUE; + vm_object_template.temporary = TRUE; + vm_object_template.alive = TRUE; + vm_object_template.lock_in_progress = FALSE; + vm_object_template.lock_restart = FALSE; + vm_object_template.use_old_pageout = TRUE; /* XXX change later */ + vm_object_template.last_alloc = (vm_offset_t) 0; #if MACH_PAGEMAP - vm_object_template->existence_info = VM_EXTERNAL_NULL; + vm_object_template.existence_info = VM_EXTERNAL_NULL; #endif /* MACH_PAGEMAP */ /* * Initialize the "kernel object" */ - kernel_object = _vm_object_allocate( + _vm_object_setup(kernel_object, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS); /* @@ -315,7 +318,7 @@ void vm_object_bootstrap(void) * kernel object so that no limit is imposed on submap sizes. */ - vm_submap_object = _vm_object_allocate( + _vm_object_setup(vm_submap_object, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS); #if MACH_PAGEMAP @@ -325,7 +328,6 @@ void vm_object_bootstrap(void) void vm_object_init(void) { -#if !NORMA_VM /* * Finish initializing the kernel object. * The submap object doesn't need a name port. @@ -335,7 +337,6 @@ void vm_object_init(void) ipc_kobject_set(kernel_object->pager_name, (ipc_kobject_t) kernel_object, IKOT_PAGING_NAME); -#endif /* !NORMA_VM */ } /* @@ -344,7 +345,7 @@ void vm_object_init(void) * Gets another reference to the given object. */ void vm_object_reference( - register vm_object_t object) + vm_object_t object) { if (object == VM_OBJECT_NULL) return; @@ -367,7 +368,7 @@ void vm_object_reference( * No object may be locked. */ void vm_object_deallocate( - register vm_object_t object) + vm_object_t object) { vm_object_t temp; @@ -414,6 +415,7 @@ void vm_object_deallocate( queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list); overflow = (++vm_object_cached_count > vm_object_cached_max); + vm_object_cached_pages_update(object->resident_page_count); vm_object_cache_unlock(); vm_object_deactivate_pages(object); @@ -520,8 +522,6 @@ void vm_object_deallocate( } } -boolean_t vm_object_terminate_remove_all = FALSE; - /* * Routine: vm_object_terminate * Purpose: @@ -536,10 +536,10 @@ boolean_t vm_object_terminate_remove_all * object will cease to exist. */ void vm_object_terminate( - register vm_object_t object) + vm_object_t object) { - register vm_page_t p; - vm_object_t shadow_object; + vm_page_t p; + vm_object_t shadow_object; /* * Make sure the object isn't already being terminated @@ -656,11 +656,7 @@ void vm_object_terminate( object->pager_name); } else if (object->pager_name != IP_NULL) { /* consumes our right for pager_name */ -#if NORMA_VM - ipc_port_release_send(object->pager_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(object->pager_name); -#endif /* NORMA_VM */ } #if MACH_PAGEMAP @@ -671,7 +667,7 @@ void vm_object_terminate( * Free the space for the object. */ - zfree(vm_object_zone, (vm_offset_t) object); + kmem_cache_free(&vm_object_cache, (vm_offset_t) object); } /* @@ -746,7 +742,6 @@ void memory_object_release( void vm_object_abort_activity( vm_object_t object) { - register vm_page_t p; vm_page_t next; @@ -800,17 +795,12 @@ void vm_object_abort_activity( * or from port destruction handling (via vm_object_destroy). */ kern_return_t memory_object_destroy( - register vm_object_t object, kern_return_t reason) { ipc_port_t old_object, old_name; pager_request_t old_control; -#ifdef lint - reason++; -#endif /* lint */ - if (object == VM_OBJECT_NULL) return KERN_SUCCESS; @@ -865,11 +855,7 @@ kern_return_t memory_object_destroy( old_name); } else if (old_name != IP_NULL) { /* consumes our right for name */ -#if NORMA_VM - ipc_port_release_send(object->pager_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(object->pager_name); -#endif /* NORMA_VM */ } /* @@ -890,9 +876,9 @@ kern_return_t memory_object_destroy( * The object must be locked. */ void vm_object_deactivate_pages( - register vm_object_t object) + vm_object_t object) { - register vm_page_t p; + vm_page_t p; queue_iterate(&object->memq, p, vm_page_t, listq) { vm_page_lock_queues(); @@ -929,9 +915,9 @@ void vm_object_deactivate_pages( boolean_t vm_object_pmap_protect_by_page = FALSE; void vm_object_pmap_protect( - register vm_object_t object, - register vm_offset_t offset, - vm_offset_t size, + vm_object_t object, + vm_offset_t offset, + vm_size_t size, pmap_t pmap, vm_offset_t pmap_start, vm_prot_t prot) @@ -952,8 +938,8 @@ void vm_object_pmap_protect( } { - register vm_page_t p; - register vm_offset_t end; + vm_page_t p; + vm_offset_t end; end = offset + size; @@ -984,7 +970,7 @@ void vm_object_pmap_protect( * Must follow shadow chain to remove access * to pages in shadowed objects. */ - register vm_object_t next_object; + vm_object_t next_object; next_object = object->shadow; if (next_object != VM_OBJECT_NULL) { @@ -1021,11 +1007,11 @@ void vm_object_pmap_protect( * The object must *not* be locked. */ void vm_object_pmap_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p; + vm_page_t p; if (object == VM_OBJECT_NULL) return; @@ -1071,7 +1057,6 @@ void vm_object_pmap_remove( * VM_OBJECT_NULL. */ kern_return_t vm_object_copy_slowly( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1125,7 +1110,6 @@ kern_return_t vm_object_copy_slowly( vm_prot_t prot = VM_PROT_READ; vm_page_t _result_page; vm_page_t top_page; - register vm_page_t result_page; vm_object_lock(src_object); @@ -1245,8 +1229,6 @@ kern_return_t vm_object_copy_slowly( * The object should be unlocked on entry and exit. */ -vm_object_t vm_object_copy_delayed(); /* forward declaration */ - boolean_t vm_object_copy_temporary( vm_object_t *_object, /* INOUT */ vm_offset_t *_offset, /* INOUT */ @@ -1255,10 +1237,6 @@ boolean_t vm_object_copy_temporary( { vm_object_t object = *_object; -#ifdef lint - ++*_offset; -#endif /* lint */ - if (object == VM_OBJECT_NULL) { *_src_needs_copy = FALSE; *_dst_needs_copy = FALSE; @@ -1488,7 +1466,7 @@ vm_object_t vm_object_copy_delayed( * synchronization required in the "push" * operation described above. * - * The copy-on-write is said to be assymetric because + * The copy-on-write is said to be asymmetric because * the original object is *not* marked copy-on-write. * A copied page is pushed to the copy object, regardless * which party attempted to modify the page. @@ -1621,7 +1599,6 @@ vm_object_t vm_object_copy_delayed( * and may be interrupted. */ kern_return_t vm_object_copy_strategically( - register vm_object_t src_object, vm_offset_t src_offset, vm_size_t size, @@ -1734,8 +1711,8 @@ void vm_object_shadow( vm_offset_t *offset, /* IN/OUT */ vm_size_t length) { - register vm_object_t source; - register vm_object_t result; + vm_object_t source; + vm_object_t result; source = *object; @@ -1861,11 +1838,7 @@ vm_object_t vm_object_lookup( if (IP_VALID(port)) { ip_lock(port); if (ip_active(port) && -#if NORMA_VM - (ip_kotype(port) == IKOT_PAGER)) { -#else /* NORMA_VM */ (ip_kotype(port) == IKOT_PAGING_REQUEST)) { -#endif /* NORMA_VM */ vm_object_cache_lock(); object = (vm_object_t) port->ip_kobject; vm_object_lock(object); @@ -1876,6 +1849,7 @@ vm_object_t vm_object_lookup( queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); vm_object_cached_count--; + vm_object_cached_pages_update(-object->resident_page_count); } object->ref_count++; @@ -1907,6 +1881,7 @@ vm_object_t vm_object_lookup_name( queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); vm_object_cached_count--; + vm_object_cached_pages_update(-object->resident_page_count); } object->ref_count++; @@ -1943,6 +1918,7 @@ void vm_object_destroy( queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); vm_object_cached_count--; + vm_object_cached_pages_update(-object->resident_page_count); } object->ref_count++; @@ -1976,16 +1952,10 @@ void vm_object_destroy( */ ipc_port_release_send(pager); -#if !NORMA_VM if (old_request != IP_NULL) ipc_port_dealloc_kernel(old_request); -#endif /* !NORMA_VM */ if (old_name != IP_NULL) -#if NORMA_VM - ipc_port_release_send(old_name); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(old_name); -#endif /* NORMA_VM */ /* * Restart pending page requests @@ -2014,7 +1984,6 @@ vm_object_t vm_object_enter( vm_size_t size, boolean_t internal) { - register vm_object_t object; vm_object_t new_object; boolean_t must_init; @@ -2102,6 +2071,7 @@ restart: queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); vm_object_cached_count--; + vm_object_cached_pages_update(-object->resident_page_count); } object->ref_count++; vm_object_unlock(object); @@ -2138,27 +2108,6 @@ restart: object->pager_created = TRUE; object->pager = pager; -#if NORMA_VM - - /* - * Let the xmm system know that we want to use the pager. - * - * Name port will be provided by the xmm system - * when set_attributes_common is called. - */ - - object->internal = internal; - object->pager_ready = internal; - if (internal) { - assert(object->temporary); - } else { - object->temporary = FALSE; - } - object->pager_name = IP_NULL; - - (void) xmm_memory_object_init(object); -#else /* NORMA_VM */ - /* * Allocate request port. */ @@ -2207,7 +2156,6 @@ restart: PAGE_SIZE); } -#endif /* NORMA_VM */ vm_object_lock(object); object->pager_initialized = TRUE; @@ -2253,7 +2201,6 @@ restart: * daemon will be using this routine. */ void vm_object_pager_create( - register vm_object_t object) { ipc_port_t pager; @@ -2359,7 +2306,6 @@ void vm_object_remove( else if (ip_kotype(port) != IKOT_NONE) panic("vm_object_remove: bad object port"); } -#if !NORMA_VM if ((port = object->pager_request) != IP_NULL) { if (ip_kotype(port) == IKOT_PAGING_REQUEST) ipc_kobject_set(port, IKO_NULL, IKOT_NONE); @@ -2372,7 +2318,6 @@ void vm_object_remove( else if (ip_kotype(port) != IKOT_NONE) panic("vm_object_remove: bad name port"); } -#endif /* !NORMA_VM */ } /* @@ -2400,14 +2345,14 @@ boolean_t vm_object_collapse_bypass_allo * so the caller should hold a reference for the object. */ void vm_object_collapse( - register vm_object_t object) + vm_object_t object) { - register vm_object_t backing_object; - register vm_offset_t backing_offset; - register vm_size_t size; - register vm_offset_t new_offset; - register vm_page_t p, pp; - ipc_port_t old_name_port; + vm_object_t backing_object; + vm_offset_t backing_offset; + vm_size_t size; + vm_offset_t new_offset; + vm_page_t p, pp; + ipc_port_t old_name_port; if (!vm_object_collapse_allowed) return; @@ -2520,9 +2465,7 @@ void vm_object_collapse( if (p->offset < backing_offset || new_offset >= size) { - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } else { pp = vm_page_lookup(object, new_offset); if (pp != VM_PAGE_NULL && !pp->absent) { @@ -2531,9 +2474,7 @@ void vm_object_collapse( * Throw away the backing object's * page. */ - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } else { if (pp != VM_PAGE_NULL) { @@ -2557,9 +2498,7 @@ void vm_object_collapse( */ panic("vm_object_collapse: bad case"); - vm_page_lock_queues(); - vm_page_free(pp); - vm_page_unlock_queues(); + VM_PAGE_FREE(pp); /* * Fall through to move the backing @@ -2594,11 +2533,11 @@ void vm_object_collapse( /* Fall through to... */ default: - printf("vm_object_collapse: %#x (pager %#x, request %#x) up to %#x\n", + printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n", backing_object, backing_object->pager, backing_object->pager_request, object); if (vm_object_collapse_debug > 2) - Debugger("vm_object_collapse"); + SoftDebugger("vm_object_collapse"); } object->pager = backing_object->pager; @@ -2611,10 +2550,6 @@ void vm_object_collapse( object->pager_created = backing_object->pager_created; object->pager_request = backing_object->pager_request; -#if NORMA_VM - old_name_port = object->pager_name; - object->pager_name = backing_object->pager_name; -#else /* NORMA_VM */ if (object->pager_request != IP_NULL) ipc_kobject_set(object->pager_request, (ipc_kobject_t) object, @@ -2628,7 +2563,6 @@ void vm_object_collapse( ipc_kobject_set(object->pager_name, (ipc_kobject_t) object, IKOT_PAGING_NAME); -#endif /* NORMA_VM */ vm_object_cache_unlock(); @@ -2677,12 +2611,8 @@ void vm_object_collapse( vm_object_unlock(object); if (old_name_port != IP_NULL) -#if NORMA_VM - ipc_port_release_send(old_name_port); -#else /* NORMA_VM */ ipc_port_dealloc_kernel(old_name_port); -#endif /* NORMA_VM */ - zfree(vm_object_zone, (vm_offset_t) backing_object); + kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object); vm_object_lock(object); object_collapses++; @@ -2792,11 +2722,11 @@ unsigned int vm_object_page_remove_looku unsigned int vm_object_page_remove_iterate = 0; void vm_object_page_remove( - register vm_object_t object, - register vm_offset_t start, - register vm_offset_t end) + vm_object_t object, + vm_offset_t start, + vm_offset_t end) { - register vm_page_t p, next; + vm_page_t p, next; /* * One and two page removals are most popular. @@ -2813,9 +2743,7 @@ void vm_object_page_remove( if (!p->fictitious) pmap_page_protect(p->phys_addr, VM_PROT_NONE); - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } } } else { @@ -2828,9 +2756,7 @@ void vm_object_page_remove( if (!p->fictitious) pmap_page_protect(p->phys_addr, VM_PROT_NONE); - vm_page_lock_queues(); - vm_page_free(p); - vm_page_unlock_queues(); + VM_PAGE_FREE(p); } p = next; } @@ -2861,7 +2787,7 @@ void vm_object_page_remove( */ boolean_t vm_object_coalesce( - register vm_object_t prev_object, + vm_object_t prev_object, vm_object_t next_object, vm_offset_t prev_offset, vm_offset_t next_offset, @@ -2870,10 +2796,6 @@ boolean_t vm_object_coalesce( { vm_size_t newsize; -#ifdef lint - next_offset++; -#endif /* lint */ - if (next_object != VM_OBJECT_NULL) { return FALSE; } @@ -2959,11 +2881,7 @@ ipc_port_t vm_object_name( p = object->pager_name; if (p != IP_NULL) -#if NORMA_VM - p = ipc_port_copy_send(p); -#else /* NORMA_VM */ p = ipc_port_make_send(p); -#endif /* NORMA_VM */ vm_object_unlock(object); return p; @@ -3003,9 +2921,7 @@ vm_object_page_map( if ((old_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { - vm_page_lock_queues(); - vm_page_free(old_page); - vm_page_unlock_queues(); + VM_PAGE_FREE(old_page); } vm_page_init(m, addr); @@ -3020,10 +2936,9 @@ vm_object_page_map( } } -#include - #if MACH_KDB +#include #define printf kdbprintf boolean_t vm_object_print_pages = FALSE; @@ -3034,10 +2949,9 @@ boolean_t vm_object_print_pages = FALSE; void vm_object_print( vm_object_t object) { - register vm_page_t p; - extern indent; + vm_page_t p; - register int count; + int count; if (object == VM_OBJECT_NULL) return;