--- Gnu-Mach/vm/vm_fault.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_fault.c 2020/09/02 04:50:01 1.1.1.5 @@ -32,15 +32,13 @@ * * Page fault handling module. */ -#include -#include -#include - +#include #include #include #include /* for error codes */ #include +#include #include #include #include @@ -51,11 +49,10 @@ #include #include #include -#include "memory_object_user.h" +#include /* For memory_object_data_{request,unlock} */ -#include -#include -#include +#include +#include #if MACH_PCSAMPLE #include @@ -87,12 +84,10 @@ typedef struct vm_fault_state { vm_prot_t vmfp_access; } vm_fault_state_t; -zone_t vm_fault_state_zone = 0; +struct kmem_cache vm_fault_state_cache; int vm_object_absent_max = 50; -int vm_fault_debug = 0; - boolean_t vm_fault_dirty_handling = FALSE; boolean_t vm_fault_interruptible = TRUE; @@ -100,19 +95,17 @@ boolean_t software_reference_bits = TRUE #if MACH_KDB extern struct db_watchpoint *db_watchpoint_list; -#endif MACH_KDB +#endif /* MACH_KDB */ /* * Routine: vm_fault_init * Purpose: * Initialize our private data structures. */ -void vm_fault_init() +void vm_fault_init(void) { - vm_fault_state_zone = zinit(sizeof(vm_fault_state_t), - THREAD_MAX * sizeof(vm_fault_state_t), - sizeof(vm_fault_state_t), - 0, "vm fault state"); + kmem_cache_init(&vm_fault_state_cache, "vm_fault_state", + sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0); } /* @@ -130,9 +123,9 @@ void vm_fault_init() * "object" must be locked. */ void -vm_fault_cleanup(object, top_page) - register vm_object_t object; - register vm_page_t top_page; +vm_fault_cleanup( + vm_object_t object, + vm_page_t top_page) { vm_object_paging_end(object); vm_object_unlock(object); @@ -209,33 +202,26 @@ vm_fault_cleanup(object, top_page) * The "result_page" is also left busy. It is not removed * from the pageout queues. */ -vm_fault_return_t vm_fault_page(first_object, first_offset, - fault_type, must_be_resident, interruptible, - protection, - result_page, top_page, - resume, continuation) +vm_fault_return_t vm_fault_page( /* Arguments: */ - vm_object_t first_object; /* Object to begin search */ - vm_offset_t first_offset; /* Offset into object */ - vm_prot_t fault_type; /* What access is requested */ - boolean_t must_be_resident;/* Must page be resident? */ - boolean_t interruptible; /* May fault be interrupted? */ + vm_object_t first_object, /* Object to begin search */ + vm_offset_t first_offset, /* Offset into object */ + vm_prot_t fault_type, /* What access is requested */ + boolean_t must_be_resident,/* Must page be resident? */ + boolean_t interruptible, /* May fault be interrupted? */ /* Modifies in place: */ - vm_prot_t *protection; /* Protection for mapping */ + vm_prot_t *protection, /* Protection for mapping */ /* Returns: */ - vm_page_t *result_page; /* Page found, if successful */ - vm_page_t *top_page; /* Page in top object, if + vm_page_t *result_page, /* Page found, if successful */ + vm_page_t *top_page, /* Page in top object, if * not result_page. */ /* More arguments: */ - boolean_t resume; /* We are restarting. */ - void (*continuation)(); /* Continuation for blocking. */ + boolean_t resume, /* We are restarting. */ + void (*continuation)()) /* Continuation for blocking. */ { - register vm_page_t m; - register vm_object_t object; - register vm_offset_t offset; vm_page_t first_m; vm_object_t next_object; @@ -243,9 +229,8 @@ vm_fault_return_t vm_fault_page(first_ob boolean_t look_for_page; vm_prot_t access_required; -#ifdef CONTINUATIONS if (resume) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; if (state->vmfp_backoff) @@ -257,13 +242,10 @@ vm_fault_return_t vm_fault_page(first_ob access_required = state->vmfp_access; goto after_thread_block; } -#else /* not CONTINUATIONS */ - assert(continuation == 0); - assert(!resume); -#endif /* not CONTINUATIONS */ vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY); vm_stat.faults++; /* needs lock XXX */ + current_task()->faults++; /* * Recovery actions @@ -286,7 +268,7 @@ vm_fault_return_t vm_fault_page(first_ob * so that the watchpoint code notices the access. */ || db_watchpoint_list -#endif MACH_KDB +#endif /* MACH_KDB */ ) { /* * If we aren't asking for write permission, @@ -365,9 +347,8 @@ vm_fault_return_t vm_fault_page(first_ob PAGE_ASSERT_WAIT(m, interruptible); vm_object_unlock(object); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; /* @@ -387,7 +368,6 @@ vm_fault_return_t vm_fault_page(first_ob counter(c_vm_fault_page_block_busy_user++); thread_block(continuation); } else -#endif /* CONTINUATIONS */ { counter(c_vm_fault_page_block_busy_kernel++); thread_block((void (*)()) 0); @@ -443,7 +423,7 @@ vm_fault_return_t vm_fault_page(first_ob * need to allocate a real page. */ - real_m = vm_page_grab(); + real_m = vm_page_grab(!object->internal); if (real_m == VM_PAGE_NULL) { vm_fault_cleanup(object, first_m); return(VM_FAULT_MEMORY_SHORTAGE); @@ -481,8 +461,9 @@ vm_fault_return_t vm_fault_page(first_ob vm_page_zero_fill(m); vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); - + vm_stat.zero_fill_count++; + current_task()->zero_fills++; vm_object_lock(object); pmap_clear_modify(m->phys_addr); break; @@ -519,7 +500,7 @@ vm_fault_return_t vm_fault_page(first_ob if ((access_required & m->unlock_request) != access_required) { vm_prot_t new_unlock_request; kern_return_t rc; - + if (!object->pager_ready) { vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, @@ -564,6 +545,7 @@ vm_fault_return_t vm_fault_page(first_ob if (m->inactive) { vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS); vm_stat.reactivations++; + current_task()->reactivations++; } VM_PAGE_QUEUES_REMOVE(m); @@ -581,7 +563,7 @@ vm_fault_return_t vm_fault_page(first_ob #if MACH_PAGEMAP && (vm_external_state_get(object->existence_info, offset + object->paging_offset) != VM_EXTERNAL_STATE_ABSENT) -#endif MACH_PAGEMAP +#endif /* MACH_PAGEMAP */ ; if ((look_for_page || (object == first_object)) @@ -625,7 +607,7 @@ vm_fault_return_t vm_fault_page(first_ob * won't block for pages. */ - if (m->fictitious && !vm_page_convert(m)) { + if (m->fictitious && !vm_page_convert(m, FALSE)) { VM_PAGE_FREE(m); vm_fault_cleanup(object, first_m); return(VM_FAULT_MEMORY_SHORTAGE); @@ -663,17 +645,18 @@ vm_fault_return_t vm_fault_page(first_ob vm_stat.pageins++; vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS); + current_task()->pageins++; - if ((rc = memory_object_data_request(object->pager, + if ((rc = memory_object_data_request(object->pager, object->pager_request, - m->offset + object->paging_offset, + m->offset + object->paging_offset, PAGE_SIZE, access_required)) != KERN_SUCCESS) { if (rc != MACH_SEND_INTERRUPTED) - printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n", + printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n", "memory_object_data_request", object->pager, object->pager_request, - m->offset + object->paging_offset, + m->offset + object->paging_offset, PAGE_SIZE, access_required, rc); /* * Don't want to leave a busy page around, @@ -689,7 +672,7 @@ vm_fault_return_t vm_fault_page(first_ob VM_FAULT_INTERRUPTED : VM_FAULT_MEMORY_ERROR); } - + /* * Retry with same object/offset, since new data may * be in a different page (i.e., m is meaningless at @@ -742,7 +725,7 @@ vm_fault_return_t vm_fault_page(first_ob assert(m->object == object); first_m = VM_PAGE_NULL; - if (m->fictitious && !vm_page_convert(m)) { + if (m->fictitious && !vm_page_convert(m, !object->internal)) { VM_PAGE_FREE(m); vm_fault_cleanup(object, VM_PAGE_NULL); return(VM_FAULT_MEMORY_SHORTAGE); @@ -752,6 +735,7 @@ vm_fault_return_t vm_fault_page(first_ob vm_page_zero_fill(m); vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); vm_stat.zero_fill_count++; + current_task()->zero_fills++; vm_object_lock(object); pmap_clear_modify(m->phys_addr); break; @@ -784,12 +768,10 @@ vm_fault_return_t vm_fault_page(first_ob * objects. */ -#if EXTRA_ASSERTIONS assert(m->busy && !m->absent); assert((first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)); -#endif EXTRA_ASSERTIONS /* * If the page is being written, but isn't @@ -828,7 +810,7 @@ vm_fault_return_t vm_fault_page(first_ob /* * Allocate a page for the copy */ - copy_m = vm_page_grab(); + copy_m = vm_page_grab(!first_object->internal); if (copy_m == VM_PAGE_NULL) { RELEASE_PAGE(m); vm_fault_cleanup(object, first_m); @@ -867,6 +849,7 @@ vm_fault_return_t vm_fault_page(first_ob vm_stat.cow_faults++; vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS); + current_task()->cow_faults++; object = first_object; offset = first_offset; @@ -983,7 +966,7 @@ vm_fault_return_t vm_fault_page(first_ob */ vm_page_copy(m, copy_m); - + /* * If the old page was in use by any users * of the copy-object, it must be removed @@ -1059,7 +1042,7 @@ vm_fault_return_t vm_fault_page(first_ob * wait result]. Can't turn off the page's * busy bit because we're not done with it. */ - + if (m->wanted) { m->wanted = FALSE; thread_wakeup_with_result((event_t) m, @@ -1099,9 +1082,8 @@ vm_fault_return_t vm_fault_page(first_ob block_and_backoff: vm_fault_cleanup(object, first_m); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; /* @@ -1114,7 +1096,6 @@ vm_fault_return_t vm_fault_page(first_ob counter(c_vm_fault_page_block_backoff_user++); thread_block(continuation); } else -#endif /* CONTINUATIONS */ { counter(c_vm_fault_page_block_backoff_kernel++); thread_block((void (*)()) 0); @@ -1148,11 +1129,10 @@ vm_fault_return_t vm_fault_page(first_ob * and deallocated when leaving vm_fault. */ -#ifdef CONTINUATIONS void -vm_fault_continue() +vm_fault_continue(void) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; (void) vm_fault(state->vmf_map, @@ -1162,16 +1142,14 @@ vm_fault_continue() TRUE, state->vmf_continuation); /*NOTREACHED*/ } -#endif /* CONTINUATIONS */ -kern_return_t vm_fault(map, vaddr, fault_type, change_wiring, - resume, continuation) - vm_map_t map; - vm_offset_t vaddr; - vm_prot_t fault_type; - boolean_t change_wiring; - boolean_t resume; - void (*continuation)(); +kern_return_t vm_fault( + vm_map_t map, + vm_offset_t vaddr, + vm_prot_t fault_type, + boolean_t change_wiring, + boolean_t resume, + void (*continuation)()) { vm_map_version_t version; /* Map version for verificiation */ boolean_t wired; /* Should mapping be wired down? */ @@ -1183,12 +1161,10 @@ kern_return_t vm_fault(map, vaddr, fault vm_page_t top_page; /* Placeholder page */ kern_return_t kr; - register vm_page_t m; /* Fast access to result_page */ -#ifdef CONTINUATIONS if (resume) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; /* @@ -1220,19 +1196,15 @@ kern_return_t vm_fault(map, vaddr, fault /* * if this assignment stmt is written as - * 'active_threads[cpu_number()] = zalloc()', - * cpu_number may be evaluated before zalloc; - * if zalloc blocks, cpu_number will be wrong + * 'active_threads[cpu_number()] = kmem_cache_alloc()', + * cpu_number may be evaluated before kmem_cache_alloc; + * if kmem_cache_alloc blocks, cpu_number will be wrong */ - state = (char *) zalloc(vm_fault_state_zone); + state = (char *) kmem_cache_alloc(&vm_fault_state_cache); current_thread()->ith_other = state; } -#else /* not CONTINUATIONS */ - assert(continuation == 0); - assert(!resume); -#endif /* not CONTINUATIONS */ RetryFault: ; @@ -1267,9 +1239,8 @@ kern_return_t vm_fault(map, vaddr, fault object->ref_count++; vm_object_paging_begin(object); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; /* @@ -1294,7 +1265,6 @@ kern_return_t vm_fault(map, vaddr, fault &prot, &result_page, &top_page, FALSE, vm_fault_continue); } else -#endif /* CONTINUATIONS */ { kr = vm_fault_page(object, offset, fault_type, (change_wiring && !wired), !change_wiring, @@ -1323,9 +1293,8 @@ kern_return_t vm_fault(map, vaddr, fault kr = KERN_SUCCESS; goto done; case VM_FAULT_MEMORY_SHORTAGE: -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; /* @@ -1342,7 +1311,6 @@ kern_return_t vm_fault(map, vaddr, fault VM_PAGE_WAIT(vm_fault_continue); } else -#endif /* CONTINUATIONS */ VM_PAGE_WAIT((void (*)()) 0); goto RetryFault; case VM_FAULT_FICTITIOUS_SHORTAGE: @@ -1508,35 +1476,31 @@ kern_return_t vm_fault(map, vaddr, fault #undef RELEASE_PAGE done: -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { - register vm_fault_state_t *state = + vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; - zfree(vm_fault_state_zone, (vm_offset_t) state); + kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state); (*continuation)(kr); /*NOTREACHED*/ } -#endif /* CONTINUATIONS */ return(kr); } -kern_return_t vm_fault_wire_fast(); - /* * vm_fault_wire: * * Wire down a range of virtual addresses in a map. */ -void vm_fault_wire(map, entry) - vm_map_t map; - vm_map_entry_t entry; +void vm_fault_wire( + vm_map_t map, + vm_map_entry_t entry) { - register vm_offset_t va; - register pmap_t pmap; - register vm_offset_t end_addr = entry->vme_end; + vm_offset_t va; + pmap_t pmap; + vm_offset_t end_addr = entry->vme_end; pmap = vm_map_pmap(map); @@ -1565,14 +1529,14 @@ void vm_fault_wire(map, entry) * * Unwire a range of virtual addresses in a map. */ -void vm_fault_unwire(map, entry) - vm_map_t map; - vm_map_entry_t entry; +void vm_fault_unwire( + vm_map_t map, + vm_map_entry_t entry) { - register vm_offset_t va; - register pmap_t pmap; - register vm_offset_t end_addr = entry->vme_end; - vm_object_t object; + vm_offset_t va; + pmap_t pmap; + vm_offset_t end_addr = entry->vme_end; + vm_object_t object; pmap = vm_map_pmap(map); @@ -1654,17 +1618,18 @@ void vm_fault_unwire(map, entry) * other than the common case will return KERN_FAILURE, and the caller * is expected to call vm_fault(). */ -kern_return_t vm_fault_wire_fast(map, va, entry) - vm_map_t map; - vm_offset_t va; - vm_map_entry_t entry; +kern_return_t vm_fault_wire_fast( + vm_map_t map, + vm_offset_t va, + vm_map_entry_t entry) { vm_object_t object; vm_offset_t offset; - register vm_page_t m; + vm_page_t m; vm_prot_t prot; vm_stat.faults++; /* needs lock XXX */ + current_task()->faults++; /* * Recovery actions */ @@ -1749,7 +1714,7 @@ kern_return_t vm_fault_wire_fast(map, va /* * Wire the page down now. All bail outs beyond this - * point must unwire the page. + * point must unwire the page. */ vm_page_lock_queues(); @@ -1774,7 +1739,7 @@ kern_return_t vm_fault_wire_fast(map, va /* * Put this page into the physical map. * We have to unlock the object because pmap_enter - * may cause other faults. + * may cause other faults. */ vm_object_unlock(object); @@ -1802,9 +1767,9 @@ kern_return_t vm_fault_wire_fast(map, va * Release a page used by vm_fault_copy. */ -void vm_fault_copy_cleanup(page, top_page) - vm_page_t page; - vm_page_t top_page; +void vm_fault_copy_cleanup( + vm_page_t page, + vm_page_t top_page) { vm_object_t object = page->object; @@ -1845,27 +1810,18 @@ void vm_fault_copy_cleanup(page, top_pag * requested. */ kern_return_t vm_fault_copy( - src_object, - src_offset, - src_size, - dst_object, - dst_offset, - dst_map, - dst_version, - interruptible - ) - vm_object_t src_object; - vm_offset_t src_offset; - vm_size_t *src_size; /* INOUT */ - vm_object_t dst_object; - vm_offset_t dst_offset; - vm_map_t dst_map; - vm_map_version_t *dst_version; - boolean_t interruptible; + vm_object_t src_object, + vm_offset_t src_offset, + vm_size_t *src_size, /* INOUT */ + vm_object_t dst_object, + vm_offset_t dst_offset, + vm_map_t dst_map, + vm_map_version_t *dst_version, + boolean_t interruptible) { vm_page_t result_page; vm_prot_t prot; - + vm_page_t src_page; vm_page_t src_top_page; @@ -2020,7 +1976,7 @@ kern_return_t vm_fault_copy( RETURN(KERN_SUCCESS); #undef RETURN - /*NOTREACHED*/ + /*NOTREACHED*/ } @@ -2042,13 +1998,11 @@ kern_return_t vm_fault_copy( * XXX Untested. Also unused. Eventually, this technology * could be used in vm_fault_copy() to advantage. */ -vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page) - register - vm_object_t dst_object; - vm_offset_t dst_offset; - vm_page_t *result_page; /* OUT */ +vm_fault_return_t vm_fault_page_overwrite( + vm_object_t dst_object, + vm_offset_t dst_offset, + vm_page_t *result_page) /* OUT */ { - register vm_page_t dst_page; #define interruptible FALSE /* XXX */ @@ -2179,4 +2133,4 @@ vm_fault_return_t vm_fault_page_overwrit #undef DISCARD_PAGE } -#endif notdef +#endif /* notdef */