--- Gnu-Mach/vm/vm_fault.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_fault.c 2020/09/02 04:45:24 1.1.1.3 @@ -32,15 +32,13 @@ * * Page fault handling module. */ -#include -#include -#include - +#include #include #include #include /* for error codes */ #include +#include #include #include #include @@ -51,11 +49,10 @@ #include #include #include -#include "memory_object_user.h" +#include /* For memory_object_data_{request,unlock} */ -#include #include -#include +#include #if MACH_PCSAMPLE #include @@ -87,7 +84,7 @@ typedef struct vm_fault_state { vm_prot_t vmfp_access; } vm_fault_state_t; -zone_t vm_fault_state_zone = 0; +struct kmem_cache vm_fault_state_cache; int vm_object_absent_max = 50; @@ -100,19 +97,17 @@ boolean_t software_reference_bits = TRUE #if MACH_KDB extern struct db_watchpoint *db_watchpoint_list; -#endif MACH_KDB +#endif /* MACH_KDB */ /* * Routine: vm_fault_init * Purpose: * Initialize our private data structures. */ -void vm_fault_init() +void vm_fault_init(void) { - vm_fault_state_zone = zinit(sizeof(vm_fault_state_t), - THREAD_MAX * sizeof(vm_fault_state_t), - sizeof(vm_fault_state_t), - 0, "vm fault state"); + kmem_cache_init(&vm_fault_state_cache, "vm_fault_state", + sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0); } /* @@ -243,7 +238,6 @@ vm_fault_return_t vm_fault_page(first_ob boolean_t look_for_page; vm_prot_t access_required; -#ifdef CONTINUATIONS if (resume) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -257,13 +251,10 @@ vm_fault_return_t vm_fault_page(first_ob access_required = state->vmfp_access; goto after_thread_block; } -#else /* not CONTINUATIONS */ - assert(continuation == 0); - assert(!resume); -#endif /* not CONTINUATIONS */ vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY); vm_stat.faults++; /* needs lock XXX */ + current_task()->faults++; /* * Recovery actions @@ -286,7 +277,7 @@ vm_fault_return_t vm_fault_page(first_ob * so that the watchpoint code notices the access. */ || db_watchpoint_list -#endif MACH_KDB +#endif /* MACH_KDB */ ) { /* * If we aren't asking for write permission, @@ -365,7 +356,6 @@ vm_fault_return_t vm_fault_page(first_ob PAGE_ASSERT_WAIT(m, interruptible); vm_object_unlock(object); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -387,7 +377,6 @@ vm_fault_return_t vm_fault_page(first_ob counter(c_vm_fault_page_block_busy_user++); thread_block(continuation); } else -#endif /* CONTINUATIONS */ { counter(c_vm_fault_page_block_busy_kernel++); thread_block((void (*)()) 0); @@ -443,7 +432,7 @@ vm_fault_return_t vm_fault_page(first_ob * need to allocate a real page. */ - real_m = vm_page_grab(); + real_m = vm_page_grab(!object->internal); if (real_m == VM_PAGE_NULL) { vm_fault_cleanup(object, first_m); return(VM_FAULT_MEMORY_SHORTAGE); @@ -481,8 +470,9 @@ vm_fault_return_t vm_fault_page(first_ob vm_page_zero_fill(m); vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); - + vm_stat.zero_fill_count++; + current_task()->zero_fills++; vm_object_lock(object); pmap_clear_modify(m->phys_addr); break; @@ -519,7 +509,7 @@ vm_fault_return_t vm_fault_page(first_ob if ((access_required & m->unlock_request) != access_required) { vm_prot_t new_unlock_request; kern_return_t rc; - + if (!object->pager_ready) { vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, @@ -564,6 +554,7 @@ vm_fault_return_t vm_fault_page(first_ob if (m->inactive) { vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS); vm_stat.reactivations++; + current_task()->reactivations++; } VM_PAGE_QUEUES_REMOVE(m); @@ -581,7 +572,7 @@ vm_fault_return_t vm_fault_page(first_ob #if MACH_PAGEMAP && (vm_external_state_get(object->existence_info, offset + object->paging_offset) != VM_EXTERNAL_STATE_ABSENT) -#endif MACH_PAGEMAP +#endif /* MACH_PAGEMAP */ ; if ((look_for_page || (object == first_object)) @@ -625,7 +616,7 @@ vm_fault_return_t vm_fault_page(first_ob * won't block for pages. */ - if (m->fictitious && !vm_page_convert(m)) { + if (m->fictitious && !vm_page_convert(m, FALSE)) { VM_PAGE_FREE(m); vm_fault_cleanup(object, first_m); return(VM_FAULT_MEMORY_SHORTAGE); @@ -663,17 +654,18 @@ vm_fault_return_t vm_fault_page(first_ob vm_stat.pageins++; vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS); + current_task()->pageins++; - if ((rc = memory_object_data_request(object->pager, + if ((rc = memory_object_data_request(object->pager, object->pager_request, - m->offset + object->paging_offset, + m->offset + object->paging_offset, PAGE_SIZE, access_required)) != KERN_SUCCESS) { if (rc != MACH_SEND_INTERRUPTED) - printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n", + printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n", "memory_object_data_request", object->pager, object->pager_request, - m->offset + object->paging_offset, + m->offset + object->paging_offset, PAGE_SIZE, access_required, rc); /* * Don't want to leave a busy page around, @@ -689,7 +681,7 @@ vm_fault_return_t vm_fault_page(first_ob VM_FAULT_INTERRUPTED : VM_FAULT_MEMORY_ERROR); } - + /* * Retry with same object/offset, since new data may * be in a different page (i.e., m is meaningless at @@ -742,7 +734,7 @@ vm_fault_return_t vm_fault_page(first_ob assert(m->object == object); first_m = VM_PAGE_NULL; - if (m->fictitious && !vm_page_convert(m)) { + if (m->fictitious && !vm_page_convert(m, !object->internal)) { VM_PAGE_FREE(m); vm_fault_cleanup(object, VM_PAGE_NULL); return(VM_FAULT_MEMORY_SHORTAGE); @@ -752,6 +744,7 @@ vm_fault_return_t vm_fault_page(first_ob vm_page_zero_fill(m); vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); vm_stat.zero_fill_count++; + current_task()->zero_fills++; vm_object_lock(object); pmap_clear_modify(m->phys_addr); break; @@ -789,7 +782,7 @@ vm_fault_return_t vm_fault_page(first_ob assert((first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)); -#endif EXTRA_ASSERTIONS +#endif /* EXTRA_ASSERTIONS */ /* * If the page is being written, but isn't @@ -828,7 +821,7 @@ vm_fault_return_t vm_fault_page(first_ob /* * Allocate a page for the copy */ - copy_m = vm_page_grab(); + copy_m = vm_page_grab(!first_object->internal); if (copy_m == VM_PAGE_NULL) { RELEASE_PAGE(m); vm_fault_cleanup(object, first_m); @@ -867,6 +860,7 @@ vm_fault_return_t vm_fault_page(first_ob vm_stat.cow_faults++; vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS); + current_task()->cow_faults++; object = first_object; offset = first_offset; @@ -983,7 +977,7 @@ vm_fault_return_t vm_fault_page(first_ob */ vm_page_copy(m, copy_m); - + /* * If the old page was in use by any users * of the copy-object, it must be removed @@ -1059,7 +1053,7 @@ vm_fault_return_t vm_fault_page(first_ob * wait result]. Can't turn off the page's * busy bit because we're not done with it. */ - + if (m->wanted) { m->wanted = FALSE; thread_wakeup_with_result((event_t) m, @@ -1099,7 +1093,6 @@ vm_fault_return_t vm_fault_page(first_ob block_and_backoff: vm_fault_cleanup(object, first_m); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -1114,7 +1107,6 @@ vm_fault_return_t vm_fault_page(first_ob counter(c_vm_fault_page_block_backoff_user++); thread_block(continuation); } else -#endif /* CONTINUATIONS */ { counter(c_vm_fault_page_block_backoff_kernel++); thread_block((void (*)()) 0); @@ -1148,7 +1140,6 @@ vm_fault_return_t vm_fault_page(first_ob * and deallocated when leaving vm_fault. */ -#ifdef CONTINUATIONS void vm_fault_continue() { @@ -1162,7 +1153,6 @@ vm_fault_continue() TRUE, state->vmf_continuation); /*NOTREACHED*/ } -#endif /* CONTINUATIONS */ kern_return_t vm_fault(map, vaddr, fault_type, change_wiring, resume, continuation) @@ -1186,7 +1176,6 @@ kern_return_t vm_fault(map, vaddr, fault register vm_page_t m; /* Fast access to result_page */ -#ifdef CONTINUATIONS if (resume) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -1220,19 +1209,15 @@ kern_return_t vm_fault(map, vaddr, fault /* * if this assignment stmt is written as - * 'active_threads[cpu_number()] = zalloc()', - * cpu_number may be evaluated before zalloc; - * if zalloc blocks, cpu_number will be wrong + * 'active_threads[cpu_number()] = kmem_cache_alloc()', + * cpu_number may be evaluated before kmem_cache_alloc; + * if kmem_cache_alloc blocks, cpu_number will be wrong */ - state = (char *) zalloc(vm_fault_state_zone); + state = (char *) kmem_cache_alloc(&vm_fault_state_cache); current_thread()->ith_other = state; } -#else /* not CONTINUATIONS */ - assert(continuation == 0); - assert(!resume); -#endif /* not CONTINUATIONS */ RetryFault: ; @@ -1267,7 +1252,6 @@ kern_return_t vm_fault(map, vaddr, fault object->ref_count++; vm_object_paging_begin(object); -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -1294,7 +1278,6 @@ kern_return_t vm_fault(map, vaddr, fault &prot, &result_page, &top_page, FALSE, vm_fault_continue); } else -#endif /* CONTINUATIONS */ { kr = vm_fault_page(object, offset, fault_type, (change_wiring && !wired), !change_wiring, @@ -1323,7 +1306,6 @@ kern_return_t vm_fault(map, vaddr, fault kr = KERN_SUCCESS; goto done; case VM_FAULT_MEMORY_SHORTAGE: -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; @@ -1342,7 +1324,6 @@ kern_return_t vm_fault(map, vaddr, fault VM_PAGE_WAIT(vm_fault_continue); } else -#endif /* CONTINUATIONS */ VM_PAGE_WAIT((void (*)()) 0); goto RetryFault; case VM_FAULT_FICTITIOUS_SHORTAGE: @@ -1508,16 +1489,14 @@ kern_return_t vm_fault(map, vaddr, fault #undef RELEASE_PAGE done: -#ifdef CONTINUATIONS if (continuation != (void (*)()) 0) { register vm_fault_state_t *state = (vm_fault_state_t *) current_thread()->ith_other; - zfree(vm_fault_state_zone, (vm_offset_t) state); + kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state); (*continuation)(kr); /*NOTREACHED*/ } -#endif /* CONTINUATIONS */ return(kr); } @@ -1665,6 +1644,7 @@ kern_return_t vm_fault_wire_fast(map, va vm_prot_t prot; vm_stat.faults++; /* needs lock XXX */ + current_task()->faults++; /* * Recovery actions */ @@ -1749,7 +1729,7 @@ kern_return_t vm_fault_wire_fast(map, va /* * Wire the page down now. All bail outs beyond this - * point must unwire the page. + * point must unwire the page. */ vm_page_lock_queues(); @@ -1774,7 +1754,7 @@ kern_return_t vm_fault_wire_fast(map, va /* * Put this page into the physical map. * We have to unlock the object because pmap_enter - * may cause other faults. + * may cause other faults. */ vm_object_unlock(object); @@ -1865,7 +1845,7 @@ kern_return_t vm_fault_copy( { vm_page_t result_page; vm_prot_t prot; - + vm_page_t src_page; vm_page_t src_top_page; @@ -2020,7 +2000,7 @@ kern_return_t vm_fault_copy( RETURN(KERN_SUCCESS); #undef RETURN - /*NOTREACHED*/ + /*NOTREACHED*/ } @@ -2179,4 +2159,4 @@ vm_fault_return_t vm_fault_page_overwrit #undef DISCARD_PAGE } -#endif notdef +#endif /* notdef */