--- Gnu-Mach/vm/vm_pageout.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_pageout.c 2020/09/02 04:52:07 1.1.1.6 @@ -34,45 +34,46 @@ * The proverbial page-out daemon. */ -#include -#include - +#include #include #include -#include "memory_object_default.h" -#include "memory_object_user.h" +#include +#include #include #include #include +#include +#include +#include #include #include #include #include #include #include -#include +#include #ifndef VM_PAGEOUT_BURST_MAX #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */ -#endif VM_PAGEOUT_BURST_MAX +#endif /* VM_PAGEOUT_BURST_MAX */ #ifndef VM_PAGEOUT_BURST_MIN #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */ -#endif VM_PAGEOUT_BURST_MIN +#endif /* VM_PAGEOUT_BURST_MIN */ #ifndef VM_PAGEOUT_BURST_WAIT -#define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */ -#endif VM_PAGEOUT_BURST_WAIT +#define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds per page */ +#endif /* VM_PAGEOUT_BURST_WAIT */ #ifndef VM_PAGEOUT_EMPTY_WAIT -#define VM_PAGEOUT_EMPTY_WAIT 200 /* milliseconds */ -#endif VM_PAGEOUT_EMPTY_WAIT +#define VM_PAGEOUT_EMPTY_WAIT 75 /* milliseconds */ +#endif /* VM_PAGEOUT_EMPTY_WAIT */ #ifndef VM_PAGEOUT_PAUSE_MAX #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */ -#endif VM_PAGEOUT_PAUSE_MAX +#endif /* VM_PAGEOUT_PAUSE_MAX */ /* * To obtain a reasonable LRU approximation, the inactive queue @@ -81,47 +82,61 @@ * of active+inactive pages that should be inactive. * The pageout daemon uses it to update vm_page_inactive_target. * - * If vm_page_free_count falls below vm_page_free_target and + * If the number of free pages falls below vm_page_free_target and * vm_page_inactive_count is below vm_page_inactive_target, * then the pageout daemon starts running. */ #ifndef VM_PAGE_INACTIVE_TARGET #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3) -#endif VM_PAGE_INACTIVE_TARGET +#endif /* VM_PAGE_INACTIVE_TARGET */ /* * Once the pageout daemon starts running, it keeps going - * until vm_page_free_count meets or exceeds vm_page_free_target. + * until the number of free pages meets or exceeds vm_page_free_target. */ #ifndef VM_PAGE_FREE_TARGET -#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) -#endif VM_PAGE_FREE_TARGET +#define VM_PAGE_FREE_TARGET(free) (150 + (free) * 10 / 100) +#endif /* VM_PAGE_FREE_TARGET */ /* - * The pageout daemon always starts running once vm_page_free_count + * The pageout daemon always starts running once the number of free pages * falls below vm_page_free_min. */ #ifndef VM_PAGE_FREE_MIN -#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) -#endif VM_PAGE_FREE_MIN +#define VM_PAGE_FREE_MIN(free) (100 + (free) * 8 / 100) +#endif /* VM_PAGE_FREE_MIN */ + +/* When vm_page_external_count exceeds vm_page_external_limit, + * allocations of externally paged pages stops. + */ + +#ifndef VM_PAGE_EXTERNAL_LIMIT +#define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2) +#endif /* VM_PAGE_EXTERNAL_LIMIT */ + +/* Attempt to keep the number of externally paged pages less + * than vm_pages_external_target. + */ +#ifndef VM_PAGE_EXTERNAL_TARGET +#define VM_PAGE_EXTERNAL_TARGET(free) ((free) / 4) +#endif /* VM_PAGE_EXTERNAL_TARGET */ /* - * When vm_page_free_count falls below vm_page_free_reserved, + * When the number of free pages falls below vm_page_free_reserved, * only vm-privileged threads can allocate pages. vm-privilege * allows the pageout daemon and default pager (and any other * associated threads needed for default pageout) to continue - * operation by dipping into the reserved pool of pages. - */ + * operation by dipping into the reserved pool of pages. */ #ifndef VM_PAGE_FREE_RESERVED -#define VM_PAGE_FREE_RESERVED 15 -#endif VM_PAGE_FREE_RESERVED +#define VM_PAGE_FREE_RESERVED 500 +#endif /* VM_PAGE_FREE_RESERVED */ /* - * When vm_page_free_count falls below vm_pageout_reserved_internal, + * When the number of free pages falls below vm_pageout_reserved_internal, * the pageout daemon no longer trusts external pagers to clean pages. * External pagers are probably all wedged waiting for a free page. * It forcibly double-pages dirty pages belonging to external objects, @@ -129,11 +144,11 @@ */ #ifndef VM_PAGEOUT_RESERVED_INTERNAL -#define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 5) -#endif VM_PAGEOUT_RESERVED_INTERNAL +#define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 250) +#endif /* VM_PAGEOUT_RESERVED_INTERNAL */ /* - * When vm_page_free_count falls below vm_pageout_reserved_really, + * When the number of free pages falls below vm_pageout_reserved_really, * the pageout daemon stops work entirely to let the default pager * catch up (assuming the default pager has pages to clean). * Beyond this point, it is too dangerous to consume memory @@ -141,15 +156,14 @@ */ #ifndef VM_PAGEOUT_RESERVED_REALLY -#define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 10) -#endif VM_PAGEOUT_RESERVED_REALLY - -extern void vm_pageout_continue(); -extern void vm_pageout_scan_continue(); +#define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 400) +#endif /* VM_PAGEOUT_RESERVED_REALLY */ unsigned int vm_pageout_reserved_internal = 0; unsigned int vm_pageout_reserved_really = 0; +unsigned int vm_page_external_target = 0; + unsigned int vm_pageout_burst_max = 0; unsigned int vm_pageout_burst_min = 0; unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */ @@ -172,14 +186,7 @@ unsigned int vm_pageout_inactive_used = unsigned int vm_pageout_inactive_clean = 0; /* debugging */ unsigned int vm_pageout_inactive_dirty = 0; /* debugging */ unsigned int vm_pageout_inactive_double = 0; /* debugging */ - -#if NORMA_VM -/* - * Define them here, since they won't be defined by memory_object_user.h. - */ -extern kern_return_t memory_object_data_initialize(); -extern kern_return_t memory_object_data_write(); -#endif NORMA_VM +unsigned int vm_pageout_inactive_cleaned_external = 0; /* * Routine: vm_pageout_setup @@ -219,16 +226,16 @@ extern kern_return_t memory_object_data_ * not busy on exit. */ vm_page_t -vm_pageout_setup(m, paging_offset, new_object, new_offset, flush) - register vm_page_t m; - vm_offset_t paging_offset; - register vm_object_t new_object; - vm_offset_t new_offset; - boolean_t flush; +vm_pageout_setup( + vm_page_t m, + vm_offset_t paging_offset, + vm_object_t new_object, + vm_offset_t new_offset, + boolean_t flush) { - register vm_object_t old_object = m->object; - register vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/ - register vm_page_t new_m; + vm_object_t old_object = m->object; + vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/ + vm_page_t new_m; assert(m->busy && !m->absent && !m->fictitious); @@ -272,7 +279,7 @@ vm_pageout_setup(m, paging_offset, new_o vm_external_state_set(old_object->existence_info, paging_offset, VM_EXTERNAL_STATE_EXISTS); -#endif MACH_PAGEMAP +#endif /* MACH_PAGEMAP */ vm_object_unlock(old_object); @@ -319,7 +326,7 @@ vm_pageout_setup(m, paging_offset, new_o vm_external_state_set(old_object->existence_info, paging_offset, VM_EXTERNAL_STATE_EXISTS); -#endif MACH_PAGEMAP +#endif /* MACH_PAGEMAP */ vm_object_unlock(old_object); @@ -406,15 +413,15 @@ vm_pageout_setup(m, paging_offset, new_o * copy to a new page in a new object, if not. */ void -vm_pageout_page(m, initial, flush) - register vm_page_t m; - boolean_t initial; - boolean_t flush; +vm_pageout_page( + vm_page_t m, + boolean_t initial, + boolean_t flush) { vm_map_copy_t copy; - register vm_object_t old_object; - register vm_object_t new_object; - register vm_page_t holding_page; + vm_object_t old_object; + vm_object_t new_object; + vm_page_t holding_page; vm_offset_t paging_offset; kern_return_t rc; boolean_t precious_clean; @@ -454,6 +461,7 @@ vm_pageout_page(m, initial, flush) * Allocate a new object into which we can put the page. */ new_object = vm_object_allocate(PAGE_SIZE); + new_object->used_for_pageout = TRUE; /* * Move the page into the new object. @@ -500,9 +508,10 @@ vm_pageout_page(m, initial, flush) * vm_page_free_wanted == 0. */ -void vm_pageout_scan() +void vm_pageout_scan(void) { unsigned int burst_count; + unsigned int want_pages; /* * We want to gradually dribble pages from the active queue @@ -533,24 +542,25 @@ void vm_pageout_scan() * into an internal object and then immediately double-page it, * sending it to the default pager. * - * consider_zone_gc should be last, because the other operations - * might return memory to zones. When we pause we use + * slab_collect should be last, because the other operations + * might return memory to caches. When we pause we use * vm_pageout_scan_continue as our continuation, so we will * reenter vm_pageout_scan periodically and attempt to reclaim * internal memory even if we never reach vm_page_free_target. */ - Restart: stack_collect(); net_kmsg_collect(); consider_task_collect(); + if (0) /* XXX: pcb_collect doesn't do anything yet, so it is + pointless to call consider_thread_collect. */ consider_thread_collect(); - consider_zone_gc(); + slab_collect(); for (burst_count = 0;;) { - register vm_page_t m; - register vm_object_t object; - unsigned int free_count; + vm_page_t m; + vm_object_t object; + unsigned long free_count; /* * Recalculate vm_page_inactivate_target. @@ -567,7 +577,7 @@ void vm_pageout_scan() while ((vm_page_inactive_count < vm_page_inactive_target) && !queue_empty(&vm_page_queue_active)) { - register vm_object_t obj; + vm_object_t obj; vm_pageout_active++; m = (vm_page_t) queue_first(&vm_page_queue_active); @@ -616,17 +626,20 @@ void vm_pageout_scan() } /* - * We are done if we have met our target *and* + * We are done if we have met our targets *and* * nobody is still waiting for a page. */ simple_lock(&vm_page_queue_free_lock); - free_count = vm_page_free_count; - if ((free_count >= vm_page_free_target) & + free_count = vm_page_mem_free(); + if ((free_count >= vm_page_free_target) && + (vm_page_external_count <= vm_page_external_target) && (vm_page_free_wanted == 0)) { vm_page_unlock_queues(); break; } + want_pages = ((free_count < vm_page_free_target) || + vm_page_free_wanted); simple_unlock(&vm_page_queue_free_lock); /* @@ -637,7 +650,7 @@ void vm_pageout_scan() * consumes memory. We don't take the risk of doing * this if the default pager already has work to do. */ - + pause: if (queue_empty(&vm_page_queue_inactive) || (burst_count >= vm_pageout_burst_max) || (vm_page_laundry_count >= vm_pageout_burst_max) || @@ -665,23 +678,27 @@ void vm_pageout_scan() thread_will_wait_with_timeout(current_thread(), msecs); counter(c_vm_pageout_scan_block++); thread_block(vm_pageout_scan_continue); -#ifndef CONTINUATIONS - /* - * Unfortunately, we don't have call_continuation - * so we can't rely on tail-recursion. - */ - - vm_pageout_scan_continue(); - goto Restart; -#else /* CONTINUATIONS */ call_continuation(vm_pageout_scan_continue); /*NOTREACHED*/ -#endif /* CONTINUATIONS */ } vm_pageout_inactive++; - m = (vm_page_t) queue_first(&vm_page_queue_inactive); - assert(!m->active && m->inactive); + + /* Find a page we are interested in paging out. If we + need pages, then we'll page anything out; otherwise + we only page out external pages. */ + m = (vm_page_t) queue_first (&vm_page_queue_inactive); + while (1) + { + assert (!m->active && m->inactive); + if (want_pages || m->external) + break; + + m = (vm_page_t) queue_next (&m->pageq); + if (!m) + goto pause; + } + object = m->object; /* @@ -727,12 +744,17 @@ void vm_pageout_scan() * If it's absent, we can reclaim the page. */ - if (m->absent) { + if (want_pages && m->absent) { vm_pageout_inactive_absent++; reclaim_page: vm_page_free(m); vm_page_unlock_queues(); - vm_object_unlock(object); + + if (vm_object_collectable(object)) + vm_object_collect(object); + else + vm_object_unlock(object); + continue; } @@ -746,6 +768,7 @@ void vm_pageout_scan() vm_object_unlock(object); vm_page_activate(m); vm_stat.reactivations++; + current_task()->reactivations++; vm_page_unlock_queues(); vm_pageout_inactive_used++; continue; @@ -760,6 +783,34 @@ void vm_pageout_scan() if (!m->dirty) m->dirty = pmap_is_modified(m->phys_addr); + if (m->external) { + /* Figure out if we still care about this + page in the limit of externally managed pages. + Clean pages don't actually cause system hosage, + so it's ok to stop considering them as + "consumers" of memory. */ + if (m->dirty && !m->extcounted) { + m->extcounted = TRUE; + vm_page_external_count++; + } else if (!m->dirty && m->extcounted) { + m->extcounted = FALSE; + vm_page_external_count--; + } + } + + /* If we don't actually need more memory, and the page + is not dirty, put it on the tail of the inactive queue + and move on to the next page. */ + if (!want_pages && !m->dirty) { + queue_remove (&vm_page_queue_inactive, m, + vm_page_t, pageq); + queue_enter (&vm_page_queue_inactive, m, + vm_page_t, pageq); + vm_page_unlock_queues(); + vm_pageout_inactive_cleaned_external++; + continue; + } + /* * If it's clean and not precious, we can free the page. */ @@ -810,7 +861,7 @@ void vm_pageout_scan() } } -void vm_pageout_scan_continue() +void vm_pageout_scan_continue(void) { /* * We just paused to let the pagers catch up. @@ -833,17 +884,15 @@ void vm_pageout_scan_continue() } vm_page_unlock_queues(); -#ifdef CONTINUATIONS vm_pageout_continue(); /*NOTREACHED*/ -#endif /* CONTINUATIONS */ } /* * vm_pageout is the high level pageout daemon. */ -void vm_pageout_continue() +void vm_pageout_continue(void) { /* * The pageout daemon is never done, so loop forever. @@ -865,12 +914,13 @@ void vm_pageout_continue() } } -void vm_pageout() +void vm_pageout(void) { - int free_after_reserve; + unsigned long free_after_reserve; current_thread()->vm_privilege = TRUE; stack_privilege(current_thread()); + thread_set_own_priority(0); /* * Initialize some paging parameters. @@ -902,7 +952,15 @@ void vm_pageout() vm_pageout_reserved_really = VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved); - free_after_reserve = vm_page_free_count - vm_page_free_reserved; + free_after_reserve = vm_page_mem_free() - vm_page_free_reserved; + + if (vm_page_external_limit == 0) + vm_page_external_limit = + VM_PAGE_EXTERNAL_LIMIT (free_after_reserve); + + if (vm_page_external_target == 0) + vm_page_external_target = + VM_PAGE_EXTERNAL_TARGET (free_after_reserve); if (vm_page_free_min == 0) vm_page_free_min = vm_page_free_reserved +