--- Gnu-Mach/vm/vm_resident.c 2020/09/02 04:36:57 1.1 +++ Gnu-Mach/vm/vm_resident.c 2020/09/02 04:47:44 1.1.1.4 @@ -27,41 +27,45 @@ * the rights to redistribute these changes. */ /* - * File: vm/vm_page.c + * File: vm/vm_resident.c * Author: Avadis Tevanian, Jr., Michael Wayne Young * * Resident memory management module. */ -#include + +#include +#include #include #include +#include #include #include #include #include -#include "vm_param.h" +#include #include -#include +#include #include #include #include #include #include -#include #if MACH_VM_DEBUG #include #include #include #endif -/* in zalloc.c XXX */ -extern vm_offset_t zdata; -extern vm_size_t zdata_size; +#if MACH_KDB +#include +#include +#endif /* MACH_KDB */ + /* - * Associated with eacn page of user-allocatable memory is a + * Associated with each page of user-allocatable memory is a * page structure. */ @@ -111,6 +115,7 @@ decl_simple_lock_data(,vm_page_queue_fre unsigned int vm_page_free_wanted; int vm_page_free_count; int vm_page_fictitious_count; +int vm_page_external_count; unsigned int vm_page_free_count_minimum; /* debugging */ @@ -123,7 +128,7 @@ unsigned int vm_page_free_count_minimum; * These page structures are allocated the way * most other kernel structures are. */ -zone_t vm_page_zone; +struct kmem_cache vm_page_cache; /* * Fictitious pages don't have a physical address, @@ -158,6 +163,8 @@ int vm_page_free_min = 0; int vm_page_inactive_target = 0; int vm_page_free_reserved = 0; int vm_page_laundry_count = 0; +int vm_page_external_limit = 0; + /* * The VM system has a couple of heuristics for deciding @@ -185,7 +192,7 @@ void vm_page_bootstrap( vm_offset_t *startp, vm_offset_t *endp) { - register vm_page_t m; + vm_page_t m; int i; /* @@ -201,6 +208,7 @@ void vm_page_bootstrap( m->active = FALSE; m->laundry = FALSE; m->free = FALSE; + m->external = FALSE; m->busy = TRUE; m->wanted = FALSE; @@ -233,14 +241,11 @@ void vm_page_bootstrap( vm_page_free_wanted = 0; /* - * Steal memory for the zone system. + * Steal memory for the kernel map entries. */ - kentry_data_size = kentry_count * sizeof(struct vm_map_entry); kentry_data = pmap_steal_memory(kentry_data_size); - zdata = pmap_steal_memory(zdata_size); - /* * Allocate (and initialize) the virtual-to-physical * table hash buckets. @@ -269,7 +274,7 @@ void vm_page_bootstrap( sizeof(vm_page_bucket_t)); for (i = 0; i < vm_page_bucket_count; i++) { - register vm_page_bucket_t *bucket = &vm_page_buckets[i]; + vm_page_bucket_t *bucket = &vm_page_buckets[i]; bucket->pages = VM_PAGE_NULL; simple_lock_init(&bucket->lock); @@ -387,6 +392,11 @@ void pmap_startup( vm_page_init(&pages[i], paddr); pages_initialized++; } + i = 0; + while (pmap_next_page(&paddr)) + i++; + if (i) + printf("%u memory page(s) left away\n", i); /* * Release pages in reverse order so that physical pages @@ -396,7 +406,7 @@ void pmap_startup( */ for (i = pages_initialized; i > 0; i--) { - vm_page_release(&pages[i - 1]); + vm_page_release(&pages[i - 1], FALSE); } /* @@ -419,10 +429,8 @@ void pmap_startup( */ void vm_page_module_init(void) { - vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page), - VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, - PAGE_SIZE, - 0, "vm pages"); + kmem_cache_init(&vm_page_cache, "vm_page", sizeof(struct vm_page), 0, + NULL, NULL, NULL, 0); } /* @@ -444,12 +452,12 @@ void vm_page_create( for (paddr = round_page(start); paddr < trunc_page(end); paddr += PAGE_SIZE) { - m = (vm_page_t) zalloc(vm_page_zone); + m = (vm_page_t) kmem_cache_alloc(&vm_page_cache); if (m == VM_PAGE_NULL) panic("vm_page_create"); vm_page_init(m, paddr); - vm_page_release(m); + vm_page_release(m, FALSE); } } @@ -475,11 +483,11 @@ void vm_page_create( */ void vm_page_insert( - register vm_page_t mem, - register vm_object_t object, - register vm_offset_t offset) + vm_page_t mem, + vm_object_t object, + vm_offset_t offset) { - register vm_page_bucket_t *bucket; + vm_page_bucket_t *bucket; VM_PAGE_CHECK(mem); @@ -515,6 +523,10 @@ void vm_page_insert( */ object->resident_page_count++; + assert(object->resident_page_count >= 0); + + if (object->can_persist && (object->ref_count == 0)) + vm_object_cached_pages_update(1); /* * Detect sequential access and inactivate previous page. @@ -543,11 +555,11 @@ void vm_page_insert( */ void vm_page_replace( - register vm_page_t mem, - register vm_object_t object, - register vm_offset_t offset) + vm_page_t mem, + vm_object_t object, + vm_offset_t offset) { - register vm_page_bucket_t *bucket; + vm_page_bucket_t *bucket; VM_PAGE_CHECK(mem); @@ -570,7 +582,7 @@ void vm_page_replace( simple_lock(&bucket->lock); if (bucket->pages) { vm_page_t *mp = &bucket->pages; - register vm_page_t m = *mp; + vm_page_t m = *mp; do { if (m->object == object && m->offset == offset) { /* @@ -583,6 +595,10 @@ void vm_page_replace( m->tabled = FALSE; object->resident_page_count--; + if (object->can_persist + && (object->ref_count == 0)) + vm_object_cached_pages_update(-1); + /* * Return page to the free list. * Note the page is not tabled now, so this @@ -614,6 +630,10 @@ void vm_page_replace( */ object->resident_page_count++; + assert(object->resident_page_count >= 0); + + if (object->can_persist && (object->ref_count == 0)) + vm_object_cached_pages_update(1); } /* @@ -626,10 +646,10 @@ void vm_page_replace( */ void vm_page_remove( - register vm_page_t mem) + vm_page_t mem) { - register vm_page_bucket_t *bucket; - register vm_page_t this; + vm_page_bucket_t *bucket; + vm_page_t this; assert(mem->tabled); VM_PAGE_CHECK(mem); @@ -645,7 +665,7 @@ void vm_page_remove( bucket->pages = mem->next; } else { - register vm_page_t *prev; + vm_page_t *prev; for (prev = &this->next; (this = *prev) != mem; @@ -669,6 +689,9 @@ void vm_page_remove( mem->object->resident_page_count--; mem->tabled = FALSE; + + if (mem->object->can_persist && (mem->object->ref_count == 0)) + vm_object_cached_pages_update(-1); } /* @@ -681,11 +704,11 @@ void vm_page_remove( */ vm_page_t vm_page_lookup( - register vm_object_t object, - register vm_offset_t offset) + vm_object_t object, + vm_offset_t offset) { - register vm_page_t mem; - register vm_page_bucket_t *bucket; + vm_page_t mem; + vm_page_bucket_t *bucket; /* * Search the hash table for this object/offset pair @@ -712,9 +735,9 @@ vm_page_t vm_page_lookup( * The object must be locked. */ void vm_page_rename( - register vm_page_t mem, - register vm_object_t new_object, - vm_offset_t new_offset) + vm_page_t mem, + vm_object_t new_object, + vm_offset_t new_offset) { /* * Changes to mem->object require the page lock because @@ -751,7 +774,7 @@ void vm_page_init( vm_page_t vm_page_grab_fictitious(void) { - register vm_page_t m; + vm_page_t m; simple_lock(&vm_page_queue_free_lock); m = vm_page_queue_fictitious; @@ -772,7 +795,7 @@ vm_page_t vm_page_grab_fictitious(void) */ void vm_page_release_fictitious( - register vm_page_t m) + vm_page_t m) { simple_lock(&vm_page_queue_free_lock); if (m->free) @@ -795,11 +818,11 @@ int vm_page_fictitious_quantum = 5; void vm_page_more_fictitious(void) { - register vm_page_t m; + vm_page_t m; int i; for (i = 0; i < vm_page_fictitious_quantum; i++) { - m = (vm_page_t) zalloc(vm_page_zone); + m = (vm_page_t) kmem_cache_alloc(&vm_page_cache); if (m == VM_PAGE_NULL) panic("vm_page_more_fictitious"); @@ -816,11 +839,12 @@ void vm_page_more_fictitious(void) */ boolean_t vm_page_convert( - register vm_page_t m) + vm_page_t m, + boolean_t external) { - register vm_page_t real_m; + vm_page_t real_m; - real_m = vm_page_grab(); + real_m = vm_page_grab(external); if (real_m == VM_PAGE_NULL) return FALSE; @@ -841,19 +865,23 @@ boolean_t vm_page_convert( * Returns VM_PAGE_NULL if the free list is too small. */ -vm_page_t vm_page_grab(void) +vm_page_t vm_page_grab( + boolean_t external) { - register vm_page_t mem; + vm_page_t mem; simple_lock(&vm_page_queue_free_lock); /* * Only let privileged threads (involved in pageout) - * dip into the reserved pool. + * dip into the reserved pool or exceed the limit + * for externally-managed pages. */ - if ((vm_page_free_count < vm_page_free_reserved) && - !current_thread()->vm_privilege) { + if (((vm_page_free_count < vm_page_free_reserved) + || (external + && (vm_page_external_count > vm_page_external_limit))) + && !current_thread()->vm_privilege) { simple_unlock(&vm_page_queue_free_lock); return VM_PAGE_NULL; } @@ -863,9 +891,12 @@ vm_page_t vm_page_grab(void) if (--vm_page_free_count < vm_page_free_count_minimum) vm_page_free_count_minimum = vm_page_free_count; + if (external) + vm_page_external_count++; mem = vm_page_queue_free; vm_page_queue_free = (vm_page_t) mem->pageq.next; mem->free = FALSE; + mem->extcounted = mem->external = external; simple_unlock(&vm_page_queue_free_lock); /* @@ -889,7 +920,7 @@ vm_page_t vm_page_grab(void) vm_offset_t vm_page_grab_phys_addr(void) { - vm_page_t p = vm_page_grab(); + vm_page_t p = vm_page_grab(FALSE); if (p == VM_PAGE_NULL) return -1; else @@ -915,12 +946,13 @@ kern_return_t vm_page_grab_contiguous_pages( int npages, vm_page_t pages[], - natural_t *bits) + natural_t *bits, + boolean_t external) { - register int first_set; + int first_set; int size, alloc_size; kern_return_t ret; - vm_page_t mem, prevmem; + vm_page_t mem, *prevmemp; #ifndef NBBY #define NBBY 8 /* size in bits of sizeof()`s unity */ @@ -948,7 +980,7 @@ vm_page_grab_contiguous_pages( } else alloc_size = 0; - bzero(bits, size); + memset(bits, 0, size); /* * A very large granularity call, its rare so that is ok @@ -959,7 +991,9 @@ vm_page_grab_contiguous_pages( * Do not dip into the reserved pool. */ - if (vm_page_free_count < vm_page_free_reserved) { + if ((vm_page_free_count < vm_page_free_reserved) + || (vm_page_external_count >= vm_page_external_limit)) { + printf_once("no more room for vm_page_grab_contiguous_pages"); simple_unlock(&vm_page_queue_free_lock); return KERN_RESOURCE_SHORTAGE; } @@ -972,7 +1006,7 @@ vm_page_grab_contiguous_pages( */ mem = vm_page_queue_free; while (mem) { - register int word_index, bit_index; + int word_index, bit_index; bit_index = (mem->phys_addr >> PAGE_SHIFT); word_index = bit_index / NBPEL; @@ -989,14 +1023,14 @@ vm_page_grab_contiguous_pages( * the free list. */ { - register int bits_so_far = 0, i; + int bits_so_far = 0, i; first_set = 0; for (i = 0; i < size; i += sizeof(natural_t)) { - register natural_t v = bits[i / sizeof(natural_t)]; - register int bitpos; + natural_t v = bits[i / sizeof(natural_t)]; + int bitpos; /* * Bitscan this one word @@ -1027,7 +1061,6 @@ count_ones: * search for beans here */ bits_so_far = 0; -count_zeroes: while ((bitpos < NBPEL) && ((v & 1) == 0)) { bitpos++; v >>= 1; @@ -1047,9 +1080,9 @@ count_zeroes: /* * We could not find enough contiguous pages. */ -not_found_em: simple_unlock(&vm_page_queue_free_lock); + printf_once("no contiguous room for vm_page_grab_contiguous_pages"); ret = KERN_RESOURCE_SHORTAGE; goto out; @@ -1063,9 +1096,10 @@ found_em: vm_page_free_count -= npages; if (vm_page_free_count < vm_page_free_count_minimum) vm_page_free_count_minimum = vm_page_free_count; - + if (external) + vm_page_external_count += npages; { - register vm_offset_t first_phys, last_phys; + vm_offset_t first_phys, last_phys; /* cache values for compare */ first_phys = first_set << PAGE_SHIFT; @@ -1073,26 +1107,26 @@ found_em: /* running pointers */ mem = vm_page_queue_free; - prevmem = VM_PAGE_NULL; + prevmemp = &vm_page_queue_free; while (mem) { - register vm_offset_t addr; + vm_offset_t addr; addr = mem->phys_addr; if ((addr >= first_phys) && (addr < last_phys)) { - if (prevmem) - prevmem->pageq.next = mem->pageq.next; + *prevmemp = (vm_page_t) mem->pageq.next; pages[(addr - first_phys) >> PAGE_SHIFT] = mem; mem->free = FALSE; + mem->extcounted = mem->external = external; /* * Got them all ? */ if (--npages == 0) break; } else - prevmem = mem; + prevmemp = (vm_page_t *) &mem->pageq.next; mem = (vm_page_t) mem->pageq.next; } @@ -1131,7 +1165,8 @@ out: */ void vm_page_release( - register vm_page_t mem) + vm_page_t mem, + boolean_t external) { simple_lock(&vm_page_queue_free_lock); if (mem->free) @@ -1140,6 +1175,8 @@ void vm_page_release( mem->pageq.next = (queue_entry_t) vm_page_queue_free; vm_page_queue_free = mem; vm_page_free_count++; + if (external) + vm_page_external_count--; /* * Check if we should wake up someone waiting for page. @@ -1181,10 +1218,6 @@ void vm_page_wait( void (*continuation)(void)) { -#ifndef CONTINUATIONS - assert (continuation == 0); -#endif - /* * We can't use vm_page_free_reserved to make this * determination. Consider: some thread might @@ -1194,7 +1227,8 @@ void vm_page_wait( */ simple_lock(&vm_page_queue_free_lock); - if (vm_page_free_count < vm_page_free_target) { + if ((vm_page_free_count < vm_page_free_target) + || (vm_page_external_count > vm_page_external_limit)) { if (vm_page_free_wanted++ == 0) thread_wakeup((event_t)&vm_page_free_wanted); assert_wait((event_t)&vm_page_free_count, FALSE); @@ -1223,9 +1257,9 @@ vm_page_t vm_page_alloc( vm_object_t object, vm_offset_t offset) { - register vm_page_t mem; + vm_page_t mem; - mem = vm_page_grab(); + mem = vm_page_grab(!object->internal); if (mem == VM_PAGE_NULL) return VM_PAGE_NULL; @@ -1245,7 +1279,7 @@ vm_page_t vm_page_alloc( * Object and page queues must be locked prior to entry. */ void vm_page_free( - register vm_page_t mem) + vm_page_t mem) { if (mem->free) panic("vm_page_free"); @@ -1280,8 +1314,9 @@ void vm_page_free( mem->fictitious = TRUE; vm_page_release_fictitious(mem); } else { + int external = mem->external && mem->extcounted; vm_page_init(mem, mem->phys_addr); - vm_page_release(mem); + vm_page_release(mem, external); } } @@ -1295,7 +1330,7 @@ void vm_page_free( * The page's object and the page queues must be locked. */ void vm_page_wire( - register vm_page_t mem) + vm_page_t mem) { VM_PAGE_CHECK(mem); @@ -1316,7 +1351,7 @@ void vm_page_wire( * The page's object and the page queues must be locked. */ void vm_page_unwire( - register vm_page_t mem) + vm_page_t mem) { VM_PAGE_CHECK(mem); @@ -1339,7 +1374,7 @@ void vm_page_unwire( * The page queues must be locked. */ void vm_page_deactivate( - register vm_page_t m) + vm_page_t m) { VM_PAGE_CHECK(m); @@ -1373,7 +1408,7 @@ void vm_page_deactivate( */ void vm_page_activate( - register vm_page_t m) + vm_page_t m) { VM_PAGE_CHECK(m); @@ -1462,7 +1497,7 @@ vm_page_info( } #endif /* MACH_VM_DEBUG */ -#include + #if MACH_KDB #define printf kdbprintf @@ -1470,10 +1505,10 @@ vm_page_info( * Routine: vm_page_print [exported] */ void vm_page_print(p) - vm_page_t p; + const vm_page_t p; { iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object); - printf(" offset 0x%X", (vm_offset_t) p->offset); + printf(" offset 0x%X", p->offset); printf("wire_count %d,", p->wire_count); printf(" %s", (p->active ? "active" : (p->inactive ? "inactive" : "loose"))); @@ -1498,7 +1533,7 @@ void vm_page_print(p) printf("%s,", (p->tabled ? "" : "not_tabled")); printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n", - (vm_offset_t) p->phys_addr, + p->phys_addr, (vm_offset_t) p->page_lock, (vm_offset_t) p->unlock_request); }