--- Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:46:55 1.1.1.3 +++ Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:55:41 1.1.1.7 @@ -83,6 +83,8 @@ #include #include #include +#include +#include #ifdef MACH_PSEUDO_PHYS #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0; @@ -157,16 +159,16 @@ vm_offset_t kernel_virtual_end; /* * Index into pv_head table, its lock bits, and the modify/reference - * bits starting at phys_first_addr. + * bits. */ -#define pa_index(pa) (atop(pa - phys_first_addr)) +#define pa_index(pa) vm_page_table_index(pa) #define pai_to_pvh(pai) (&pv_head_table[pai]) #define lock_pvh_pai(pai) (bit_lock(pai, pv_lock_table)) #define unlock_pvh_pai(pai) (bit_unlock(pai, pv_lock_table)) /* - * Array of physical page attribites for managed pages. + * Array of physical page attributes for managed pages. * One byte per physical page. */ char *pmap_phys_attributes; @@ -325,12 +327,7 @@ lock_data_t pmap_system_lock; #endif /* NCPUS > 1 */ -#define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */ - #ifdef MACH_PV_PAGETABLES -#if 1 -#define INVALIDATE_TLB(pmap, s, e) hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL) -#else #define INVALIDATE_TLB(pmap, s, e) do { \ if (__builtin_constant_p((e) - (s)) \ && (e) - (s) == PAGE_SIZE) \ @@ -338,26 +335,16 @@ lock_data_t pmap_system_lock; else \ hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \ } while(0) -#endif #else /* MACH_PV_PAGETABLES */ -#if 0 /* It is hard to know when a TLB flush becomes less expensive than a bunch of * invlpgs. But it surely is more expensive than just one invlpg. */ -#define INVALIDATE_TLB(pmap, s, e) { \ +#define INVALIDATE_TLB(pmap, s, e) do { \ if (__builtin_constant_p((e) - (s)) \ && (e) - (s) == PAGE_SIZE) \ - invlpg_linear(s); \ + invlpg_linear((pmap) == kernel_pmap ? kvtolin(s) : (s)); \ else \ flush_tlb(); \ -} -#else -#define INVALIDATE_TLB(pmap, s, e) { \ - (void) (pmap); \ - (void) (s); \ - (void) (e); \ - flush_tlb(); \ -} -#endif +} while (0) #endif /* MACH_PV_PAGETABLES */ @@ -401,8 +388,14 @@ struct pmap kernel_pmap_store; pmap_t kernel_pmap; struct kmem_cache pmap_cache; /* cache of pmap structures */ +struct kmem_cache pd_cache; /* cache of page directories */ +#if PAE +struct kmem_cache pdpt_cache; /* cache of page + directory pointer + tables */ +#endif -int pmap_debug = 0; /* flag for debugging prints */ +boolean_t pmap_debug = FALSE; /* flag for debugging prints */ #if 0 int ptes_per_vm_page; /* number of hardware ptes needed @@ -413,25 +406,30 @@ int ptes_per_vm_page; /* number of hard unsigned int inuse_ptepages_count = 0; /* debugging */ -extern char end; - /* * Pointer to the basic page directory for the kernel. * Initialized by pmap_bootstrap(). */ pt_entry_t *kernel_page_dir; -void pmap_remove_range(); /* forward */ -#if NCPUS > 1 -void signal_cpus(); /* forward */ -#endif /* NCPUS > 1 */ +/* + * Two slots for temporary physical page mapping, to allow for + * physical-to-physical transfers. + */ +static pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS]; static inline pt_entry_t * -pmap_pde(pmap_t pmap, vm_offset_t addr) +pmap_pde(const pmap_t pmap, vm_offset_t addr) { + pt_entry_t *page_dir; if (pmap == kernel_pmap) addr = kvtolin(addr); - return &pmap->dirbase[lin2pdenum(addr)]; +#if PAE + page_dir = (pt_entry_t *) ptetokv(pmap->pdpbase[lin2pdpnum(addr)]); +#else + page_dir = pmap->dirbase; +#endif + return &page_dir[lin2pdenum(addr)]; } /* @@ -442,13 +440,18 @@ pmap_pde(pmap_t pmap, vm_offset_t addr) * This is only used internally. */ pt_entry_t * -pmap_pte(pmap_t pmap, vm_offset_t addr) +pmap_pte(const pmap_t pmap, vm_offset_t addr) { pt_entry_t *ptp; pt_entry_t pte; +#if PAE + if (pmap->pdpbase == 0) + return(PT_ENTRY_NULL); +#else if (pmap->dirbase == 0) return(PT_ENTRY_NULL); +#endif pte = *pmap_pde(pmap, addr); if ((pte & INTEL_PTE_VALID) == 0) return(PT_ENTRY_NULL); @@ -459,10 +462,9 @@ pmap_pte(pmap_t pmap, vm_offset_t addr) #define DEBUG_PTE_PAGE 0 #if DEBUG_PTE_PAGE -void ptep_check(ptep) - ptep_t ptep; +void ptep_check(ptep_t ptep) { - register pt_entry_t *pte, *epte; + pt_entry_t *pte, *epte; int ctu, ctw; /* check the use and wired counts */ @@ -496,13 +498,13 @@ void ptep_check(ptep) * For now, VM is already on, we only need to map the * specified memory. */ -vm_offset_t pmap_map(virt, start, end, prot) - register vm_offset_t virt; - register vm_offset_t start; - register vm_offset_t end; - register int prot; +vm_offset_t pmap_map( + vm_offset_t virt, + phys_addr_t start, + phys_addr_t end, + int prot) { - register int ps; + int ps; ps = PAGE_SIZE; while (start < end) { @@ -515,19 +517,19 @@ vm_offset_t pmap_map(virt, start, end, p /* * Back-door routine for mapping kernel VM at initialization. - * Useful for mapping memory outside the range - * [phys_first_addr, phys_last_addr) (i.e., devices). + * Useful for mapping memory outside the range of direct mapped + * physical memory (i.e., devices). * Otherwise like pmap_map. */ -vm_offset_t pmap_map_bd(virt, start, end, prot) - register vm_offset_t virt; - register vm_offset_t start; - register vm_offset_t end; - vm_prot_t prot; +vm_offset_t pmap_map_bd( + vm_offset_t virt, + phys_addr_t start, + phys_addr_t end, + vm_prot_t prot) { - register pt_entry_t template; - register pt_entry_t *pte; - int spl; + pt_entry_t template; + pt_entry_t *pte; + int spl; #ifdef MACH_PV_PAGETABLES int n, i = 0; struct mmu_update update[HYP_BATCH_MMU_UPDATES]; @@ -580,7 +582,7 @@ vm_offset_t pmap_map_bd(virt, start, end * and direct-map all physical memory. * Called with mapping off. */ -void pmap_bootstrap() +void pmap_bootstrap(void) { /* * Mapping is turned off; we must reference only physical addresses. @@ -616,8 +618,8 @@ void pmap_bootstrap() * mapped into the kernel address space, * and extends to a stupid arbitrary limit beyond that. */ - kernel_virtual_start = phystokv(phys_last_addr); - kernel_virtual_end = phystokv(phys_last_addr) + VM_KERNEL_MAP_SIZE; + kernel_virtual_start = phystokv(biosmem_directmap_end()); + kernel_virtual_end = kernel_virtual_start + VM_KERNEL_MAP_SIZE; if (kernel_virtual_end < kernel_virtual_start || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS) @@ -632,21 +634,24 @@ void pmap_bootstrap() { vm_offset_t addr; init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr); - kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr); + kernel_page_dir = (pt_entry_t*)phystokv(addr); } kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page()); { int i; for (i = 0; i < PDPNUM; i++) - WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + WRITE_PTE(&kernel_pmap->pdpbase[i], + pa_to_pte(_kvtophys((void *) kernel_page_dir + + i * INTEL_PGBYTES)) + | INTEL_PTE_VALID); } #else /* PAE */ kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page()); #endif /* PAE */ { - int i; + unsigned i; for (i = 0; i < NPDES; i++) - kernel_pmap->dirbase[i] = 0; + kernel_page_dir[i] = 0; } #ifdef MACH_PV_PAGETABLES @@ -679,7 +684,7 @@ void pmap_bootstrap() l2_map += (la >> PDESHIFT) & PDEMASK; if (!(*l2_map & INTEL_PTE_VALID)) { struct mmu_update update; - int j, n; + unsigned j, n; l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page()); for (j = 0; j < NPTES; j++) @@ -708,8 +713,7 @@ void pmap_bootstrap() pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0; /* - * Map virtual memory for all known physical memory, 1-1, - * from phys_first_addr to phys_last_addr. + * Map virtual memory for all directly mappable physical memory, 1-1, * Make any mappings completely in the kernel's text segment read-only. * * Also allocate some additional all-null page tables afterwards @@ -718,9 +722,9 @@ void pmap_bootstrap() * to allocate new kernel page tables later. * XX fix this */ - for (va = phystokv(phys_first_addr); va >= phystokv(phys_first_addr) && va < kernel_virtual_end; ) + for (va = phystokv(0); va >= phystokv(0) && va < kernel_virtual_end; ) { - pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va)); + pt_entry_t *pde = kernel_page_dir + lin2pdenum_cont(kvtolin(va)); pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page()); pt_entry_t *pte; @@ -729,7 +733,7 @@ void pmap_bootstrap() | INTEL_PTE_VALID | INTEL_PTE_WRITE); /* Initialize the page table. */ - for (pte = ptable; (va < phystokv(phys_last_addr)) && (pte < ptable+NPTES); pte++) + for (pte = ptable; (va < phystokv(biosmem_directmap_end())) && (pte < ptable+NPTES); pte++) { if ((pte - ptable) < ptenum(va)) { @@ -781,6 +785,12 @@ void pmap_bootstrap() } for (; pte < ptable+NPTES; pte++) { + if (va >= kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE && va < kernel_virtual_end) + { + pmap_mapwindow_t *win = &mapwindows[atop(va - (kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE))]; + win->entry = pte; + win->vaddr = va; + } WRITE_PTE(pte, 0); va += INTEL_PGBYTES; } @@ -852,11 +862,11 @@ void pmap_set_page_readonly_init(void *_ } void pmap_clear_bootstrap_pagetable(pt_entry_t *base) { - int i; + unsigned i; pt_entry_t *dir; vm_offset_t va = 0; #if PAE - int j; + unsigned j; #endif /* PAE */ if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base))) panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base)); @@ -891,12 +901,53 @@ void pmap_clear_bootstrap_pagetable(pt_e } #endif /* MACH_PV_PAGETABLES */ -void pmap_virtual_space(startp, endp) - vm_offset_t *startp; - vm_offset_t *endp; +/* + * Create a temporary mapping for a given physical entry + * + * This can be used to access physical pages which are not mapped 1:1 by + * phystokv(). + */ +pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry) +{ + pmap_mapwindow_t *map; + + assert(entry != 0); + + /* Find an empty one. */ + for (map = &mapwindows[0]; map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]; map++) + if (!(*map->entry)) + break; + assert(map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]); + +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(map->entry), pa_to_ma(entry))) + panic("pmap_get_mapwindow"); +#else /* MACH_PV_PAGETABLES */ + WRITE_PTE(map->entry, entry); +#endif /* MACH_PV_PAGETABLES */ + return map; +} + +/* + * Destroy a temporary mapping for a physical entry + */ +void pmap_put_mapwindow(pmap_mapwindow_t *map) +{ +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(map->entry), 0)) + panic("pmap_put_mapwindow"); +#else /* MACH_PV_PAGETABLES */ + WRITE_PTE(map->entry, 0); +#endif /* MACH_PV_PAGETABLES */ + PMAP_UPDATE_TLBS(kernel_pmap, map->vaddr, map->vaddr + PAGE_SIZE); +} + +void pmap_virtual_space( + vm_offset_t *startp, + vm_offset_t *endp) { *startp = kernel_virtual_start; - *endp = kernel_virtual_end; + *endp = kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE; } /* @@ -904,11 +955,11 @@ void pmap_virtual_space(startp, endp) * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ -void pmap_init() +void pmap_init(void) { - register long npages; + unsigned long npages; vm_offset_t addr; - register vm_size_t s; + vm_size_t s; #if NCPUS > 1 int i; #endif /* NCPUS > 1 */ @@ -918,7 +969,7 @@ void pmap_init() * the modify bit array, and the pte_page table. */ - npages = atop(phys_last_addr - phys_first_addr); + npages = vm_page_table_size(); s = (vm_size_t) (sizeof(struct pv_entry) * npages + pv_lock_table_size(npages) + npages); @@ -926,7 +977,7 @@ void pmap_init() s = round_page(s); if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS) panic("pmap_init"); - memset((char *) addr, 0, s); + memset((void *) addr, 0, s); /* * Allocate the structures first to preserve word-alignment. @@ -944,9 +995,18 @@ void pmap_init() * and of the physical-to-virtual entries. */ s = (vm_size_t) sizeof(struct pmap); - kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0); + kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, 0); + kmem_cache_init(&pd_cache, "pd", + INTEL_PGBYTES, INTEL_PGBYTES, NULL, + KMEM_CACHE_PHYSMEM); +#if PAE + kmem_cache_init(&pdpt_cache, "pdpt", + PDPNUM * sizeof(pt_entry_t), + PDPNUM * sizeof(pt_entry_t), NULL, + KMEM_CACHE_PHYSMEM); +#endif s = (vm_size_t) sizeof(struct pv_entry); - kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0); + kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, 0); #if NCPUS > 1 /* @@ -966,32 +1026,16 @@ void pmap_init() pmap_initialized = TRUE; } -#define valid_page(x) (pmap_initialized && pmap_valid_page(x)) - -boolean_t pmap_verify_free(phys) - vm_offset_t phys; +static inline boolean_t +valid_page(phys_addr_t addr) { - pv_entry_t pv_h; - int pai; - int spl; - boolean_t result; + struct vm_page *p; - assert(phys != vm_page_fictitious_addr); if (!pmap_initialized) - return(TRUE); + return FALSE; - if (!pmap_valid_page(phys)) - return(FALSE); - - PMAP_WRITE_LOCK(spl); - - pai = pa_index(phys); - pv_h = pai_to_pvh(pai); - - result = (pv_h->pmap == PMAP_NULL); - PMAP_WRITE_UNLOCK(spl); - - return(result); + p = vm_page_lookup_pa(addr); + return (p != NULL); } /* @@ -1003,10 +1047,10 @@ boolean_t pmap_verify_free(phys) * since these must be unlocked to use vm_page_grab. */ vm_offset_t -pmap_page_table_page_alloc() +pmap_page_table_page_alloc(void) { - register vm_page_t m; - register vm_offset_t pa; + vm_page_t m; + vm_offset_t pa; check_simple_locks(); @@ -1016,12 +1060,12 @@ pmap_page_table_page_alloc() * Allocate it now if it is missing. */ if (pmap_object == VM_OBJECT_NULL) - pmap_object = vm_object_allocate(phys_last_addr - phys_first_addr); + pmap_object = vm_object_allocate(vm_page_table_size() * PAGE_SIZE); /* * Allocate a VM page for the level 2 page table entries. */ - while ((m = vm_page_grab(FALSE)) == VM_PAGE_NULL) + while ((m = vm_page_grab()) == VM_PAGE_NULL) VM_PAGE_WAIT((void (*)()) 0); /* @@ -1092,8 +1136,7 @@ void pmap_map_mfn(void *_addr, unsigned * and be removed from its page directory. */ void -pmap_page_table_page_dealloc(pa) - vm_offset_t pa; +pmap_page_table_page_dealloc(vm_offset_t pa) { vm_page_t m; @@ -1118,11 +1161,12 @@ pmap_page_table_page_dealloc(pa) * the map will be used in software only, and * is bounded by that size. */ -pmap_t pmap_create(size) - vm_size_t size; +pmap_t pmap_create(vm_size_t size) { - register pmap_t p; - register pmap_statistics_t stats; + pt_entry_t *page_dir[PDPNUM]; + int i; + pmap_t p; + pmap_statistics_t stats; /* * A software use-only map doesn't even need a map. @@ -1139,41 +1183,65 @@ pmap_t pmap_create(size) p = (pmap_t) kmem_cache_alloc(&pmap_cache); if (p == PMAP_NULL) - panic("pmap_create"); + return PMAP_NULL; - if (kmem_alloc_wired(kernel_map, - (vm_offset_t *)&p->dirbase, PDPNUM * INTEL_PGBYTES) - != KERN_SUCCESS) - panic("pmap_create"); + for (i = 0; i < PDPNUM; i++) { + page_dir[i] = (pt_entry_t *) kmem_cache_alloc(&pd_cache); + if (page_dir[i] == NULL) { + i -= 1; + while (i >= 0) { + kmem_cache_free(&pd_cache, + (vm_address_t) page_dir[i]); + i -= 1; + } + kmem_cache_free(&pmap_cache, (vm_address_t) p); + return PMAP_NULL; + } + memcpy(page_dir[i], + (void *) kernel_page_dir + i * INTEL_PGBYTES, + INTEL_PGBYTES); + } - memcpy(p->dirbase, kernel_page_dir, PDPNUM * INTEL_PGBYTES); #ifdef LINUX_DEV #if VM_MIN_KERNEL_ADDRESS != 0 /* Do not map BIOS in user tasks */ - p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] = 0; + page_dir +#if PAE + [lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] +#else + [0] +#endif + [lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] + = 0; #endif #endif #ifdef MACH_PV_PAGETABLES { - int i; for (i = 0; i < PDPNUM; i++) - pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES); + pmap_set_page_readonly((void *) page_dir[i]); } #endif /* MACH_PV_PAGETABLES */ #if PAE - if (kmem_alloc_wired(kernel_map, - (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES) - != KERN_SUCCESS) - panic("pmap_create"); + p->pdpbase = (pt_entry_t *) kmem_cache_alloc(&pdpt_cache); + if (p->pdpbase == NULL) { + for (i = 0; i < PDPNUM; i++) + kmem_cache_free(&pd_cache, (vm_address_t) page_dir[i]); + kmem_cache_free(&pmap_cache, (vm_address_t) p); + return PMAP_NULL; + } + { - int i; for (i = 0; i < PDPNUM; i++) - WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + WRITE_PTE(&p->pdpbase[i], + pa_to_pte(kvtophys((vm_offset_t) page_dir[i])) + | INTEL_PTE_VALID); } #ifdef MACH_PV_PAGETABLES pmap_set_page_readonly(p->pdpbase); #endif /* MACH_PV_PAGETABLES */ +#else /* PAE */ + p->dirbase = page_dir[0]; #endif /* PAE */ p->ref_count = 1; @@ -1198,13 +1266,17 @@ pmap_t pmap_create(size) * no valid mappings. */ -void pmap_destroy(p) - register pmap_t p; +void pmap_destroy(pmap_t p) { - register pt_entry_t *pdep; - register vm_offset_t pa; - register int c, s; - register vm_page_t m; +#if PAE + int i; +#endif + boolean_t free_all; + pt_entry_t *page_dir; + pt_entry_t *pdep; + phys_addr_t pa; + int c, s; + vm_page_t m; if (p == PMAP_NULL) return; @@ -1219,44 +1291,53 @@ void pmap_destroy(p) return; /* still in use */ } - /* - * Free the memory maps, then the - * pmap structure. - */ - for (pdep = p->dirbase; - pdep < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]; - pdep += ptes_per_vm_page) { - if (*pdep & INTEL_PTE_VALID) { - pa = pte_to_pa(*pdep); - vm_object_lock(pmap_object); - m = vm_page_lookup(pmap_object, pa); - if (m == VM_PAGE_NULL) - panic("pmap_destroy: pte page not in object"); - vm_page_lock_queues(); -#ifdef MACH_PV_PAGETABLES - if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa))) - panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa)); - pmap_set_page_readwrite((void*) phystokv(pa)); -#endif /* MACH_PV_PAGETABLES */ - vm_page_free(m); - inuse_ptepages_count--; - vm_page_unlock_queues(); - vm_object_unlock(pmap_object); +#if PAE + for (i = 0; i <= lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); i++) { + free_all = i < lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); + page_dir = (pt_entry_t *) ptetokv(p->pdpbase[i]); +#else + free_all = FALSE; + page_dir = p->dirbase; +#endif + + /* + * Free the memory maps, then the + * pmap structure. + */ + for (pdep = page_dir; + (free_all + || pdep < &page_dir[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]) + && pdep < &page_dir[NPTES]; + pdep += ptes_per_vm_page) { + if (*pdep & INTEL_PTE_VALID) { + pa = pte_to_pa(*pdep); + vm_object_lock(pmap_object); + m = vm_page_lookup(pmap_object, pa); + if (m == VM_PAGE_NULL) + panic("pmap_destroy: pte page not in object"); + vm_page_lock_queues(); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa))) + panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa)); + pmap_set_page_readwrite((void*) phystokv(pa)); +#endif /* MACH_PV_PAGETABLES */ + vm_page_free(m); + inuse_ptepages_count--; + vm_page_unlock_queues(); + vm_object_unlock(pmap_object); + } } - } #ifdef MACH_PV_PAGETABLES - { - int i; - for (i = 0; i < PDPNUM; i++) - pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES); - } + pmap_set_page_readwrite((void*) page_dir); #endif /* MACH_PV_PAGETABLES */ - kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM * INTEL_PGBYTES); + kmem_cache_free(&pd_cache, (vm_offset_t) page_dir); #if PAE + } + #ifdef MACH_PV_PAGETABLES pmap_set_page_readwrite(p->pdpbase); #endif /* MACH_PV_PAGETABLES */ - kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES); + kmem_cache_free(&pdpt_cache, (vm_offset_t) p->pdpbase); #endif /* PAE */ kmem_cache_free(&pmap_cache, (vm_offset_t) p); } @@ -1265,8 +1346,7 @@ void pmap_destroy(p) * Add a reference to the specified pmap. */ -void pmap_reference(p) - register pmap_t p; +void pmap_reference(pmap_t p) { int s; if (p != PMAP_NULL) { @@ -1291,16 +1371,16 @@ void pmap_reference(p) */ /* static */ -void pmap_remove_range(pmap, va, spte, epte) - pmap_t pmap; - vm_offset_t va; - pt_entry_t *spte; - pt_entry_t *epte; -{ - register pt_entry_t *cpte; - int num_removed, num_unwired; - int pai; - vm_offset_t pa; +void pmap_remove_range( + pmap_t pmap, + vm_offset_t va, + pt_entry_t *spte, + pt_entry_t *epte) +{ + pt_entry_t *cpte; + unsigned long num_removed, num_unwired; + unsigned long pai; + phys_addr_t pa; #ifdef MACH_PV_PAGETABLES int n, ii = 0; struct mmu_update update[HYP_BATCH_MMU_UPDATES]; @@ -1330,8 +1410,8 @@ void pmap_remove_range(pmap, va, spte, e * Outside range of managed physical memory. * Just remove the mappings. */ - register int i = ptes_per_vm_page; - register pt_entry_t *lpte = cpte; + int i = ptes_per_vm_page; + pt_entry_t *lpte = cpte; do { #ifdef MACH_PV_PAGETABLES update[ii].ptr = kv_to_ma(lpte); @@ -1358,8 +1438,8 @@ void pmap_remove_range(pmap, va, spte, e * Get the modify and reference bits. */ { - register int i; - register pt_entry_t *lpte; + int i; + pt_entry_t *lpte; i = ptes_per_vm_page; lpte = cpte; @@ -1388,7 +1468,7 @@ void pmap_remove_range(pmap, va, spte, e * this physical page. */ { - register pv_entry_t pv_h, prev, cur; + pv_entry_t pv_h, prev, cur; pv_h = pai_to_pvh(pai); if (pv_h->pmap == PMAP_NULL) { @@ -1447,13 +1527,14 @@ void pmap_remove_range(pmap, va, spte, e * rounded to the hardware page size. */ -void pmap_remove(map, s, e) - pmap_t map; - vm_offset_t s, e; +void pmap_remove( + pmap_t map, + vm_offset_t s, + vm_offset_t e) { int spl; - register pt_entry_t *pde; - register pt_entry_t *spte, *epte; + pt_entry_t *pde; + pt_entry_t *spte, *epte; vm_offset_t l; vm_offset_t _s = s; @@ -1488,15 +1569,15 @@ void pmap_remove(map, s, e) * Lower the permission for all mappings to a given * page. */ -void pmap_page_protect(phys, prot) - vm_offset_t phys; - vm_prot_t prot; +void pmap_page_protect( + phys_addr_t phys, + vm_prot_t prot) { pv_entry_t pv_h, prev; - register pv_entry_t pv_e; - register pt_entry_t *pte; - int pai; - register pmap_t pmap; + pv_entry_t pv_e; + pt_entry_t *pte; + unsigned long pai; + pmap_t pmap; int spl; boolean_t remove; @@ -1542,7 +1623,7 @@ void pmap_page_protect(phys, prot) prev = pv_e = pv_h; do { - register vm_offset_t va; + vm_offset_t va; pmap = pv_e->pmap; /* @@ -1556,8 +1637,8 @@ void pmap_page_protect(phys, prot) /* * Consistency checks. */ - /* assert(*pte & INTEL_PTE_VALID); XXX */ - /* assert(pte_to_phys(*pte) == phys); */ + assert(*pte & INTEL_PTE_VALID); + assert(pte_to_pa(*pte) == phys); /* * Remove the mapping if new protection is NONE @@ -1568,10 +1649,10 @@ void pmap_page_protect(phys, prot) * Remove the mapping, collecting any modify bits. */ if (*pte & INTEL_PTE_WIRED) - panic("pmap_remove_all removing a wired page"); + panic("pmap_page_protect removing a wired page"); { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { pmap_phys_attributes[pai] |= @@ -1608,7 +1689,7 @@ void pmap_page_protect(phys, prot) /* * Write-protect. */ - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { #ifdef MACH_PV_PAGETABLES @@ -1651,14 +1732,15 @@ void pmap_page_protect(phys, prot) * specified range of this map as requested. * Will not increase permissions. */ -void pmap_protect(map, s, e, prot) - pmap_t map; - vm_offset_t s, e; - vm_prot_t prot; -{ - register pt_entry_t *pde; - register pt_entry_t *spte, *epte; - vm_offset_t l; +void pmap_protect( + pmap_t map, + vm_offset_t s, + vm_offset_t e, + vm_prot_t prot) +{ + pt_entry_t *pde; + pt_entry_t *spte, *epte; + vm_offset_t l; int spl; vm_offset_t _s = s; @@ -1757,20 +1839,21 @@ void pmap_protect(map, s, e, prot) * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ -void pmap_enter(pmap, v, pa, prot, wired) - register pmap_t pmap; - vm_offset_t v; - register vm_offset_t pa; - vm_prot_t prot; - boolean_t wired; -{ - register pt_entry_t *pte; - register pv_entry_t pv_h; - register int i, pai; +void pmap_enter( + pmap_t pmap, + vm_offset_t v, + phys_addr_t pa, + vm_prot_t prot, + boolean_t wired) +{ + boolean_t is_physmem; + pt_entry_t *pte; + pv_entry_t pv_h; + unsigned long i, pai; pv_entry_t pv_e; pt_entry_t template; int spl; - vm_offset_t old_pa; + phys_addr_t old_pa; assert(pa != vm_page_fictitious_addr); if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa); @@ -1895,6 +1978,11 @@ Retry: continue; } + if (vm_page_ready()) + is_physmem = (vm_page_lookup_pa(pa) != NULL); + else + is_physmem = (pa < biosmem_directmap_end()); + /* * Special case if the physical page is already mapped * at this address. @@ -1916,7 +2004,7 @@ Retry: if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 - && pa >= phys_last_addr) + && !is_physmem) template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; @@ -2028,7 +2116,7 @@ Retry: if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 - && pa >= phys_last_addr) + && !is_physmem) template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; @@ -2059,14 +2147,14 @@ Retry: * In/out conditions: * The mapping must already exist in the pmap. */ -void pmap_change_wiring(map, v, wired) - register pmap_t map; - vm_offset_t v; - boolean_t wired; +void pmap_change_wiring( + pmap_t map, + vm_offset_t v, + boolean_t wired) { - register pt_entry_t *pte; - register int i; - int spl; + pt_entry_t *pte; + int i; + int spl; /* * We must grab the pmap system lock because we may @@ -2114,20 +2202,20 @@ void pmap_change_wiring(map, v, wired) * with the given map/virtual_address pair. */ -vm_offset_t pmap_extract(pmap, va) - register pmap_t pmap; - vm_offset_t va; +phys_addr_t pmap_extract( + pmap_t pmap, + vm_offset_t va) { - register pt_entry_t *pte; - register vm_offset_t pa; - int spl; + pt_entry_t *pte; + phys_addr_t pa; + int spl; SPLVM(spl); simple_lock(&pmap->lock); if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL) - pa = (vm_offset_t) 0; + pa = 0; else if (!(*pte & INTEL_PTE_VALID)) - pa = (vm_offset_t) 0; + pa = 0; else pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK); simple_unlock(&pmap->lock); @@ -2150,9 +2238,6 @@ void pmap_copy(dst_pmap, src_pmap, dst_a vm_size_t len; vm_offset_t src_addr; { -#ifdef lint - dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++; -#endif /* lint */ } #endif /* 0 */ @@ -2167,12 +2252,14 @@ void pmap_copy(dst_pmap, src_pmap, dst_a * Usage: * Called by the pageout daemon when pages are scarce. */ -void pmap_collect(p) - pmap_t p; +void pmap_collect(pmap_t p) { - register pt_entry_t *pdp, *ptp; + int i; + boolean_t free_all; + pt_entry_t *page_dir; + pt_entry_t *pdp, *ptp; pt_entry_t *eptp; - vm_offset_t pa; + phys_addr_t pa; int spl, wired; if (p == PMAP_NULL) @@ -2181,92 +2268,107 @@ void pmap_collect(p) if (p == kernel_pmap) return; - /* - * Garbage collect map. - */ - PMAP_READ_LOCK(p, spl); - for (pdp = p->dirbase; - pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]; - pdp += ptes_per_vm_page) - { - if (*pdp & INTEL_PTE_VALID) { +#if PAE + for (i = 0; i <= lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); i++) { + free_all = i < lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); + page_dir = (pt_entry_t *) ptetokv(p->pdpbase[i]); +#else + i = 0; + free_all = FALSE; + page_dir = p->dirbase; +#endif - pa = pte_to_pa(*pdp); - ptp = (pt_entry_t *)phystokv(pa); - eptp = ptp + NPTES*ptes_per_vm_page; + /* + * Garbage collect map. + */ + PMAP_READ_LOCK(p, spl); + for (pdp = page_dir; + (free_all + || pdp < &page_dir[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]) + && pdp < &page_dir[NPTES]; + pdp += ptes_per_vm_page) { + if (*pdp & INTEL_PTE_VALID) { + + pa = pte_to_pa(*pdp); + ptp = (pt_entry_t *)phystokv(pa); + eptp = ptp + NPTES*ptes_per_vm_page; - /* - * If the pte page has any wired mappings, we cannot - * free it. - */ - wired = 0; - { - register pt_entry_t *ptep; - for (ptep = ptp; ptep < eptp; ptep++) { - if (*ptep & INTEL_PTE_WIRED) { - wired = 1; - break; - } - } - } - if (!wired) { /* - * Remove the virtual addresses mapped by this pte page. + * If the pte page has any wired mappings, we cannot + * free it. */ - { /*XXX big hack*/ - vm_offset_t va = pdenum2lin(pdp - p->dirbase); - if (p == kernel_pmap) - va = lintokv(va); - pmap_remove_range(p, - va, - ptp, - eptp); + wired = 0; + { + pt_entry_t *ptep; + for (ptep = ptp; ptep < eptp; ptep++) { + if (*ptep & INTEL_PTE_WIRED) { + wired = 1; + break; + } + } } + if (!wired) { + /* + * Remove the virtual addresses mapped by this pte page. + */ + { /*XXX big hack*/ + vm_offset_t va = pdenum2lin(pdp - page_dir + + i * NPTES); + if (p == kernel_pmap) + va = lintokv(va); + pmap_remove_range(p, + va, + ptp, + eptp); + } - /* - * Invalidate the page directory pointer. - */ - { - register int i = ptes_per_vm_page; - register pt_entry_t *pdep = pdp; - do { -#ifdef MACH_PV_PAGETABLES - unsigned long pte = *pdep; - void *ptable = (void*) ptetokv(pte); - if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0))) - panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1); - if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable))) - panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable))); - pmap_set_page_readwrite(ptable); + /* + * Invalidate the page directory pointer. + */ + { + int i = ptes_per_vm_page; + pt_entry_t *pdep = pdp; + do { +#ifdef MACH_PV_PAGETABLES + unsigned long pte = *pdep; + void *ptable = (void*) ptetokv(pte); + if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0))) + panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1); + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable))) + panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable))); + pmap_set_page_readwrite(ptable); #else /* MACH_PV_PAGETABLES */ - *pdep++ = 0; + *pdep++ = 0; #endif /* MACH_PV_PAGETABLES */ - } while (--i > 0); - } + } while (--i > 0); + } - PMAP_READ_UNLOCK(p, spl); + PMAP_READ_UNLOCK(p, spl); - /* - * And free the pte page itself. - */ - { - register vm_page_t m; + /* + * And free the pte page itself. + */ + { + vm_page_t m; - vm_object_lock(pmap_object); - m = vm_page_lookup(pmap_object, pa); - if (m == VM_PAGE_NULL) - panic("pmap_collect: pte page not in object"); - vm_page_lock_queues(); - vm_page_free(m); - inuse_ptepages_count--; - vm_page_unlock_queues(); - vm_object_unlock(pmap_object); - } + vm_object_lock(pmap_object); + m = vm_page_lookup(pmap_object, pa); + if (m == VM_PAGE_NULL) + panic("pmap_collect: pte page not in object"); + vm_page_lock_queues(); + vm_page_free(m); + inuse_ptepages_count--; + vm_page_unlock_queues(); + vm_object_unlock(pmap_object); + } - PMAP_READ_LOCK(p, spl); + PMAP_READ_LOCK(p, spl); + } } } +#if PAE } +#endif PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS); PMAP_READ_UNLOCK(p, spl); @@ -2303,9 +2405,6 @@ void pmap_deactivate(pmap, th, which_cpu thread_t th; int which_cpu; { -#ifdef lint - pmap++; th++; which_cpu++; -#endif /* lint */ PMAP_DEACTIVATE(pmap, th, which_cpu); } #endif /* 0 */ @@ -2378,30 +2477,27 @@ pmap_copy_page(src, dst) * down (or not) as appropriate. */ void -pmap_pageable(pmap, start, end, pageable) - pmap_t pmap; - vm_offset_t start; - vm_offset_t end; - boolean_t pageable; -{ -#ifdef lint - pmap++; start++; end++; pageable++; -#endif /* lint */ +pmap_pageable( + pmap_t pmap, + vm_offset_t start, + vm_offset_t end, + boolean_t pageable) +{ } /* * Clear specified attribute bits. */ void -phys_attribute_clear(phys, bits) - vm_offset_t phys; - int bits; +phys_attribute_clear( + phys_addr_t phys, + int bits) { pv_entry_t pv_h; - register pv_entry_t pv_e; - register pt_entry_t *pte; - int pai; - register pmap_t pmap; + pv_entry_t pv_e; + pt_entry_t *pte; + unsigned long pai; + pmap_t pmap; int spl; assert(phys != vm_page_fictitious_addr); @@ -2432,7 +2528,7 @@ phys_attribute_clear(phys, bits) * There are some mappings. */ for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) { - register vm_offset_t va; + vm_offset_t va; pmap = pv_e->pmap; /* @@ -2443,19 +2539,17 @@ phys_attribute_clear(phys, bits) va = pv_e->va; pte = pmap_pte(pmap, va); -#if 0 /* * Consistency checks. */ assert(*pte & INTEL_PTE_VALID); - /* assert(pte_to_phys(*pte) == phys); */ -#endif + assert(pte_to_pa(*pte) == phys); /* * Clear modify or reference bits. */ { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { #ifdef MACH_PV_PAGETABLES if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits))) @@ -2479,15 +2573,15 @@ phys_attribute_clear(phys, bits) * Check specified attribute bits. */ boolean_t -phys_attribute_test(phys, bits) - vm_offset_t phys; - int bits; +phys_attribute_test( + phys_addr_t phys, + int bits) { pv_entry_t pv_h; - register pv_entry_t pv_e; - register pt_entry_t *pte; - int pai; - register pmap_t pmap; + pv_entry_t pv_e; + pt_entry_t *pte; + unsigned long pai; + pmap_t pmap; int spl; assert(phys != vm_page_fictitious_addr); @@ -2531,25 +2625,23 @@ phys_attribute_test(phys, bits) simple_lock(&pmap->lock); { - register vm_offset_t va; + vm_offset_t va; va = pv_e->va; pte = pmap_pte(pmap, va); -#if 0 /* * Consistency checks. */ assert(*pte & INTEL_PTE_VALID); - /* assert(pte_to_phys(*pte) == phys); */ -#endif + assert(pte_to_pa(*pte) == phys); } /* * Check modify or reference bits. */ { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { if (*pte & bits) { @@ -2570,8 +2662,7 @@ phys_attribute_test(phys, bits) * Clear the modify bits on the specified physical page. */ -void pmap_clear_modify(phys) - register vm_offset_t phys; +void pmap_clear_modify(phys_addr_t phys) { phys_attribute_clear(phys, PHYS_MODIFIED); } @@ -2583,8 +2674,7 @@ void pmap_clear_modify(phys) * by any physical maps. */ -boolean_t pmap_is_modified(phys) - register vm_offset_t phys; +boolean_t pmap_is_modified(phys_addr_t phys) { return (phys_attribute_test(phys, PHYS_MODIFIED)); } @@ -2595,8 +2685,7 @@ boolean_t pmap_is_modified(phys) * Clear the reference bit on the specified physical page. */ -void pmap_clear_reference(phys) - vm_offset_t phys; +void pmap_clear_reference(phys_addr_t phys) { phys_attribute_clear(phys, PHYS_REFERENCED); } @@ -2608,8 +2697,7 @@ void pmap_clear_reference(phys) * by any physical maps. */ -boolean_t pmap_is_referenced(phys) - vm_offset_t phys; +boolean_t pmap_is_referenced(phys_addr_t phys) { return (phys_attribute_test(phys, PHYS_REFERENCED)); } @@ -2678,13 +2766,14 @@ boolean_t pmap_is_referenced(phys) /* * Signal another CPU that it must flush its TLB */ -void signal_cpus(use_list, pmap, start, end) - cpu_set use_list; - pmap_t pmap; - vm_offset_t start, end; +void signal_cpus( + cpu_set use_list, + pmap_t pmap, + vm_offset_t start, + vm_offset_t end) { - register int which_cpu, j; - register pmap_update_list_t update_list_p; + int which_cpu, j; + pmap_update_list_t update_list_p; while ((which_cpu = ffs(use_list)) != 0) { which_cpu -= 1; /* convert to 0 origin */ @@ -2717,13 +2806,12 @@ void signal_cpus(use_list, pmap, star } } -void process_pmap_updates(my_pmap) - register pmap_t my_pmap; +void process_pmap_updates(pmap_t my_pmap) { - register int my_cpu = cpu_number(); - register pmap_update_list_t update_list_p; - register int j; - register pmap_t pmap; + int my_cpu = cpu_number(); + pmap_update_list_t update_list_p; + int j; + pmap_t pmap; update_list_p = &cpu_update_list[my_cpu]; simple_lock(&update_list_p->lock); @@ -2748,9 +2836,9 @@ void process_pmap_updates(my_pmap) */ void pmap_update_interrupt(void) { - register int my_cpu; - register pmap_t my_pmap; - int s; + int my_cpu; + pmap_t my_pmap; + int s; my_cpu = cpu_number(); @@ -2806,7 +2894,7 @@ void pmap_update_interrupt(void) /* * Dummy routine to satisfy external reference. */ -void pmap_update_interrupt() +void pmap_update_interrupt(void) { /* should never be called. */ } @@ -2815,10 +2903,11 @@ void pmap_update_interrupt() #if defined(__i386__) /* Unmap page 0 to trap NULL references. */ void -pmap_unmap_page_zero () +pmap_unmap_page_zero (void) { int *pte; + printf("Unmapping the zero page. Some BIOS functions may not be working any more.\n"); pte = (int *) pmap_pte (kernel_pmap, 0); if (!pte) return;