--- Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:44:40 1.1.1.2 +++ Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:46:55 1.1.1.3 @@ -28,7 +28,7 @@ * Author: Avadis Tevanian, Jr., Michael Wayne Young * (These guys wrote the Vax version) * - * Physical Map management code for Intel i386, i486, and i860. + * Physical Map management code for Intel i386, and i486. * * Manages physical address maps. * @@ -55,77 +55,40 @@ * and to when physical maps must be made correct. */ -#include +#include #include #include +#include +#include #include -#include +#include #include #include #include #include -#include "vm_param.h" +#include #include #include #include #include #include +#include #include -#include "cpu_number.h" -#if i860 -#include -#endif - -#ifdef ORC -#define OLIVETTICACHE 1 -#endif /* ORC */ - -#ifndef OLIVETTICACHE +#include +#include +#include +#include + +#ifdef MACH_PSEUDO_PHYS +#define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0; +#else /* MACH_PSEUDO_PHYS */ #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry); -#define WRITE_PTE_FAST(pte_p, pte_entry) *(pte_p) = (pte_entry); -#else /* OLIVETTICACHE */ -#error might not work anymore - -/* This gross kludgery is needed for Olivetti XP7 & XP9 boxes to get - * around an apparent hardware bug. Other than at startup it doesn't - * affect run-time performacne very much, so we leave it in for all - * machines. - */ -extern unsigned *pstart(); -#define CACHE_LINE 8 -#define CACHE_SIZE 512 -#define CACHE_PAGE 0x1000; - -#define WRITE_PTE(pte_p, pte_entry) { write_pte(pte_p, pte_entry); } - -write_pte(pte_p, pte_entry) -pt_entry_t *pte_p, pte_entry; -{ - unsigned long count; - volatile unsigned long hold, *addr1, *addr2; - - if ( pte_entry != *pte_p ) - *pte_p = pte_entry; - else { - /* This isn't necessarily the optimal algorithm */ - addr1 = (unsigned long *)pstart; - for (count = 0; count < CACHE_SIZE; count++) { - addr2 = addr1 + CACHE_PAGE; - hold = *addr1; /* clear cache bank - A - */ - hold = *addr2; /* clear cache bank - B - */ - addr1 += CACHE_LINE; - } - } -} - -#define WRITE_PTE_FAST(pte_p, pte_entry)*pte_p = pte_entry; - -#endif /* OLIVETTICACHE */ +#endif /* MACH_PSEUDO_PHYS */ /* * Private data structures. @@ -150,7 +113,7 @@ pv_entry_t pv_head_table; /* array of e /* * pv_list entries are kept on a list that can only be accessed * with the pmap system locked (at SPLVM, not in the cpus_active set). - * The list is refilled from the pv_list_zone if it becomes empty. + * The list is refilled from the pv_list_cache if it becomes empty. */ pv_entry_t pv_free_list; /* free list at SPLVM */ decl_simple_lock_data(, pv_free_list_lock) @@ -170,7 +133,7 @@ decl_simple_lock_data(, pv_free_list_loc simple_unlock(&pv_free_list_lock); \ } -zone_t pv_list_zone; /* zone of pv_entry structures */ +struct kmem_cache pv_list_cache; /* cache of pv_entry structures */ /* * Each entry in the pv_head_table is locked by a bit in the @@ -185,13 +148,6 @@ char *pv_lock_table; /* pointer to arra boolean_t pmap_initialized = FALSE; /* - * More-specific code provides these; - * they indicate the total extent of physical memory - * that we know about and might ever have to manage. - */ -extern vm_offset_t phys_first_addr, phys_last_addr; - -/* * Range of kernel virtual addresses available for kernel memory mapping. * Does not include the virtual addresses used to map physical memory 1-1. * Initialized by pmap_bootstrap. @@ -199,9 +155,6 @@ extern vm_offset_t phys_first_addr, phys vm_offset_t kernel_virtual_start; vm_offset_t kernel_virtual_end; -/* XXX stupid fixed limit - get rid */ -vm_size_t morevm = 40 * 1024 * 1024; /* VM space for kernel map */ - /* * Index into pv_head table, its lock bits, and the modify/reference * bits starting at phys_first_addr. @@ -345,14 +298,14 @@ lock_data_t pmap_system_lock; \ /* invalidate our own TLB if pmap is in use */ \ if ((pmap)->cpus_using & cpu_mask) { \ - INVALIDATE_TLB((s), (e)); \ + INVALIDATE_TLB((pmap), (s), (e)); \ } \ } #else /* NCPUS > 1 */ -#define SPLVM(spl) -#define SPLX(spl) +#define SPLVM(spl) ((void)(spl)) +#define SPLX(spl) ((void)(spl)) #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl) #define PMAP_WRITE_LOCK(spl) SPLVM(spl) @@ -366,7 +319,7 @@ lock_data_t pmap_system_lock; #define PMAP_UPDATE_TLBS(pmap, s, e) { \ /* invalidate our own TLB if pmap is in use */ \ if ((pmap)->cpus_using) { \ - INVALIDATE_TLB((s), (e)); \ + INVALIDATE_TLB((pmap), (s), (e)); \ } \ } @@ -374,17 +327,38 @@ lock_data_t pmap_system_lock; #define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */ -#if i860 -/* Do a data cache flush until we find the caching bug XXX prp */ -#define INVALIDATE_TLB(s, e) { \ - flush(); \ - flush_tlb(); \ +#ifdef MACH_PV_PAGETABLES +#if 1 +#define INVALIDATE_TLB(pmap, s, e) hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL) +#else +#define INVALIDATE_TLB(pmap, s, e) do { \ + if (__builtin_constant_p((e) - (s)) \ + && (e) - (s) == PAGE_SIZE) \ + hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s) : (s)); \ + else \ + hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \ +} while(0) +#endif +#else /* MACH_PV_PAGETABLES */ +#if 0 +/* It is hard to know when a TLB flush becomes less expensive than a bunch of + * invlpgs. But it surely is more expensive than just one invlpg. */ +#define INVALIDATE_TLB(pmap, s, e) { \ + if (__builtin_constant_p((e) - (s)) \ + && (e) - (s) == PAGE_SIZE) \ + invlpg_linear(s); \ + else \ + flush_tlb(); \ } -#else /* i860 */ -#define INVALIDATE_TLB(s, e) { \ +#else +#define INVALIDATE_TLB(pmap, s, e) { \ + (void) (pmap); \ + (void) (s); \ + (void) (e); \ flush_tlb(); \ } -#endif /* i860 */ +#endif +#endif /* MACH_PV_PAGETABLES */ #if NCPUS > 1 @@ -426,7 +400,7 @@ struct pmap_update_list cpu_update_list[ struct pmap kernel_pmap_store; pmap_t kernel_pmap; -struct zone *pmap_zone; /* zone of pmap structures */ +struct kmem_cache pmap_cache; /* cache of pmap structures */ int pmap_debug = 0; /* flag for debugging prints */ @@ -452,13 +426,6 @@ void pmap_remove_range(); /* forward */ void signal_cpus(); /* forward */ #endif /* NCPUS > 1 */ -#if i860 -/* - * Paging flag - */ -int paging_enabled = 0; -#endif - static inline pt_entry_t * pmap_pde(pmap_t pmap, vm_offset_t addr) { @@ -551,9 +518,6 @@ vm_offset_t pmap_map(virt, start, end, p * Useful for mapping memory outside the range * [phys_first_addr, phys_last_addr) (i.e., devices). * Otherwise like pmap_map. -#if i860 - * Sets no-cache bit. -#endif */ vm_offset_t pmap_map_bd(virt, start, end, prot) register vm_offset_t virt; @@ -563,24 +527,50 @@ vm_offset_t pmap_map_bd(virt, start, end { register pt_entry_t template; register pt_entry_t *pte; + int spl; +#ifdef MACH_PV_PAGETABLES + int n, i = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ template = pa_to_pte(start) -#if i860 - | INTEL_PTE_NCACHE -#endif + | INTEL_PTE_NCACHE|INTEL_PTE_WTHRU | INTEL_PTE_VALID; + if (CPU_HAS_FEATURE(CPU_FEATURE_PGE)) + template |= INTEL_PTE_GLOBAL; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + PMAP_READ_LOCK(pmap, spl); while (start < end) { pte = pmap_pte(kernel_pmap, virt); if (pte == PT_ENTRY_NULL) panic("pmap_map_bd: Invalid kernel address\n"); - WRITE_PTE_FAST(pte, template) +#ifdef MACH_PV_PAGETABLES + update[i].ptr = kv_to_ma(pte); + update[i].val = pa_to_ma(template); + i++; + if (i == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_map_bd\n"); + i = 0; + } +#else /* MACH_PV_PAGETABLES */ + WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte_increment_pa(template); virt += PAGE_SIZE; start += PAGE_SIZE; } +#ifdef MACH_PV_PAGETABLES + if (i > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_map_bd"); + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_map_bd\n"); +#endif /* MACH_PV_PAGETABLES */ + PMAP_READ_UNLOCK(pmap, spl); return(virt); } @@ -626,24 +616,96 @@ void pmap_bootstrap() * mapped into the kernel address space, * and extends to a stupid arbitrary limit beyond that. */ - kernel_virtual_start = phys_last_addr; - kernel_virtual_end = phys_last_addr + morevm; + kernel_virtual_start = phystokv(phys_last_addr); + kernel_virtual_end = phystokv(phys_last_addr) + VM_KERNEL_MAP_SIZE; + + if (kernel_virtual_end < kernel_virtual_start + || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS) + kernel_virtual_end = VM_MAX_KERNEL_ADDRESS; /* * Allocate and clear a kernel page directory. */ - kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)pmap_grab_page(); + /* Note: initial Xen mapping holds at least 512kB free mapped page. + * We use that for directly building our linear mapping. */ +#if PAE + { + vm_offset_t addr; + init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr); + kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr); + } + kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page()); + { + int i; + for (i = 0; i < PDPNUM; i++) + WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + } +#else /* PAE */ + kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page()); +#endif /* PAE */ { int i; for (i = 0; i < NPDES; i++) kernel_pmap->dirbase[i] = 0; } +#ifdef MACH_PV_PAGETABLES + /* We don't actually deal with the CR3 register content at all */ + hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3); + /* + * Xen may only provide as few as 512KB extra bootstrap linear memory, + * which is far from enough to map all available memory, so we need to + * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE) + * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is + * enough for a pagetable mapping 4GiB. + */ +#ifdef PAE +#define NSUP_L1 4 +#else +#define NSUP_L1 1 +#endif + pt_entry_t *l1_map[NSUP_L1]; + { + pt_entry_t *base = (pt_entry_t*) boot_info.pt_base; + vm_offset_t la; + int n_l1map; + for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS; la >= VM_MIN_KERNEL_ADDRESS; la += NPTES * PAGE_SIZE) { +#ifdef PAE + pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)]); +#else /* PAE */ + pt_entry_t *l2_map = base; +#endif /* PAE */ + /* Like lin2pdenum, but works with non-contiguous boot L3 */ + l2_map += (la >> PDESHIFT) & PDEMASK; + if (!(*l2_map & INTEL_PTE_VALID)) { + struct mmu_update update; + int j, n; + + l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page()); + for (j = 0; j < NPTES; j++) + l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS) * NPTES + j)) << PAGE_SHIFT) | INTEL_PTE_VALID | INTEL_PTE_WRITE; + pmap_set_page_readonly_init(l1_map[n_l1map]); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map]))) + panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map])); + update.ptr = kv_to_ma(l2_map); + update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID | INTEL_PTE_WRITE; + hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF); + if (n != 1) + panic("couldn't complete bootstrap map"); + /* added the last L1 table, can stop */ + if (++n_l1map >= NSUP_L1) + break; + } + } + } +#endif /* MACH_PV_PAGETABLES */ + /* * Allocate and set up the kernel page tables. */ { vm_offset_t va; + pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0; /* * Map virtual memory for all known physical memory, 1-1, @@ -656,136 +718,178 @@ void pmap_bootstrap() * to allocate new kernel page tables later. * XX fix this */ - for (va = phys_first_addr; va < phys_last_addr + morevm; ) + for (va = phystokv(phys_first_addr); va >= phystokv(phys_first_addr) && va < kernel_virtual_end; ) { pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va)); - pt_entry_t *ptable = (pt_entry_t*)pmap_grab_page(); + pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page()); pt_entry_t *pte; - vm_offset_t pteva; /* Initialize the page directory entry. */ - *pde = pa_to_pte((vm_offset_t)ptable) - | INTEL_PTE_VALID | INTEL_PTE_WRITE; + WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable)) + | INTEL_PTE_VALID | INTEL_PTE_WRITE); /* Initialize the page table. */ - for (pte = ptable; (va < phys_last_addr) && (pte < ptable+NPTES); pte++) + for (pte = ptable; (va < phystokv(phys_last_addr)) && (pte < ptable+NPTES); pte++) { if ((pte - ptable) < ptenum(va)) { - WRITE_PTE_FAST(pte, 0); + WRITE_PTE(pte, 0); } else +#ifdef MACH_PV_PAGETABLES + if (va == (vm_offset_t) &hyp_shared_info) + { + *pte = boot_info.shared_info | INTEL_PTE_VALID | INTEL_PTE_WRITE; + va += INTEL_PGBYTES; + } + else +#endif /* MACH_PV_PAGETABLES */ { extern char _start[], etext[]; - if ((va >= (vm_offset_t)_start) + if (((va >= (vm_offset_t) _start) && (va + INTEL_PGBYTES <= (vm_offset_t)etext)) +#ifdef MACH_PV_PAGETABLES + || (va >= (vm_offset_t) boot_info.pt_base + && (va + INTEL_PGBYTES <= + (vm_offset_t) ptable + INTEL_PGBYTES)) +#endif /* MACH_PV_PAGETABLES */ + ) { - WRITE_PTE_FAST(pte, pa_to_pte(va) - | INTEL_PTE_VALID); + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | global); } else { - WRITE_PTE_FAST(pte, pa_to_pte(va) - | INTEL_PTE_VALID | INTEL_PTE_WRITE); +#ifdef MACH_PV_PAGETABLES + /* Keep supplementary L1 pages read-only */ + int i; + for (i = 0; i < NSUP_L1; i++) + if (va == (vm_offset_t) l1_map[i]) { + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | global); + break; + } + if (i == NSUP_L1) +#endif /* MACH_PV_PAGETABLES */ + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | INTEL_PTE_WRITE | global) + } va += INTEL_PGBYTES; } } for (; pte < ptable+NPTES; pte++) { - WRITE_PTE_FAST(pte, 0); + WRITE_PTE(pte, 0); va += INTEL_PGBYTES; } +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly_init(ptable); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable))) + panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable)); +#endif /* MACH_PV_PAGETABLES */ } } -#if i860 -#error probably doesnt work anymore - XXX move to architecture-specific code just after the pmap_bootstrap call. - - /* kvtophys should now work in phys range */ - - /* - * Mark page table pages non-cacheable - */ - - pt_pte = (pt_entry_t *)pte_to_pa(*(kpde + pdenum(sva))) + ptenum(sva); - - for (va = load_start; va < tva; va += INTEL_PGBYTES*NPTES) { - /* Mark page table non-cacheable */ - *pt_pte |= INTEL_PTE_NCACHE; - pt_pte++; - } - - /* - * Map I/O space - */ - - ppde = kpde; - ppde += pdenum(IO_BASE); - - if (pte_to_pa(*ppde) == 0) { - /* This pte has not been allocated */ - ppte = (pt_entry_t *)kvtophys(virtual_avail); - ptend = ppte + NPTES; - virtual_avail = phystokv((vm_offset_t)ptend); - *ppde = pa_to_pte((vm_offset_t)ppte) - | INTEL_PTE_VALID - | INTEL_PTE_WRITE; - pte = ptend; + /* Architecture-specific code will turn on paging + soon after we return from here. */ +} - /* Mark page table non-cacheable */ - *pt_pte |= INTEL_PTE_NCACHE; - pt_pte++; +#ifdef MACH_PV_PAGETABLES +/* These are only required because of Xen security policies */ - bzero((char *)ppte, INTEL_PGBYTES); - } else { - ppte = (pt_entry_t *)(*ppde); /* first pte of page */ +/* Set back a page read write */ +void pmap_set_page_readwrite(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; + vm_offset_t paddr = kvtophys(vaddr); + vm_offset_t canon_vaddr = phystokv(paddr); + if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE)) + panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr)); + if (canon_vaddr != vaddr) + if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE)) + panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr)); +} + +/* Set a page read only (so as to pin it for instance) */ +void pmap_set_page_readonly(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; + vm_offset_t paddr = kvtophys(vaddr); + vm_offset_t canon_vaddr = phystokv(paddr); + if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr)); + } + if (canon_vaddr != vaddr && + *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr)); } - *ppde |= INTEL_PTE_USER; - - - WRITE_PTE(ppte + ptenum(FIFO_ADDR), - pa_to_pte(FIFO_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); - - WRITE_PTE(ppte + ptenum(FIFO_ADDR + XEOD_OFF), - pa_to_pte(FIFO_ADDR_PH + XEOD_OFF_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); - -/* XXX Allowed user access to control reg - cfj */ - WRITE_PTE(ppte + ptenum(CSR_ADDR), - pa_to_pte(CSR_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE | INTEL_PTE_USER); - -/* XXX Allowed user access to perf reg - cfj */ - WRITE_PTE(ppte + ptenum(PERFCNT_ADDR), - pa_to_pte(PERFCNT_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_USER | INTEL_PTE_NCACHE | INTEL_PTE_USER); - - WRITE_PTE(ppte + ptenum(UART_ADDR), - pa_to_pte(UART_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); - - WRITE_PTE(ppte + ptenum(0xFFFFF000), - pa_to_pte(avail_end) - | INTEL_PTE_VALID | INTEL_PTE_WRITE); - avail_start = kvtophys(virtual_avail); - -/* - * Turn on mapping - */ - - flush_and_ctxsw(kernel_pmap->dirbase); - paging_enabled = 1; +} - printf("Paging enabled.\n"); +/* This needs to be called instead of pmap_set_page_readonly as long as RC3 + * still points to the bootstrap dirbase, to also fix the bootstrap table. */ +void pmap_set_page_readonly_init(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; +#if PAE + pt_entry_t *pdpbase = (void*) boot_info.pt_base; + /* The bootstrap table does not necessarily use contiguous pages for the pde tables */ + pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)]); +#else + pt_entry_t *dirbase = (void*) boot_info.pt_base; #endif + pt_entry_t *pte = &dirbase[lin2pdenum(vaddr) & PTEMASK]; + /* Modify our future kernel map (can't use update_va_mapping for this)... */ + if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) { + if (!hyp_mmu_update_la (kvtolin(vaddr), pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID)) + panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr)); + } + /* ... and the bootstrap map. */ + if (*pte & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr)); + } +} - /* Architecture-specific code will turn on paging - soon after we return from here. */ +void pmap_clear_bootstrap_pagetable(pt_entry_t *base) { + int i; + pt_entry_t *dir; + vm_offset_t va = 0; +#if PAE + int j; +#endif /* PAE */ + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base))) + panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base)); +#if PAE + for (j = 0; j < PDPNUM; j++) + { + pt_entry_t pdpe = base[j]; + if (pdpe & INTEL_PTE_VALID) { + dir = (pt_entry_t *) ptetokv(pdpe); +#else /* PAE */ + dir = base; +#endif /* PAE */ + for (i = 0; i < NPTES; i++) { + pt_entry_t pde = dir[i]; + unsigned long pfn = atop(pte_to_pa(pde)); + void *pgt = (void*) phystokv(ptoa(pfn)); + if (pde & INTEL_PTE_VALID) + hyp_free_page(pfn, pgt); + va += NPTES * INTEL_PGBYTES; + if (va >= HYP_VIRT_START) + break; + } +#if PAE + hyp_free_page(atop(_kvtophys(dir)), dir); + } else + va += NPTES * NPTES * INTEL_PGBYTES; + if (va >= HYP_VIRT_START) + break; + } +#endif /* PAE */ + hyp_free_page(atop(_kvtophys(base)), base); } +#endif /* MACH_PV_PAGETABLES */ void pmap_virtual_space(startp, endp) vm_offset_t *startp; @@ -805,7 +909,9 @@ void pmap_init() register long npages; vm_offset_t addr; register vm_size_t s; +#if NCPUS > 1 int i; +#endif /* NCPUS > 1 */ /* * Allocate memory for the pv_head_table and its lock bits, @@ -820,7 +926,7 @@ void pmap_init() s = round_page(s); if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS) panic("pmap_init"); - bzero((char *) addr, s); + memset((char *) addr, 0, s); /* * Allocate the structures first to preserve word-alignment. @@ -834,13 +940,13 @@ void pmap_init() pmap_phys_attributes = (char *) addr; /* - * Create the zone of physical maps, + * Create the cache of physical maps, * and of the physical-to-virtual entries. */ s = (vm_size_t) sizeof(struct pmap); - pmap_zone = zinit(s, 400*s, 4096, 0, "pmap"); /* XXX */ + kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0); s = (vm_size_t) sizeof(struct pv_entry); - pv_list_zone = zinit(s, 10000*s, 4096, 0, "pv_list"); /* XXX */ + kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0); #if NCPUS > 1 /* @@ -906,7 +1012,7 @@ pmap_page_table_page_alloc() /* * We cannot allocate the pmap_object in pmap_init, - * because it is called before the zone package is up. + * because it is called before the cache package is up. * Allocate it now if it is missing. */ if (pmap_object == VM_OBJECT_NULL) @@ -934,26 +1040,52 @@ pmap_page_table_page_alloc() /* * Zero the page. */ - bzero(phystokv(pa), PAGE_SIZE); + memset((void *)phystokv(pa), 0, PAGE_SIZE); -#if i860 - /* - * Mark the page table page(s) non-cacheable. - */ - { - int i = ptes_per_vm_page; - pt_entry_t *pdp; - - pdp = pmap_pte(kernel_pmap, pa); - do { - *pdp |= INTEL_PTE_NCACHE; - pdp++; - } while (--i > 0); - } -#endif return pa; } +#ifdef MACH_XEN +void pmap_map_mfn(void *_addr, unsigned long mfn) { + vm_offset_t addr = (vm_offset_t) _addr; + pt_entry_t *pte, *pdp; + vm_offset_t ptp; + pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT; + + /* Add a ptp if none exist yet for this pte */ + if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) { + ptp = phystokv(pmap_page_table_page_alloc()); +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly((void*) ptp); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp))) + panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp)); +#endif /* MACH_PV_PAGETABLES */ + pdp = pmap_pde(kernel_pmap, addr); + +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pdp), + pa_to_pte(kv_to_ma(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE)) + panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__,__LINE__,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp)); +#else /* MACH_PV_PAGETABLES */ + *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ + pte = pmap_pte(kernel_pmap, addr); + } + +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID | INTEL_PTE_WRITE)) + panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__,__LINE__,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma)); +#else /* MACH_PV_PAGETABLES */ + /* Note: in this case, mfn is actually a pfn. */ + WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE); +#endif /* MACH_PV_PAGETABLES */ +} +#endif /* MACH_XEN */ + /* * Deallocate a page-table page. * The page-table page must have all mappings removed, @@ -1001,20 +1133,49 @@ pmap_t pmap_create(size) } /* - * Allocate a pmap struct from the pmap_zone. Then allocate - * the page descriptor table from the pd_zone. + * Allocate a pmap struct from the pmap_cache. Then allocate + * the page descriptor table. */ - p = (pmap_t) zalloc(pmap_zone); + p = (pmap_t) kmem_cache_alloc(&pmap_cache); if (p == PMAP_NULL) panic("pmap_create"); if (kmem_alloc_wired(kernel_map, - (vm_offset_t *)&p->dirbase, INTEL_PGBYTES) + (vm_offset_t *)&p->dirbase, PDPNUM * INTEL_PGBYTES) != KERN_SUCCESS) panic("pmap_create"); - bcopy(kernel_page_dir, p->dirbase, INTEL_PGBYTES); + memcpy(p->dirbase, kernel_page_dir, PDPNUM * INTEL_PGBYTES); +#ifdef LINUX_DEV +#if VM_MIN_KERNEL_ADDRESS != 0 + /* Do not map BIOS in user tasks */ + p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] = 0; +#endif +#endif +#ifdef MACH_PV_PAGETABLES + { + int i; + for (i = 0; i < PDPNUM; i++) + pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES); + } +#endif /* MACH_PV_PAGETABLES */ + +#if PAE + if (kmem_alloc_wired(kernel_map, + (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES) + != KERN_SUCCESS) + panic("pmap_create"); + { + int i; + for (i = 0; i < PDPNUM; i++) + WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + } +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly(p->pdpbase); +#endif /* MACH_PV_PAGETABLES */ +#endif /* PAE */ + p->ref_count = 1; simple_lock_init(&p->lock); @@ -1072,14 +1233,32 @@ void pmap_destroy(p) if (m == VM_PAGE_NULL) panic("pmap_destroy: pte page not in object"); vm_page_lock_queues(); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa))) + panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa)); + pmap_set_page_readwrite((void*) phystokv(pa)); +#endif /* MACH_PV_PAGETABLES */ vm_page_free(m); inuse_ptepages_count--; vm_page_unlock_queues(); vm_object_unlock(pmap_object); } } - kmem_free(kernel_map, p->dirbase, INTEL_PGBYTES); - zfree(pmap_zone, (vm_offset_t) p); +#ifdef MACH_PV_PAGETABLES + { + int i; + for (i = 0; i < PDPNUM; i++) + pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES); + } +#endif /* MACH_PV_PAGETABLES */ + kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM * INTEL_PGBYTES); +#if PAE +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readwrite(p->pdpbase); +#endif /* MACH_PV_PAGETABLES */ + kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES); +#endif /* PAE */ + kmem_cache_free(&pmap_cache, (vm_offset_t) p); } /* @@ -1122,6 +1301,10 @@ void pmap_remove_range(pmap, va, spte, e int num_removed, num_unwired; int pai; vm_offset_t pa; +#ifdef MACH_PV_PAGETABLES + int n, ii = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ #if DEBUG_PTE_PAGE if (pmap != kernel_pmap) @@ -1150,7 +1333,19 @@ void pmap_remove_range(pmap, va, spte, e register int i = ptes_per_vm_page; register pt_entry_t *lpte = cpte; do { +#ifdef MACH_PV_PAGETABLES + update[ii].ptr = kv_to_ma(lpte); + update[ii].val = 0; + ii++; + if (ii == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); + ii = 0; + } +#else /* MACH_PV_PAGETABLES */ *lpte = 0; +#endif /* MACH_PV_PAGETABLES */ lpte++; } while (--i > 0); continue; @@ -1171,7 +1366,19 @@ void pmap_remove_range(pmap, va, spte, e do { pmap_phys_attributes[pai] |= *lpte & (PHYS_MODIFIED|PHYS_REFERENCED); +#ifdef MACH_PV_PAGETABLES + update[ii].ptr = kv_to_ma(lpte); + update[ii].val = 0; + ii++; + if (ii == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); + ii = 0; + } +#else /* MACH_PV_PAGETABLES */ *lpte = 0; +#endif /* MACH_PV_PAGETABLES */ lpte++; } while (--i > 0); } @@ -1217,6 +1424,14 @@ void pmap_remove_range(pmap, va, spte, e } } +#ifdef MACH_PV_PAGETABLES + if (ii > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_remove_range"); + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); +#endif /* MACH_PV_PAGETABLES */ + /* * Update the counts */ @@ -1240,17 +1455,13 @@ void pmap_remove(map, s, e) register pt_entry_t *pde; register pt_entry_t *spte, *epte; vm_offset_t l; + vm_offset_t _s = s; if (map == PMAP_NULL) return; PMAP_READ_LOCK(map, spl); - /* - * Invalidate the translation buffer first - */ - PMAP_UPDATE_TLBS(map, s, e); - pde = pmap_pde(map, s); while (s < e) { l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1); @@ -1265,6 +1476,7 @@ void pmap_remove(map, s, e) s = l; pde++; } + PMAP_UPDATE_TLBS(map, _s, e); PMAP_READ_UNLOCK(map, spl); } @@ -1330,29 +1542,22 @@ void pmap_page_protect(phys, prot) prev = pv_e = pv_h; do { + register vm_offset_t va; + pmap = pv_e->pmap; /* * Lock the pmap to block pmap_extract and similar routines. */ simple_lock(&pmap->lock); - { - register vm_offset_t va; + va = pv_e->va; + pte = pmap_pte(pmap, va); - va = pv_e->va; - pte = pmap_pte(pmap, va); - - /* - * Consistency checks. - */ - /* assert(*pte & INTEL_PTE_VALID); XXX */ - /* assert(pte_to_phys(*pte) == phys); */ - - /* - * Invalidate TLBs for all CPUs using this mapping. - */ - PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); - } + /* + * Consistency checks. + */ + /* assert(*pte & INTEL_PTE_VALID); XXX */ + /* assert(pte_to_phys(*pte) == phys); */ /* * Remove the mapping if new protection is NONE @@ -1371,7 +1576,12 @@ void pmap_page_protect(phys, prot) do { pmap_phys_attributes[pai] |= *pte & (PHYS_MODIFIED|PHYS_REFERENCED); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0)) + panic("%s:%d could not clear pte %p\n",__FILE__,__LINE__,pte-1); +#else /* MACH_PV_PAGETABLES */ *pte++ = 0; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } @@ -1401,7 +1611,12 @@ void pmap_page_protect(phys, prot) register int i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE)) + panic("%s:%d could not disable write on pte %p\n",__FILE__,__LINE__,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ pte++; } while (--i > 0); @@ -1410,6 +1625,7 @@ void pmap_page_protect(phys, prot) */ prev = pv_e; } + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); simple_unlock(&pmap->lock); @@ -1444,6 +1660,7 @@ void pmap_protect(map, s, e, prot) register pt_entry_t *spte, *epte; vm_offset_t l; int spl; + vm_offset_t _s = s; if (map == PMAP_NULL) return; @@ -1478,11 +1695,6 @@ void pmap_protect(map, s, e, prot) SPLVM(spl); simple_lock(&map->lock); - /* - * Invalidate the translation buffer first - */ - PMAP_UPDATE_TLBS(map, s, e); - pde = pmap_pde(map, s); while (s < e) { l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1); @@ -1493,15 +1705,41 @@ void pmap_protect(map, s, e, prot) spte = &spte[ptenum(s)]; epte = &spte[intel_btop(l-s)]; +#ifdef MACH_PV_PAGETABLES + int n, i = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ + while (spte < epte) { - if (*spte & INTEL_PTE_VALID) + if (*spte & INTEL_PTE_VALID) { +#ifdef MACH_PV_PAGETABLES + update[i].ptr = kv_to_ma(spte); + update[i].val = *spte & ~INTEL_PTE_WRITE; + i++; + if (i == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_protect\n"); + i = 0; + } +#else /* MACH_PV_PAGETABLES */ *spte &= ~INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ + } spte++; } +#ifdef MACH_PV_PAGETABLES + if (i > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_protect"); + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_protect\n"); +#endif /* MACH_PV_PAGETABLES */ } s = l; pde++; } + PMAP_UPDATE_TLBS(map, _s, e); simple_unlock(&map->lock); SPLX(spl); @@ -1535,10 +1773,14 @@ void pmap_enter(pmap, v, pa, prot, wired vm_offset_t old_pa; assert(pa != vm_page_fictitious_addr); -if (pmap_debug) printf("pmap(%x, %x)\n", v, pa); +if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa); if (pmap == PMAP_NULL) return; +#if !MACH_KDB + if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end)) + panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa); +#endif if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0 && !wired /* hack for io_wire */ ) { /* @@ -1556,9 +1798,9 @@ if (pmap_debug) printf("pmap(%x, %x)\n", * Invalidate the translation buffer, * then remove the mapping. */ - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); pmap_remove_range(pmap, v, pte, pte + ptes_per_vm_page); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } PMAP_READ_UNLOCK(pmap, spl); return; @@ -1566,7 +1808,7 @@ if (pmap_debug) printf("pmap(%x, %x)\n", /* * Must allocate a new pvlist entry while we're unlocked; - * zalloc may cause pageout (which will lock the pmap system). + * Allocating may cause pageout (which will lock the pmap system). * If we determine we need a pvlist entry, we will unlock * and allocate one. Then we will retry, throughing away * the allocated entry later (if we no longer need it). @@ -1602,7 +1844,7 @@ Retry: */ PMAP_READ_UNLOCK(pmap, spl); - ptp = pmap_page_table_page_alloc(); + ptp = phystokv(pmap_page_table_page_alloc()); /* * Re-lock the pmap and check that another thread has @@ -1617,7 +1859,7 @@ Retry: * Oops... */ PMAP_READ_UNLOCK(pmap, spl); - pmap_page_table_page_dealloc(ptp); + pmap_page_table_page_dealloc(kvtophys(ptp)); PMAP_READ_LOCK(pmap, spl); continue; } @@ -1629,18 +1871,23 @@ Retry: /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/ pdp = pmap_pde(pmap, v); do { - *pdp = pa_to_pte(ptp) | INTEL_PTE_VALID - | INTEL_PTE_USER - | INTEL_PTE_WRITE; +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly((void *) ptp); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp))) + panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp)); + if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)), + pa_to_pte(pa_to_ma(kvtophys(ptp))) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE)) + panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp))); +#else /* MACH_PV_PAGETABLES */ + *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ pdp++; ptp += INTEL_PGBYTES; } while (--i > 0); -#if i860 - /* - * Flush the data cache. - */ - flush(); -#endif /* i860 */ /* * Now, get the address of the page-table entry. @@ -1668,17 +1915,25 @@ Retry: template |= INTEL_PTE_USER; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 + && pa >= phys_last_addr) + template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); i = ptes_per_vm_page; do { if (*pte & INTEL_PTE_MOD) template |= INTEL_PTE_MOD; +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))) + panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template); +#else /* MACH_PV_PAGETABLES */ WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte++; pte_increment_pa(template); } while (--i > 0); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } else { @@ -1687,17 +1942,12 @@ Retry: */ if (*pte) { /* - * Invalidate the translation buffer, - * then remove the mapping. - */ - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); - - /* * Don't free the pte page if removing last * mapping - we will immediately replace it. */ pmap_remove_range(pmap, v, pte, pte + ptes_per_vm_page); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } if (valid_page(pa)) { @@ -1742,9 +1992,9 @@ Retry: PMAP_READ_UNLOCK(pmap, spl); /* - * Refill from zone. + * Refill from cache. */ - pv_e = (pv_entry_t) zalloc(pv_list_zone); + pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache); goto Retry; } } @@ -1777,11 +2027,19 @@ Retry: template |= INTEL_PTE_USER; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 + && pa >= phys_last_addr) + template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))) + panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template); +#else /* MACH_PV_PAGETABLES */ WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte++; pte_increment_pa(template); } while (--i > 0); @@ -1836,7 +2094,13 @@ void pmap_change_wiring(map, v, wired) map->stats.wired_count--; i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED))) + panic("%s:%d could not wire down pte %p\n",__FILE__,__LINE__,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~INTEL_PTE_WIRED; +#endif /* MACH_PV_PAGETABLES */ + pte++; } while (--i > 0); } @@ -1921,8 +2185,6 @@ void pmap_collect(p) * Garbage collect map. */ PMAP_READ_LOCK(p, spl); - PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS); - for (pdp = p->dirbase; pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]; pdp += ptes_per_vm_page) @@ -1968,7 +2230,17 @@ void pmap_collect(p) register int i = ptes_per_vm_page; register pt_entry_t *pdep = pdp; do { +#ifdef MACH_PV_PAGETABLES + unsigned long pte = *pdep; + void *ptable = (void*) ptetokv(pte); + if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0))) + panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1); + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable))) + panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable))); + pmap_set_page_readwrite(ptable); +#else /* MACH_PV_PAGETABLES */ *pdep++ = 0; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } @@ -1995,6 +2267,8 @@ void pmap_collect(p) } } } + PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS); + PMAP_READ_UNLOCK(p, spl); return; @@ -2103,6 +2377,7 @@ pmap_copy_page(src, dst) * will specify that these pages are to be wired * down (or not) as appropriate. */ +void pmap_pageable(pmap, start, end, pageable) pmap_t pmap; vm_offset_t start; @@ -2157,6 +2432,7 @@ phys_attribute_clear(phys, bits) * There are some mappings. */ for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) { + register vm_offset_t va; pmap = pv_e->pmap; /* @@ -2164,35 +2440,32 @@ phys_attribute_clear(phys, bits) */ simple_lock(&pmap->lock); - { - register vm_offset_t va; - - va = pv_e->va; - pte = pmap_pte(pmap, va); + va = pv_e->va; + pte = pmap_pte(pmap, va); #if 0 - /* - * Consistency checks. - */ - assert(*pte & INTEL_PTE_VALID); - /* assert(pte_to_phys(*pte) == phys); */ + /* + * Consistency checks. + */ + assert(*pte & INTEL_PTE_VALID); + /* assert(pte_to_phys(*pte) == phys); */ #endif - /* - * Invalidate TLBs for all CPUs using this mapping. - */ - PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); - } - /* * Clear modify or reference bits. */ { register int i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits))) + panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__,__LINE__,bits,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~bits; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); simple_unlock(&pmap->lock); } } @@ -2460,7 +2733,8 @@ void process_pmap_updates(my_pmap) if (pmap == my_pmap || pmap == kernel_pmap) { - INVALIDATE_TLB(update_list_p->item[j].start, + INVALIDATE_TLB(pmap, + update_list_p->item[j].start, update_list_p->item[j].end); } } @@ -2472,7 +2746,7 @@ void process_pmap_updates(my_pmap) /* * Interrupt routine for TBIA requested from other processor. */ -void pmap_update_interrupt() +void pmap_update_interrupt(void) { register int my_cpu; register pmap_t my_pmap; @@ -2538,17 +2812,7 @@ void pmap_update_interrupt() } #endif /* NCPUS > 1 */ -#if i860 /* akp */ -void set_dirbase(dirbase) - register vm_offset_t dirbase; -{ - /*flush();*/ - /*flush_tlb();*/ - flush_and_ctxsw(dirbase); -} -#endif /* i860 */ - -#ifdef i386 +#if defined(__i386__) /* Unmap page 0 to trap NULL references. */ void pmap_unmap_page_zero () @@ -2556,8 +2820,15 @@ pmap_unmap_page_zero () int *pte; pte = (int *) pmap_pte (kernel_pmap, 0); + if (!pte) + return; assert (pte); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), 0)) + printf("couldn't unmap page 0\n"); +#else /* MACH_PV_PAGETABLES */ *pte = 0; - asm volatile ("movl %%cr3,%%eax; movl %%eax,%%cr3" ::: "ax"); + INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE); +#endif /* MACH_PV_PAGETABLES */ } -#endif /* i386 */ +#endif /* __i386__ */