--- Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:36:59 1.1 +++ Gnu-Mach/i386/intel/pmap.c 2020/09/02 04:49:25 1.1.1.4 @@ -2,24 +2,24 @@ * Mach Operating System * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -28,7 +28,7 @@ * Author: Avadis Tevanian, Jr., Michael Wayne Young * (These guys wrote the Vax version) * - * Physical Map management code for Intel i386, i486, and i860. + * Physical Map management code for Intel i386, and i486. * * Manages physical address maps. * @@ -55,77 +55,40 @@ * and to when physical maps must be made correct. */ -#include +#include #include #include +#include +#include #include -#include +#include #include #include #include #include -#include "vm_param.h" +#include #include #include #include #include #include +#include #include -#include "cpu_number.h" -#if i860 -#include -#endif - -#ifdef ORC -#define OLIVETTICACHE 1 -#endif ORC - -#ifndef OLIVETTICACHE +#include +#include +#include +#include + +#ifdef MACH_PSEUDO_PHYS +#define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0; +#else /* MACH_PSEUDO_PHYS */ #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry); -#define WRITE_PTE_FAST(pte_p, pte_entry) *(pte_p) = (pte_entry); -#else OLIVETTICACHE -#error might not work anymore - -/* This gross kludgery is needed for Olivetti XP7 & XP9 boxes to get - * around an apparent hardware bug. Other than at startup it doesn't - * affect run-time performacne very much, so we leave it in for all - * machines. - */ -extern unsigned *pstart(); -#define CACHE_LINE 8 -#define CACHE_SIZE 512 -#define CACHE_PAGE 0x1000; - -#define WRITE_PTE(pte_p, pte_entry) { write_pte(pte_p, pte_entry); } - -write_pte(pte_p, pte_entry) -pt_entry_t *pte_p, pte_entry; -{ - unsigned long count; - volatile unsigned long hold, *addr1, *addr2; - - if ( pte_entry != *pte_p ) - *pte_p = pte_entry; - else { - /* This isn't necessarily the optimal algorithm */ - addr1 = (unsigned long *)pstart; - for (count = 0; count < CACHE_SIZE; count++) { - addr2 = addr1 + CACHE_PAGE; - hold = *addr1; /* clear cache bank - A - */ - hold = *addr2; /* clear cache bank - B - */ - addr1 += CACHE_LINE; - } - } -} - -#define WRITE_PTE_FAST(pte_p, pte_entry)*pte_p = pte_entry; - -#endif OLIVETTICACHE +#endif /* MACH_PSEUDO_PHYS */ /* * Private data structures. @@ -150,7 +113,7 @@ pv_entry_t pv_head_table; /* array of e /* * pv_list entries are kept on a list that can only be accessed * with the pmap system locked (at SPLVM, not in the cpus_active set). - * The list is refilled from the pv_list_zone if it becomes empty. + * The list is refilled from the pv_list_cache if it becomes empty. */ pv_entry_t pv_free_list; /* free list at SPLVM */ decl_simple_lock_data(, pv_free_list_lock) @@ -170,7 +133,7 @@ decl_simple_lock_data(, pv_free_list_loc simple_unlock(&pv_free_list_lock); \ } -zone_t pv_list_zone; /* zone of pv_entry structures */ +struct kmem_cache pv_list_cache; /* cache of pv_entry structures */ /* * Each entry in the pv_head_table is locked by a bit in the @@ -185,13 +148,6 @@ char *pv_lock_table; /* pointer to arra boolean_t pmap_initialized = FALSE; /* - * More-specific code provides these; - * they indicate the total extent of physical memory - * that we know about and might ever have to manage. - */ -extern vm_offset_t phys_first_addr, phys_last_addr; - -/* * Range of kernel virtual addresses available for kernel memory mapping. * Does not include the virtual addresses used to map physical memory 1-1. * Initialized by pmap_bootstrap. @@ -199,9 +155,6 @@ extern vm_offset_t phys_first_addr, phys vm_offset_t kernel_virtual_start; vm_offset_t kernel_virtual_end; -/* XXX stupid fixed limit - get rid */ -vm_size_t morevm = 40 * 1024 * 1024; /* VM space for kernel map */ - /* * Index into pv_head table, its lock bits, and the modify/reference * bits starting at phys_first_addr. @@ -345,14 +298,14 @@ lock_data_t pmap_system_lock; \ /* invalidate our own TLB if pmap is in use */ \ if ((pmap)->cpus_using & cpu_mask) { \ - INVALIDATE_TLB((s), (e)); \ + INVALIDATE_TLB((pmap), (s), (e)); \ } \ } -#else NCPUS > 1 +#else /* NCPUS > 1 */ -#define SPLVM(spl) -#define SPLX(spl) +#define SPLVM(spl) ((void)(spl)) +#define SPLX(spl) ((void)(spl)) #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl) #define PMAP_WRITE_LOCK(spl) SPLVM(spl) @@ -366,25 +319,46 @@ lock_data_t pmap_system_lock; #define PMAP_UPDATE_TLBS(pmap, s, e) { \ /* invalidate our own TLB if pmap is in use */ \ if ((pmap)->cpus_using) { \ - INVALIDATE_TLB((s), (e)); \ + INVALIDATE_TLB((pmap), (s), (e)); \ } \ } -#endif NCPUS > 1 +#endif /* NCPUS > 1 */ #define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */ -#if i860 -/* Do a data cache flush until we find the caching bug XXX prp */ -#define INVALIDATE_TLB(s, e) { \ - flush(); \ - flush_tlb(); \ +#ifdef MACH_PV_PAGETABLES +#if 1 +#define INVALIDATE_TLB(pmap, s, e) hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL) +#else +#define INVALIDATE_TLB(pmap, s, e) do { \ + if (__builtin_constant_p((e) - (s)) \ + && (e) - (s) == PAGE_SIZE) \ + hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s) : (s)); \ + else \ + hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \ +} while(0) +#endif +#else /* MACH_PV_PAGETABLES */ +#if 0 +/* It is hard to know when a TLB flush becomes less expensive than a bunch of + * invlpgs. But it surely is more expensive than just one invlpg. */ +#define INVALIDATE_TLB(pmap, s, e) { \ + if (__builtin_constant_p((e) - (s)) \ + && (e) - (s) == PAGE_SIZE) \ + invlpg_linear(s); \ + else \ + flush_tlb(); \ } -#else i860 -#define INVALIDATE_TLB(s, e) { \ +#else +#define INVALIDATE_TLB(pmap, s, e) { \ + (void) (pmap); \ + (void) (s); \ + (void) (e); \ flush_tlb(); \ } -#endif i860 +#endif +#endif /* MACH_PV_PAGETABLES */ #if NCPUS > 1 @@ -415,7 +389,7 @@ typedef struct pmap_update_list *pmap_up struct pmap_update_list cpu_update_list[NCPUS]; -#endif NCPUS > 1 +#endif /* NCPUS > 1 */ /* * Other useful macros. @@ -426,9 +400,9 @@ struct pmap_update_list cpu_update_list[ struct pmap kernel_pmap_store; pmap_t kernel_pmap; -struct zone *pmap_zone; /* zone of pmap structures */ +struct kmem_cache pmap_cache; /* cache of pmap structures */ -int pmap_debug = 0; /* flag for debugging prints */ +boolean_t pmap_debug = FALSE; /* flag for debugging prints */ #if 0 int ptes_per_vm_page; /* number of hardware ptes needed @@ -439,28 +413,20 @@ int ptes_per_vm_page; /* number of hard unsigned int inuse_ptepages_count = 0; /* debugging */ -extern char end; - /* * Pointer to the basic page directory for the kernel. * Initialized by pmap_bootstrap(). */ pt_entry_t *kernel_page_dir; -void pmap_remove_range(); /* forward */ -#if NCPUS > 1 -void signal_cpus(); /* forward */ -#endif NCPUS > 1 - -#if i860 /* - * Paging flag + * Two slots for temporary physical page mapping, to allow for + * physical-to-physical transfers. */ -int paging_enabled = 0; -#endif +static pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS]; static inline pt_entry_t * -pmap_pde(pmap_t pmap, vm_offset_t addr) +pmap_pde(const pmap_t pmap, vm_offset_t addr) { if (pmap == kernel_pmap) addr = kvtolin(addr); @@ -475,7 +441,7 @@ pmap_pde(pmap_t pmap, vm_offset_t addr) * This is only used internally. */ pt_entry_t * -pmap_pte(pmap_t pmap, vm_offset_t addr) +pmap_pte(const pmap_t pmap, vm_offset_t addr) { pt_entry_t *ptp; pt_entry_t pte; @@ -492,10 +458,9 @@ pmap_pte(pmap_t pmap, vm_offset_t addr) #define DEBUG_PTE_PAGE 0 #if DEBUG_PTE_PAGE -void ptep_check(ptep) - ptep_t ptep; +void ptep_check(ptep_t ptep) { - register pt_entry_t *pte, *epte; + pt_entry_t *pte, *epte; int ctu, ctw; /* check the use and wired counts */ @@ -520,7 +485,7 @@ void ptep_check(ptep) panic("pte count"); } } -#endif DEBUG_PTE_PAGE +#endif /* DEBUG_PTE_PAGE */ /* * Map memory at initialization. The physical addresses being @@ -529,13 +494,13 @@ void ptep_check(ptep) * For now, VM is already on, we only need to map the * specified memory. */ -vm_offset_t pmap_map(virt, start, end, prot) - register vm_offset_t virt; - register vm_offset_t start; - register vm_offset_t end; - register int prot; +vm_offset_t pmap_map( + vm_offset_t virt, + vm_offset_t start, + vm_offset_t end, + int prot) { - register int ps; + int ps; ps = PAGE_SIZE; while (start < end) { @@ -547,40 +512,63 @@ vm_offset_t pmap_map(virt, start, end, p } /* - * Back-door routine for mapping kernel VM at initialization. + * Back-door routine for mapping kernel VM at initialization. * Useful for mapping memory outside the range * [phys_first_addr, phys_last_addr) (i.e., devices). * Otherwise like pmap_map. -#if i860 - * Sets no-cache bit. -#endif */ -vm_offset_t pmap_map_bd(virt, start, end, prot) - register vm_offset_t virt; - register vm_offset_t start; - register vm_offset_t end; - vm_prot_t prot; +vm_offset_t pmap_map_bd( + vm_offset_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot) { - register pt_entry_t template; - register pt_entry_t *pte; + pt_entry_t template; + pt_entry_t *pte; + int spl; +#ifdef MACH_PV_PAGETABLES + int n, i = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ template = pa_to_pte(start) -#if i860 - | INTEL_PTE_NCACHE -#endif + | INTEL_PTE_NCACHE|INTEL_PTE_WTHRU | INTEL_PTE_VALID; + if (CPU_HAS_FEATURE(CPU_FEATURE_PGE)) + template |= INTEL_PTE_GLOBAL; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + PMAP_READ_LOCK(pmap, spl); while (start < end) { pte = pmap_pte(kernel_pmap, virt); if (pte == PT_ENTRY_NULL) panic("pmap_map_bd: Invalid kernel address\n"); - WRITE_PTE_FAST(pte, template) +#ifdef MACH_PV_PAGETABLES + update[i].ptr = kv_to_ma(pte); + update[i].val = pa_to_ma(template); + i++; + if (i == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_map_bd\n"); + i = 0; + } +#else /* MACH_PV_PAGETABLES */ + WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte_increment_pa(template); virt += PAGE_SIZE; start += PAGE_SIZE; } +#ifdef MACH_PV_PAGETABLES + if (i > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_map_bd"); + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_map_bd\n"); +#endif /* MACH_PV_PAGETABLES */ + PMAP_READ_UNLOCK(pmap, spl); return(virt); } @@ -590,7 +578,7 @@ vm_offset_t pmap_map_bd(virt, start, end * and direct-map all physical memory. * Called with mapping off. */ -void pmap_bootstrap() +void pmap_bootstrap(void) { /* * Mapping is turned off; we must reference only physical addresses. @@ -614,7 +602,7 @@ void pmap_bootstrap() #if NCPUS > 1 lock_init(&pmap_system_lock, FALSE); /* NOT a sleep lock */ -#endif NCPUS > 1 +#endif /* NCPUS > 1 */ simple_lock_init(&kernel_pmap->lock); @@ -626,24 +614,96 @@ void pmap_bootstrap() * mapped into the kernel address space, * and extends to a stupid arbitrary limit beyond that. */ - kernel_virtual_start = phys_last_addr; - kernel_virtual_end = phys_last_addr + morevm; + kernel_virtual_start = phystokv(phys_last_addr); + kernel_virtual_end = phystokv(phys_last_addr) + VM_KERNEL_MAP_SIZE; + + if (kernel_virtual_end < kernel_virtual_start + || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS) + kernel_virtual_end = VM_MAX_KERNEL_ADDRESS; /* * Allocate and clear a kernel page directory. */ - kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)pmap_grab_page(); + /* Note: initial Xen mapping holds at least 512kB free mapped page. + * We use that for directly building our linear mapping. */ +#if PAE + { + vm_offset_t addr; + init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr); + kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr); + } + kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page()); + { + int i; + for (i = 0; i < PDPNUM; i++) + WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + } +#else /* PAE */ + kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page()); +#endif /* PAE */ { int i; for (i = 0; i < NPDES; i++) kernel_pmap->dirbase[i] = 0; } +#ifdef MACH_PV_PAGETABLES + /* We don't actually deal with the CR3 register content at all */ + hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3); + /* + * Xen may only provide as few as 512KB extra bootstrap linear memory, + * which is far from enough to map all available memory, so we need to + * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE) + * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is + * enough for a pagetable mapping 4GiB. + */ +#ifdef PAE +#define NSUP_L1 4 +#else +#define NSUP_L1 1 +#endif + pt_entry_t *l1_map[NSUP_L1]; + { + pt_entry_t *base = (pt_entry_t*) boot_info.pt_base; + vm_offset_t la; + int n_l1map; + for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS; la >= VM_MIN_KERNEL_ADDRESS; la += NPTES * PAGE_SIZE) { +#ifdef PAE + pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)]); +#else /* PAE */ + pt_entry_t *l2_map = base; +#endif /* PAE */ + /* Like lin2pdenum, but works with non-contiguous boot L3 */ + l2_map += (la >> PDESHIFT) & PDEMASK; + if (!(*l2_map & INTEL_PTE_VALID)) { + struct mmu_update update; + int j, n; + + l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page()); + for (j = 0; j < NPTES; j++) + l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS) * NPTES + j)) << PAGE_SHIFT) | INTEL_PTE_VALID | INTEL_PTE_WRITE; + pmap_set_page_readonly_init(l1_map[n_l1map]); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map]))) + panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map])); + update.ptr = kv_to_ma(l2_map); + update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID | INTEL_PTE_WRITE; + hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF); + if (n != 1) + panic("couldn't complete bootstrap map"); + /* added the last L1 table, can stop */ + if (++n_l1map >= NSUP_L1) + break; + } + } + } +#endif /* MACH_PV_PAGETABLES */ + /* * Allocate and set up the kernel page tables. */ { vm_offset_t va; + pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0; /* * Map virtual memory for all known physical memory, 1-1, @@ -656,143 +716,220 @@ void pmap_bootstrap() * to allocate new kernel page tables later. * XX fix this */ - for (va = phys_first_addr; va < phys_last_addr + morevm; ) + for (va = phystokv(phys_first_addr); va >= phystokv(phys_first_addr) && va < kernel_virtual_end; ) { pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va)); - pt_entry_t *ptable = (pt_entry_t*)pmap_grab_page(); + pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page()); pt_entry_t *pte; - vm_offset_t pteva; /* Initialize the page directory entry. */ - *pde = pa_to_pte((vm_offset_t)ptable) - | INTEL_PTE_VALID | INTEL_PTE_WRITE; + WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable)) + | INTEL_PTE_VALID | INTEL_PTE_WRITE); /* Initialize the page table. */ - for (pte = ptable; (va < phys_last_addr) && (pte < ptable+NPTES); pte++) + for (pte = ptable; (va < phystokv(phys_last_addr)) && (pte < ptable+NPTES); pte++) { if ((pte - ptable) < ptenum(va)) { - WRITE_PTE_FAST(pte, 0); + WRITE_PTE(pte, 0); + } + else +#ifdef MACH_PV_PAGETABLES + if (va == (vm_offset_t) &hyp_shared_info) + { + *pte = boot_info.shared_info | INTEL_PTE_VALID | INTEL_PTE_WRITE; + va += INTEL_PGBYTES; } - else + else +#endif /* MACH_PV_PAGETABLES */ { - extern char start[], etext[]; + extern char _start[], etext[]; - if ((va >= (vm_offset_t)start) + if (((va >= (vm_offset_t) _start) && (va + INTEL_PGBYTES <= (vm_offset_t)etext)) +#ifdef MACH_PV_PAGETABLES + || (va >= (vm_offset_t) boot_info.pt_base + && (va + INTEL_PGBYTES <= + (vm_offset_t) ptable + INTEL_PGBYTES)) +#endif /* MACH_PV_PAGETABLES */ + ) { - WRITE_PTE_FAST(pte, pa_to_pte(va) - | INTEL_PTE_VALID); + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | global); } else { - WRITE_PTE_FAST(pte, pa_to_pte(va) - | INTEL_PTE_VALID | INTEL_PTE_WRITE); +#ifdef MACH_PV_PAGETABLES + /* Keep supplementary L1 pages read-only */ + int i; + for (i = 0; i < NSUP_L1; i++) + if (va == (vm_offset_t) l1_map[i]) { + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | global); + break; + } + if (i == NSUP_L1) +#endif /* MACH_PV_PAGETABLES */ + WRITE_PTE(pte, pa_to_pte(_kvtophys(va)) + | INTEL_PTE_VALID | INTEL_PTE_WRITE | global) + } va += INTEL_PGBYTES; } } for (; pte < ptable+NPTES; pte++) { - WRITE_PTE_FAST(pte, 0); + if (va >= kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE && va < kernel_virtual_end) + { + pmap_mapwindow_t *win = &mapwindows[atop(va - (kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE))]; + win->entry = pte; + win->vaddr = va; + } + WRITE_PTE(pte, 0); va += INTEL_PGBYTES; } +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly_init(ptable); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable))) + panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable)); +#endif /* MACH_PV_PAGETABLES */ } } -#if i860 -#error probably doesnt work anymore - XXX move to architecture-specific code just after the pmap_bootstrap call. - - /* kvtophys should now work in phys range */ - - /* - * Mark page table pages non-cacheable - */ + /* Architecture-specific code will turn on paging + soon after we return from here. */ +} - pt_pte = (pt_entry_t *)pte_to_pa(*(kpde + pdenum(sva))) + ptenum(sva); +#ifdef MACH_PV_PAGETABLES +/* These are only required because of Xen security policies */ - for (va = load_start; va < tva; va += INTEL_PGBYTES*NPTES) { - /* Mark page table non-cacheable */ - *pt_pte |= INTEL_PTE_NCACHE; - pt_pte++; +/* Set back a page read write */ +void pmap_set_page_readwrite(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; + vm_offset_t paddr = kvtophys(vaddr); + vm_offset_t canon_vaddr = phystokv(paddr); + if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE)) + panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr)); + if (canon_vaddr != vaddr) + if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE)) + panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr)); +} + +/* Set a page read only (so as to pin it for instance) */ +void pmap_set_page_readonly(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; + vm_offset_t paddr = kvtophys(vaddr); + vm_offset_t canon_vaddr = phystokv(paddr); + if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr)); + } + if (canon_vaddr != vaddr && + *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr)); } +} - /* - * Map I/O space - */ - - ppde = kpde; - ppde += pdenum(IO_BASE); - - if (pte_to_pa(*ppde) == 0) { - /* This pte has not been allocated */ - ppte = (pt_entry_t *)kvtophys(virtual_avail); - ptend = ppte + NPTES; - virtual_avail = phystokv((vm_offset_t)ptend); - *ppde = pa_to_pte((vm_offset_t)ppte) - | INTEL_PTE_VALID - | INTEL_PTE_WRITE; - pte = ptend; - - /* Mark page table non-cacheable */ - *pt_pte |= INTEL_PTE_NCACHE; - pt_pte++; - - bzero((char *)ppte, INTEL_PGBYTES); - } else { - ppte = (pt_entry_t *)(*ppde); /* first pte of page */ +/* This needs to be called instead of pmap_set_page_readonly as long as RC3 + * still points to the bootstrap dirbase, to also fix the bootstrap table. */ +void pmap_set_page_readonly_init(void *_vaddr) { + vm_offset_t vaddr = (vm_offset_t) _vaddr; +#if PAE + pt_entry_t *pdpbase = (void*) boot_info.pt_base; + /* The bootstrap table does not necessarily use contiguous pages for the pde tables */ + pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)]); +#else + pt_entry_t *dirbase = (void*) boot_info.pt_base; +#endif + pt_entry_t *pte = &dirbase[lin2pdenum(vaddr) & PTEMASK]; + /* Modify our future kernel map (can't use update_va_mapping for this)... */ + if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) { + if (!hyp_mmu_update_la (kvtolin(vaddr), pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID)) + panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr)); + } + /* ... and the bootstrap map. */ + if (*pte & INTEL_PTE_VALID) { + if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID, UVMF_NONE)) + panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr)); } - *ppde |= INTEL_PTE_USER; - - - WRITE_PTE(ppte + ptenum(FIFO_ADDR), - pa_to_pte(FIFO_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); - - WRITE_PTE(ppte + ptenum(FIFO_ADDR + XEOD_OFF), - pa_to_pte(FIFO_ADDR_PH + XEOD_OFF_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); - -/* XXX Allowed user access to control reg - cfj */ - WRITE_PTE(ppte + ptenum(CSR_ADDR), - pa_to_pte(CSR_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE | INTEL_PTE_USER); - -/* XXX Allowed user access to perf reg - cfj */ - WRITE_PTE(ppte + ptenum(PERFCNT_ADDR), - pa_to_pte(PERFCNT_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_USER | INTEL_PTE_NCACHE | INTEL_PTE_USER); - - WRITE_PTE(ppte + ptenum(UART_ADDR), - pa_to_pte(UART_ADDR_PH) - | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE); +} - WRITE_PTE(ppte + ptenum(0xFFFFF000), - pa_to_pte(avail_end) - | INTEL_PTE_VALID | INTEL_PTE_WRITE); - avail_start = kvtophys(virtual_avail); +void pmap_clear_bootstrap_pagetable(pt_entry_t *base) { + int i; + pt_entry_t *dir; + vm_offset_t va = 0; +#if PAE + int j; +#endif /* PAE */ + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base))) + panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base)); +#if PAE + for (j = 0; j < PDPNUM; j++) + { + pt_entry_t pdpe = base[j]; + if (pdpe & INTEL_PTE_VALID) { + dir = (pt_entry_t *) ptetokv(pdpe); +#else /* PAE */ + dir = base; +#endif /* PAE */ + for (i = 0; i < NPTES; i++) { + pt_entry_t pde = dir[i]; + unsigned long pfn = atop(pte_to_pa(pde)); + void *pgt = (void*) phystokv(ptoa(pfn)); + if (pde & INTEL_PTE_VALID) + hyp_free_page(pfn, pgt); + va += NPTES * INTEL_PGBYTES; + if (va >= HYP_VIRT_START) + break; + } +#if PAE + hyp_free_page(atop(_kvtophys(dir)), dir); + } else + va += NPTES * NPTES * INTEL_PGBYTES; + if (va >= HYP_VIRT_START) + break; + } +#endif /* PAE */ + hyp_free_page(atop(_kvtophys(base)), base); +} +#endif /* MACH_PV_PAGETABLES */ /* - * Turn on mapping + * Create a temporary mapping for a given physical entry + * + * This can be used to access physical pages which are not mapped 1:1 by + * phystokv(). */ +pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry) +{ + pmap_mapwindow_t *map; - flush_and_ctxsw(kernel_pmap->dirbase); - paging_enabled = 1; + /* Find an empty one. */ + for (map = &mapwindows[0]; map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]; map++) + if (!(*map->entry)) + break; + assert(map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]); - printf("Paging enabled.\n"); -#endif + WRITE_PTE(map->entry, entry); + return map; +} - /* Architecture-specific code will turn on paging - soon after we return from here. */ +/* + * Destroy a temporary mapping for a physical entry + */ +void pmap_put_mapwindow(pmap_mapwindow_t *map) +{ + WRITE_PTE(map->entry, 0); + PMAP_UPDATE_TLBS(kernel_pmap, map->vaddr, map->vaddr + PAGE_SIZE); } -void pmap_virtual_space(startp, endp) - vm_offset_t *startp; - vm_offset_t *endp; +void pmap_virtual_space( + vm_offset_t *startp, + vm_offset_t *endp) { *startp = kernel_virtual_start; - *endp = kernel_virtual_end; + *endp = kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE; } /* @@ -800,12 +937,14 @@ void pmap_virtual_space(startp, endp) * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ -void pmap_init() +void pmap_init(void) { - register long npages; + long npages; vm_offset_t addr; - register vm_size_t s; + vm_size_t s; +#if NCPUS > 1 int i; +#endif /* NCPUS > 1 */ /* * Allocate memory for the pv_head_table and its lock bits, @@ -820,7 +959,7 @@ void pmap_init() s = round_page(s); if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS) panic("pmap_init"); - bzero((char *) addr, s); + memset((void *) addr, 0, s); /* * Allocate the structures first to preserve word-alignment. @@ -834,13 +973,13 @@ void pmap_init() pmap_phys_attributes = (char *) addr; /* - * Create the zone of physical maps, + * Create the cache of physical maps, * and of the physical-to-virtual entries. */ s = (vm_size_t) sizeof(struct pmap); - pmap_zone = zinit(s, 400*s, 4096, 0, "pmap"); /* XXX */ + kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0); s = (vm_size_t) sizeof(struct pv_entry); - pv_list_zone = zinit(s, 10000*s, 4096, 0, "pv_list"); /* XXX */ + kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0); #if NCPUS > 1 /* @@ -852,7 +991,7 @@ void pmap_init() simple_lock_init(&up->lock); up->count = 0; } -#endif NCPUS > 1 +#endif /* NCPUS > 1 */ /* * Indicate that the PMAP module is now fully initialized. @@ -862,8 +1001,7 @@ void pmap_init() #define valid_page(x) (pmap_initialized && pmap_valid_page(x)) -boolean_t pmap_verify_free(phys) - vm_offset_t phys; +boolean_t pmap_verify_free(vm_offset_t phys) { pv_entry_t pv_h; int pai; @@ -897,16 +1035,16 @@ boolean_t pmap_verify_free(phys) * since these must be unlocked to use vm_page_grab. */ vm_offset_t -pmap_page_table_page_alloc() +pmap_page_table_page_alloc(void) { - register vm_page_t m; - register vm_offset_t pa; + vm_page_t m; + vm_offset_t pa; check_simple_locks(); /* * We cannot allocate the pmap_object in pmap_init, - * because it is called before the zone package is up. + * because it is called before the cache package is up. * Allocate it now if it is missing. */ if (pmap_object == VM_OBJECT_NULL) @@ -915,7 +1053,7 @@ pmap_page_table_page_alloc() /* * Allocate a VM page for the level 2 page table entries. */ - while ((m = vm_page_grab()) == VM_PAGE_NULL) + while ((m = vm_page_grab(FALSE)) == VM_PAGE_NULL) VM_PAGE_WAIT((void (*)()) 0); /* @@ -934,34 +1072,59 @@ pmap_page_table_page_alloc() /* * Zero the page. */ - bzero(phystokv(pa), PAGE_SIZE); + memset((void *)phystokv(pa), 0, PAGE_SIZE); -#if i860 - /* - * Mark the page table page(s) non-cacheable. - */ - { - int i = ptes_per_vm_page; - pt_entry_t *pdp; - - pdp = pmap_pte(kernel_pmap, pa); - do { - *pdp |= INTEL_PTE_NCACHE; - pdp++; - } while (--i > 0); - } -#endif return pa; } +#ifdef MACH_XEN +void pmap_map_mfn(void *_addr, unsigned long mfn) { + vm_offset_t addr = (vm_offset_t) _addr; + pt_entry_t *pte, *pdp; + vm_offset_t ptp; + pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT; + + /* Add a ptp if none exist yet for this pte */ + if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) { + ptp = phystokv(pmap_page_table_page_alloc()); +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly((void*) ptp); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp))) + panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp)); +#endif /* MACH_PV_PAGETABLES */ + pdp = pmap_pde(kernel_pmap, addr); + +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pdp), + pa_to_pte(kv_to_ma(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE)) + panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__,__LINE__,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp)); +#else /* MACH_PV_PAGETABLES */ + *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ + pte = pmap_pte(kernel_pmap, addr); + } + +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID | INTEL_PTE_WRITE)) + panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__,__LINE__,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma)); +#else /* MACH_PV_PAGETABLES */ + /* Note: in this case, mfn is actually a pfn. */ + WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE); +#endif /* MACH_PV_PAGETABLES */ +} +#endif /* MACH_XEN */ + /* * Deallocate a page-table page. * The page-table page must have all mappings removed, * and be removed from its page directory. */ void -pmap_page_table_page_dealloc(pa) - vm_offset_t pa; +pmap_page_table_page_dealloc(vm_offset_t pa) { vm_page_t m; @@ -986,11 +1149,10 @@ pmap_page_table_page_dealloc(pa) * the map will be used in software only, and * is bounded by that size. */ -pmap_t pmap_create(size) - vm_size_t size; +pmap_t pmap_create(vm_size_t size) { - register pmap_t p; - register pmap_statistics_t stats; + pmap_t p; + pmap_statistics_t stats; /* * A software use-only map doesn't even need a map. @@ -1001,20 +1163,49 @@ pmap_t pmap_create(size) } /* - * Allocate a pmap struct from the pmap_zone. Then allocate - * the page descriptor table from the pd_zone. + * Allocate a pmap struct from the pmap_cache. Then allocate + * the page descriptor table. */ - p = (pmap_t) zalloc(pmap_zone); + p = (pmap_t) kmem_cache_alloc(&pmap_cache); if (p == PMAP_NULL) panic("pmap_create"); if (kmem_alloc_wired(kernel_map, - (vm_offset_t *)&p->dirbase, INTEL_PGBYTES) + (vm_offset_t *)&p->dirbase, PDPNUM * INTEL_PGBYTES) + != KERN_SUCCESS) + panic("pmap_create"); + + memcpy(p->dirbase, kernel_page_dir, PDPNUM * INTEL_PGBYTES); +#ifdef LINUX_DEV +#if VM_MIN_KERNEL_ADDRESS != 0 + /* Do not map BIOS in user tasks */ + p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] = 0; +#endif +#endif +#ifdef MACH_PV_PAGETABLES + { + int i; + for (i = 0; i < PDPNUM; i++) + pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES); + } +#endif /* MACH_PV_PAGETABLES */ + +#if PAE + if (kmem_alloc_wired(kernel_map, + (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES) != KERN_SUCCESS) panic("pmap_create"); + { + int i; + for (i = 0; i < PDPNUM; i++) + WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID); + } +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly(p->pdpbase); +#endif /* MACH_PV_PAGETABLES */ +#endif /* PAE */ - bcopy(kernel_page_dir, p->dirbase, INTEL_PGBYTES); p->ref_count = 1; simple_lock_init(&p->lock); @@ -1037,13 +1228,12 @@ pmap_t pmap_create(size) * no valid mappings. */ -void pmap_destroy(p) - register pmap_t p; +void pmap_destroy(pmap_t p) { - register pt_entry_t *pdep; - register vm_offset_t pa; - register int c, s; - register vm_page_t m; + pt_entry_t *pdep; + vm_offset_t pa; + int c, s; + vm_page_t m; if (p == PMAP_NULL) return; @@ -1072,22 +1262,39 @@ void pmap_destroy(p) if (m == VM_PAGE_NULL) panic("pmap_destroy: pte page not in object"); vm_page_lock_queues(); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa))) + panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa)); + pmap_set_page_readwrite((void*) phystokv(pa)); +#endif /* MACH_PV_PAGETABLES */ vm_page_free(m); inuse_ptepages_count--; vm_page_unlock_queues(); vm_object_unlock(pmap_object); } } - kmem_free(kernel_map, p->dirbase, INTEL_PGBYTES); - zfree(pmap_zone, (vm_offset_t) p); +#ifdef MACH_PV_PAGETABLES + { + int i; + for (i = 0; i < PDPNUM; i++) + pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES); + } +#endif /* MACH_PV_PAGETABLES */ + kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM * INTEL_PGBYTES); +#if PAE +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readwrite(p->pdpbase); +#endif /* MACH_PV_PAGETABLES */ + kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES); +#endif /* PAE */ + kmem_cache_free(&pmap_cache, (vm_offset_t) p); } /* * Add a reference to the specified pmap. */ -void pmap_reference(p) - register pmap_t p; +void pmap_reference(pmap_t p) { int s; if (p != PMAP_NULL) { @@ -1112,21 +1319,25 @@ void pmap_reference(p) */ /* static */ -void pmap_remove_range(pmap, va, spte, epte) - pmap_t pmap; - vm_offset_t va; - pt_entry_t *spte; - pt_entry_t *epte; +void pmap_remove_range( + pmap_t pmap, + vm_offset_t va, + pt_entry_t *spte, + pt_entry_t *epte) { - register pt_entry_t *cpte; + pt_entry_t *cpte; int num_removed, num_unwired; int pai; vm_offset_t pa; +#ifdef MACH_PV_PAGETABLES + int n, ii = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ #if DEBUG_PTE_PAGE if (pmap != kernel_pmap) ptep_check(get_pte_page(spte)); -#endif DEBUG_PTE_PAGE +#endif /* DEBUG_PTE_PAGE */ num_removed = 0; num_unwired = 0; @@ -1147,10 +1358,22 @@ void pmap_remove_range(pmap, va, spte, e * Outside range of managed physical memory. * Just remove the mappings. */ - register int i = ptes_per_vm_page; - register pt_entry_t *lpte = cpte; + int i = ptes_per_vm_page; + pt_entry_t *lpte = cpte; do { +#ifdef MACH_PV_PAGETABLES + update[ii].ptr = kv_to_ma(lpte); + update[ii].val = 0; + ii++; + if (ii == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); + ii = 0; + } +#else /* MACH_PV_PAGETABLES */ *lpte = 0; +#endif /* MACH_PV_PAGETABLES */ lpte++; } while (--i > 0); continue; @@ -1163,15 +1386,27 @@ void pmap_remove_range(pmap, va, spte, e * Get the modify and reference bits. */ { - register int i; - register pt_entry_t *lpte; + int i; + pt_entry_t *lpte; i = ptes_per_vm_page; lpte = cpte; do { pmap_phys_attributes[pai] |= *lpte & (PHYS_MODIFIED|PHYS_REFERENCED); +#ifdef MACH_PV_PAGETABLES + update[ii].ptr = kv_to_ma(lpte); + update[ii].val = 0; + ii++; + if (ii == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); + ii = 0; + } +#else /* MACH_PV_PAGETABLES */ *lpte = 0; +#endif /* MACH_PV_PAGETABLES */ lpte++; } while (--i > 0); } @@ -1181,7 +1416,7 @@ void pmap_remove_range(pmap, va, spte, e * this physical page. */ { - register pv_entry_t pv_h, prev, cur; + pv_entry_t pv_h, prev, cur; pv_h = pai_to_pvh(pai); if (pv_h->pmap == PMAP_NULL) { @@ -1217,6 +1452,14 @@ void pmap_remove_range(pmap, va, spte, e } } +#ifdef MACH_PV_PAGETABLES + if (ii > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_remove_range"); + hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF); + if (n != ii) + panic("couldn't pmap_remove_range\n"); +#endif /* MACH_PV_PAGETABLES */ + /* * Update the counts */ @@ -1232,25 +1475,22 @@ void pmap_remove_range(pmap, va, spte, e * rounded to the hardware page size. */ -void pmap_remove(map, s, e) - pmap_t map; - vm_offset_t s, e; +void pmap_remove( + pmap_t map, + vm_offset_t s, + vm_offset_t e) { int spl; - register pt_entry_t *pde; - register pt_entry_t *spte, *epte; + pt_entry_t *pde; + pt_entry_t *spte, *epte; vm_offset_t l; + vm_offset_t _s = s; if (map == PMAP_NULL) return; PMAP_READ_LOCK(map, spl); - /* - * Invalidate the translation buffer first - */ - PMAP_UPDATE_TLBS(map, s, e); - pde = pmap_pde(map, s); while (s < e) { l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1); @@ -1265,6 +1505,7 @@ void pmap_remove(map, s, e) s = l; pde++; } + PMAP_UPDATE_TLBS(map, _s, e); PMAP_READ_UNLOCK(map, spl); } @@ -1276,15 +1517,15 @@ void pmap_remove(map, s, e) * Lower the permission for all mappings to a given * page. */ -void pmap_page_protect(phys, prot) - vm_offset_t phys; - vm_prot_t prot; +void pmap_page_protect( + vm_offset_t phys, + vm_prot_t prot) { pv_entry_t pv_h, prev; - register pv_entry_t pv_e; - register pt_entry_t *pte; + pv_entry_t pv_e; + pt_entry_t *pte; int pai; - register pmap_t pmap; + pmap_t pmap; int spl; boolean_t remove; @@ -1330,29 +1571,22 @@ void pmap_page_protect(phys, prot) prev = pv_e = pv_h; do { + vm_offset_t va; + pmap = pv_e->pmap; /* * Lock the pmap to block pmap_extract and similar routines. */ simple_lock(&pmap->lock); - { - register vm_offset_t va; - - va = pv_e->va; - pte = pmap_pte(pmap, va); - - /* - * Consistency checks. - */ - /* assert(*pte & INTEL_PTE_VALID); XXX */ - /* assert(pte_to_phys(*pte) == phys); */ + va = pv_e->va; + pte = pmap_pte(pmap, va); - /* - * Invalidate TLBs for all CPUs using this mapping. - */ - PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); - } + /* + * Consistency checks. + */ + /* assert(*pte & INTEL_PTE_VALID); XXX */ + /* assert(pte_to_phys(*pte) == phys); */ /* * Remove the mapping if new protection is NONE @@ -1366,12 +1600,17 @@ void pmap_page_protect(phys, prot) panic("pmap_remove_all removing a wired page"); { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { pmap_phys_attributes[pai] |= *pte & (PHYS_MODIFIED|PHYS_REFERENCED); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0)) + panic("%s:%d could not clear pte %p\n",__FILE__,__LINE__,pte-1); +#else /* MACH_PV_PAGETABLES */ *pte++ = 0; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } @@ -1398,10 +1637,15 @@ void pmap_page_protect(phys, prot) /* * Write-protect. */ - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE)) + panic("%s:%d could not disable write on pte %p\n",__FILE__,__LINE__,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ pte++; } while (--i > 0); @@ -1410,6 +1654,7 @@ void pmap_page_protect(phys, prot) */ prev = pv_e; } + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); simple_unlock(&pmap->lock); @@ -1435,15 +1680,17 @@ void pmap_page_protect(phys, prot) * specified range of this map as requested. * Will not increase permissions. */ -void pmap_protect(map, s, e, prot) - pmap_t map; - vm_offset_t s, e; - vm_prot_t prot; -{ - register pt_entry_t *pde; - register pt_entry_t *spte, *epte; - vm_offset_t l; +void pmap_protect( + pmap_t map, + vm_offset_t s, + vm_offset_t e, + vm_prot_t prot) +{ + pt_entry_t *pde; + pt_entry_t *spte, *epte; + vm_offset_t l; int spl; + vm_offset_t _s = s; if (map == PMAP_NULL) return; @@ -1478,11 +1725,6 @@ void pmap_protect(map, s, e, prot) SPLVM(spl); simple_lock(&map->lock); - /* - * Invalidate the translation buffer first - */ - PMAP_UPDATE_TLBS(map, s, e); - pde = pmap_pde(map, s); while (s < e) { l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1); @@ -1493,15 +1735,41 @@ void pmap_protect(map, s, e, prot) spte = &spte[ptenum(s)]; epte = &spte[intel_btop(l-s)]; +#ifdef MACH_PV_PAGETABLES + int n, i = 0; + struct mmu_update update[HYP_BATCH_MMU_UPDATES]; +#endif /* MACH_PV_PAGETABLES */ + while (spte < epte) { - if (*spte & INTEL_PTE_VALID) + if (*spte & INTEL_PTE_VALID) { +#ifdef MACH_PV_PAGETABLES + update[i].ptr = kv_to_ma(spte); + update[i].val = *spte & ~INTEL_PTE_WRITE; + i++; + if (i == HYP_BATCH_MMU_UPDATES) { + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_protect\n"); + i = 0; + } +#else /* MACH_PV_PAGETABLES */ *spte &= ~INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ + } spte++; } +#ifdef MACH_PV_PAGETABLES + if (i > HYP_BATCH_MMU_UPDATES) + panic("overflowed array in pmap_protect"); + hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF); + if (n != i) + panic("couldn't pmap_protect\n"); +#endif /* MACH_PV_PAGETABLES */ } s = l; pde++; } + PMAP_UPDATE_TLBS(map, _s, e); simple_unlock(&map->lock); SPLX(spl); @@ -1519,26 +1787,30 @@ void pmap_protect(map, s, e, prot) * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ -void pmap_enter(pmap, v, pa, prot, wired) - register pmap_t pmap; - vm_offset_t v; - register vm_offset_t pa; - vm_prot_t prot; - boolean_t wired; -{ - register pt_entry_t *pte; - register pv_entry_t pv_h; - register int i, pai; +void pmap_enter( + pmap_t pmap, + vm_offset_t v, + vm_offset_t pa, + vm_prot_t prot, + boolean_t wired) +{ + pt_entry_t *pte; + pv_entry_t pv_h; + int i, pai; pv_entry_t pv_e; pt_entry_t template; int spl; vm_offset_t old_pa; assert(pa != vm_page_fictitious_addr); -if (pmap_debug) printf("pmap(%x, %x)\n", v, pa); +if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa); if (pmap == PMAP_NULL) return; +#if !MACH_KDB + if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end)) + panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa); +#endif if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0 && !wired /* hack for io_wire */ ) { /* @@ -1556,9 +1828,9 @@ if (pmap_debug) printf("pmap(%x, %x)\n", * Invalidate the translation buffer, * then remove the mapping. */ - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); pmap_remove_range(pmap, v, pte, pte + ptes_per_vm_page); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } PMAP_READ_UNLOCK(pmap, spl); return; @@ -1566,7 +1838,7 @@ if (pmap_debug) printf("pmap(%x, %x)\n", /* * Must allocate a new pvlist entry while we're unlocked; - * zalloc may cause pageout (which will lock the pmap system). + * Allocating may cause pageout (which will lock the pmap system). * If we determine we need a pvlist entry, we will unlock * and allocate one. Then we will retry, throughing away * the allocated entry later (if we no longer need it). @@ -1602,7 +1874,7 @@ Retry: */ PMAP_READ_UNLOCK(pmap, spl); - ptp = pmap_page_table_page_alloc(); + ptp = phystokv(pmap_page_table_page_alloc()); /* * Re-lock the pmap and check that another thread has @@ -1617,7 +1889,7 @@ Retry: * Oops... */ PMAP_READ_UNLOCK(pmap, spl); - pmap_page_table_page_dealloc(ptp); + pmap_page_table_page_dealloc(kvtophys(ptp)); PMAP_READ_LOCK(pmap, spl); continue; } @@ -1629,18 +1901,23 @@ Retry: /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/ pdp = pmap_pde(pmap, v); do { - *pdp = pa_to_pte(ptp) | INTEL_PTE_VALID - | INTEL_PTE_USER - | INTEL_PTE_WRITE; +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readonly((void *) ptp); + if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp))) + panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp)); + if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)), + pa_to_pte(pa_to_ma(kvtophys(ptp))) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE)) + panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp))); +#else /* MACH_PV_PAGETABLES */ + *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID + | INTEL_PTE_USER + | INTEL_PTE_WRITE; +#endif /* MACH_PV_PAGETABLES */ pdp++; ptp += INTEL_PGBYTES; } while (--i > 0); -#if i860 - /* - * Flush the data cache. - */ - flush(); -#endif /* i860 */ /* * Now, get the address of the page-table entry. @@ -1657,7 +1934,7 @@ Retry: /* * May be changing its wired attribute or protection */ - + if (wired && !(*pte & INTEL_PTE_WIRED)) pmap->stats.wired_count++; else if (!wired && (*pte & INTEL_PTE_WIRED)) @@ -1668,17 +1945,25 @@ Retry: template |= INTEL_PTE_USER; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 + && pa >= phys_last_addr) + template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); i = ptes_per_vm_page; do { if (*pte & INTEL_PTE_MOD) template |= INTEL_PTE_MOD; +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))) + panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template); +#else /* MACH_PV_PAGETABLES */ WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte++; pte_increment_pa(template); } while (--i > 0); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } else { @@ -1687,17 +1972,12 @@ Retry: */ if (*pte) { /* - * Invalidate the translation buffer, - * then remove the mapping. - */ - PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); - - /* * Don't free the pte page if removing last * mapping - we will immediately replace it. */ pmap_remove_range(pmap, v, pte, pte + ptes_per_vm_page); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); } if (valid_page(pa)) { @@ -1730,8 +2010,8 @@ Retry: e = e->next; } } -#endif DEBUG - +#endif /* DEBUG */ + /* * Add new pv_entry after header. */ @@ -1742,9 +2022,9 @@ Retry: PMAP_READ_UNLOCK(pmap, spl); /* - * Refill from zone. + * Refill from cache. */ - pv_e = (pv_entry_t) zalloc(pv_list_zone); + pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache); goto Retry; } } @@ -1777,11 +2057,19 @@ Retry: template |= INTEL_PTE_USER; if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; + if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486 + && pa >= phys_last_addr) + template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU; if (wired) template |= INTEL_PTE_WIRED; i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))) + panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template); +#else /* MACH_PV_PAGETABLES */ WRITE_PTE(pte, template) +#endif /* MACH_PV_PAGETABLES */ pte++; pte_increment_pa(template); } while (--i > 0); @@ -1801,14 +2089,14 @@ Retry: * In/out conditions: * The mapping must already exist in the pmap. */ -void pmap_change_wiring(map, v, wired) - register pmap_t map; - vm_offset_t v; - boolean_t wired; +void pmap_change_wiring( + pmap_t map, + vm_offset_t v, + boolean_t wired) { - register pt_entry_t *pte; - register int i; - int spl; + pt_entry_t *pte; + int i; + int spl; /* * We must grab the pmap system lock because we may @@ -1836,7 +2124,13 @@ void pmap_change_wiring(map, v, wired) map->stats.wired_count--; i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED))) + panic("%s:%d could not wire down pte %p\n",__FILE__,__LINE__,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~INTEL_PTE_WIRED; +#endif /* MACH_PV_PAGETABLES */ + pte++; } while (--i > 0); } @@ -1850,13 +2144,13 @@ void pmap_change_wiring(map, v, wired) * with the given map/virtual_address pair. */ -vm_offset_t pmap_extract(pmap, va) - register pmap_t pmap; - vm_offset_t va; +vm_offset_t pmap_extract( + pmap_t pmap, + vm_offset_t va) { - register pt_entry_t *pte; - register vm_offset_t pa; - int spl; + pt_entry_t *pte; + vm_offset_t pa; + int spl; SPLVM(spl); simple_lock(&pmap->lock); @@ -1886,11 +2180,8 @@ void pmap_copy(dst_pmap, src_pmap, dst_a vm_size_t len; vm_offset_t src_addr; { -#ifdef lint - dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++; -#endif lint } -#endif 0 +#endif /* 0 */ /* * Routine: pmap_collect @@ -1903,10 +2194,9 @@ void pmap_copy(dst_pmap, src_pmap, dst_a * Usage: * Called by the pageout daemon when pages are scarce. */ -void pmap_collect(p) - pmap_t p; +void pmap_collect(pmap_t p) { - register pt_entry_t *pdp, *ptp; + pt_entry_t *pdp, *ptp; pt_entry_t *eptp; vm_offset_t pa; int spl, wired; @@ -1921,8 +2211,6 @@ void pmap_collect(p) * Garbage collect map. */ PMAP_READ_LOCK(p, spl); - PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS); - for (pdp = p->dirbase; pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)]; pdp += ptes_per_vm_page) @@ -1939,7 +2227,7 @@ void pmap_collect(p) */ wired = 0; { - register pt_entry_t *ptep; + pt_entry_t *ptep; for (ptep = ptp; ptep < eptp; ptep++) { if (*ptep & INTEL_PTE_WIRED) { wired = 1; @@ -1965,10 +2253,20 @@ void pmap_collect(p) * Invalidate the page directory pointer. */ { - register int i = ptes_per_vm_page; - register pt_entry_t *pdep = pdp; + int i = ptes_per_vm_page; + pt_entry_t *pdep = pdp; do { +#ifdef MACH_PV_PAGETABLES + unsigned long pte = *pdep; + void *ptable = (void*) ptetokv(pte); + if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0))) + panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1); + if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable))) + panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable))); + pmap_set_page_readwrite(ptable); +#else /* MACH_PV_PAGETABLES */ *pdep++ = 0; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } @@ -1978,7 +2276,7 @@ void pmap_collect(p) * And free the pte page itself. */ { - register vm_page_t m; + vm_page_t m; vm_object_lock(pmap_object); m = vm_page_lookup(pmap_object, pa); @@ -1995,6 +2293,8 @@ void pmap_collect(p) } } } + PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS); + PMAP_READ_UNLOCK(p, spl); return; @@ -2014,7 +2314,7 @@ void pmap_activate(my_pmap, th, my_cpu) { PMAP_ACTIVATE(my_pmap, th, my_cpu); } -#endif 0 +#endif /* 0 */ /* * Routine: pmap_deactivate @@ -2029,12 +2329,9 @@ void pmap_deactivate(pmap, th, which_cpu thread_t th; int which_cpu; { -#ifdef lint - pmap++; th++; which_cpu++; -#endif lint PMAP_DEACTIVATE(pmap, th, which_cpu); } -#endif 0 +#endif /* 0 */ /* * Routine: pmap_kernel @@ -2046,7 +2343,7 @@ pmap_t pmap_kernel() { return (kernel_pmap); } -#endif 0 +#endif /* 0 */ /* * pmap_zero_page zeros the specified (machine independent) page. @@ -2065,7 +2362,7 @@ pmap_zero_page(phys) while (i--) zero_phys(phys++); } -#endif 0 +#endif /* 0 */ /* * pmap_copy_page copies the specified (machine independent) page. @@ -2087,7 +2384,7 @@ pmap_copy_page(src, dst) dst += INTEL_PGBYTES; } } -#endif 0 +#endif /* 0 */ /* * Routine: pmap_pageable @@ -2103,30 +2400,28 @@ pmap_copy_page(src, dst) * will specify that these pages are to be wired * down (or not) as appropriate. */ -pmap_pageable(pmap, start, end, pageable) - pmap_t pmap; - vm_offset_t start; - vm_offset_t end; - boolean_t pageable; -{ -#ifdef lint - pmap++; start++; end++; pageable++; -#endif lint +void +pmap_pageable( + pmap_t pmap, + vm_offset_t start, + vm_offset_t end, + boolean_t pageable) +{ } /* * Clear specified attribute bits. */ void -phys_attribute_clear(phys, bits) - vm_offset_t phys; - int bits; +phys_attribute_clear( + vm_offset_t phys, + int bits) { pv_entry_t pv_h; - register pv_entry_t pv_e; - register pt_entry_t *pte; + pv_entry_t pv_e; + pt_entry_t *pte; int pai; - register pmap_t pmap; + pmap_t pmap; int spl; assert(phys != vm_page_fictitious_addr); @@ -2157,6 +2452,7 @@ phys_attribute_clear(phys, bits) * There are some mappings. */ for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) { + vm_offset_t va; pmap = pv_e->pmap; /* @@ -2164,35 +2460,32 @@ phys_attribute_clear(phys, bits) */ simple_lock(&pmap->lock); - { - register vm_offset_t va; - - va = pv_e->va; - pte = pmap_pte(pmap, va); + va = pv_e->va; + pte = pmap_pte(pmap, va); #if 0 - /* - * Consistency checks. - */ - assert(*pte & INTEL_PTE_VALID); - /* assert(pte_to_phys(*pte) == phys); */ + /* + * Consistency checks. + */ + assert(*pte & INTEL_PTE_VALID); + /* assert(pte_to_phys(*pte) == phys); */ #endif - /* - * Invalidate TLBs for all CPUs using this mapping. - */ - PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); - } - /* * Clear modify or reference bits. */ { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { +#ifdef MACH_PV_PAGETABLES + if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits))) + panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__,__LINE__,bits,pte); +#else /* MACH_PV_PAGETABLES */ *pte &= ~bits; +#endif /* MACH_PV_PAGETABLES */ } while (--i > 0); } + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); simple_unlock(&pmap->lock); } } @@ -2206,15 +2499,15 @@ phys_attribute_clear(phys, bits) * Check specified attribute bits. */ boolean_t -phys_attribute_test(phys, bits) - vm_offset_t phys; - int bits; +phys_attribute_test( + vm_offset_t phys, + int bits) { pv_entry_t pv_h; - register pv_entry_t pv_e; - register pt_entry_t *pte; + pv_entry_t pv_e; + pt_entry_t *pte; int pai; - register pmap_t pmap; + pmap_t pmap; int spl; assert(phys != vm_page_fictitious_addr); @@ -2258,7 +2551,7 @@ phys_attribute_test(phys, bits) simple_lock(&pmap->lock); { - register vm_offset_t va; + vm_offset_t va; va = pv_e->va; pte = pmap_pte(pmap, va); @@ -2276,7 +2569,7 @@ phys_attribute_test(phys, bits) * Check modify or reference bits. */ { - register int i = ptes_per_vm_page; + int i = ptes_per_vm_page; do { if (*pte & bits) { @@ -2297,8 +2590,7 @@ phys_attribute_test(phys, bits) * Clear the modify bits on the specified physical page. */ -void pmap_clear_modify(phys) - register vm_offset_t phys; +void pmap_clear_modify(vm_offset_t phys) { phys_attribute_clear(phys, PHYS_MODIFIED); } @@ -2310,8 +2602,7 @@ void pmap_clear_modify(phys) * by any physical maps. */ -boolean_t pmap_is_modified(phys) - register vm_offset_t phys; +boolean_t pmap_is_modified(vm_offset_t phys) { return (phys_attribute_test(phys, PHYS_MODIFIED)); } @@ -2322,8 +2613,7 @@ boolean_t pmap_is_modified(phys) * Clear the reference bit on the specified physical page. */ -void pmap_clear_reference(phys) - vm_offset_t phys; +void pmap_clear_reference(vm_offset_t phys) { phys_attribute_clear(phys, PHYS_REFERENCED); } @@ -2335,8 +2625,7 @@ void pmap_clear_reference(phys) * by any physical maps. */ -boolean_t pmap_is_referenced(phys) - vm_offset_t phys; +boolean_t pmap_is_referenced(vm_offset_t phys) { return (phys_attribute_test(phys, PHYS_REFERENCED)); } @@ -2344,7 +2633,7 @@ boolean_t pmap_is_referenced(phys) #if NCPUS > 1 /* * TLB Coherence Code (TLB "shootdown" code) -* +* * Threads that belong to the same task share the same address space and * hence share a pmap. However, they may run on distinct cpus and thus * have distinct TLBs that cache page table entries. In order to guarantee @@ -2357,7 +2646,7 @@ boolean_t pmap_is_referenced(phys) * flush its own TLB; a processor that needs to invalidate another TLB * needs to interrupt the processor that owns that TLB to signal the * update. -* +* * Whenever a pmap is updated, the lock on that pmap is locked, and all * cpus using the pmap are signaled to invalidate. All threads that need * to activate a pmap must wait for the lock to clear to await any updates @@ -2366,7 +2655,7 @@ boolean_t pmap_is_referenced(phys) * throughout the TLB code is that all kernel code that runs at or higher * than splvm blocks out update interrupts, and that such code does not * touch pageable pages. -* +* * A shootdown interrupt serves another function besides signaling a * processor to invalidate. The interrupt routine (pmap_update_interrupt) * waits for the both the pmap lock (and the kernel pmap lock) to clear, @@ -2379,17 +2668,17 @@ boolean_t pmap_is_referenced(phys) * Spinning on the VALUES of the locks is sufficient (rather than * having to acquire the locks) because any updates that occur subsequent * to finding the lock unlocked will be signaled via another interrupt. -* (This assumes the interrupt is cleared before the low level interrupt code -* calls pmap_update_interrupt()). -* +* (This assumes the interrupt is cleared before the low level interrupt code +* calls pmap_update_interrupt()). +* * The signaling processor must wait for any implicit updates in progress * to terminate before continuing with its update. Thus it must wait for an * acknowledgement of the interrupt from each processor for which such * references could be made. For maintaining this information, a set -* cpus_active is used. A cpu is in this set if and only if it can +* cpus_active is used. A cpu is in this set if and only if it can * use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from * this set; when all such cpus are removed, it is safe to update. -* +* * Before attempting to acquire the update lock on a pmap, a cpu (A) must * be at least at the priority of the interprocessor interrupt * (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a @@ -2405,13 +2694,14 @@ boolean_t pmap_is_referenced(phys) /* * Signal another CPU that it must flush its TLB */ -void signal_cpus(use_list, pmap, start, end) - cpu_set use_list; - pmap_t pmap; - vm_offset_t start, end; +void signal_cpus( + cpu_set use_list, + pmap_t pmap, + vm_offset_t start, + vm_offset_t end) { - register int which_cpu, j; - register pmap_update_list_t update_list_p; + int which_cpu, j; + pmap_update_list_t update_list_p; while ((which_cpu = ffs(use_list)) != 0) { which_cpu -= 1; /* convert to 0 origin */ @@ -2444,13 +2734,12 @@ void signal_cpus(use_list, pmap, star } } -void process_pmap_updates(my_pmap) - register pmap_t my_pmap; +void process_pmap_updates(pmap_t my_pmap) { - register int my_cpu = cpu_number(); - register pmap_update_list_t update_list_p; - register int j; - register pmap_t pmap; + int my_cpu = cpu_number(); + pmap_update_list_t update_list_p; + int j; + pmap_t pmap; update_list_p = &cpu_update_list[my_cpu]; simple_lock(&update_list_p->lock); @@ -2460,7 +2749,8 @@ void process_pmap_updates(my_pmap) if (pmap == my_pmap || pmap == kernel_pmap) { - INVALIDATE_TLB(update_list_p->item[j].start, + INVALIDATE_TLB(pmap, + update_list_p->item[j].start, update_list_p->item[j].end); } } @@ -2472,11 +2762,11 @@ void process_pmap_updates(my_pmap) /* * Interrupt routine for TBIA requested from other processor. */ -void pmap_update_interrupt() +void pmap_update_interrupt(void) { - register int my_cpu; - register pmap_t my_pmap; - int s; + int my_cpu; + pmap_t my_pmap; + int s; my_cpu = cpu_number(); @@ -2525,39 +2815,37 @@ void pmap_update_interrupt() i_bit_set(my_cpu, &cpus_active); } while (cpu_update_needed[my_cpu]); - + splx(s); } -#else NCPUS > 1 +#else /* NCPUS > 1 */ /* * Dummy routine to satisfy external reference. */ -void pmap_update_interrupt() +void pmap_update_interrupt(void) { /* should never be called. */ } -#endif NCPUS > 1 - -#if i860 /* akp */ -void set_dirbase(dirbase) - register vm_offset_t dirbase; -{ - /*flush();*/ - /*flush_tlb();*/ - flush_and_ctxsw(dirbase); -} -#endif i860 +#endif /* NCPUS > 1 */ -#ifdef i386 +#if defined(__i386__) /* Unmap page 0 to trap NULL references. */ void -pmap_unmap_page_zero () +pmap_unmap_page_zero (void) { int *pte; + printf("Unmapping the zero page. Some BIOS functions may not be working any more.\n"); pte = (int *) pmap_pte (kernel_pmap, 0); + if (!pte) + return; assert (pte); +#ifdef MACH_PV_PAGETABLES + if (!hyp_mmu_update_pte(kv_to_ma(pte), 0)) + printf("couldn't unmap page 0\n"); +#else /* MACH_PV_PAGETABLES */ *pte = 0; - asm volatile ("movl %%cr3,%%eax; movl %%eax,%%cr3" ::: "ax"); + INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE); +#endif /* MACH_PV_PAGETABLES */ } -#endif /* i386 */ +#endif /* __i386__ */