Source to machdep/i386/pmap.c


Enter a symbol's name here to quickly find it.

/*
 * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
 *
 * @[email protected]
 * 
 * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
 * Reserved.  This file contains Original Code and/or Modifications of
 * Original Code as defined in and that are subject to the Apple Public
 * Source License Version 1.0 (the 'License').  You may not use this file
 * except in compliance with the License.  Please obtain a copy of the
 * License at http://www.apple.com/publicsource and read it before using
 * this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License."
 * 
 * @[email protected]
 */

/*
 * Copyright (c) 1992 NeXT Computer, Inc.
 *
 * Intel386 Family:	Hardware page mapping.
 *
 * HISTORY
 *
 * 9 April 1992 ? at NeXT
 *	Created.
 */
 
#import <cpus.h>

#import <mach/mach_types.h>

#import <vm/vm_kern.h>
#import <vm/vm_page.h>

#import <machdep/i386/pmap_private.h>
#import <machdep/i386/pmap_inline.h>
#import <machdep/i386/cpu_inline.h>

/*
 * Setup structures to map from mach vm_prot_t
 * to machine protections.
 */
unsigned int			user_prot_codes[8];
unsigned int			kernel_prot_codes[8];
#define kernel_pmap_prot(x)	(kernel_prot_codes[(x)])

static
void
pte_prot_init(
    void
)
{
    unsigned int	*kp, *up;
    int			prot;

    kp = kernel_prot_codes;
    up = user_prot_codes;
    for (prot = 0; prot < 8; prot++) {
	switch ((vm_prot_t)prot) {
	case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
	    *kp++ = 0;
	    *up++ = 0;
	    break;
	case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
	case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
	case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
	    *kp++ = PT_PROT_KR;
	    *up++ = PT_PROT_UR;
	    break;
	case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
	case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
	case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
	case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
	    *kp++ = PT_PROT_KRW;
	    *up++ = PT_PROT_URW;
	    break;
	}
    }
}

/*
 * Given a map and a machine independent protection code,
 * set the protection code in the given pte
 */
static inline
void
pte_prot(
    pmap_t		pmap,
    vm_prot_t		prot,
    int			valid,		/* valid bit for pte */
    pt_entry_t *	pte		/* IN/OUT */
)
{
    if (pmap == kernel_pmap)
	pte->prot = kernel_prot_codes[prot];
    else
	pte->prot = user_prot_codes[prot];

    pte->valid = valid;
}

/*
 * Return a ptr to the page
 * table entry at the indicated
 * offset in the pmap.  Return
 * PT_ENTRY_NULL if the page table
 * does not exist.
 */
inline
pt_entry_t *
pmap_pt_entry(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    pd_entry_t *	pde;
    
    pde = pd_to_pd_entry(pmap->root, va);
    
    if (!pde->valid)
	return (PT_ENTRY_NULL);
	
    return (pd_entry_to_pt_entry(pde, va));
}

/*
 * Return a ptr to the page
 * directory entry at the indicated
 * offset in the pmap.
 */
inline
pd_entry_t *
pmap_pd_entry(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    return (pd_to_pd_entry(pmap->root, va));
}

/*
 * Return the physical address
 * corresponding to the indicated
 * offset in the pmap.  Only used
 * internally since no locking is
 * done.
 */
static inline
unsigned int
pmap_phys(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    pt_entry_t		*pte;
    
    pte = pmap_pt_entry(pmap, va);
    if (pte == PT_ENTRY_NULL || !pte->valid)
	return (0);
	    
    return (pfn_to_phys(pte->pfn) + page_offset(va));
}

static inline
void
pmap_update_tlbs(
    pmap_t		pmap,
    vm_offset_t		start,
    vm_offset_t		end
)
{
    if (pmap == kernel_pmap || pmap->cpus_using) {
	tlb_stat.total++;
	if (end - start > PAGE_SIZE)
	    flush_tlb();
	else {
	    for (; start < end; start += I386_PGBYTES)
		invlpg(start, pmap == kernel_pmap);
	  
	    tlb_stat.single++;
	}
    }
}

/*
 * Allocate a new page table
 * in the kernel pmap at the
 * given offset when one does
 * not already exist.
 */
static
void
pmap_kernel_pt_alloc(
    vm_offset_t		addr
)
{
    pd_entry_t		template, *pde;
    vm_offset_t		pt;
    unsigned int	phys;
    int			i;
    
    pde = pmap_pd_entry(kernel_pmap, trunc_section(addr));
    if (pde->valid)
	panic("pmap_kernel_pt_alloc");
	
    if (!pmap_initialized) {
	pt = alloc_pages(PAGE_SIZE);
	if (!pt)
	    panic("pmap_kernel_pt_alloc 2");

	bzero(pt, PAGE_SIZE);

	phys = pt - VM_MIN_KERNEL_ADDRESS;	
    }
    else {
	if (kmem_alloc_wired(kernel_map, &pt, PAGE_SIZE) != KERN_SUCCESS)
	    panic("pmap_kernel_pt_alloc 3");
	    
	phys = pmap_phys(kernel_pmap, pt);
    }
    
    template = (pd_entry_t) { 0 };
    template.valid = 1;
    template.prot = PT_PROT_KRW;
    template.pfn = phys_to_pfn(phys);
    
    for (i = ptes_per_vm_page; i-- > 0; pde++) {
	*pde = template;
	template.pfn++;
    }
}

/*
 * Map memory at initialization.  The physical addresses being
 * mapped are not managed and are never unmapped.
 */
vm_offset_t
pmap_map(
    vm_offset_t		virt,
    vm_offset_t		start,
    vm_offset_t		end,
    vm_prot_t		prot
)
{
    pt_entry_t		template, *pte;
    
    template = (pt_entry_t) { 0 };

    if (prot != VM_PROT_NONE) {
	template.valid = 1;
	template.prot =  kernel_pmap_prot(prot);
	template.pfn = phys_to_pfn(start);
    }

    while (start < end) {
	pte = pmap_pt_entry(kernel_pmap, virt);
	if (pte == PT_ENTRY_NULL) {
	    pmap_kernel_pt_alloc(virt);
	    pte = pmap_pt_entry(kernel_pmap, virt);
	}

	/*
	 * Is this necessary ??
	 */	
	if (	start >= (640 * 1024) &&
		start < (1024 * 1024))
	    template.cachewrt = 1;
	else
	    template.cachewrt = 0;
	
	*pte = template;
	if (prot != VM_PROT_NONE)
	    template.pfn++;

	virt += I386_PGBYTES;
	start += I386_PGBYTES;
    }

    flush_tlb();

    return (virt);
}

static
void
pmap_enable_pg(
    vm_offset_t		kernel_pd
)
{
    pd_entry_t		*kpde, *end_kpde;
    pd_entry_t		*pde;
    cr0_t		_cr0 = cr0();

    /*
     * Double map the kernel memory
     * into the low end of the kernel
     * pmap linear space.  This is
     * necessary in order to enable
     * paging.
     */
    pde = (pd_entry_t *)kernel_pd;

    kpde =
    	pmap_pd_entry(kernel_pmap,
			VM_MIN_KERNEL_ADDRESS);
    end_kpde =
    	pmap_pd_entry(kernel_pmap,
			VM_MAX_KERNEL_ADDRESS);
			
    while (kpde < end_kpde)
	*pde++ = *kpde++;

    /*
     * Use the kernel pmap
     * as our initial translation
     * tree.
     */ 
    kernel_pmap->cr3 =
	    pmap_phys(kernel_pmap, kernel_pd);
    
    set_cr3(kernel_pmap->cr3);

    /*
     * Now, enable paging by
     * turning on the PG bit
     * in CR0.  Also turn on
     * the WP bit to allow
     * the write protecting of
     * memory with repect to
     * the kernel.
     */ 
    _cr0.pg = _cr0.wp = TRUE;
    set_cr0(_cr0);
}

/*
 */
void
pmap_bootstrap(
    mem_region_t	mem_region,
    int			num_regions,
    vm_offset_t *	virt_avail,	/* OUT */
    vm_offset_t *	virt_end	/* OUT */
)
{
    vm_offset_t		va;
    unsigned int	phys_end = mem_region[0].last_phys_addr;
    vm_offset_t		kernel_pd;
    
    /*
     * Setup section_size variable.
     */
    section_size = SECTION_SIZE;
    	
    /*
     *	Set ptes_per_vm_page for general use.
     */
    ptes_per_vm_page = PAGE_SIZE / I386_PGBYTES;

    /*
     *	Initialize pte protection arrays.
     */
    pte_prot_init();

    /*
     *	The kernel's pmap is statically allocated so we don't
     *	have to use pmap_create, which does not work
     *	correctly at this part of the boot sequence.
     */
    kernel_pmap = &kernel_pmap_store;

    simple_lock_init(&kernel_pmap->lock);

    /*
     * Allocate a page directory for the
     * kernel pmap.
     */
    (vm_offset_t)kernel_pmap->root =
	    kernel_pd = alloc_cnvmem(I386_PGBYTES, I386_PGBYTES);

    bzero((vm_offset_t)kernel_pmap->root, I386_PGBYTES);

    kernel_pmap->ref_count = 1;

    kernel_pmap->root +=
    	(KERNEL_LINEAR_BASE - VM_MIN_KERNEL_ADDRESS) / I386_SECTBYTES;
    
    /*
     * Map all physical memory V == P.
     */
    va = pmap_map(VM_MIN_KERNEL_ADDRESS,
		  0,
		  phys_end,
		  VM_PROT_READ | VM_PROT_WRITE);

    /*
     * Allocate additional kernel page tables
     * needed for allocating kernel virtual memory later.
     */
    *virt_avail = va;
     va = pmap_map(va,
     		   0,
		   64*1024*1024	+	/* base size */
		   zone_map_sizer() +	/* zone allocator */
		   buffer_map_sizer(),	/* buffer cache */
		   VM_PROT_NONE);
		   
    *virt_end = va;

    /*
     * Finish initialization
     * of the kernel pmap, and
     * then enable paging.
     */
    pmap_enable_pg(kernel_pd);
}

/*
 * Initialize the pmap module.
 * Called by vm_init, to initialize any structures that the pmap
 * system needs to map virtual memory.
 */
void
pmap_init(
    mem_region_t	mem_region,
    int			num_regions
)
{
    unsigned int	phys_start, phys_end;
    int			npages;
    vm_size_t		s;

    phys_start	= mem_region[0].first_phys_addr;
    phys_end	= mem_region[0].last_phys_addr;

    npages	= mem_region[0].num_pages;

    /*
     * Allocate memory for the page descriptor
     * table.
     */
    s = sizeof(struct pg_desc) * npages;
    (void) kmem_alloc_wired(kernel_map, (vm_offset_t *)&pg_desc_tbl, s);
    pg_first_phys = phys_start;

    /*
     * Create the zone of physical maps,
     * and of the physical-to-virtual entries.
     */
    s = sizeof (struct pmap);
    pmap_zone = zinit(s, 400*s, 0, FALSE, "pmap");		/* XXX */

    s = sizeof (struct pv_entry);
    pv_entry_zone = zinit(s, 10000*s, 0, FALSE, "pv_entry");	/* XXX */
    
    /*
     * Create the zone of page
     * extensions.
     */
    s = sizeof (struct pg_exten);
    pg_exten_zone = zinit(s, npages*s, 0, FALSE, "pg_exten");

    /*
     * Initialize the queues of active and
     * free page tables and free page directories.
     */	
    queue_init(&pt_active_queue);
    queue_init(&pt_free_queue);
    queue_init(&pd_free_queue);

    /*
     * Only now, when all of the data structures are allocated,
     * can we set vm_first_phys and vm_last_phys.  If we set them
     * too soon, the kmem_alloc above will try to use these
     * data structures and blow up.
     */

    vm_first_phys	= phys_start;
    vm_last_phys	= phys_end;

    pmap_initialized = TRUE;
}

static
pg_exten_t *
pg_exten_alloc(
    vm_offset_t		page
)
{
    pg_exten_t		*pe;
    pg_desc_t		*pd;
        
    (vm_offset_t)pe = zalloc(pg_exten_zone);
    
    pe->phys = pmap_phys(kernel_pmap, page);
    
    pd = pg_desc(pe->phys);

    pd->pg_exten = pe; pe->pg_desc = pd;
    
    pe->alloc_count = pe->wired_count = pe->unrefd_age = pe->alloc_bmap = 0;
    
    return (pe);
}

static
void
pg_exten_free(
    pg_exten_t		*pe
)
{
    pe->pg_desc->pg_exten = PG_EXTEN_NULL;
    
    zfree(pg_exten_zone, (vm_offset_t)pe);
}

static
void
pmap_alloc_pd(
    pmap_t		pmap
)
{
    pg_exten_t		*pe;
    int			i;
   
    if (pe = pd_free_obtain()) {
	if (++pe->alloc_count == ptes_per_vm_page)
	    pd_free_remove(pe);

	i = ffs(~pe->alloc_bmap) - 1;
	pe->alloc_bmap |= (1 << i);

	(vm_offset_t)pmap->root = pe->pg_desc->pv_list.va + (I386_PGBYTES * i);
    }
    else {
    	if (kmem_alloc_wired(kernel_map,
			(vm_offset_t *)&pmap->root, PAGE_SIZE) != KERN_SUCCESS)
	    panic("pmap_alloc_pd");

	pd_alloc_count++;
	
	pe = pg_exten_alloc((vm_offset_t)pmap->root);

	if (++pe->alloc_count < ptes_per_vm_page)
	    pd_free_add(pe);

	pe->alloc_bmap |= 1;
    }
}

static
void
pmap_free_pd(
    pmap_t		pmap
)
{
    pg_exten_t		*pe;
    pg_desc_t		*pd;
    int			i;
    
    pd = pg_desc(pmap_phys(kernel_pmap, (vm_offset_t)pmap->root));
    
    pe = pd->pg_exten;

    if (pe->alloc_count-- == ptes_per_vm_page)
    	pd_free_add(pe);

    i = ((vm_offset_t)pmap->root - pd->pv_list.va) / I386_PGBYTES;

    pe->alloc_bmap &= ~(1 << i);
}

/*
 * Allocate and setup the page directory
 * for a new pmap.  The pd_entries that
 * correspond to the kernel address space
 * are initialized by copying them from
 * the kernel pmap.  This works correctly
 * since we never expand the kernel pmap.
 */
static
void
pmap_create_pd(
    pmap_t		pmap
)
{
    pd_entry_t		*pde;
    pd_entry_t		*kpde, *end_kpde;
    
    pmap_alloc_pd(pmap);
    
    if (pmap->root == PD_ENTRY_NULL)
	panic("pmap_create_page_directory");
    
    if (i386_trunc_page(pmap->root) != (vm_offset_t)pmap->root)
    	panic("pmap_create_page_directory 1");
    
    pmap->cr3 = pmap_phys(kernel_pmap, (vm_offset_t)pmap->root);
	
    kpde =
    	pmap_pd_entry(kernel_pmap,
			VM_MIN_KERNEL_ADDRESS);
    end_kpde =
    	pmap_pd_entry(kernel_pmap,
			VM_MAX_KERNEL_ADDRESS);
		    
    pde = pmap_pd_entry(pmap, KERNEL_LINEAR_BASE);
    
    while (kpde < end_kpde)
	*pde++ = *kpde++;
}

/*
 * Create and return a physical map.
 *
 * If the size specified for the map
 * is zero, the map is an actual physical
 * map, and may be referenced by the
 * hardware.
 *
 * If the size specified is non-zero,
 * the map will be used in software only, and
 * is bounded by that size.
 */
pmap_t
pmap_create(
    vm_size_t		size
)
{
    pmap_t		pmap;

    /*
     *	A software use-only map doesn't even need a pmap.
     */
    if (size != 0)
	return (PMAP_NULL);

    /*
     * Allocate a pmap struct from the pmap_zone.
     */
    pmap = (pmap_t) zalloc(pmap_zone);
    if (pmap == PMAP_NULL)
	panic("pmap_create pmap");

    bzero((vm_offset_t)pmap, sizeof (struct pmap));

    /*
     * Create the page directory.
     */
    pmap_create_pd(pmap);

    pmap->ref_count = 1;

    simple_lock_init(&pmap->lock);
	
    return (pmap);
}

/*
 * Retire the given physical map from service.
 * Should only be called if the map contains
 * no valid mappings.
 */
void
pmap_destroy(
    pmap_t		pmap
)
{
    int			c, s;

    if (pmap == PMAP_NULL)
	return;

    SPLVM(s);
    simple_lock(&pmap->lock);

    c = --pmap->ref_count;

    simple_unlock(&pmap->lock);
    SPLX(s);

    if (c != 0)
	return;

    pmap_free_pd(pmap);

    zfree(pmap_zone, (vm_offset_t)pmap);
}

/*
 *	Add a reference to the specified pmap.
 */

void
pmap_reference(
    pmap_t		pmap
)
{
    int			s;

    if (pmap != PMAP_NULL) {
	SPLVM(s);
	simple_lock(&pmap->lock);

	pmap->ref_count++;

	simple_unlock(&pmap->lock);
	SPLX(s);
    }
}

/*
 * Remove a range of mappings from
 * a pmap.  The indicated range must
 * lie completely within one section,
 * i.e. the ptes must be within one
 * page table.
 */
static
void
pmap_remove_range(
    pmap_t		pmap,
    vm_offset_t		start,
    vm_offset_t		end,
    boolean_t		free_table
)
{
    pt_entry_t		*pte, *epte;
    pg_desc_t		*pd;
    pv_entry_t		pv_h;
    pv_entry_t		cur, prev;
    unsigned int	pa;
    vm_offset_t		va = start;
    int			i, num_removed = 0, num_unwired = 0;
    
    if ((pte = pmap_pt_entry(pmap, start)) == PT_ENTRY_NULL)
	return;
	
    epte = pmap_pt_entry(pmap, end);
    if (trunc_page(pte) != trunc_page(epte))
	epte = (pt_entry_t *)round_page(pte + ptes_per_vm_page);
    
    for (; pte < epte; va += PAGE_SIZE) {
	if (!pte->valid) {
	    pte += ptes_per_vm_page;
	    continue;
	}
	    
	num_removed++;
	if (pte->wired)
	    num_unwired++;
	    
	pa = pfn_to_phys(pte->pfn);
	if (!managed_page(pa)) {
	    for (i = ptes_per_vm_page; i-- > 0; pte++)
		*pte = (pt_entry_t) { 0 };
	    continue;
	}
	    
	pd = pg_desc(pa);
	LOCK_PVH(pd);
	
	/*
	 * Collect the referenced & dirty bits
	 * and clear the mapping.
	 */
	for (i = ptes_per_vm_page; i-- > 0; pte++) {
	    if (pte->dirty) {
		vm_page_t	m = PHYS_TO_VM_PAGE(pa);
		
		vm_page_set_modified(m);
		pd->page_attrib |= PG_DIRTY;
	    }
	    
	    if (pte->refer)
		pd->page_attrib |= PG_REFER;
		
	    *pte = (pt_entry_t) { 0 };
	}

	/*
	 * Remove the mapping from the pvlist for
	 * this physical page.
	 */
	pv_h = pg_desc_pvh(pd);
	if (pv_h->pmap == PMAP_NULL)
	    panic("pmap_remove_range");

	if (pv_h->va == va && pv_h->pmap == pmap) {
	    /*
	     * Header is the pv_entry.  Copy the next one
	     * to header and free the next one (we can't
	     * free the header)
	     */
	    cur = pv_h->next;
	    if (cur != PV_ENTRY_NULL) {
		*pv_h = *cur;
		zfree(pv_entry_zone, (vm_offset_t) cur);
	    }
	    else
		pv_h->pmap = PMAP_NULL;
	}
	else {
	    prev = pv_h;
	    while ((cur = prev->next) != PV_ENTRY_NULL) {
		if (cur->va == va && cur->pmap == pmap)
		    break;
		prev = cur;
	    }
	    if (cur == PV_ENTRY_NULL)
		panic("pmap_remove_range 2");

	    prev->next = cur->next;
	    zfree(pv_entry_zone, (vm_offset_t) cur);
	}
	
	UNLOCK_PVH(pd);
    }

    /*
     * Free the mappings from the page table.
     */	
    pmap_deallocate_mappings(
			    pmap, start,
			    num_removed, num_unwired, free_table);
}

/*
 * Remove the given range of addresses
 * from the specified pmap.
 *
 * It is assumed that the start and end are properly
 * rounded to the page size.
 */
void
pmap_remove(
    pmap_t		pmap,
    vm_offset_t		start,
    vm_offset_t		end
)
{
    vm_offset_t		sect_end;
    int			s;

    if (pmap == PMAP_NULL)
	return;

    PMAP_READ_LOCK(pmap, s);

    /*
     * Invalidate the translation buffer first
     */
    PMAP_UPDATE_TLBS(pmap, start, end);
    
    while (start < end) {
	sect_end = round_section(start + PAGE_SIZE);
	if (sect_end > end)
	    sect_end = end;

	pmap_remove_range(pmap, start, sect_end, TRUE);
	start = sect_end;
    }

    PMAP_READ_UNLOCK(pmap, s);
}

/*
 * Remove all references to 
 * the indicated page from
 * all pmaps.
 */
void
pmap_remove_all(
    unsigned int	pa
)
{
    pt_entry_t		*pte;
    pv_entry_t		pv_h, cur;
    pg_desc_t		*pd;
    vm_offset_t		va;
    pmap_t		pmap;
    int			s, i;
    
    if (!managed_page(pa))
	return;

    /*
     * Lock the pmap system first, since we will be changing
     * several pmaps.
     */
    PMAP_WRITE_LOCK(s);
    
    /*
     * Walk down PV list, removing all mappings.
     * We have to do the same work as in pmap_remove_range
     * since that routine locks the pv_head.  We don't have
     * to lock the pv_head, since we have the entire pmap system.
     */
    pd = pg_desc(pa);
    pv_h = pg_desc_pvh(pd);
    
    while ((pmap = pv_h->pmap) != PMAP_NULL) {
	va = pv_h->va;

	simple_lock(&pmap->lock);

	pte = pmap_pt_entry(pmap, va);
	if (pte == PT_ENTRY_NULL || !pte->valid)
	    panic("pmap_remove_all");
	    
	if (pfn_to_phys(pte->pfn) != pa)
	    panic("pmap_remove_all 2");
	    
	if (pte->wired)
	    panic("pmap_remove_all 3");
	
	/*
	 * Tell CPU using pmap to invalidate its TLB
	 */
	PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);

	if ((cur = pv_h->next) != PV_ENTRY_NULL) {
	    *pv_h = *cur;
	    zfree(pv_entry_zone, (vm_offset_t) cur);
	}
	else
	    pv_h->pmap = PMAP_NULL;	
	
	/*
	 * Collect the referenced & dirty bits
	 * and clear the mapping.
	 */
	for (i = ptes_per_vm_page; i-- > 0; pte++) {
	    if (pte->dirty) {
		vm_page_t	m = PHYS_TO_VM_PAGE(pa);
		
		vm_page_set_modified(m);
		pd->page_attrib |= PG_DIRTY;
	    }
	    
	    if (pte->refer)
		pd->page_attrib |= PG_REFER;
		
	    *pte = (pt_entry_t) { 0 };
	}
	
	pmap_deallocate_mappings(pmap, va, 1, 0, TRUE);
	
	simple_unlock(&pmap->lock);
    }
    
    PMAP_WRITE_UNLOCK(s);
}

/*
 * Remove write access to the
 * indicated page from all pmaps.
 */
void
pmap_copy_on_write(
    unsigned int	pa
)
{
    pt_entry_t		*pte;
    pv_entry_t		pv_e;
    int			i, s;

    if (!managed_page(pa))
	return;
	
    /*
     * Lock the entire pmap system, since we may be changing
     * several maps.
     */
    PMAP_WRITE_LOCK(s);
    
    pv_e = pg_desc_pvh(pg_desc(pa));
    if (pv_e->pmap == PMAP_NULL) {
	PMAP_WRITE_UNLOCK(s);
	return;
    }    

    /*
     * Run down the list of mappings to this physical page,
     * disabling write privileges on each one.
     */
    while (pv_e != PV_ENTRY_NULL) {
	pmap_t		pmap;
	vm_offset_t	va;

	pmap = pv_e->pmap;
	va = pv_e->va;

	simple_lock(&pmap->lock);
	
	pte = pmap_pt_entry(pmap, va);
	
	if (pte == PT_ENTRY_NULL || !pte->valid)
	    panic("pmap_copy_on_write");

	/*
	 * Ask cpus using pmap to invalidate their TLBs
	 */
	PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
	
	if (pte->prot == PT_PROT_URW || pte->prot == PT_PROT_KRW)
	    for (i = ptes_per_vm_page; i-- > 0; pte++)
		pte_prot(pmap, VM_PROT_READ, pte->valid, pte);
	    
	simple_unlock(&pmap->lock);
	
	pv_e = pv_e->next;
    }
    
    PMAP_WRITE_UNLOCK(s);
}

/*
 * Change the page protection
 * on a range of addresses in
 * the indicated pmap.  If protect
 * is being changed to VM_PROT_NONE,
 * remove the mappings.
 */
void
pmap_protect(
    pmap_t		pmap,
    vm_offset_t		start,
    vm_offset_t		end,
    vm_prot_t		prot
)
{
    pt_entry_t		*pte, *epte;
    vm_offset_t		sect_end;
    int			i, s;
    
    if (pmap == PMAP_NULL)
	return;
	
    if (prot == VM_PROT_NONE) {
	pmap_remove(pmap, start, end);
	return;
    }
    
    SPLVM(s);
    simple_lock(&pmap->lock);

    /*
     * Invalidate the translation buffer first
     */
    PMAP_UPDATE_TLBS(pmap, start, end);    
    
    while (start < end) {
	sect_end = round_section(start + PAGE_SIZE);
	if (sect_end > end)
	    sect_end = end;
	    
	pte = pmap_pt_entry(pmap, start);
	if (pte != PT_ENTRY_NULL) {
	    epte = pmap_pt_entry(pmap, sect_end);
	    if (trunc_page(pte) != trunc_page(epte))
		epte = (pt_entry_t *)round_page(pte + ptes_per_vm_page);
		
	    while (pte < epte) {
		if (!pte->valid) {
		    pte += ptes_per_vm_page;
		    continue;
		}
		
		for (i = ptes_per_vm_page; i-- > 0; pte++)
		    pte_prot(pmap, prot, pte->valid, pte);
	    }
	}

	start = sect_end;
    }
    
    simple_unlock(&pmap->lock);
    SPLX(s);
}

/*
 * Insert the given physical page (p) at
 * the specified virtual address (v) in the
 * target physical map with the protection requested.
 *
 * If specified, the page will be wired down, meaning
 * that the related pte can not be reclaimed.
 *
 * NB:  This is the only routine which MAY NOT lazy-evaluate
 * or lose information.  That is, this routine must actually
 * insert this page into the given map NOW.
 */
void inline
pmap_enter_cache_spec(
    pmap_t		pmap,
    vm_offset_t		va,
    vm_offset_t		pa,
    vm_prot_t		prot,
    boolean_t		wired,
    cache_spec_t	caching
)
{
    pt_entry_t		*pte;
    pv_entry_t		pv_h;
    pg_desc_t		*pd;
    int			i, s;
    pv_entry_t		pv_e;
    pt_entry_t		template;
    vm_offset_t		old_pa;
    
    if (pmap == PMAP_NULL)
	return;
	
    if (prot == VM_PROT_NONE) {
	pmap_remove(pmap, va, va + PAGE_SIZE);
	return;
    }

    /*
     * Must allocate a new pvlist entry while we're unlocked;
     * zalloc may cause pageout (which will lock the pmap system).
     * If we determine we need a pvlist entry, we will unlock
     * and allocate one.  Then we will retry, throwing away
     * the allocated entry later (if we no longer need it).
     */
    pv_e = PV_ENTRY_NULL;
    template = (pt_entry_t) { 0 };

Retry:
    PMAP_READ_LOCK(pmap, s);

    /*
     * Expand pmap to include this pte.  Assume that
     * pmap is always expanded to include enough
     * pages to map one VM page.
     */
    while ((pte = pmap_pt_entry(pmap, va)) == PT_ENTRY_NULL) {
	/*
	 * Must unlock to expand the pmap.
	 */
	PMAP_READ_UNLOCK(pmap, s);

	pmap_expand(pmap, va);

	PMAP_READ_LOCK(pmap, s);
    }

    /*
     * Special case if the physical page is already mapped
     * at this address.
     */
    old_pa = pfn_to_phys(pte->pfn);
    if (pte->valid && old_pa == pa) {
	/*
	 * May be changing its wired attribute or protection
	 */
	if (wired && !pte->wired)
	    pmap_wire_mapping(pmap, va);
	else
	if (!wired && pte->wired)
	    pmap_unwire_mapping(pmap, va);

	pte_prot(pmap, prot, 1, &template);
	template.pfn = phys_to_pfn(pa);
	if (wired)
	    template.wired = 1;
	    
	if (caching == cache_disable)
	    template.cachedis = 1;
	else
	if (caching == cache_writethrough)
	    template.cachewrt = 1;

	PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
	
	for (i = ptes_per_vm_page; i-- > 0; pte++) {
	    if (pte->dirty)
		template.dirty = 1;
	    *pte = template;
	    template.pfn++;
	}
    }
    else {

	/*
	 * Remove old mapping from the PV list if necessary.
	 */
	if (pte->valid) {
	    /*
	     * Invalidate the translation buffer,
	     * then remove the mapping.
	     */
	    PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
	    
	    pmap_remove_range(pmap, va, va + PAGE_SIZE, FALSE);
	}
	
	if (managed_page(pa)) {

	    /*
	     * Enter the mapping in the PV list for this
	     * physical page.
	     */
	    pd = pg_desc(pa);
	    LOCK_PVH(pd);
	    pv_h = pg_desc_pvh(pd);

	    if (pv_h->pmap == PMAP_NULL) {

		/*
		 * No mappings yet
		 */
		pv_h->va = va;
		pv_h->pmap = pmap;
		pv_h->next = PV_ENTRY_NULL;
	    }
	    else {
		    
		/*
		 * Add new pv_entry after header.
		 */
		if (pv_e == PV_ENTRY_NULL) {
		    UNLOCK_PVH(pd);
		    PMAP_READ_UNLOCK(pmap, s);
		    pv_e = (pv_entry_t) zalloc(pv_entry_zone);
		    goto Retry;
		}
		pv_e->va = va;
		pv_e->pmap = pmap;
		pv_e->next = pv_h->next;
		pv_h->next = pv_e;
		/*
		 * Remember that we used the pvlist entry.
		 */
		pv_e = PV_ENTRY_NULL;
	    }
	    
	    UNLOCK_PVH(pd);
	}
	    
	pmap_allocate_mapping(pmap, va, wired);

	/*
	 * Build a template to speed up entering -
	 * only the pfn changes.
	 */
	pte_prot(pmap, prot, 1, &template);
	template.pfn = phys_to_pfn(pa);
	if (wired)
	    template.wired = 1;
	    
	if (caching == cache_disable)
	    template.cachedis = 1;
	else
	if (caching == cache_writethrough)
	    template.cachewrt = 1;
	
	for (i = ptes_per_vm_page; i-- > 0; pte++) {
	    *pte = template;
	    template.pfn++;
	}
    }

    PMAP_READ_UNLOCK(pmap, s);

    if (pv_e != PV_ENTRY_NULL)
	zfree(pv_entry_zone, (vm_offset_t) pv_e);
}

void
pmap_enter(
    pmap_t		pmap,
    vm_offset_t		va,
    vm_offset_t		pa,
    vm_prot_t		prot,
    boolean_t		wired
)
{
    pmap_enter_cache_spec(
			pmap,
			va,
			pa,
			prot,
			wired,
			cache_default);
}

void
pmap_enter_shared_range(
    pmap_t		pmap,
    vm_offset_t		va,
    vm_size_t		size,
    vm_offset_t		kern
)
{
    vm_offset_t		end = round_page(va + size);
    
    while (va < end) {
    	pmap_enter(
		    pmap,
		    va,
		    pmap_resident_extract(kernel_pmap, kern),
		    VM_PROT_READ|VM_PROT_WRITE,
		    TRUE);
		    
	va += PAGE_SIZE; kern += PAGE_SIZE;
    }
}

/*
 * Change the wiring attribute for a
 * pmap/virtual-address	pair.
 *
 * The mapping must already exist in the pmap.
 */
void
pmap_change_wiring(
    pmap_t		pmap,
    vm_offset_t		va,
    boolean_t		wired
)
{
    pt_entry_t		*pte;
    int			i, s;
    
    /*
     * We must grab the pmap system lock because we may
     * change a pte_page queue.
     */
    PMAP_READ_LOCK(pmap, s);
    
    if ((pte = pmap_pt_entry(pmap, va)) == PT_ENTRY_NULL)
	panic("pmap_change_wiring");

    if (wired && !pte->wired) {
	/*
	 * wiring down mapping
	 */
	pmap_wire_mapping(pmap, va);
    }
    else
    if (!wired && pte->wired) {
	/*
	 * unwiring mapping
	 */
	pmap_unwire_mapping(pmap, va);
    }
    
    for (i = ptes_per_vm_page; i-- > 0; pte++)
	pte->wired = wired;

    PMAP_READ_UNLOCK(map, s);
}

vm_offset_t
pmap_extract(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    vm_offset_t		pa;
    int			s;
    
    SPLVM(s);
    simple_lock(&pmap->lock);
    
    pa = pmap_phys(pmap, va);
    
    simple_unlock(&pmap->lock);
    SPLX(s);
    
    return (pa);
}

vm_offset_t
pmap_resident_extract(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    return ((vm_offset_t) pmap_phys(pmap, va));
}

/*
 * Expand a pmap to be able to map the
 * specified virtual address by allocating
 * a single page table either from the list 
 * of free page tables or directly from kernel
 * memory.
 */
static
void
pmap_expand(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    pd_entry_t		template, *pde;
    pg_exten_t		*pe;
    vm_offset_t		pt;
    int			i, s;

    if (pmap == kernel_pmap)
	panic("pmap_expand");
	    
    if (pe = pt_free_obtain())
	pt = pe->pg_desc->pv_list.va;
    else {
	if (kmem_alloc_wired(kernel_map, &pt, PAGE_SIZE) != KERN_SUCCESS)
	    return;

	pt_alloc_count++;
	
	pe = pg_exten_alloc(pt);
    }

    PMAP_READ_LOCK(pmap, s);	

    /*
     * See if someone else expanded us first.
     */
    if (pmap_pt_entry(pmap, va) != PT_ENTRY_NULL) {
	PMAP_READ_UNLOCK(pmap, s);
	
	pg_exten_free(pe);

	kmem_free(kernel_map, pt, PAGE_SIZE);
	
	pt_alloc_count--;

	return;
    }

    /*
     * What virtual memory does this
     * page table map ?
     */
    pe->offset = trunc_section(va);
    pe->pmap = pmap;

    /*
     * Clear the reference aging
     * tick count.
     */
    pe->unrefd_age = 0;

    /*
     * Add this page table
     * to the active queue.
     */	
    pt_active_add(pe);

    /*
     * Setup entry template.
     */
    template = (pd_entry_t) { 0 };
    template.valid = 1;
    template.prot = PT_PROT_URW;
    template.pfn = phys_to_pfn(pe->phys);

    /*
     * Set the page directory entries for this page table
     */
    pde = pmap_pd_entry(pmap, pe->offset);
    for (i = ptes_per_vm_page; i-- > 0; pde++) {
	*pde = template;
	template.pfn++;
    }
    
    PMAP_READ_UNLOCK(pmap, s);
}

/*
 * Allocate one additional mapping
 * in a page table.  Note: pmap is
 * already locked.
 */
static
void
pmap_allocate_mapping(
    pmap_t		pmap,
    vm_offset_t		va,
    boolean_t		wired
)
{
    pd_entry_t		*pde;
    pg_exten_t		*pe;
    
    pmap->stats.resident_count++;
    if (wired)
    	pmap->stats.wired_count++;
    
    if (pmap == kernel_pmap)
	return;
	
    pde = pmap_pd_entry(pmap, trunc_section(va));
    if (pde->valid) {
	pe = pg_desc(pfn_to_phys(pde->pfn))->pg_exten;
	pe->alloc_count++;
	if (wired)
	    pe->wired_count++;
    }
}

/*
 * Deallocate one or more mappings
 * in a page table.  The page table
 * is freed if no mappings remain.
 * Note: pmap is already locked.
 */
static
void
pmap_deallocate_mappings(
    pmap_t		pmap,
    vm_offset_t		va,
    int			count,
    int			unwire_count,
    boolean_t		free_table
)
{
    pd_entry_t		*pde;
    pg_exten_t		*pe;
    int			i;
    
    pmap->stats.wired_count -= unwire_count;
    pmap->stats.resident_count -= count;

    if (pmap == kernel_pmap)
	return;
	
    pde = pmap_pd_entry(pmap, trunc_section(va));
    if (pde->valid) {	
	pe = pg_desc(pfn_to_phys(pde->pfn))->pg_exten;
	if (pe->wired_count < unwire_count)
	    panic("pmap_deallocate_mappings unwire");
	pe->wired_count -= unwire_count;
	if (pe->alloc_count < count)
	    panic("pmap_deallocate_mappings");
	pe->alloc_count -= count;

	if (free_table && pe->alloc_count == 0) {
	    for (i = ptes_per_vm_page; i-- > 0; pde++)
		pde->valid = 0;
		
	    pt_active_remove(pe);
	    
	    pt_free_add(pe);
	}
    }
}

static
void
pmap_wire_mapping(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    pd_entry_t		*pde;
    pg_exten_t		*pe;
    
    pmap->stats.wired_count++;
    
    if (pmap == kernel_pmap)
	return;
	
    pde = pmap_pd_entry(pmap, trunc_section(va));
    if (pde->valid) {
	pe = pg_desc(pfn_to_phys(pde->pfn))->pg_exten;
	pe->wired_count++;
    }
}

static
void
pmap_unwire_mapping(
    pmap_t		pmap,
    vm_offset_t		va
)
{
    pd_entry_t		*pde;
    pg_exten_t		*pe;
    
    pmap->stats.wired_count--;
    
    if (pmap == kernel_pmap)
	return;
	
    pde = pmap_pd_entry(pmap, trunc_section(va));
    if (pde->valid) {
	pe = pg_desc(pfn_to_phys(pde->pfn))->pg_exten;
	pe->wired_count--;
    }
}

/*
 * Copy the range specified by src_addr/len
 * from the source map to the range dst_addr/len
 * in the destination map.
 *
 * This routine is only advisory and need not do anything.
 */
void
pmap_copy(
    pmap_t		dst_pmap,
    pmap_t		src_pmap,
    vm_offset_t		dst_addr,
    vm_size_t		len,
    vm_offset_t		src_addr
)
{
    /* OPTIONAL */
}

/*
 * Require that all active physical maps contain no
 * incorrect entries NOW.  [This update includes
 * forcing updates of any address map caching.]
 *
 * Generally used to insure that a thread about
 * to run will see a semantically correct world.
 */
void
pmap_update(
    void
)
{
    pg_exten_t		*pe, *npe;
    vm_offset_t		page;
    pd_entry_t		*pde;
    int			tick_delta;
    static unsigned int	last_tick;

    if (!last_tick)
    	last_tick = sched_tick;	/* initialization */

    tick_delta = sched_tick - last_tick;

    if (tick_delta > 1) {
	/*
	 * Free all of the pages in
	 * the free page table list.
	 */	
	while (pe = pt_free_obtain()) {
	    page = pe->pg_desc->pv_list.va;
    
	    pg_exten_free(pe);
    
	    kmem_free(kernel_map, page, PAGE_SIZE);
	    pt_alloc_count--;
	}

	/*
	 * Run through the page directory
	 * free list, freeing any that are
	 * not in use.
	 */
	(queue_entry_t)pe = queue_first(&pd_free_queue);
	while (!queue_end(&pd_free_queue, (queue_entry_t)pe)) {
	    if (pe->alloc_count == 0) {
		remqueue(&pd_free_queue, (queue_entry_t)pe);
		
		pd_free_count--;
		
		page = pe->pg_desc->pv_list.va;
		
		pg_exten_free(pe);
		
		kmem_free(kernel_map, page, PAGE_SIZE);
		pd_alloc_count--;
		
		(queue_entry_t)pe = queue_first(&pd_free_queue);
	    }
	    else
		(queue_entry_t)pe = queue_next((queue_entry_t)pe);
	}
    }

    /*
     * Age the active page tables.  Remove
     * all of the mappings from the tables
     * which have not been used for address
     * translation recently (excluding those
     * with wired mappings).  Removing all of
     * a table's mappings causes the table to
     * be removed from the pmap and added to
     * the free page table list.
     */
    {
	static int	pt_aging_k[] = { 8, 12, 16, 24, 32 },
			pt_aging_n = sizeof pt_aging_k / sizeof pt_aging_k[0];
	int		tick_age;
	
	tick_age = ((tick_age = tick_delta >> 3) > (pt_aging_n - 1)) ?
						pt_aging_k[pt_aging_n - 1] :
						pt_aging_k[tick_age];	    

	(queue_entry_t)pe = queue_first(&pt_active_queue);
	while (!queue_end(&pt_active_queue, (queue_entry_t)pe)) {
	    pde = pmap_pd_entry(pe->pmap, pe->offset);
	    if (pe->wired_count == 0 && pde->valid) {
		(queue_entry_t)npe = queue_next((queue_entry_t)pe);
    
		if (tick_delta > 0 && !pde->refer) {
		    int		old_age = pe->unrefd_age;
    
		    pe->unrefd_age += tick_delta;
		    if (old_age > pe->unrefd_age || pe->unrefd_age > tick_age)
			pmap_remove(
				pe->pmap,
				pe->offset,
				pe->offset + section_size);
		}
		else if (pde->refer) {
		    pde->refer = FALSE;
		    pe->unrefd_age = 0;
		}
		    
		pe = npe;
	    }
	    else {
		pe->unrefd_age = 0;
		(queue_entry_t)pe = queue_next((queue_entry_t)pe);
	    }
	}
    }

    last_tick += tick_delta;
}

/*
 * Garbage collects the physical map system for
 * pages which are no longer used.
 * Success need not be guaranteed -- that is, there
 * may well be pages which are not referenced, but
 * others may be collected.
 *
 * Called by the pageout daemon when pages are scarce.
 */
void
pmap_collect(
    pmap_t		pmap
)
{
    /* OPTIONAL */
}

/*
 * Bind the given pmap to the given
 * processor.
 */
void
pmap_activate(
    pmap_t		pmap,
    thread_t		thread,
    int			cpu
)
{
    PMAP_ACTIVATE(pmap, thread, cpu);
}


/*
 * Indicates that the given physical map is no longer
 * in use on the specified processor.
 */
void
pmap_deactivate(
    pmap_t		pmap,
    thread_t		thread,
    int			cpu
)
{
    PMAP_DEACTIVATE(pmap, thread, cpu);
}

/*
 * Return the pmap handle for the kernel.
 */
pmap_t
pmap_kernel(
    void
)
{
    return (kernel_pmap);
}

/*
 * pmap_zero_page zeros the specified (machine independent)
 * page.
 */
void
pmap_zero_page(
    vm_offset_t		pa
)
{
    page_set(pmap_phys_to_kern(pa), 0, PAGE_SIZE);
}

/*
 * pmap_copy_page copies the specified (machine independent)
 * pages.
 */
void
pmap_copy_page(
    vm_offset_t		src,
    vm_offset_t		dst
)
{
    page_copy(pmap_phys_to_kern(dst), pmap_phys_to_kern(src), PAGE_SIZE);
}

void
copy_to_phys(
    vm_offset_t		src,
    vm_offset_t		dst,
    vm_size_t		count
)
{
    bcopy(src, pmap_phys_to_kern(dst), count);
}

void
copy_from_phys(
    vm_offset_t		src,
    vm_offset_t		dst,
    vm_size_t		count
)
{
    bcopy(pmap_phys_to_kern(src), dst, count);
}

/*
 * Make the specified pages (by pmap, offset)
 * pageable (or not) as requested.
 *
 * A page which is not pageable may not take
 * a fault; therefore, its page table entry
 * must remain valid for the duration.
 *
 * This routine is merely advisory; pmap_enter
 * will specify that these pages are to be wired
 * down (or not) as appropriate.
 */
pmap_pageable(
    pmap_t		pmap,
    vm_offset_t		start,
    vm_offset_t		end,
    boolean_t		pageable
)
{
    /* OPTIONAL */
}

kern_return_t
pmap_attribute(
    pmap_t			pmap,
    vm_offset_t			address,
    vm_size_t			size,
    vm_machine_attribute_t	attribute,
    vm_machine_attribute_val_t	*value
)
{
	kern_return_t ret;

	if (attribute != MATTR_CACHE)
		return KERN_INVALID_ARGUMENT;

	/*
	** We can't get the caching attribute for more than one page
	** at a time
	*/
	if ((*value == MATTR_VAL_GET) &&
	    (trunc_page(address) != trunc_page(address+size-1)))
		return KERN_INVALID_ARGUMENT;

	if (pmap == PMAP_NULL)
		return KERN_SUCCESS;

	ret = KERN_SUCCESS;

        simple_lock(&pmap->lock);

	switch (*value) {
	case MATTR_VAL_CACHE_FLUSH:     /* flush from all caches */
	case MATTR_VAL_DCACHE_FLUSH:    /* flush from data cache(s) */
	case MATTR_VAL_ICACHE_FLUSH:    /* flush from instr cache(s) */
		break;

	case MATTR_VAL_GET:             /* return current value */
	case MATTR_VAL_OFF:             /* turn attribute off */
	case MATTR_VAL_ON:              /* turn attribute on */
	default:
		ret = KERN_INVALID_ARGUMENT;
		break;
	}
	simple_unlock(&pmap->lock);

	return ret;

}

/*
 * Clear the modify bits on the specified physical page.
 */
void
pmap_clear_modify(
    vm_offset_t		pa
)
{
    if (managed_page(pa))
	pmap_clear_page_attrib(pa, PG_DIRTY);
}

/*
 * Return whether or not the specified physical page is modified
 * by any pmaps.
 */
boolean_t
pmap_is_modified(
    vm_offset_t		pa
)
{
    if (managed_page(pa))
	return (pmap_check_page_attrib(pa, PG_DIRTY));
    else
	return (FALSE);
}

/*
 * Clear the reference bit on the specified physical page.
 */
void
pmap_clear_reference(
    vm_offset_t		pa
)
{
    if (managed_page(pa))
	pmap_clear_page_attrib(pa, PG_REFER);
}

/*
 * Return whether or not the specified physical page is referenced
 * by any physical maps.
 */
boolean_t
pmap_is_referenced(
    vm_offset_t		pa
)
{
    if (managed_page(pa))
	return (pmap_check_page_attrib(pa, PG_REFER));
    else
	return (FALSE);
}

/*
 * Clear the specified page attributes both in the
 * pmap_page_attributes table and the address translation
 * tables.  Note that we DO have to flush the entries from
 * the TLB since the processor uses the bits in the TLB to
 * determine whether it has to write the bits out to memory.
 */
static
void
pmap_clear_page_attrib(
    vm_offset_t			pa,
    int				attrib
)
{
    pt_entry_t			*pte;
    pv_entry_t			pv_h;
    pmap_t			pmap;
    vm_offset_t			va;
    pg_desc_t			*pd;
    int				i, s;

    pd = pg_desc(pa);

    PMAP_WRITE_LOCK(s);

    pd->page_attrib &= ~attrib;

    pv_h = pg_desc_pvh(pd);

    while ((pmap = pv_h->pmap) != PMAP_NULL) {
	va = pv_h->va;

	simple_lock(&pmap->lock);

	PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);

	pte = pmap_pt_entry(pmap, va);
	if (pte != PT_ENTRY_NULL) {
	    for (i = ptes_per_vm_page; i-- > 0; pte++) {
		if (attrib & PG_DIRTY)
		    pte->dirty = 0;
		if (attrib & PG_REFER)
		    pte->refer = 0;
	    }
	}

	simple_unlock(&pmap->lock);

	if ((pv_h = pv_h->next) == PV_ENTRY_NULL)
	    break;
    }

    PMAP_WRITE_UNLOCK(s);
}

/*
 * Check for the specified attributes for the
 * physical page.  if all bits are true in
 * the pmap_page_attributes table, we can trust
 * it.  otherwise, we must check the address
 * translation tables ourselves.  Note that we
 * DO NOT have to flush the entry from the TLB
 * before looking at the address translation
 * table since the TLB is write-through for the bits.
 */
static
boolean_t
pmap_check_page_attrib(
    vm_offset_t			pa,
    int				attrib
)
{
    pt_entry_t			*pte;
    pv_entry_t			pv_h;
    pmap_t			pmap;
    vm_offset_t			va;
    pg_desc_t			*pd;
    int				i, s;

    pd = pg_desc(pa);

    if ((pd->page_attrib & attrib) == attrib)
	return (TRUE);

    pv_h = pg_desc_pvh(pd);

    PMAP_WRITE_LOCK(s);

    while ((pmap = pv_h->pmap) != PMAP_NULL) {
	va = pv_h->va;

	simple_lock(&pmap->lock);

	pte = pmap_pt_entry(pmap, va);
	if (pte != PT_ENTRY_NULL) {
	    for (i = ptes_per_vm_page; i-- > 0; pte++) {
		if (pte->dirty)
		    pd->page_attrib |= PG_DIRTY;
		if (pte->refer)
		    pd->page_attrib |= PG_REFER;
	    }

	    if ((pd->page_attrib & attrib) == attrib) {
		simple_unlock(&pmap->lock);

		PMAP_WRITE_UNLOCK(s);

		return (TRUE);
	    }
	}

	simple_unlock(&pmap->lock);

	if ((pv_h = pv_h->next) == PV_ENTRY_NULL)
	    break;
    }

    PMAP_WRITE_UNLOCK(s);

    return (FALSE);
}

/*
 * Dummy routine to satisfy external reference.
 */
void
pmap_update_interrupt(
    void
)
{
    /* should never be called. */
    panic("pmap_update_interrupt");
}

/*
 * Lower the permission for all mappings to a given page.
 */
void
pmap_page_protect(
    vm_offset_t			pa,
    vm_prot_t			prot
)
{
    switch (prot) {
	case VM_PROT_READ:
	case VM_PROT_READ|VM_PROT_EXECUTE:
	    pmap_copy_on_write(pa);
	    break;

	case VM_PROT_ALL:
	    break;

	default:
	    pmap_remove_all(pa);
	    break;
    }
}