Return to pmap.c CVS log

Up to [Apple Darwin 0.x] / kernel / machdep / ppc

Annotation of kernel/machdep/ppc/pmap.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
        !             3:  *
        !             4:  * @APPLE_LICENSE_HEADER_START@
        !             5:  * 
        !             6:  * Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
        !             7:  * Reserved.  This file contains Original Code and/or Modifications of
        !             8:  * Original Code as defined in and that are subject to the Apple Public
        !             9:  * Source License Version 1.1 (the "License").  You may not use this file
        !            10:  * except in compliance with the License.  Please obtain a copy of the
        !            11:  * License at http://www.apple.com/publicsource and read it before using
        !            12:  * this file.
        !            13:  * 
        !            14:  * The Original Code and all software distributed under the License are
        !            15:  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
        !            16:  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
        !            17:  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
        !            18:  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
        !            19:  * License for the specific language governing rights and limitations
        !            20:  * under the License.
        !            21:  * 
        !            22:  * @APPLE_LICENSE_HEADER_END@
        !            23:  */
        !            24: 
        !            25: /*
        !            26:  * Mach Operating System
        !            27:  * Copyright (c) 1990,1991,1992 The University of Utah and
        !            28:  * the Center for Software Science (CSS).
        !            29:  * Copyright (c) 1991,1987 Carnegie Mellon University.
        !            30:  * All rights reserved.
        !            31:  *
        !            32:  * Permission to use, copy, modify and distribute this software and its
        !            33:  * documentation is hereby granted, provided that both the copyright
        !            34:  * notice and this permission notice appear in all copies of the
        !            35:  * software, derivative works or modified versions, and any portions
        !            36:  * thereof, and that both notices appear in supporting documentation,
        !            37:  * and that all advertising materials mentioning features or use of
        !            38:  * this software display the following acknowledgement: ``This product
        !            39:  * includes software developed by the Center for Software Science at
        !            40:  * the University of Utah.''
        !            41:  *
        !            42:  * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSS ALLOW FREE USE OF
        !            43:  * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
        !            44:  * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
        !            45:  * THIS SOFTWARE.
        !            46:  *
        !            47:  * CSS requests users of this software to return to [email protected] any
        !            48:  * improvements that they make and grant CSS redistribution rights.
        !            49:  *
        !            50:  * Carnegie Mellon requests users of this software to return to
        !            51:  *  Software Distribution Coordinator  or  [email protected]
        !            52:  *  School of Computer Science
        !            53:  *  Carnegie Mellon University
        !            54:  *  Pittsburgh PA 15213-3890
        !            55:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            56:  * the rights to redistribute these changes.
        !            57:  *
        !            58:  *     Utah $Hdr: pmap.c 1.28 92/06/23$
        !            59:  *     Author: Mike Hibler, Bob Wheeler, University of Utah CSS, 10/90
        !            60:  */
        !            61:  
        !            62: /*
        !            63:  *     Manages physical address maps for powerpc.
        !            64:  *
        !            65:  *     In addition to hardware address maps, this
        !            66:  *     module is called upon to provide software-use-only
        !            67:  *     maps which may or may not be stored in the same
        !            68:  *     form as hardware maps.  These pseudo-maps are
        !            69:  *     used to store intermediate results from copy
        !            70:  *     operations to and from address spaces.
        !            71:  *
        !            72:  *     Since the information managed by this module is
        !            73:  *     also stored by the logical address mapping module,
        !            74:  *     this module may throw away valid virtual-to-physical
        !            75:  *     mappings at almost any time.  However, invalidations
        !            76:  *     of virtual-to-physical mappings must be done as
        !            77:  *     requested.
        !            78:  *
        !            79:  *     In order to cope with hardware architectures which
        !            80:  *     make virtual-to-physical map invalidates expensive,
        !            81:  *     this module may delay invalidate or reduced protection
        !            82:  *     operations until such time as they are actually
        !            83:  *     necessary.  This module is given full information to
        !            84:  *     when physical maps must be made correct.
        !            85:  *     
        !            86:  */
        !            87: 
        !            88: /*
        !            89:  * CAVAETS:
        !            90:  *
        !            91:  *     Needs more work for MP support
        !            92:  */
        !            93: 
        !            94: #include <debug.h>
        !            95: #include <mach_vm_debug.h>
        !            96: 
        !            97: #include <kern/thread.h>
        !            98: #include <mach/vm_attributes.h>
        !            99: #include <mach/vm_param.h>
        !           100: #include <kernserv/machine/spl.h>
        !           101: 
        !           102: #include <machdep/ppc/proc_reg.h>
        !           103: #include <machdep/ppc/mem.h>
        !           104: #include <machdep/ppc/pmap.h>
        !           105: #include <machdep/ppc/pmap_internals.h>
        !           106: #include <machdep/ppc/powermac.h>
        !           107: 
        !           108: 
        !           109: #define NULL 0
        !           110: #define DPRINTF(x) if(0) {kprintf("%s : ", __FUNCTION__);kprintf x;}
        !           111: 
        !           112: /* forward */
        !           113: void pmap_activate(pmap_t pmap, thread_t th, int which_cpu);
        !           114: void pmap_deactivate(pmap_t pmap, thread_t th, int which_cpu);
        !           115: void copy_to_phys(vm_offset_t sva, vm_offset_t dpa, int bytecount);
        !           116: 
        !           117: static struct mapping *pmap_find_mapping(space_t space, vm_offset_t offset);
        !           118: 
        !           119: static void pmap_free_mapping(register struct mapping *mp);
        !           120: 
        !           121: static void pmap_reap_mappings(void);
        !           122: 
        !           123: static struct mapping *pmap_enter_mapping(pmap_t pmap,
        !           124:                                          space_t space,
        !           125:                                          vm_offset_t va,
        !           126:                                          vm_offset_t pa,
        !           127:                                          pte_t *pte,
        !           128:                                          unsigned prot,
        !           129:                                          struct phys_entry *pp);
        !           130: 
        !           131: #if DEBUG
        !           132: #define PDB_USER       0x01    /* exported functions */
        !           133: #define PDB_MAPPING    0x02    /* low-level mapping routines */
        !           134: #define PDB_ENTER      0x04    /* pmap_enter specifics */
        !           135: #define PDB_COPY       0x08    /* copy page debugging */
        !           136: #define PDB_ZERO       0x10    /* zero page debugging */
        !           137: #define PDB_WIRED      0x20    /* things concerning wired entries */
        !           138: #define PDB_PTEG       0x40    /* PTEG overflows */
        !           139: #define PDB_MASSIVE    0x80    /* Massive costly assert checks */
        !           140: #define PDB_IO         0x100   /* Improper use of WIMG_IO checks - PCI machines */
        !           141: int pmdebug = 0;
        !           142: 
        !           143: #define PCI_BASE       0x80000000
        !           144: #endif
        !           145: 
        !           146: struct pmap    kernel_pmap_store;
        !           147: pmap_t         kernel_pmap;
        !           148: struct zone    *pmap_zone;             /* zone of pmap structures */
        !           149: boolean_t      pmap_initialized = FALSE;
        !           150: 
        !           151: #define                HASH_TABLE_FACTLOG2     0
        !           152: int            hash_table_factlog2 = HASH_TABLE_FACTLOG2;
        !           153: 
        !           154: /*
        !           155:  * Physical-to-virtual translations are handled by inverted page table
        !           156:  * structures, phys_tables.  Multiple mappings of a single page are handled
        !           157:  * by linking the affected mapping structures. We initialise one region
        !           158:  * for phys_tables of the physical memory we know about, but more may be
        !           159:  * added as it is discovered (eg. by drivers).
        !           160:  */
        !           161: struct phys_entry *phys_table; /* For debugging */
        !           162: 
        !           163: /*
        !           164:  * XXX use mpqueue_head_t's??
        !           165:  * (no one else does, are they coming or going?)
        !           166:  */
        !           167: queue_head_t           free_mapping;      /* list of free mapping structs */
        !           168: decl_simple_lock_data(,free_mapping_lock) /* and lock */
        !           169: 
        !           170: pmap_t                 free_pmap;         /* list of free pmaps */
        !           171: decl_simple_lock_data(,free_pmap_lock)   /* and lock */
        !           172: 
        !           173: decl_simple_lock_data(,pmap_lock)         /* XXX this is all broken */
        !           174: 
        !           175: unsigned       prot_bits[8];
        !           176: 
        !           177: /*
        !           178:  * This is the master space ID counter, initially set to the kernel's
        !           179:  * SID. The counter is incremented (though not necessarily by one)
        !           180:  * whenever a new space ID is required. For PPC, the SID ranges from
        !           181:  * to 2**20 - 1.
        !           182:  */
        !           183: int            sid_counter=PPC_SID_KERNEL;     /* seed for SID generator */
        !           184: 
        !           185: 
        !           186: 
        !           187: /*
        !           188:  * pmap_find_physentry(pa)
        !           189:  *
        !           190:  * Function to get index into phys_table for a given physical address
        !           191:  */
        !           192: struct phys_entry *
        !           193: pmap_find_physentry(
        !           194:        vm_offset_t pa)
        !           195: {
        !           196:        int i;
        !           197:        struct phys_entry *entry;
        !           198: 
        !           199:        for (i = pmap_mem_regions_count-1; i >= 0; i--) {
        !           200:                if (pa < pmap_mem_regions[i].start)
        !           201:                        continue;
        !           202:                if (pa >= pmap_mem_regions[i].end)
        !           203:                {
        !           204:                        return PHYS_NULL;
        !           205:                }
        !           206:                
        !           207:                entry = &pmap_mem_regions[i].
        !           208:                    phys_table[(pa -
        !           209:                                pmap_mem_regions[i].start) >> PPC_PGSHIFT];
        !           210:                return entry;
        !           211:        }
        !           212:        DPRINTF(("NMGS DEBUG : pmap_find_physentry 0x%08x out of range\n",pa));
        !           213:        return (struct phys_entry *)PHYS_NULL;
        !           214: }
        !           215: 
        !           216: 
        !           217: 
        !           218: /*
        !           219:  * kern_return_t
        !           220:  * pmap_add_physical_memory(vm_offset_t spa, vm_offset_t epa,
        !           221:  *                          boolean_t available, unsigned int attr)
        !           222:  *     Allocate some extra physentries for the physical addresses given,
        !           223:  *     specifying some default attribute that on the powerpc specifies
        !           224:  *      the default cachability for any mappings using these addresses
        !           225:  *     If the memory is marked as available, it is added to the general
        !           226:  *     VM pool, otherwise it is not (it is reserved for card IO etc).
        !           227:  */
        !           228: kern_return_t
        !           229: pmap_add_physical_memory(
        !           230:        vm_offset_t spa,
        !           231:        vm_offset_t epa,
        !           232:        boolean_t available,
        !           233:        unsigned int attr)
        !           234: {
        !           235:        int i,j;
        !           236:        spl_t s;
        !           237:        struct phys_entry *phys_table;
        !           238: 
        !           239:        /* Only map whole pages */
        !           240: 
        !           241:        spa = trunc_page(spa);
        !           242:        epa = round_page(epa);
        !           243: 
        !           244:        /* First check that the region doesn't already exist */
        !           245: 
        !           246:        assert (epa >= spa);
        !           247:        for (i = 0; i < pmap_mem_regions_count; i++) {
        !           248:                /* If we're below the next region, then no conflict */
        !           249:                if (epa <= pmap_mem_regions[i].start)
        !           250:                        break;
        !           251:                if (spa < pmap_mem_regions[i].end) {
        !           252: #if DEBUG
        !           253:                        DPRINTF(("(0x%08x,0x%08x,0x%08x) - memory already present\n",spa,epa,attr));
        !           254: #endif /* DEBUG */
        !           255:                        return KERN_NO_SPACE;
        !           256:                }
        !           257:        }
        !           258: 
        !           259:        /* Check that we've got enough space for another region */
        !           260:        if (pmap_mem_regions_count == MEM_REGION_MAX)
        !           261:                return KERN_RESOURCE_SHORTAGE;
        !           262: 
        !           263:        /* Once here, i points to the mem_region above ours in physical mem */
        !           264: 
        !           265:        /* allocate a new phys_table for this new region */
        !           266: 
        !           267:        phys_table =  (struct phys_entry *)
        !           268:                kalloc(sizeof(struct phys_entry) * atop(epa-spa));
        !           269: 
        !           270:        /* Initialise the new phys_table entries */
        !           271:        for (j = 0; j < atop(epa-spa); j++) {
        !           272:                queue_init(&phys_table[j].phys_link);
        !           273:                /* We currently only support these two attributes */
        !           274:                assert((attr == PTE_WIMG_DEFAULT) ||
        !           275:                       (attr == PTE_WIMG_IO));
        !           276:                phys_table[j].pte1.bits.wimg = attr;
        !           277:        }
        !           278:        s = splhigh();
        !           279:        
        !           280:        /* Move all the phys_table entries up some to make room in
        !           281:         * the ordered list.
        !           282:         */
        !           283:        for (j = pmap_mem_regions_count; j > i ; j--)
        !           284:                pmap_mem_regions[j] = pmap_mem_regions[j-1];
        !           285: 
        !           286:        /* Insert a new entry with some memory to back it */
        !           287: 
        !           288:        pmap_mem_regions[i].start            = spa;
        !           289:        pmap_mem_regions[i].end           = epa;
        !           290:        pmap_mem_regions[i].phys_table    = phys_table;
        !           291: 
        !           292:        pmap_mem_regions_count++;
        !           293:        splx(s);
        !           294: 
        !           295:        if (available) {
        !           296:                DPRINTF(("warning : pmap_add_physical_mem() "
        !           297:                       "available not yet supported\n"));
        !           298:        }
        !           299: 
        !           300:        return KERN_SUCCESS;
        !           301: }
        !           302: 
        !           303: 
        !           304: 
        !           305: /*
        !           306:  * ppc_protection_init()
        !           307:  *     Initialise the user/kern_prot_codes[] arrays which are 
        !           308:  *     used to translate machine independent protection codes
        !           309:  *     to powerpc protection codes. The PowerPc can only provide
        !           310:  *      [no rights, read-only, read-write]. Read implies execute.
        !           311:  * See PowerPC 601 User's Manual Table 6.9
        !           312:  */
        !           313: void
        !           314: ppc_protection_init(void)
        !           315: {
        !           316:        prot_bits[VM_PROT_NONE | VM_PROT_NONE  | VM_PROT_NONE]     = 0;
        !           317:        prot_bits[VM_PROT_READ | VM_PROT_NONE  | VM_PROT_NONE]     = 3;
        !           318:        prot_bits[VM_PROT_READ | VM_PROT_NONE  | VM_PROT_EXECUTE]  = 3;
        !           319:        prot_bits[VM_PROT_NONE | VM_PROT_NONE  | VM_PROT_EXECUTE]  = 3;
        !           320:        prot_bits[VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE]     = 2;
        !           321:        prot_bits[VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE]     = 2;
        !           322:        prot_bits[VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE]  = 2;
        !           323:        prot_bits[VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE]  = 2;
        !           324: }
        !           325: 
        !           326: 
        !           327: 
        !           328: /*
        !           329:  * pmap_map(va, spa, epa, prot)
        !           330:  *     is called during boot to map memory in the kernel's address map.
        !           331:  *     A virtual address range starting at "va" is mapped to the physical
        !           332:  *     address range "spa" to "epa" with machine independent protection
        !           333:  *     "prot".
        !           334:  *
        !           335:  *     "va", "spa", and "epa" are byte addresses and must be on machine
        !           336:  *     independent page boundaries.
        !           337:  */
        !           338: vm_offset_t
        !           339: pmap_map(
        !           340:        vm_offset_t va,
        !           341:        vm_offset_t spa,
        !           342:        vm_offset_t epa,
        !           343:        vm_prot_t prot)
        !           344: {
        !           345: 
        !           346:        if (spa == epa)
        !           347:                return(va);
        !           348: 
        !           349:        assert(epa > spa);
        !           350: 
        !           351:        while (spa < epa) {
        !           352:                pmap_enter(kernel_pmap, va, spa, prot, TRUE);
        !           353: 
        !           354:                va += PAGE_SIZE;
        !           355:                spa += PAGE_SIZE;
        !           356:        }
        !           357:        return(va);
        !           358: }
        !           359: 
        !           360: 
        !           361: 
        !           362: /*
        !           363:  * pmap_map_bd(va, spa, epa, prot)
        !           364:  *     Back-door routine for mapping kernel VM at initialisation.
        !           365:  *     Used for mapping memory outside the known physical memory
        !           366:  *      space, with caching disabled. Designed for use by device probes.
        !           367:  * 
        !           368:  *     A virtual address range starting at "va" is mapped to the physical
        !           369:  *     address range "spa" to "epa" with machine independent protection
        !           370:  *     "prot".
        !           371:  *
        !           372:  *     "va", "spa", and "epa" are byte addresses and must be on machine
        !           373:  *     independent page boundaries.
        !           374:  *
        !           375:  * WARNING: The current version of memcpy() can use the dcbz instruction
        !           376:  * on the destination addresses.  This will cause an alignment exception
        !           377:  * and consequent overhead if the destination is caching-disabled.  So
        !           378:  * avoid memcpy()ing into the memory mapped by this function.
        !           379:  *
        !           380:  * also, many other pmap_ routines will misbehave if you try and change
        !           381:  * protections or remove these mappings, they are designed to be permanent.
        !           382:  */
        !           383: vm_offset_t
        !           384: pmap_map_bd(
        !           385:        vm_offset_t va,
        !           386:        vm_offset_t spa,
        !           387:        vm_offset_t epa,
        !           388:        vm_prot_t prot)
        !           389: {
        !           390:        pte_t  *pte;
        !           391:        struct phys_entry* pp;
        !           392:        struct mapping *mp;
        !           393:        spl_t s;
        !           394: 
        !           395:        if (spa == epa)
        !           396:                return(va);
        !           397: 
        !           398:        assert(epa > spa);
        !           399: 
        !           400: #if DEBUG
        !           401:        DPRINTF(("va=0x%08X, spa=0x%08X, epa=0x%08X, prot=0x%x\n\n",va,spa,epa,prot));
        !           402: #endif
        !           403:        s = splhigh();
        !           404:        while (spa < epa) {
        !           405:                
        !           406:                pte = find_or_allocate_pte(PPC_SID_KERNEL, va, TRUE);
        !           407: 
        !           408:                assert(pte != PTE_NULL);
        !           409: 
        !           410:                /* remapping of already-mapped addresses is OK, we just
        !           411:                 * trample on the PTE, but make sure we're mapped to same
        !           412:                 * address
        !           413:                 */
        !           414:                if (pte->pte0.bits.valid)
        !           415:                        assert(trunc_page(pte->pte1.word) == spa);
        !           416: 
        !           417:                /*      Perform a general case PTE update designed to work
        !           418:                        in SMP configurations. TODO - locks and tlbsync
        !           419:                */
        !           420:                assert(pte->pte0.bits.valid == TRUE);
        !           421:                pte->pte0.bits.valid = FALSE;   /* Invalidate pte entry */
        !           422:                sync();                 /* Force updates to complete */
        !           423:                tlbie(va);              /* Wipe out this virtual address */
        !           424:                eieio();                /* Enforce ordering of v bit */
        !           425:                tlbsync();
        !           426:                sync();
        !           427: 
        !           428:                /*      Allowable early PTE updates: RPN,R,C,WIMG,PP
        !           429:                 */
        !           430:                pte->pte1.bits.protection = prot_bits[prot];
        !           431:                pte->pte1.bits.phys_page = spa >> PPC_PGSHIFT;
        !           432:                pte->pte1.bits.wimg       = PTE_WIMG_IO;
        !           433:                eieio();
        !           434: 
        !           435:                /*      Post tlbie updates: VSID,H,API,V
        !           436:                */
        !           437:                pte->pte0.bits.valid = TRUE;    /* Validate pte entry */
        !           438:                sync();
        !           439: 
        !           440: 
        !           441:                /* If memory is mappable (is in phys table) then set
        !           442:                 * default cach attributes to non-cached and verify
        !           443:                 * that there aren't any non-cached mappings which would
        !           444:                 * break the processor spec (and hang the machine).
        !           445:                 */
        !           446:                pp = pmap_find_physentry(spa);
        !           447:                if (pp != PHYS_NULL) {
        !           448:                        /* Set the default mapping type */
        !           449:                        pp->pte1.bits.wimg = PTE_WIMG_IO;
        !           450: #if DEBUG
        !           451:                        /* Belt and braces check on mappings */
        !           452:                        queue_iterate(&pp->phys_link, mp,
        !           453:                                      struct mapping *, phys_link) {
        !           454:                                assert(mp->pte->pte1.bits.wimg ==
        !           455:                                       PTE_WIMG_IO);
        !           456:                        }
        !           457: #endif /* DEBUG */
        !           458:                }
        !           459:                va += PAGE_SIZE;
        !           460:                spa += PAGE_SIZE;
        !           461:        }
        !           462:        splx(s);
        !           463:        return(va);
        !           464: }
        !           465: 
        !           466: 
        !           467: 
        !           468: /*
        !           469:  *     Bootstrap the system enough to run with virtual memory.
        !           470:  *     Map the kernel's code and data, and allocate the system page table.
        !           471:  *     Called with mapping done by BATs. Page_size must already be set.
        !           472:  *
        !           473:  *     Parameters:
        !           474:  *     first_avail     PA of address where we can allocate structures.
        !           475:  */
        !           476: void
        !           477: pmap_bootstrap(
        !           478:        unsigned int mem_size,
        !           479:        vm_offset_t *first_avail)
        !           480: {
        !           481:        struct mapping          *mp;
        !           482:        vm_offset_t             struct_addr;
        !           483:        vm_size_t               struct_size, hash_table_align, phys_table_size,
        !           484:                                        mapping_table_size;
        !           485:        unsigned int            mapping_table_num, phys_pages_num;
        !           486:        int                     i;
        !           487: 
        !           488:        *first_avail = round_page(*first_avail);
        !           489: 
        !           490:        assert(PAGE_SIZE == PPC_PGBYTES);
        !           491: 
        !           492:        ppc_protection_init();
        !           493: 
        !           494:        /*
        !           495:         * Initialize kernel pmap
        !           496:         */
        !           497:        kernel_pmap = &kernel_pmap_store;
        !           498: #if    NCPUS > 1
        !           499:        lock_init(&pmap_lock, FALSE, ETAP_VM_PMAP_SYS, ETAP_VM_PMAP_SYS_I);
        !           500: #endif /* NCPUS > 1 */
        !           501:        simple_lock_init(&kernel_pmap->lock);
        !           502:        simple_lock_init(&free_pmap_lock);
        !           503: 
        !           504:        kernel_pmap->ref_count = 1;
        !           505:        kernel_pmap->space = PPC_SID_KERNEL;
        !           506:        kernel_pmap->next = PMAP_NULL;
        !           507: 
        !           508:        /*
        !           509:         * Allocate: (from first_avail up)
        !           510:         *      aligned to its own size:
        !           511:          *        hash table (for mem size 2**x, allocate 2**(x-10) entries)
        !           512:         *      physical page entries (1 per physical page)
        !           513:         *      physical -> virtual mappings (for multi phys->virt mappings)
        !           514:         */
        !           515: 
        !           516:        /*
        !           517:         * Table 7-21 on page 7-52 of the PowerPC Programming
        !           518:         * Environments book (32-bit) doesn't tell you how to
        !           519:         * size the hashed page table for strange memory sizes
        !           520:         * (i.e. not a power of 2).  It has been empirically
        !           521:         * determined that splitting the difference and rounding
        !           522:         * can be used effectively in these circumstances.
        !           523:         */
        !           524:        {
        !           525:            vm_size_t   rounded_mem_size;
        !           526:            int         factor;
        !           527: 
        !           528:            /*
        !           529:             * The minimum size for the hash table is 64K
        !           530:             * and is bounded by a memory size of 8MB.
        !           531:             */
        !           532:            hash_table_size = 64*1024; rounded_mem_size = 8*1024*1024;
        !           533:            while (hash_table_size < 32*1024*104 &&
        !           534:                   rounded_mem_size < mem_size) {
        !           535:                hash_table_size *= 2; rounded_mem_size *= 2;
        !           536:            }
        !           537: 
        !           538:            /*
        !           539:             * If we need to allocate more than a minimum
        !           540:             * sized hash table and our memory size is not
        !           541:             * a power of 2
        !           542:             *
        !           543:             * AND
        !           544:             *
        !           545:             * if we are less than half way to next higher
        !           546:             * power of 2, then use the next lower value.
        !           547:             */
        !           548:            if (hash_table_size > 64*1024 && rounded_mem_size != mem_size) {
        !           549:                if (
        !           550:                    /*
        !           551:                     * Special case for mem_size
        !           552:                     * greater than 2G since
        !           553:                     * rounded_mem_size is zero
        !           554:                     * in this case.
        !           555:                     */
        !           556:                    (mem_size > 2*1024*1024*1024UL &&
        !           557:                     mem_size < 3*1024*1024*1024UL)
        !           558:                    ||
        !           559:                    (mem_size < ((rounded_mem_size / 2) +
        !           560:                                 (rounded_mem_size / 4)))
        !           561:                    ) hash_table_size /= 2;
        !           562:            }
        !           563: 
        !           564: #ifdef notdef
        !           565:            printf("rounded_mem_size %x\n", rounded_mem_size);
        !           566: #endif
        !           567: 
        !           568:            /*
        !           569:             * Determine the value of any additional
        !           570:             * factor to apply to the size of the
        !           571:             * hash table.
        !           572:             */
        !           573:            if (hash_table_factlog2 < 0)
        !           574:                factor = -1;
        !           575:            else
        !           576:                factor = 1;
        !           577:            
        !           578:            hash_table_factlog2 *= factor;
        !           579: 
        !           580:            while (hash_table_factlog2 > 0) {
        !           581:                factor *= 2; hash_table_factlog2--;
        !           582:            }
        !           583: 
        !           584:            if (factor > 0)
        !           585:                printf("hash table factor is %d\n", factor);
        !           586:            else
        !           587:                printf("hash table factor is 1/%d\n", -factor);
        !           588: 
        !           589:            /*
        !           590:             * Apply the factor, and catch out
        !           591:             * of bound sizes.
        !           592:             */
        !           593:            if (factor > 1) {
        !           594:                hash_table_size *= factor;
        !           595: 
        !           596:                if (hash_table_size < 64*1024)
        !           597:                    hash_table_size = 64*1024;
        !           598:            }
        !           599:            else if (factor < -1) {
        !           600:                hash_table_size /= -factor;
        !           601: 
        !           602:                if (hash_table_size > 32*1024*1024 || hash_table_size == 0)
        !           603:                    hash_table_size = 32*1024*1024;
        !           604:            }
        !           605: 
        !           606:            /*
        !           607:             * Notice a bit of handwaving here: since we
        !           608:             * use a BAT to statically map the kernel,
        !           609:             * the hashed page table and the ancillary
        !           610:             * mapping structures, using mem_size is a
        !           611:             * bit generous.  However, we do need to allow
        !           612:             * for some extra mappings for I/O regions, so
        !           613:             * we'll just call it even.
        !           614:             */
        !           615:        }
        !           616: 
        !           617:        /* HASH TABLE MUST BE aligned to its size */
        !           618: 
        !           619:        struct_addr = (*first_avail + (hash_table_size - 1)) &~
        !           620:                                                (hash_table_size - 1);
        !           621: 
        !           622:        hash_table_align = struct_addr - *first_avail;
        !           623: 
        !           624:        if (round_page(*first_avail) + PPC_PGBYTES < round_page(struct_addr)) {
        !           625:                free_regions[free_regions_count].start = 
        !           626:                        round_page(*first_avail) + PPC_PGBYTES;
        !           627:                free_regions[free_regions_count].end = round_page(struct_addr);
        !           628:                free_regions_count++;
        !           629:        }
        !           630: 
        !           631:        phys_pages_num = atop(mem_size);
        !           632: 
        !           633:        phys_table_size = sizeof(struct phys_entry) * phys_pages_num;
        !           634:        mapping_table_size =
        !           635:            sizeof(struct mapping) * (hash_table_size/sizeof(pte_t));
        !           636: 
        !           637:        /* size of all structures that we're going to allocate */
        !           638: 
        !           639:        struct_size = (vm_size_t) (hash_table_size +
        !           640:                                   phys_table_size +
        !           641:                                   mapping_table_size);
        !           642: 
        !           643: #ifdef notdef
        !           644:        printf("pmap static allocation:\n");
        !           645:        printf("hash table: %x (alignment) + %x (size) +\n",
        !           646:               hash_table_align, hash_table_size);
        !           647:        printf("ptov table: %x (phys entries) + %x (mappings) =\n",
        !           648:               phys_table_size, mapping_table_size);
        !           649:        printf("total: %x\n", hash_table_align + struct_size);
        !           650: #endif
        !           651: 
        !           652:        struct_size = round_page(struct_size);
        !           653: 
        !           654:        /* Zero everything - this also invalidates the hash table entries */
        !           655:        bzero((char *)struct_addr, struct_size);
        !           656: 
        !           657:        /* Set up some pointers to our new structures */
        !           658:        
        !           659:        /*
        !           660:         * Set up hash table address, keeping alignment. These
        !           661:         * mappings are 1-1.
        !           662:         */
        !           663:        hash_table_base = struct_addr;
        !           664:        struct_addr += hash_table_size;
        !           665: 
        !           666:        /*
        !           667:         * phys_table is static to help debugging,
        !           668:         * this variable is no longer actually used
        !           669:         * outside of this scope
        !           670:         */
        !           671:        phys_table = (struct phys_entry *) struct_addr;
        !           672: 
        !           673:        for (i = 0; i < pmap_mem_regions_count; i++) {
        !           674:                pmap_mem_regions[i].phys_table = phys_table;
        !           675:                phys_table = phys_table +
        !           676:                        atop(pmap_mem_regions[i].end -
        !           677:                             pmap_mem_regions[i].start);
        !           678:        }
        !           679: 
        !           680:        /* restore phys_table for debug */
        !           681:        phys_table = (struct phys_entry *) struct_addr;
        !           682:        struct_addr += phys_table_size;
        !           683: 
        !           684:        /* Initialise the registers necessary for supporting the hashtable */
        !           685:        hash_table_init(hash_table_base, hash_table_size);
        !           686: 
        !           687:        /* Initialise the physical table mappings */ 
        !           688:        for (i = 0; i < phys_pages_num; i++) {
        !           689:                queue_init(&phys_table[i].phys_link);
        !           690:                phys_table[i].pte1.bits.wimg = PTE_WIMG_DEFAULT;
        !           691:        }
        !           692: 
        !           693:        /*
        !           694:         * Remaining space is for mapping entries.  Chain them
        !           695:         * together (XXX can't use a zone
        !           696:         * since zone package hasn't been initialized yet).
        !           697:         */
        !           698:        mapping_table_num = mapping_table_size /
        !           699:                sizeof(struct mapping);
        !           700: 
        !           701:        mp = (struct mapping *) struct_addr;
        !           702: 
        !           703:        queue_init(&free_mapping);
        !           704:        simple_lock_init(&free_mapping_lock);
        !           705: 
        !           706:        while (mapping_table_num-- > 0) {
        !           707:                queue_enter(&free_mapping, mp, struct mapping *, phys_link);
        !           708:                mp++;
        !           709:        }
        !           710: 
        !           711:        *first_avail = round_page(mp);
        !           712: 
        !           713:        /* All the rest of memory is free - add it to the free
        !           714:         * regions so that it can be allocated by pmap_steal
        !           715:         */
        !           716:        free_regions[free_regions_count].start = *first_avail;
        !           717:        free_regions[free_regions_count].end = pmap_mem_regions[0].end;
        !           718:        free_regions_count++;
        !           719: }
        !           720: 
        !           721: 
        !           722: 
        !           723: /*
        !           724:  * pmap_init()
        !           725:  *     finishes the initialization of the pmap module.
        !           726:  *     This procedure is called from vm_mem_init() in vm/vm_init.c
        !           727:  *     to initialize any remaining data structures that the pmap module
        !           728:  *     needs to map virtual memory (VM is already ON).
        !           729:  */
        !           730: /*ARGSUSED*/
        !           731: void
        !           732: pmap_init(void)
        !           733: {
        !           734:        vm_size_t s;
        !           735: 
        !           736:        s = sizeof(struct pmap);
        !           737:        pmap_zone = zinit(s, 400*s, 4096, FALSE, "pmap");       /* XXX */
        !           738: 
        !           739:        pmap_initialized = TRUE;
        !           740: }
        !           741: 
        !           742: 
        !           743: /*
        !           744:  * pmap_create
        !           745:  *
        !           746:  * Create and return a physical map.
        !           747:  *
        !           748:  * If the size specified for the map is zero, the map is an actual physical
        !           749:  * map, and may be referenced by the hardware.
        !           750:  *
        !           751:  * If the size specified is non-zero, the map will be used in software 
        !           752:  * only, and is bounded by that size.
        !           753:  */
        !           754: pmap_t
        !           755: pmap_create(
        !           756:        vm_size_t size)
        !           757: {
        !           758:        pmap_t pmap;
        !           759:        spl_t s;
        !           760: 
        !           761: #if DEBUG
        !           762:        if (pmdebug & PDB_USER)
        !           763:                DPRINTF(("(size=%x)%c", size, size ? '\n' : ' '));
        !           764: #endif
        !           765: 
        !           766:        /*
        !           767:         * A software use-only map doesn't even need a pmap structure.
        !           768:         */
        !           769:        if (size)
        !           770:                return(PMAP_NULL);
        !           771: 
        !           772:        /* 
        !           773:         * If there is a pmap in the pmap free list, reuse it. 
        !           774:         */
        !           775:        s = splhigh();
        !           776:        simple_lock(&free_pmap_lock);
        !           777:        if (free_pmap != PMAP_NULL) {
        !           778:                pmap = free_pmap;
        !           779:                free_pmap = pmap->next;
        !           780:        } else
        !           781:                pmap = PMAP_NULL;
        !           782:        simple_unlock(&free_pmap_lock);
        !           783: 
        !           784:        /*
        !           785:         * Couldn't find a pmap on the free list, try to allocate a new one.
        !           786:         */
        !           787:        if (pmap == PMAP_NULL) {
        !           788:                pmap = (pmap_t) zalloc(pmap_zone);
        !           789:                if (pmap == PMAP_NULL)
        !           790:                {
        !           791:                        splx(s);
        !           792:                        return(PMAP_NULL);
        !           793:                }
        !           794: 
        !           795:                /*
        !           796:                 * Allocate space IDs for the pmap.
        !           797:                 * If all are allocated, there is nothing we can do.
        !           798:                 */
        !           799: 
        !           800:                /* If sid_counter == MAX_SID, we've allocated
        !           801:                 * all the possible SIDs and they're all live,
        !           802:                 * we can't carry on, but this is HUGE, so don't
        !           803:                 * expect it in normal operation.
        !           804:                 */
        !           805:                assert(sid_counter != SID_MAX);
        !           806: 
        !           807:                /* Try to spread out the sid's through the possible
        !           808:                 * name space to improve the hashing heuristics
        !           809:                 */
        !           810: #warning Get a better VSID allocation algorithm from the AIX folks.
        !           811: /*             sid_counter = (sid_counter + PPC_SID_PRIME) & PPC_SID_MASK; */
        !           812:                sid_counter ++; /* Replaced by a simpler algorithm */
        !           813: 
        !           814: 
        !           815:                /* 
        !           816:                 * Initialize the sids
        !           817:                 */
        !           818:                pmap->space = sid_counter;
        !           819:                simple_lock_init(&pmap->lock);
        !           820:        }
        !           821:        pmap->ref_count = 1;
        !           822:        pmap->next = PMAP_NULL;
        !           823:        pmap->stats.resident_count = 0;
        !           824:        pmap->stats.wired_count = 0;
        !           825: #if DEBUG
        !           826:        if (pmdebug & PDB_USER)
        !           827:                DPRINTF(("-> %x, space id = %d\n", pmap, pmap->space));
        !           828: #endif
        !           829:        splx(s);
        !           830:        return(pmap);
        !           831: }
        !           832: 
        !           833: 
        !           834: 
        !           835: /* 
        !           836:  * pmap_destroy
        !           837:  * 
        !           838:  * Gives up a reference to the specified pmap.  When the reference count 
        !           839:  * reaches zero the pmap structure is added to the pmap free list.
        !           840:  *
        !           841:  * Should only be called if the map contains no valid mappings.
        !           842:  */
        !           843: void
        !           844: pmap_destroy(
        !           845:        pmap_t pmap)
        !           846: {
        !           847:        int ref_count;
        !           848:        spl_t s;
        !           849: 
        !           850: #if DEBUG
        !           851:        if (pmdebug & PDB_USER)
        !           852:                DPRINTF(("(pmap=%x)\n", pmap));
        !           853: #endif
        !           854: 
        !           855:        if (pmap == PMAP_NULL)
        !           856:                return;
        !           857: 
        !           858:        s = splhigh();
        !           859:        simple_lock(&pmap->lock);
        !           860:        ref_count = --pmap->ref_count;
        !           861:        simple_unlock(&pmap->lock);
        !           862: 
        !           863:        if (ref_count < 0)
        !           864:                panic("pmap_destroy(): ref_count < 0");
        !           865:        if (ref_count > 0)
        !           866:        {
        !           867:                splx(s);
        !           868:                return;
        !           869:        }
        !           870: 
        !           871: #if DEBUG
        !           872:        if (pmap->stats.resident_count != 0 || pmap->stats.wired_count != 0) {
        !           873:                kprintf("pmap_destroy: non_empty pmap\n");
        !           874:        }
        !           875: #endif /* DEBUG */
        !           876: 
        !           877:        /* 
        !           878:         * Add the pmap to the pmap free list. 
        !           879:         */
        !           880:        simple_lock(&free_pmap_lock);
        !           881:        pmap->next = free_pmap;
        !           882:        free_pmap = pmap;
        !           883:        simple_unlock(&free_pmap_lock);
        !           884:        splx(s);
        !           885: }
        !           886: 
        !           887: 
        !           888: 
        !           889: /*
        !           890:  * pmap_reference(pmap)
        !           891:  *
        !           892:  *     gains a reference to the specified pmap.
        !           893:  *
        !           894:  *     This is used to indicate that the pmap is in use by multiple
        !           895:  *     machine-independent maps and will prevent early deallocation
        !           896:  *     of the pmap.
        !           897:  */
        !           898: void
        !           899: pmap_reference(
        !           900:        pmap_t pmap)
        !           901: {
        !           902:        spl_t s;
        !           903: 
        !           904: #if DEBUG
        !           905:        if (pmdebug & PDB_USER)
        !           906:                DPRINTF(("(pmap=%x)\n", pmap));
        !           907: #endif
        !           908: 
        !           909:        if (pmap != PMAP_NULL) {
        !           910:                s = splhigh();
        !           911:                simple_lock(&pmap->lock);
        !           912:                pmap->ref_count++;
        !           913:                simple_unlock(&pmap->lock);
        !           914:                splx(s);
        !           915:        }
        !           916: }
        !           917: 
        !           918: /*
        !           919:  * pmap_remove(pmap, s, e)
        !           920:  *     unmaps all virtual addresses v in the virtual address
        !           921:  *     range determined by [s, e) and pmap.
        !           922:  *     s and e must be on machine independent page boundaries and
        !           923:  *     s must be less than or equal to e.
        !           924:  */
        !           925: void
        !           926: pmap_remove(
        !           927:        pmap_t pmap,
        !           928:        vm_offset_t sva,
        !           929:        vm_offset_t eva)
        !           930: {
        !           931:        space_t space;
        !           932:        struct mapping *mp;
        !           933:        spl_t s;
        !           934: 
        !           935: #if DEBUG
        !           936:        if (pmdebug & PDB_USER)
        !           937:                DPRINTF(("(pmap=%x, sva=%x, eva=%x)\n",
        !           938:                       pmap, sva, eva));
        !           939: #endif
        !           940: 
        !           941:        if (pmap == PMAP_NULL)
        !           942:                return;
        !           943: 
        !           944:        s = splhigh();
        !           945:        space = pmap->space;
        !           946: 
        !           947:        /* It is just possible that eva might have wrapped around to zero,
        !           948:         * and sometimes we get asked to liberate something of size zero
        !           949:         * even though it's dumb (eg. after zero length read_overwrites)
        !           950:         */
        !           951:        assert(eva >= sva);
        !           952: 
        !           953:        /* We liberate addresses from high to low, since the stack grows
        !           954:         * down. This means that we won't need to test addresses below
        !           955:         * the limit of stack growth
        !           956:         */
        !           957:        while ((pmap->stats.resident_count > 0) && (eva > sva)) {
        !           958:                eva -= PAGE_SIZE;
        !           959:                if ((mp = pmap_find_mapping(space, eva)) != MAPPING_NULL) {
        !           960:                        pmap->stats.resident_count--;
        !           961:                        pmap_free_mapping(mp);
        !           962:                }
        !           963:        }
        !           964:        splx(s);
        !           965: }
        !           966: 
        !           967: 
        !           968: 
        !           969: /*
        !           970:  *     Routine:
        !           971:  *             pmap_page_protect
        !           972:  *
        !           973:  *     Function:
        !           974:  *             Lower the permission for all mappings to a given page.
        !           975:  */
        !           976: void
        !           977: pmap_page_protect(
        !           978:        vm_offset_t pa,
        !           979:        vm_prot_t prot)
        !           980: {
        !           981:        register struct phys_entry *pp;
        !           982:        register struct mapping *mp;
        !           983:        register struct mapping *mp2;
        !           984:        unsigned pteprot;
        !           985:        boolean_t remove;
        !           986:        spl_t s;
        !           987: 
        !           988: #if DEBUG
        !           989:        if (pmdebug & PDB_USER)
        !           990:                DPRINTF(("(pa=%x, prot=%x)\n", pa, prot));
        !           991: #endif
        !           992: 
        !           993:        switch (prot) 
        !           994:        {
        !           995:        case VM_PROT_READ:
        !           996:        case VM_PROT_READ|VM_PROT_EXECUTE:
        !           997:                remove = FALSE;
        !           998:                break;
        !           999:        case VM_PROT_ALL:
        !          1000:                return;
        !          1001:        default:
        !          1002:                remove = TRUE;
        !          1003:                break;
        !          1004:        }
        !          1005: 
        !          1006:        pp = pmap_find_physentry(pa);
        !          1007:        if (pp == PHYS_NULL)
        !          1008:                return;
        !          1009: 
        !          1010:        s = splhigh();
        !          1011:        if (remove) 
        !          1012:        {
        !          1013:                mp2 = MAPPING_NULL;
        !          1014:                while (!queue_empty(&pp->phys_link)) 
        !          1015:                {
        !          1016:                        mp = (struct mapping *) queue_first(&pp->phys_link);
        !          1017:                        if (mp->vm_info.bits.phys) 
        !          1018:                        {
        !          1019:                                // Don't zap the phys page
        !          1020:                                mp2 = mp;
        !          1021:                                queue_remove(&pp->phys_link, mp2, struct mapping *, phys_link);
        !          1022:                        } else 
        !          1023:                        {
        !          1024:                                mp->pmap->stats.resident_count--;
        !          1025: #if    DEBUG
        !          1026:                                if (mp->vm_info.bits.wired) 
        !          1027:                                {
        !          1028:                                        DPRINTF((": removing WIRED page!!\n"));
        !          1029:                                }
        !          1030: #endif /* DEBUG */
        !          1031:                                pmap_free_mapping(mp);
        !          1032:                        }
        !          1033:                }
        !          1034: #if    DEBUG
        !          1035:                if (queue_empty(&pp->phys_link) && (mp2 == MAPPING_NULL))
        !          1036:                {
        !          1037:                        DPRINTF(("WHOOPS! removed and entire map\n"));
        !          1038:                }
        !          1039: #endif /* DEBUG */
        !          1040: 
        !          1041:                if (mp2 != MAPPING_NULL) 
        !          1042:                {
        !          1043:                    // Restore the phys page
        !          1044:                    queue_enter_first(&pp->phys_link, mp2, struct mapping *, phys_link);
        !          1045:                }
        !          1046:                splx(s);
        !          1047:                return;
        !          1048:        }
        !          1049:        
        !          1050:        /*
        !          1051:         * Modify mappings if necessary.
        !          1052:         */
        !          1053:        queue_iterate(&pp->phys_link, mp, struct mapping *, phys_link) 
        !          1054:        {
        !          1055:                /*      Compare the new protection bits with the old
        !          1056:                        one to see if anything needs to be changed.
        !          1057:                        Note: In mach 2.5, we must not reduce the 
        !          1058:                        protection status of the static logical mapping
        !          1059:                        (the phys page), but we *do* want to record RC
        !          1060:                        bits.
        !          1061:                */
        !          1062:                if (mp->vm_info.bits.phys) 
        !          1063:                {
        !          1064:                        // Phys page mapping - don't alter protection
        !          1065:                        // Do not merge RC bits into pp.
        !          1066:                        continue;
        !          1067:                }
        !          1068: 
        !          1069:                pteprot = prot_bits[prot];
        !          1070:                if ((mp->pte->pte1.bits.protection) != pteprot)
        !          1071:                {
        !          1072:                        /*
        !          1073:                         * Perform a sync to force saving
        !          1074:                         * of change/reference bits, followed by a
        !          1075:                         * fault and reload with the new protection.
        !          1076:                         */
        !          1077:                        assert(mp->pte->pte0.bits.valid == TRUE);
        !          1078:                        mp->pte->pte0.bits.valid        = FALSE;
        !          1079:                        sync();
        !          1080:                        tlbie(mp->vm_info.bits.page << PPC_PGSHIFT);
        !          1081:                        eieio();
        !          1082:                        tlbsync();
        !          1083:                        sync();
        !          1084: 
        !          1085:                        mp->pte->pte1.bits.protection = pteprot;
        !          1086:                        eieio();
        !          1087: 
        !          1088:                        mp->pte->pte0.bits.valid        = TRUE;
        !          1089:                        sync();
        !          1090:                }
        !          1091:                //pp->pte1.word |= mp->pte->pte1.word;
        !          1092:                pp->pte1.bits.protection |= mp->pte->pte1.bits.protection;
        !          1093:        }
        !          1094:        splx(s);
        !          1095: }
        !          1096: 
        !          1097: 
        !          1098: 
        !          1099: /*
        !          1100:  * pmap_protect(pmap, s, e, prot)
        !          1101:  *     changes the protection on all virtual addresses v in the 
        !          1102:  *     virtual address range determined by [s, e] and pmap to prot.
        !          1103:  *     s and e must be on machine independent page boundaries and
        !          1104:  *     s must be less than or equal to e.
        !          1105:  */
        !          1106: void
        !          1107: pmap_protect(
        !          1108:        pmap_t pmap,
        !          1109:        vm_offset_t sva, 
        !          1110:        vm_offset_t eva,
        !          1111:        vm_prot_t prot)
        !          1112: {
        !          1113:        register struct mapping *mp;
        !          1114:        pte_t  *pte;
        !          1115:        unsigned pteprot;
        !          1116:        space_t space;
        !          1117:        spl_t s;
        !          1118: 
        !          1119: #if DEBUG
        !          1120:        if (pmdebug & PDB_USER)
        !          1121:                DPRINTF(("(pmap=%x, sva=%x, eva=%x, prot=%x)\n",
        !          1122:                       pmap, sva, eva, prot));
        !          1123: #endif
        !          1124: 
        !          1125:        if (pmap == PMAP_NULL)
        !          1126:                return;
        !          1127: 
        !          1128:        assert(sva <= eva);
        !          1129: 
        !          1130:        if (prot == VM_PROT_NONE) {
        !          1131:                pmap_remove(pmap, sva, eva);
        !          1132:                return;
        !          1133:        }
        !          1134: #if MACH_30
        !          1135:        if (prot & VM_PROT_WRITE)
        !          1136:                return;
        !          1137: #endif
        !          1138: 
        !          1139:        s = splhigh();
        !          1140:        space = pmap->space;
        !          1141:        for ( ; sva < eva; sva += PAGE_SIZE) {
        !          1142:                mp = pmap_find_mapping(space, sva);
        !          1143:                if (mp == MAPPING_NULL)
        !          1144:                        continue;
        !          1145: #if DEBUG
        !          1146:                if (pmap != mp->pmap)
        !          1147:                        panic("protect: pmap mismatch");
        !          1148: #endif
        !          1149:                /*
        !          1150:                 * Determine if mapping is changing.
        !          1151:                 * If not, nothing to do.
        !          1152:                 */
        !          1153:                pte     = mp->pte;
        !          1154:                pteprot = prot_bits[prot];
        !          1155:                if (pte->pte1.bits.protection == pteprot)
        !          1156:                        continue;
        !          1157:                
        !          1158:                /*
        !          1159:                 * Purge the current TLB entry (if any) to force
        !          1160:                 * any modifications to changed/referenced bits, and so
        !          1161:                 * that future references will fault and reload with the
        !          1162:                 * new protection.
        !          1163:                 */
        !          1164: 
        !          1165:                /*      Perform a general case PTE update designed to work
        !          1166:                        in SMP configurations. TODO - locks
        !          1167:                */
        !          1168:                assert(pte->pte0.bits.valid == TRUE);
        !          1169:                pte->pte0.bits.valid = FALSE;   /* Invalidate pte entry */
        !          1170:                sync();                         /* Force updates to complete */
        !          1171:                tlbie(sva);
        !          1172:                eieio();
        !          1173:                tlbsync();
        !          1174:                sync();
        !          1175: 
        !          1176:                /*      Allowable early PTE updates: RPN,R,C,WIMG,PP
        !          1177:                */
        !          1178:                pte->pte1.bits.protection = pteprot;    /* Change just the protection */
        !          1179:                eieio();
        !          1180: 
        !          1181:                /*      Post tlbie updates: VSID,H,API,V
        !          1182:                */
        !          1183:                pte->pte0.bits.valid = TRUE;    /* Validate pte entry */
        !          1184:                sync();
        !          1185: 
        !          1186:        }
        !          1187:        splx(s);
        !          1188: }
        !          1189: 
        !          1190: 
        !          1191: 
        !          1192: void
        !          1193: pmap_enter_phys_page(
        !          1194:        vm_offset_t pa,
        !          1195:        vm_offset_t va)
        !          1196: {
        !          1197:     struct mapping *mp;
        !          1198: 
        !          1199:     pmap_enter(kernel_pmap, va, pa, VM_PROT_READ|VM_PROT_WRITE, TRUE);
        !          1200: 
        !          1201:     if (mp = pmap_find_mapping(kernel_pmap->space, va)) {
        !          1202:        mp->vm_info.bits.phys = TRUE;
        !          1203:     }
        !          1204: #if    DEBUG
        !          1205:     else
        !          1206:     {
        !          1207:        DPRINTF(("pmap_enter_phys_page: mp == NULL\n"));
        !          1208:        DPRINTF(("pmap_enter_phys_page: mp == NULL\n"));
        !          1209:        panic("pmap_enter_phys_page: can't find mapping entry");
        !          1210:     }
        !          1211: #endif /* DEBUG */
        !          1212: }
        !          1213: 
        !          1214: 
        !          1215: 
        !          1216: int pmap_enter_cnt = 0;
        !          1217: int pmap_enter_valid_cnt = 0;
        !          1218: int pmap_enter_invalid_cnt = 0;
        !          1219: /*
        !          1220:  * pmap_enter
        !          1221:  *
        !          1222:  * Create a translation for the virtual address (virt) to the physical
        !          1223:  * address (phys) in the pmap with the protection requested. If the
        !          1224:  * translation is wired then we can not allow a page fault to occur
        !          1225:  * for this mapping.
        !          1226:  *
        !          1227:  * NB: This is the only routine which MAY NOT lazy-evaluate
        !          1228:  *     or lose information.  That is, this routine must actually
        !          1229:  *     insert this page into the given map NOW.
        !          1230:  */
        !          1231: void
        !          1232: pmap_enter(
        !          1233:        pmap_t pmap,
        !          1234:        vm_offset_t va,
        !          1235:        vm_offset_t pa,
        !          1236:        vm_prot_t prot,
        !          1237:        boolean_t wired)
        !          1238: {
        !          1239:        register struct mapping *mp;
        !          1240:        struct phys_entry *pp,*old_pp;
        !          1241:        pte_t *pte;
        !          1242:        space_t space;
        !          1243:        boolean_t was_wired;
        !          1244:        spl_t s;
        !          1245: 
        !          1246: #if DEBUG
        !          1247:        if ((wired && (pmdebug & (PDB_WIRED))) ||
        !          1248:             ((pmdebug & (PDB_USER|PDB_ENTER)) == (PDB_USER | PDB_ENTER)))
        !          1249:                DPRINTF(("(pmap=%x, va=%x, pa=%x, prot=%x, wire=%x)\n", pmap, va, pa, prot, wired));
        !          1250:        pmap_enter_cnt++;
        !          1251: #endif
        !          1252:         if (pmap == PMAP_NULL)
        !          1253:                 return;
        !          1254: #if DEBUG
        !          1255:        if (prot == VM_PROT_NONE) {
        !          1256:                pmap_remove(pmap, va, va + PAGE_SIZE);
        !          1257:                return;
        !          1258:        }
        !          1259: #endif /* DEBUG */
        !          1260: 
        !          1261:        /* TODO NMGS - take lock on physentry? */
        !          1262:        pp = pmap_find_physentry(pa);
        !          1263:        if (pp == PHYS_NULL)
        !          1264:                return;
        !          1265: 
        !          1266:        /* Find the pte associated with the virtual address/space ID */
        !          1267:        space = pmap->space;
        !          1268: 
        !          1269:        /* TODO NMGS - take lock on pte?! */
        !          1270:        pte = find_or_allocate_pte(space,va,TRUE);
        !          1271: 
        !          1272:        assert(pte != PTE_NULL);
        !          1273:        /* Even if the pte is new, pte0 should be set up for the new mapping,
        !          1274:         * but with the valid bit set to FALSE
        !          1275:         */
        !          1276:        assert((pte->pte0.word & 0x7fffffff) != PTE_EMPTY);
        !          1277: 
        !          1278:        s = splhigh();
        !          1279:        if (pte->pte0.bits.valid == TRUE) {
        !          1280: 
        !          1281:                /* mapping is already valid, thus virtual address was
        !          1282:                 * already mapped somewhere...
        !          1283:                 */
        !          1284: 
        !          1285: #if    DEBUG
        !          1286:                pmap_enter_valid_cnt++;
        !          1287: #endif /* DEBUG */
        !          1288:                /*
        !          1289:                **  invalidate tlb for previous mapping 
        !          1290:                */
        !          1291:                pte->pte0.bits.valid = FALSE;/* Invalidate pte entry */
        !          1292:                sync();         /* Force updates to complete */
        !          1293:                tlbie(va);      /* Wipe out this virtual address */
        !          1294:                eieio();        /* Enforce ordering of v bit */
        !          1295:                tlbsync();
        !          1296:                sync();
        !          1297: 
        !          1298:                if (((physical_addr_t *) &pa)->bits.page_no !=
        !          1299:                    pte->pte1.bits.phys_page) {
        !          1300: 
        !          1301:                        /* The current mapping is to a different
        !          1302:                         * physical addr - release the mapping
        !          1303:                         * before mapping the new address. 
        !          1304:                         */
        !          1305: 
        !          1306: #if DEBUG
        !          1307:                        if ((pmdebug & (PDB_USER|PDB_ENTER)) == (PDB_USER|PDB_ENTER))
        !          1308:                                DPRINTF(("Virt 0x%08x was mapped to different address 0x%08x, changing mapping\n",va,((physical_addr_t *) &pa)->bits.page_no * PPC_PGBYTES));
        !          1309: #endif
        !          1310: 
        !          1311:                        /* Take advantage of the fact that pte1 can be
        !          1312:                         * considered as a pointer to an address in
        !          1313:                         * the physical page when looking for physentry.
        !          1314:                         */
        !          1315: 
        !          1316:                        old_pp = pmap_find_physentry(
        !          1317:                                             (vm_offset_t)(pte->pte1.word));
        !          1318: 
        !          1319:                        queue_iterate(&old_pp->phys_link, mp,
        !          1320:                                      struct mapping *, phys_link) {
        !          1321:                                if (mp->pte == pte)
        !          1322:                                        break;
        !          1323:                        }
        !          1324: 
        !          1325:                        /* pte entry must have good reverse mapping */
        !          1326:                        assert(mp != NULL);
        !          1327:                        assert(mp->pte == pte);
        !          1328:                        assert(((mp->vm_info.bits.page >> 10) & 0x3f) ==
        !          1329:                               mp->pte->pte0.bits.page_index);
        !          1330: 
        !          1331:                        /* Wired/not wired is taken care of later on
        !          1332:                         * in this function. We can remove wired mappings.
        !          1333:                         */
        !          1334: 
        !          1335:                        /* Do the equivalent of pmap_free_mapping followed
        !          1336:                         * by a pmap_enter_mapping, recycling the pte and
        !          1337:                         * the mapping
        !          1338:                         */
        !          1339: 
        !          1340:                        /* keep track of changed and referenced at old
        !          1341:                         * address
        !          1342:                         */
        !          1343:                        old_pp->pte1.word |= mp->pte->pte1.word;
        !          1344: 
        !          1345:                        queue_remove(&old_pp->phys_link, mp,
        !          1346:                                     struct mapping *, phys_link);
        !          1347: 
        !          1348:                        mp->vm_info.bits.page  = (va >> PPC_PGSHIFT);
        !          1349: 
        !          1350:                        assert(mp->pte == pte);
        !          1351: 
        !          1352:                        queue_enter_first(&pp->phys_link, mp,
        !          1353:                                          struct mapping *, phys_link);
        !          1354: 
        !          1355: 
        !          1356:                        assert(mp->pmap == pmap);
        !          1357: 
        !          1358:                        /* Set up pte as it would have been done by
        !          1359:                         * find_or_allocate_pte() + pmap_enter_mapping()
        !          1360:                         */
        !          1361: 
        !          1362:                        /*      Allowable early PTE updates: RPN,R,C,WIMG,PP
        !          1363:                        */
        !          1364:                        pte->pte1.word = 0;
        !          1365:                        pte->pte1.bits.phys_page  = pa >> PPC_PGSHIFT;
        !          1366:                        pte->pte1.bits.wimg       = pp->pte1.bits.wimg; /* default */
        !          1367: 
        !          1368:        
        !          1369:                        /*      Post tlbie updates: VSID,H,API,V
        !          1370:                        */
        !          1371:                        pte->pte0.bits.segment_id = (space << 4) | (va >> 28);
        !          1372:                        pte->pte0.bits.page_index = ((va_abbrev_t *)&va)->page_index;
        !          1373:                        eieio();
        !          1374: 
        !          1375: 
        !          1376:                } else {
        !          1377:                        /* The current mapping is the same as the one
        !          1378:                         * we're being asked to make, find the mapping
        !          1379:                         * structure that goes along with it.
        !          1380:                         */
        !          1381:                        queue_iterate(&pp->phys_link, mp,
        !          1382:                                      struct mapping *, phys_link) {
        !          1383:                                if (mp->pte == pte)
        !          1384:                                        break;
        !          1385:                        }
        !          1386:                        /* mapping struct must exist */
        !          1387:                        assert(mp != NULL);
        !          1388:                        assert(mp->pte == pte);
        !          1389: 
        !          1390: 
        !          1391: #if DEBUG
        !          1392: 
        !          1393: 
        !          1394:                        if ((wired && (pmdebug & (PDB_WIRED))) ||
        !          1395:                            ((pmdebug & (PDB_USER|PDB_ENTER)) == (PDB_USER|PDB_ENTER))) {
        !          1396:                                DPRINTF(("address already mapped, changing prots or wiring\n"));
        !          1397:                                if (pte->pte1.bits.protection !=
        !          1398:                                    prot_bits[prot])
        !          1399:                                        DPRINTF(("Changing PROTS\n"));
        !          1400:                                if (wired != mp->vm_info.bits.wired)
        !          1401:                                       DPRINTF(("changing WIRING\n"));
        !          1402:                        }
        !          1403: #endif
        !          1404:                }
        !          1405:        } else {
        !          1406: 
        !          1407: #if DEBUG
        !          1408:                pmap_enter_invalid_cnt++;
        !          1409: #endif /* DEBUG */
        !          1410: 
        !          1411:                /* There was no pte match, a new entry was created
        !          1412:                  * (marked as invalid). We make sure that a new mapping
        !          1413:                 * structure is allocated as it's partner.
        !          1414:                 */
        !          1415:                mp = MAPPING_NULL;
        !          1416:                assert(pte->pte0.word != PTE_EMPTY);
        !          1417:                sync();         /* Force updates to complete */
        !          1418:                tlbie(va);      /* Wipe out this virtual address */
        !          1419:                eieio();        /* Enforce ordering of v bit */
        !          1420:                tlbsync();
        !          1421:                sync();
        !          1422: 
        !          1423:                pte->pte1.bits.protection = prot_bits[prot];
        !          1424:                eieio();
        !          1425:        }
        !          1426: 
        !          1427:        /* Once here, pte contains a valid pointer to a structure
        !          1428:         * that we may use to map the virtual address (which may
        !          1429:         * already map this (and be valid) or something else (invalid)
        !          1430:         * and mp contains either MAPPING_NULL (new mapping) or
        !          1431:         * a pointer to the old mapping which may need it's
        !          1432:         * privileges modified.
        !          1433:         */
        !          1434: 
        !          1435:        /* set up the protection bits on new mapping
        !          1436:         */
        !          1437: 
        !          1438: 
        !          1439:        if (mp == MAPPING_NULL) {
        !          1440:                /*
        !          1441:                 * Mapping for this virtual address doesn't exist.
        !          1442:                 * Get a mapping structure and, and fill it in,
        !          1443:                 * updating the pte that we already have, and
        !          1444:                 * making it valid
        !          1445:                 */
        !          1446:                mp = pmap_enter_mapping(pmap, space, va, pa, pte, prot, pp);
        !          1447:                pmap->stats.resident_count++;
        !          1448:                was_wired = FALSE;/* indicate that page was never wired */
        !          1449:        } else {
        !          1450:                
        !          1451:                /*
        !          1452:                 * We are just changing the protection of a current mapping.
        !          1453:                 */
        !          1454:                was_wired = mp->vm_info.bits.wired;
        !          1455: 
        !          1456:                /*
        !          1457:                 * This is the pte that came in valid
        !          1458:                 */
        !          1459:                pte->pte1.bits.protection = prot_bits[prot];
        !          1460:                eieio();
        !          1461:                pte->pte0.bits.valid = TRUE;            /* !!! */
        !          1462:                sync();
        !          1463:        }
        !          1464: 
        !          1465:        assert(pte->pte0.bits.valid == TRUE);
        !          1466: 
        !          1467:        /* if wired status has changed, update stats and change bit */
        !          1468:        if (was_wired != wired) {
        !          1469:                pmap->stats.wired_count += wired ? 1 : -1;
        !          1470:                mp->vm_info.bits.wired = wired;
        !          1471: #if DEBUG
        !          1472:                if (pmdebug & (PDB_WIRED))
        !          1473:                        DPRINTF(("changing wired status to : %s\n",
        !          1474:                               wired ? "TRUE" : "FALSE"));
        !          1475: #endif /* DEBUG */
        !          1476:        }
        !          1477: 
        !          1478: 
        !          1479:        /* Belt and braces check to make sure we didn't give a bogus map,
        !          1480:         * this map is given once when mapping the kernel
        !          1481:         */
        !          1482:        assert((pte->pte1.bits.phys_page != 0) ||
        !          1483:               (kgdb_kernel_in_pmap == FALSE));
        !          1484:        
        !          1485: #if DEBUG
        !          1486: 
        !          1487:        /*      Assert a PTE of type WIMG_IO is not mapped to general RAM.
        !          1488:        */
        !          1489:        if (pmdebug & PDB_IO    &&
        !          1490:            pte->pte1.bits.wimg == PTE_WIMG_IO)
        !          1491:                assert (((unsigned) pa) >= PCI_BASE)
        !          1492: 
        !          1493:        /* Assert that there are no other mappings with different
        !          1494:         * cachability information, as this can freeze the machine
        !          1495:         */
        !          1496:        {
        !          1497:                struct mapping *mp2;
        !          1498:                queue_iterate(&pp->phys_link, mp2,
        !          1499:                              struct mapping *, phys_link) {
        !          1500:                        assert(mp2->pte->pte1.bits.wimg ==
        !          1501:                               pte->pte1.bits.wimg);
        !          1502:                }
        !          1503:        }
        !          1504: 
        !          1505:        if (pmdebug & PDB_MASSIVE) {
        !          1506:                register struct mapping *mp2;
        !          1507:                queue_iterate(&free_mapping, mp2,
        !          1508:                              struct mapping *, phys_link) {
        !          1509:                        assert(mp2->phys_link.next != NULL);
        !          1510:                }
        !          1511:        }
        !          1512: #endif /* DEBUG */     
        !          1513:        splx(s);
        !          1514: #if DEBUG
        !          1515:        if ((pmdebug & (PDB_USER|PDB_ENTER)) == (PDB_USER | PDB_ENTER))
        !          1516:                DPRINTF(("leaving pmap_enter\n"));
        !          1517: #endif
        !          1518: }
        !          1519: 
        !          1520: 
        !          1521: 
        !          1522: /*
        !          1523:  *     Routine:        pmap_change_wiring
        !          1524:  *     Function:       Change the wiring attribute for a map/virtual-address
        !          1525:  *                     pair.
        !          1526:  *     In/out conditions:
        !          1527:  *                     The mapping must already exist in the pmap.
        !          1528:  *
        !          1529:  * Change the wiring for a given virtual page. This routine currently is 
        !          1530:  * only used to unwire pages and hence the mapping entry will exist.
        !          1531:  */
        !          1532: void
        !          1533: pmap_change_wiring(
        !          1534:        register pmap_t pmap,
        !          1535:        vm_offset_t     va,
        !          1536:        boolean_t       wired)
        !          1537: {
        !          1538:        register struct mapping *mp;
        !          1539:        boolean_t waswired;
        !          1540:        spl_t s;
        !          1541: 
        !          1542: #if DEBUG
        !          1543:        if ((pmdebug & (PDB_USER | PDB_WIRED)) == (PDB_USER|PDB_WIRED))
        !          1544:                DPRINTF(("(pmap=%x, va=%x, wire=%x)\n",
        !          1545:                       pmap, va, wired));
        !          1546: #endif
        !          1547:        if (pmap == PMAP_NULL)
        !          1548:                return;
        !          1549: 
        !          1550:        s = splhigh();
        !          1551:        if ((mp = pmap_find_mapping(pmap->space, va)) == MAPPING_NULL)
        !          1552:                panic("pmap_change_wiring: can't find mapping entry");
        !          1553: 
        !          1554:        waswired = mp->vm_info.bits.wired;
        !          1555:        if (wired && !waswired) {
        !          1556:                mp->vm_info.bits.wired = TRUE;
        !          1557:                pmap->stats.wired_count++;
        !          1558:        } else if (!wired && waswired) {
        !          1559:                mp->vm_info.bits.wired = FALSE;
        !          1560:                pmap->stats.wired_count--;
        !          1561:        }
        !          1562:        splx(s);
        !          1563: }
        !          1564: 
        !          1565: 
        !          1566: 
        !          1567: /*
        !          1568:  * pmap_extract(pmap, va)
        !          1569:  *     returns the physical address corrsponding to the 
        !          1570:  *     virtual address specified by pmap and va if the
        !          1571:  *     virtual address is mapped and 0 if it is not.
        !          1572:  */
        !          1573: vm_offset_t
        !          1574: pmap_extract(
        !          1575:        pmap_t pmap,
        !          1576:        vm_offset_t va)
        !          1577: {
        !          1578:        pte_t *pte;
        !          1579:        spl_t s;
        !          1580: 
        !          1581: #if DEBUG
        !          1582:        if (pmdebug & PDB_USER)
        !          1583:                DPRINTF(("(pmap=%x, va=%x)\n", pmap, va));
        !          1584: #endif
        !          1585: 
        !          1586:        if (pmap == PMAP_NULL)
        !          1587:        {
        !          1588:                DPRINTF((": pmap == PMAP_NULL\n"));
        !          1589:                return 0;
        !          1590:        }
        !          1591: 
        !          1592:        if (pmap == kernel_pmap) {
        !          1593:            extern vm_offset_t virtual_avail;
        !          1594: 
        !          1595:            if (va < virtual_avail)
        !          1596:                return va;
        !          1597:        }
        !          1598: 
        !          1599:        s = splhigh();
        !          1600:        pte = find_or_allocate_pte(pmap->space, trunc_page(va), FALSE);
        !          1601:        if (pte == NULL)
        !          1602:        {
        !          1603:                splx(s);
        !          1604:                return(0);
        !          1605:        }
        !          1606:        splx(s);
        !          1607:        return trunc_page(pte->pte1.word) | (va & page_mask);
        !          1608: }
        !          1609: 
        !          1610: 
        !          1611: 
        !          1612: /*
        !          1613:  *     pmap_attributes:
        !          1614:  *
        !          1615:  *     Set/Get special memory attributes
        !          1616:  *
        !          1617:  */
        !          1618: kern_return_t
        !          1619: pmap_attribute(
        !          1620:        pmap_t          pmap,
        !          1621:        vm_offset_t     address,
        !          1622:        vm_size_t       size,
        !          1623:        vm_machine_attribute_t  attribute,
        !          1624:        vm_machine_attribute_val_t* value)      
        !          1625: {
        !          1626:        vm_offset_t     sva, eva;
        !          1627:        vm_offset_t     addr;
        !          1628:        kern_return_t   ret;
        !          1629: 
        !          1630:        if (attribute != MATTR_CACHE)
        !          1631:                return KERN_INVALID_ARGUMENT;
        !          1632: 
        !          1633:        /* We can't get the caching attribute for more than one page
        !          1634:         * at a time
        !          1635:         */
        !          1636:        if ((*value == MATTR_VAL_GET) &&
        !          1637:            (trunc_page(address) != trunc_page(address+size-1)))
        !          1638:                return KERN_INVALID_ARGUMENT;
        !          1639: 
        !          1640:        if (pmap == PMAP_NULL)
        !          1641:                return KERN_SUCCESS;
        !          1642: 
        !          1643:        sva = trunc_page(address);
        !          1644:        eva = round_page(address + size);
        !          1645:        ret = KERN_SUCCESS;
        !          1646: 
        !          1647:        simple_lock(&pmap->lock);
        !          1648: 
        !          1649:        switch (*value) {
        !          1650:        case MATTR_VAL_CACHE_FLUSH:     /* flush from all caches */
        !          1651:        case MATTR_VAL_DCACHE_FLUSH:    /* flush from data cache(s) */
        !          1652:        case MATTR_VAL_ICACHE_FLUSH:    /* flush from instr cache(s) */
        !          1653:                sva = trunc_page(sva);
        !          1654:                for (; sva < eva; sva += PAGE_SIZE) {
        !          1655:                        if ((addr = pmap_extract(pmap, sva)) == 0)
        !          1656:                                continue;
        !          1657:                        flush_cache(addr, PAGE_SIZE);
        !          1658:                }
        !          1659:                break;
        !          1660: 
        !          1661:        case MATTR_VAL_GET:             /* return current value */
        !          1662:        case MATTR_VAL_OFF:             /* turn attribute off */
        !          1663:        case MATTR_VAL_ON:              /* turn attribute on */
        !          1664:        default:
        !          1665:                ret = KERN_INVALID_ARGUMENT;
        !          1666:                break;
        !          1667:        }
        !          1668:        simple_unlock(&pmap->lock);
        !          1669: 
        !          1670:        return ret;
        !          1671: }
        !          1672: 
        !          1673: 
        !          1674: 
        !          1675: /*
        !          1676:  * pmap_collect
        !          1677:  * 
        !          1678:  * Garbage collects the physical map system for pages that are no longer used.
        !          1679:  */
        !          1680: /* ARGSUSED */
        !          1681: void
        !          1682: pmap_collect(
        !          1683:        pmap_t pmap)
        !          1684: {
        !          1685: #if DEBUG
        !          1686:        if (pmdebug & PDB_USER)
        !          1687:                DPRINTF(("(pmap=%x)\n", pmap));
        !          1688: #endif
        !          1689: }
        !          1690: 
        !          1691: 
        !          1692: 
        !          1693: /*
        !          1694:  *     Routine:        pmap_activate
        !          1695:  *     Function:
        !          1696:  *             Binds the given physical map to the given
        !          1697:  *             processor, and returns a hardware map description.
        !          1698:  */
        !          1699: /* ARGSUSED */
        !          1700: void
        !          1701: pmap_activate(
        !          1702:        pmap_t pmap,
        !          1703:        thread_t th,
        !          1704:        int which_cpu)
        !          1705: {
        !          1706: #if DEBUG
        !          1707:        if (pmdebug & PDB_USER)
        !          1708:                DPRINTF(("(pmap=%x, th=%x, cpu=%x)\n",
        !          1709:                       pmap, th, which_cpu));
        !          1710: #endif
        !          1711: }
        !          1712: 
        !          1713: 
        !          1714: 
        !          1715: /* ARGSUSED */
        !          1716: void
        !          1717: pmap_deactivate(
        !          1718:        pmap_t pmap,
        !          1719:        thread_t th,
        !          1720:        int which_cpu)
        !          1721: {
        !          1722: #if DEBUG
        !          1723:        if (pmdebug & PDB_USER)
        !          1724:                DPRINTF(("(pmap=%x, th=%x, cpu=%x)\n",
        !          1725:                       pmap, th, which_cpu));
        !          1726: #endif
        !          1727: }
        !          1728: 
        !          1729: 
        !          1730: 
        !          1731: #if DEBUG
        !          1732: 
        !          1733: /*
        !          1734:  * pmap_zero_page
        !          1735:  * pmap_copy_page
        !          1736:  * 
        !          1737:  * are implemented in movc.s, these
        !          1738:  * are just wrappers to help debugging
        !          1739:  */
        !          1740: 
        !          1741: extern void pmap_zero_page_assembler(vm_offset_t p);
        !          1742: extern void pmap_copy_page_assembler(vm_offset_t src, vm_offset_t dst);
        !          1743: 
        !          1744: 
        !          1745: 
        !          1746: /*
        !          1747:  * pmap_zero_page(pa)
        !          1748:  *
        !          1749:  * pmap_zero_page zeros the specified (machine independent) page pa.
        !          1750:  */
        !          1751: void
        !          1752: pmap_zero_page(
        !          1753:        vm_offset_t p)
        !          1754: {
        !          1755: 
        !          1756:        if ((pmdebug & (PDB_USER|PDB_ZERO)) == (PDB_USER|PDB_ZERO))
        !          1757:                DPRINTF(("(pa=%x)\n", p));
        !          1758: 
        !          1759:        /*
        !          1760:         * XXX can these happen?
        !          1761:         */
        !          1762:        if (pmap_find_physentry(p) == PHYS_NULL)
        !          1763:                panic("zero_page: physaddr out of range");
        !          1764: 
        !          1765:        /* TODO NMGS optimisation - if page has no mappings, set bit in
        !          1766:         * physentry to indicate 'needs zeroing', then zero page in
        !          1767:         * pmap_enter or pmap_copy_page if bit is set. This keeps the
        !          1768:         * cache 'hot'.
        !          1769:         */
        !          1770: 
        !          1771:        assert((p & 0xFFF) == 0);
        !          1772:        pmap_zero_page_assembler(p);
        !          1773: }
        !          1774: 
        !          1775: 
        !          1776: 
        !          1777: /*
        !          1778:  * pmap_copy_page(src, dst)
        !          1779:  *
        !          1780:  * pmap_copy_page copies the specified (machine independent)
        !          1781:  * page from physical address src to physical address dst.
        !          1782:  *
        !          1783:  * We need to invalidate the cache for address dst before
        !          1784:  * we do the copy. Apparently there won't be any mappings
        !          1785:  * to the dst address normally.
        !          1786:  */
        !          1787: void
        !          1788: pmap_copy_page(
        !          1789:        vm_offset_t src,
        !          1790:        vm_offset_t dst)
        !          1791: {
        !          1792: 
        !          1793:        if ((pmdebug & (PDB_USER|PDB_COPY)) == (PDB_USER|PDB_COPY))
        !          1794:                DPRINTF(("(spa=%x, dpa=%x)\n", src, dst));
        !          1795:        if (pmdebug & PDB_COPY)
        !          1796:                DPRINTF((": phys_copy(%x, %x, %x)\n",
        !          1797:                       src, dst, PAGE_SIZE));
        !          1798: 
        !          1799:        assert((src & 0xFFF) == 0);
        !          1800:        assert((dst & 0xFFF) == 0);
        !          1801:        pmap_copy_page_assembler(src, dst);
        !          1802: }
        !          1803: #endif /* DEBUG */
        !          1804: 
        !          1805: 
        !          1806: 
        !          1807: /*
        !          1808:  * pmap_pageable(pmap, s, e, pageable)
        !          1809:  *     Make the specified pages (by pmap, offset)
        !          1810:  *     pageable (or not) as requested.
        !          1811:  *
        !          1812:  *     A page which is not pageable may not take
        !          1813:  *     a fault; therefore, its page table entry
        !          1814:  *     must remain valid for the duration.
        !          1815:  *
        !          1816:  *     This routine is merely advisory; pmap_enter()
        !          1817:  *     will specify that these pages are to be wired
        !          1818:  *     down (or not) as appropriate.
        !          1819:  *
        !          1820:  *     (called from vm/vm_fault.c).
        !          1821:  */
        !          1822: /* ARGSUSED */
        !          1823: void
        !          1824: pmap_pageable(
        !          1825:        pmap_t          pmap,
        !          1826:        vm_offset_t     start,
        !          1827:        vm_offset_t     end,
        !          1828:        boolean_t       pageable)
        !          1829: {
        !          1830: #if DEBUG
        !          1831:        if ((pmdebug & (PDB_USER | PDB_WIRED)) == (PDB_USER|PDB_WIRED))
        !          1832:                DPRINTF(("(pmap=%x, sva=%x, eva=%x, pageable=%s)\n",
        !          1833:                       pmap, start, end, pageable ? "TRUE" : "FALSE"));
        !          1834: #endif
        !          1835: }
        !          1836: 
        !          1837: 
        !          1838: 
        !          1839: /*
        !          1840:  * pmap_clear_modify(phys)
        !          1841:  *     clears the hardware modified ("dirty") bit for one
        !          1842:  *     machine independant page starting at the given
        !          1843:  *     physical address.  phys must be aligned on a machine
        !          1844:  *     independant page boundary.
        !          1845:  */
        !          1846: void
        !          1847: pmap_clear_modify(
        !          1848:        vm_offset_t pa)
        !          1849: {
        !          1850:        register struct phys_entry *pp;
        !          1851:        register struct mapping *mp;
        !          1852:        pte_t   *pte;
        !          1853:        register vm_offset_t offset;
        !          1854:        spl_t s;
        !          1855:        boolean_t was_valid;
        !          1856: 
        !          1857: #if DEBUG
        !          1858:        if (pmdebug & PDB_USER)
        !          1859:                DPRINTF(("(pa=%x)\n", pa));
        !          1860: #endif
        !          1861: 
        !          1862:        pp = pmap_find_physentry(pa);
        !          1863:        if (pp == PHYS_NULL)
        !          1864:                return;
        !          1865: 
        !          1866:        s = splhigh();
        !          1867:        /*
        !          1868:         * invalidate any pte entries and remove them from tlb
        !          1869:         * TODO might it be faster to wipe all tlb entries outside loop? NO, rav.
        !          1870:         */
        !          1871: 
        !          1872:        /* Remove any reference to modified to the physical page */
        !          1873:        pp->pte1.bits.changed = FALSE;
        !          1874: 
        !          1875:        /* mark PTE entries as having no modifies, and flush tlbs */
        !          1876:        queue_iterate(&pp->phys_link, mp,
        !          1877:                      struct mapping *, phys_link) {
        !          1878:                pte     = mp->pte;
        !          1879:                offset  = mp->vm_info.bits.page << PPC_PGSHIFT;
        !          1880: 
        !          1881:                was_valid = FALSE;
        !          1882:                if (pte->pte0.bits.valid == TRUE)       {
        !          1883:                        /* Perform a general case PTE update designed to work
        !          1884:                        ** in SMP configurations. TODO - locks and tlbsync
        !          1885:                        */
        !          1886:                        pte->pte0.bits.valid = FALSE;   /* Invalidate PTE */
        !          1887:                        was_valid = TRUE;
        !          1888:                }
        !          1889:                sync();         /* Force updates to complete */
        !          1890:                tlbie(offset);  /* Wipe out this virtual address */
        !          1891:                eieio();        /* Enforce ordering of v bit */
        !          1892:                tlbsync();
        !          1893:                sync();
        !          1894:        
        !          1895:                /*      Allowable early PTE updates: RPN,R,C,WIMG,PP
        !          1896:                */
        !          1897:                pte->pte1.bits.changed = FALSE;
        !          1898:                eieio();
        !          1899: 
        !          1900:                /*      Post tlbie updates: VSID,H,API,V
        !          1901:                */
        !          1902:                if (was_valid == TRUE)
        !          1903:                {
        !          1904:                        pte->pte0.bits.valid = TRUE;    /* Validate pte entry */
        !          1905:                }
        !          1906:                sync();
        !          1907:        }
        !          1908: 
        !          1909: #if    DEBUG
        !          1910:        /* Assert that there are no other mappings with different
        !          1911:         * cachability information, as this can freeze the machine
        !          1912:         */
        !          1913:        {
        !          1914:                struct mapping *mp2;
        !          1915:                queue_iterate(&pp->phys_link, mp2,
        !          1916:                              struct mapping *, phys_link) {
        !          1917:                        assert(mp2->pte->pte1.bits.wimg ==
        !          1918:                               pte->pte1.bits.wimg);
        !          1919:                }
        !          1920:        }
        !          1921: #endif /* DEBUG */
        !          1922: 
        !          1923:        splx(s);
        !          1924: }
        !          1925: 
        !          1926: 
        !          1927: 
        !          1928: /*
        !          1929:  * pmap_is_modified(phys)
        !          1930:  *     returns TRUE if the given physical page has been modified 
        !          1931:  *     since the last call to pmap_clear_modify().
        !          1932:  */
        !          1933: boolean_t
        !          1934: pmap_is_modified(
        !          1935:        vm_offset_t pa)
        !          1936: {
        !          1937:        struct phys_entry *pp;
        !          1938:        struct mapping *mp;
        !          1939:        spl_t s;
        !          1940: 
        !          1941: #if DEBUG
        !          1942:        if (pmdebug & PDB_USER)
        !          1943:                DPRINTF(("(pa=%x)\n", pa));
        !          1944: #endif
        !          1945:        pp = pmap_find_physentry(pa);
        !          1946:        if (pp == PHYS_NULL)
        !          1947:                return(FALSE);
        !          1948: 
        !          1949:        /* Check to see if we've already noted this as modified */
        !          1950:        if (pp->pte1.bits.changed)
        !          1951:                return(TRUE);
        !          1952: 
        !          1953:        s = splhigh();
        !          1954:        /*      Make sure all TLB -> PTE updates have completed.
        !          1955:        */
        !          1956:        sync();
        !          1957: 
        !          1958:        queue_iterate(&pp->phys_link, mp,
        !          1959:                      struct mapping *, phys_link) {
        !          1960: 
        !          1961:                /*      Find the first changed page mapping but
        !          1962:                        do not include the phys page mapping.
        !          1963:                */
        !          1964:                if (!mp->vm_info.bits.phys      && 
        !          1965:                     mp->pte->pte1.bits.changed) 
        !          1966:                {
        !          1967:                        pp->pte1.bits.changed = TRUE;
        !          1968:                        splx(s);
        !          1969:                        return TRUE;
        !          1970:                }
        !          1971:        }
        !          1972:        
        !          1973:        splx(s);
        !          1974:        return(FALSE);
        !          1975: }
        !          1976: 
        !          1977: 
        !          1978: 
        !          1979: /*
        !          1980:  * pmap_clear_reference(phys)
        !          1981:  *     clears the hardware referenced bit in the given machine
        !          1982:  *     independant physical page.  
        !          1983:  *
        !          1984:  */
        !          1985: void
        !          1986: pmap_clear_reference(
        !          1987:        vm_offset_t pa)
        !          1988: {
        !          1989:        register struct phys_entry *pp;
        !          1990:        register struct mapping *mp;
        !          1991:        spl_t s;
        !          1992:        boolean_t was_valid;
        !          1993: 
        !          1994: #if DEBUG
        !          1995:        if (pmdebug & PDB_USER)
        !          1996:                DPRINTF(("(pa=%x)\n", pa));
        !          1997: #endif
        !          1998: 
        !          1999:        pp = pmap_find_physentry(pa);
        !          2000:        if (pp == PHYS_NULL)
        !          2001:                return;
        !          2002: 
        !          2003:        s = splhigh();
        !          2004:        /*      Run through all the virtual mappings to clear the reference
        !          2005:                bit in each PTE. Because the reference bit does not have to be
        !          2006:                maintained exactly, we can avoid issuing syncs but once.        
        !          2007:         */
        !          2008:        queue_iterate(&pp->phys_link, mp,
        !          2009:                      struct mapping *, phys_link) {
        !          2010: 
        !          2011:                was_valid = FALSE;
        !          2012:                if (mp->pte->pte0.bits.valid == TRUE) {
        !          2013:                        mp->pte->pte0.bits.valid = FALSE;
        !          2014:                        was_valid = TRUE;
        !          2015:                }
        !          2016:                /*
        !          2017:                ** Make sure there is no entry in the TBL, it is assumed
        !          2018:                ** that if there is an entry that the page has been referenced
        !          2019:                */
        !          2020:                sync();
        !          2021:                tlbie(mp->vm_info.bits.page << PPC_PGSHIFT);
        !          2022:                eieio();
        !          2023:                tlbsync();
        !          2024:                sync();
        !          2025: 
        !          2026:                /*
        !          2027:                ** Gotta drop both bits or the chip gets hosed!
        !          2028:                **      See section 7.5.3
        !          2029:                **
        !          2030:                ** Thanks Rene!
        !          2031:                */
        !          2032:                pp->pte1.bits.changed |=
        !          2033:                        mp->pte->pte1.bits.changed;
        !          2034:                mp->pte->pte1.bits.changed = FALSE;
        !          2035:                mp->pte->pte1.bits.referenced = FALSE;
        !          2036:                eieio();
        !          2037:                if (was_valid == TRUE)
        !          2038:                {
        !          2039:                        mp->pte->pte0.bits.valid = TRUE;
        !          2040:                }
        !          2041:                sync();
        !          2042:        }
        !          2043: 
        !          2044:        /*      Mark the physical entry shadow copy as non-referenced too.
        !          2045:        */
        !          2046:        pp->pte1.bits.referenced = FALSE;
        !          2047: 
        !          2048:        splx(s);
        !          2049: }
        !          2050: 
        !          2051: 
        !          2052: 
        !          2053: /*
        !          2054:  * pmap_is_referenced(phys)
        !          2055:  *     returns TRUE if the given physical page has been referenced 
        !          2056:  *     since the last call to pmap_clear_reference().
        !          2057:  */
        !          2058: boolean_t
        !          2059: pmap_is_referenced(
        !          2060:        vm_offset_t pa)
        !          2061: {
        !          2062:        struct phys_entry *pp;
        !          2063:        register struct mapping *mp;
        !          2064:        spl_t s;
        !          2065: 
        !          2066: #if DEBUG
        !          2067:        if (pmdebug & PDB_USER)
        !          2068:                DPRINTF(("(pa=%x)\n", pa));
        !          2069: #endif
        !          2070:        pp = pmap_find_physentry(pa);
        !          2071:        if (pp == PHYS_NULL)
        !          2072:                return(FALSE);
        !          2073: 
        !          2074:        if (pp->pte1.bits.referenced)
        !          2075:                return (TRUE);
        !          2076: 
        !          2077: 
        !          2078:        s = splhigh();
        !          2079:        /*      Make sure all TLB -> PTE updates have completed.
        !          2080:        */
        !          2081:        sync();
        !          2082: 
        !          2083:        /*      Check all the mappings to see if any one of them
        !          2084:                have recorded a reference to this physical page.
        !          2085:        */
        !          2086:        queue_iterate(&pp->phys_link, mp,
        !          2087:                      struct mapping *, phys_link) {
        !          2088: 
        !          2089:                /*      Find the first referenced page mapping but
        !          2090:                        do not include the phys page mapping.
        !          2091:                */
        !          2092:                if (!mp->vm_info.bits.phys      && 
        !          2093:                     mp->pte->pte1.bits.referenced) 
        !          2094:                {
        !          2095:                        pp->pte1.bits.referenced = TRUE;
        !          2096:                        splx(s);
        !          2097:                        return (TRUE);
        !          2098:                }
        !          2099:        }
        !          2100:        splx(s);
        !          2101:        return(FALSE);
        !          2102: }
        !          2103: 
        !          2104: 
        !          2105: 
        !          2106: /*
        !          2107:  * Auxilliary routines for manipulating mappings
        !          2108:  */
        !          2109: 
        !          2110: static struct mapping *
        !          2111: pmap_enter_mapping(
        !          2112:        pmap_t  pmap,
        !          2113:        space_t space,
        !          2114:        vm_offset_t     va,
        !          2115:        vm_offset_t     pa,
        !          2116:        pte_t   *pte,
        !          2117:        unsigned        prot,
        !          2118:        struct phys_entry *pp)
        !          2119: {
        !          2120:        register struct mapping *mp;
        !          2121:        spl_t s;
        !          2122: 
        !          2123: #if DEBUG
        !          2124:        if (pmdebug & PDB_MAPPING)
        !          2125:                DPRINTF(("(pmap=%x, sp=%x, off=%x, &pte=%x, prot=%x)",
        !          2126:                       pmap, space, va, pte, prot));
        !          2127: #endif /* DEBUG */
        !          2128: 
        !          2129:        s = splhigh();
        !          2130:        simple_lock(&free_mapping_lock);
        !          2131:        if (queue_empty(&free_mapping))
        !          2132:                pmap_reap_mappings();
        !          2133:        queue_remove_first(&free_mapping, mp, struct mapping *, phys_link);
        !          2134: 
        !          2135:        simple_unlock(&free_mapping_lock);
        !          2136:        assert(mp != MAPPING_NULL);
        !          2137: 
        !          2138:        mp->pmap = pmap;
        !          2139:        mp->vm_info.bits.phys   = FALSE;
        !          2140:        mp->vm_info.bits.wired  = FALSE;
        !          2141:        mp->vm_info.bits.page  = (va >> PPC_PGSHIFT);
        !          2142:        mp->pte                 = pte;
        !          2143: 
        !          2144:        /* Set up pte -
        !          2145:         *   pte0 already contains everything except the valid bit.
        !          2146:         *   pte1 already contains the protection information.
        !          2147:         */
        !          2148: 
        !          2149:        assert(pte->pte0.word != PTE_EMPTY);
        !          2150: 
        !          2151:        assert(pte->pte0.bits.valid == FALSE);
        !          2152: 
        !          2153:        sync();
        !          2154:        tlbie(mp->vm_info.bits.page << PPC_PGSHIFT);
        !          2155:        eieio();
        !          2156:        tlbsync();
        !          2157:        sync();
        !          2158: 
        !          2159:        pte->pte1.bits.phys_page = pa >> PPC_PGSHIFT;
        !          2160:        pte->pte1.bits.wimg = pp->pte1.bits.wimg; /* Set default cachability*/
        !          2161: 
        !          2162:        /*      Make sure those pte1 writes are ordered ahead of
        !          2163:                of the v bit.
        !          2164:        */
        !          2165:        eieio();
        !          2166: 
        !          2167: 
        !          2168:        pte->pte0.bits.valid = TRUE;            /* Validate pte entry */
        !          2169:        sync();
        !          2170: 
        !          2171:        queue_enter_first(&pp->phys_link, mp, struct mapping *, phys_link);
        !          2172: 
        !          2173: #if    DEBUG
        !          2174:        /*      Assert a PTE of type WIMG_IO is not mapped to general RAM.
        !          2175:        */
        !          2176:        if (pmdebug & PDB_IO    &&
        !          2177:            pte->pte1.bits.wimg == PTE_WIMG_IO)
        !          2178:                assert (((unsigned) pa) >= PCI_BASE)
        !          2179: 
        !          2180:        /* Assert that there are no other mappings with different
        !          2181:         * cachability information, as this can freeze the machine
        !          2182:         */
        !          2183:        {
        !          2184:                struct mapping *mp2;
        !          2185:                queue_iterate(&pp->phys_link, mp2,
        !          2186:                              struct mapping *, phys_link) {
        !          2187:                        assert(mp2->pte->pte1.bits.wimg ==
        !          2188:                               pte->pte1.bits.wimg);
        !          2189:                }
        !          2190:        }
        !          2191: #endif /* DEBUG */
        !          2192: #if DEBUG
        !          2193:        if (pmdebug & PDB_MAPPING)
        !          2194:                DPRINTF((" -> %x\n"));
        !          2195: #endif
        !          2196: #if DEBUG
        !          2197:        if (pmdebug & PDB_MASSIVE) {
        !          2198:                register struct mapping *mp2;
        !          2199:                queue_iterate(&free_mapping, mp2,
        !          2200:                              struct mapping *, phys_link) {
        !          2201:                        assert(mp2->phys_link.next != NULL);
        !          2202:                }
        !          2203:        }
        !          2204: #endif /* DEBUG */     
        !          2205:        splx(s);
        !          2206:        return(mp);
        !          2207: }
        !          2208: 
        !          2209: 
        !          2210: 
        !          2211: /*
        !          2212:  * pmap_find_mapping(space, offset)
        !          2213:  *     Lookup the virtual address <space,offset> in the mapping "table".
        !          2214:  *     Returns a pointer to the mapping or NULL if none.
        !          2215:  *
        !          2216:  *     XXX what about MP locking?
        !          2217:  */
        !          2218: static struct mapping *
        !          2219: pmap_find_mapping(
        !          2220:        register space_t space,
        !          2221:        vm_offset_t offset)
        !          2222: {
        !          2223:        register struct mapping *mp;
        !          2224:        register struct phys_entry *pp;
        !          2225:        register pte_t *pte;
        !          2226:        spl_t s;
        !          2227: 
        !          2228: #if DEBUG
        !          2229:        if (pmdebug & PDB_MAPPING)
        !          2230:                DPRINTF(("(sp=%x, off=%x) -> ", space, offset));
        !          2231: #endif
        !          2232:        offset = trunc_page(offset);
        !          2233: 
        !          2234:        /* Locate the pte but don't allocate a new one if not found */
        !          2235:        pte = find_or_allocate_pte(space, offset, FALSE);
        !          2236: 
        !          2237:        if (pte == NULL) {
        !          2238: #if DEBUG
        !          2239:                if (pmdebug & PDB_MAPPING)
        !          2240:                        DPRINTF(("MAPPING_NULL\n"));
        !          2241: #endif /* DEBUG */
        !          2242:                return MAPPING_NULL;
        !          2243:        }
        !          2244: 
        !          2245:        s = splhigh();
        !          2246:        /* Take advantage of the fact that pte1 can be
        !          2247:         * considered as a pointer to an address in
        !          2248:         * the physical page when looking for physentry.
        !          2249:         */
        !          2250: 
        !          2251:        pp = pmap_find_physentry((vm_offset_t)(pte->pte1.word));
        !          2252: 
        !          2253:        queue_iterate(&pp->phys_link, mp,
        !          2254:                      struct mapping *, phys_link) {
        !          2255:                if (mp->pte == pte)
        !          2256:                        break;
        !          2257:        }
        !          2258:        assert(mp != MAPPING_NULL);     /* There must be mapping for a pte */
        !          2259:        assert(mp->pte == pte);
        !          2260:        assert(((mp->vm_info.bits.page >> 10) & 0x3f) ==
        !          2261:               mp->pte->pte0.bits.page_index);
        !          2262: 
        !          2263: #if DEBUG
        !          2264:        if (pmdebug & PDB_MAPPING)
        !          2265:                DPRINTF(("%x\n", mp));
        !          2266: #endif
        !          2267:        splx(s);
        !          2268:        return(mp);
        !          2269: }
        !          2270: 
        !          2271: 
        !          2272: 
        !          2273: static void
        !          2274: pmap_free_mapping(
        !          2275:        register struct mapping *mp)
        !          2276: {
        !          2277:        register vm_offset_t offset;
        !          2278:        struct phys_entry *pp;
        !          2279:        spl_t s;
        !          2280: 
        !          2281: #if DEBUG
        !          2282:        if (pmdebug & PDB_MAPPING)
        !          2283:                DPRINTF(("(mp=%x), pte at 0x%x (0x%08x,0x%08x)\n",
        !          2284:                       mp, mp->pte, mp->pte->pte0.word,mp->pte->pte1.word));
        !          2285: #endif
        !          2286: 
        !          2287:        s = splhigh();
        !          2288: 
        !          2289:        offset = mp->vm_info.bits.page << PPC_PGSHIFT;
        !          2290: 
        !          2291:        pp = pmap_find_physentry((vm_offset_t)mp->pte->pte1.word);
        !          2292:        assert(pp != PHYS_NULL);
        !          2293: 
        !          2294:        /*      Perform a general case PTE update designed to work
        !          2295:                in SMP configurations. TODO - locks and tlbsync
        !          2296:        */
        !          2297:        mp->pte->pte0.bits.valid = FALSE;               /* Invalidate PTE */
        !          2298:        sync();                         /* Force updates to complete */
        !          2299:        tlbie(offset);                  /* Wipe out this virtual address */
        !          2300:        eieio();
        !          2301:        tlbsync();
        !          2302:        sync();                         /* force completion */
        !          2303: 
        !          2304:        /* Reflect back the modify and reference bits for the pager */
        !          2305:        if (!mp->vm_info.bits.phys) 
        !          2306:                pp->pte1.word |= mp->pte->pte1.word;
        !          2307: 
        !          2308:        /*
        !          2309:         * Now remove from the PTOV/VTOP lists and return to the free list.
        !          2310:         * Note that we must block interrupts since they might cause TLB
        !          2311:         * misses which would search (potentially inconsistant) lists.
        !          2312:         */
        !          2313: 
        !          2314: 
        !          2315:        /* for now, just remove the mapping */
        !          2316:        mp->pte->pte0.word = PTE_EMPTY; /* Remove pte (v->p) lookup */
        !          2317: #if DEBUG
        !          2318:        mp->pte->pte1.word = 0;
        !          2319: #endif 
        !          2320: 
        !          2321:        queue_remove(&pp->phys_link, mp, struct mapping *, phys_link);
        !          2322: 
        !          2323:        simple_lock(&free_mapping_lock);
        !          2324:        queue_enter(&free_mapping, mp, struct mapping *, phys_link);
        !          2325:        simple_unlock(&free_mapping_lock);
        !          2326: #if DEBUG
        !          2327:        if (pmdebug & PDB_MASSIVE) {
        !          2328:                register struct mapping *mp2;
        !          2329:                queue_iterate(&free_mapping, mp2,
        !          2330:                              struct mapping *, phys_link) {
        !          2331:                        assert(mp2->phys_link.next != NULL);
        !          2332:                }
        !          2333:        }
        !          2334: #endif /* DEBUG */     
        !          2335:        splx(s);
        !          2336: }
        !          2337: 
        !          2338: 
        !          2339: 
        !          2340: /*
        !          2341:  * Deal with a hash-table pteg overflow. pmap_pteg_overflow is upcalled from
        !          2342:  * find_or_allocate_pte(), and assumes knowledge of the pteg hashing
        !          2343:  * structure
        !          2344:  */
        !          2345: 
        !          2346: static int pteg_overflow_counter=0;
        !          2347: 
        !          2348: static pte_t *pteg_pte_exchange(pte_t *pteg, pte0_t match);
        !          2349: pte_t *
        !          2350: pmap_pteg_overflow(
        !          2351:        pte_t *primary_hash,
        !          2352:        pte0_t primary_match,
        !          2353:        pte_t *secondary_hash,
        !          2354:        pte0_t secondary_match)
        !          2355: {
        !          2356:        pte_t *pte;
        !          2357: #if DEBUG
        !          2358:        int i;
        !          2359:        if (pmdebug & PDB_PTEG) {
        !          2360:                kprintf("pteg overflow for ptegs 0x%08x and 0x%08x\n",
        !          2361:                       primary_hash,secondary_hash);
        !          2362:                kprintf("PRIMARY\t\t\t\t\tSECONDARY\n");
        !          2363:                for (i=0; i<8; i++) {
        !          2364:                        kprintf("0x%08x : (0x%08x,0x%08x)\t"
        !          2365:                               "0x%08x : (0x%08x,0x%08x)\n",
        !          2366:                               &primary_hash[i],
        !          2367:                               primary_hash[i].pte0.word,
        !          2368:                               primary_hash[i].pte1.word,
        !          2369:                               &secondary_hash[i],
        !          2370:                               secondary_hash[i].pte0.word,
        !          2371:                               secondary_hash[i].pte1.word);
        !          2372:                        assert(primary_hash[i].pte0.word != PTE_EMPTY);
        !          2373:                        assert(secondary_hash[i].pte0.word != PTE_EMPTY);
        !          2374:                }
        !          2375:        }
        !          2376: #endif
        !          2377:        /* First try to replace an entry in the primary PTEG */
        !          2378:        pte = pteg_pte_exchange(primary_hash, primary_match);
        !          2379:        if (pte != PTE_NULL)
        !          2380:                return pte;
        !          2381:        pte = pteg_pte_exchange(secondary_hash, secondary_match);
        !          2382:        if (pte != PTE_NULL)
        !          2383:                return pte;
        !          2384: 
        !          2385:        panic("both ptegs are completely wired down\n");
        !          2386:        return PTE_NULL;
        !          2387: }
        !          2388: 
        !          2389: 
        !          2390:                        
        !          2391: /*
        !          2392:  * Used by the pmap_pteg_overflow function to scan through a pteg
        !          2393: */
        !          2394: static pte_t *
        !          2395: pteg_pte_exchange(
        !          2396:        pte_t *pteg,
        !          2397:        pte0_t match)
        !          2398: {
        !          2399:        /* Try and replace an entry in a pteg */
        !          2400: 
        !          2401:        int i;
        !          2402:        register struct phys_entry *pp;
        !          2403:        register struct mapping *mp;
        !          2404:        pte_t *pte;
        !          2405: 
        !          2406:        /* TODO NMGS pteg overflow algorithm needs working on!
        !          2407:         * currently we just loop around the PTEGs rejecting
        !          2408:         * the first available entry.
        !          2409:         */
        !          2410: 
        !          2411:        for (i=0; i < 8; i++) {
        !          2412: 
        !          2413:                pte = &pteg[pteg_overflow_counter];
        !          2414:                pteg_overflow_counter = (pteg_overflow_counter+1) % 8;
        !          2415: 
        !          2416:                /* all mappings must be valid otherwise we wouldn't be here */
        !          2417:                assert(pte->pte0.bits.valid == TRUE);
        !          2418: 
        !          2419:                /* Take advantage of the fact that pte1 can be
        !          2420:                 * considered as a pointer to an address in
        !          2421:                 * the physical page when looking for physentry.
        !          2422:                 */
        !          2423:                pp = pmap_find_physentry(pte->pte1.word);
        !          2424: 
        !          2425:                /* pte entry must have reverse mapping, otherwise we
        !          2426:                 * cannot remove it (it was entered by pmap_map_bd)
        !          2427:                 */
        !          2428:                if (pp == PHYS_NULL)
        !          2429:                        continue;
        !          2430: 
        !          2431:                queue_iterate(&pp->phys_link, mp,
        !          2432:                              struct mapping *, phys_link) {
        !          2433:                        if (mp->pte == pte)
        !          2434:                                break;
        !          2435:                }
        !          2436:                assert(mp != NULL);
        !          2437:                assert(mp->pte == pte);
        !          2438: 
        !          2439:                /* if this entry isn't wired, isn't in the kernel space,
        !          2440:                 * and it doesn't have any special cachability attributes
        !          2441:                 * we can remove mapping without losing any information.
        !          2442:                 * XXX kernel space mustn't be touched as we may be doing
        !          2443:                 * XXX an I/O on a non-wired page, and we must not take
        !          2444:                 * XXX a page fault in an interrupt handler.
        !          2445:                 */
        !          2446:                /* TODO better implementation for cache info? */
        !          2447:                if (!mp->vm_info.bits.wired &&
        !          2448:                    mp->pmap != kernel_pmap &&
        !          2449:                    mp->pte->pte1.bits.wimg == PTE_WIMG_DEFAULT) {
        !          2450: #if DEBUG
        !          2451:                        if (pmdebug & PDB_PTEG) {
        !          2452:                                kprintf("entry chosen at 0x%08x in %s PTEG\n",
        !          2453:                                       pte,
        !          2454:                                       pte->pte0.bits.hash_id ?
        !          2455:                                        "SECONDARY" : "PRIMARY");
        !          2456:                                kprintf("throwing away mapping from sp 0x%x, "
        !          2457:                                       "virtual 0x%08x to physical 0x%08x\n",
        !          2458:                                       mp->pmap->space,
        !          2459:                                       mp->vm_info.bits.page << PPC_PGSHIFT,
        !          2460:                                       trunc_page(mp->pte->pte1.word));
        !          2461:                                kprintf("pteg_overflow_counter=%d\n",
        !          2462:                                       pteg_overflow_counter);
        !          2463:                                kprintf("pte0.vsid=0x%08x\n",
        !          2464:                                       pte->pte0.bits.segment_id);
        !          2465:                        }
        !          2466: #endif /* DEBUG */
        !          2467:                        mp->pmap->stats.resident_count--;
        !          2468: 
        !          2469:                        /* release mapping structure and PTE, keeping
        !          2470:                         * track of changed and referenced bits
        !          2471:                         */
        !          2472:                        pmap_free_mapping(mp);
        !          2473: 
        !          2474:                        /* Replace the pte entry by the new one */
        !          2475: 
        !          2476:                        match.bits.valid = FALSE;
        !          2477:                        sync();
        !          2478:                        pte->pte0 = match;
        !          2479:                        sync();
        !          2480: 
        !          2481:                        return pte;
        !          2482:                }
        !          2483:        }
        !          2484:        return PTE_NULL;
        !          2485: }
        !          2486: 
        !          2487: 
        !          2488: 
        !          2489: /*
        !          2490:  * Collect un-wired mappings. How best to do this??
        !          2491:  */
        !          2492: static void
        !          2493: pmap_reap_mappings(void)
        !          2494: {
        !          2495: #if DEBUG
        !          2496:        if (pmdebug & PDB_MAPPING)
        !          2497:                DPRINTF(("pmap_reap_mappings()\n"));
        !          2498: #endif
        !          2499: 
        !          2500:        panic("out of mapping structs");
        !          2501: }
        !          2502: 
        !          2503: 
        !          2504: /*
        !          2505:  *     kvtophys(addr)
        !          2506:  *
        !          2507:  *     Convert a kernel virtual address to a physical address
        !          2508:  */
        !          2509: vm_offset_t
        !          2510: kvtophys(
        !          2511:        vm_offset_t va)
        !          2512: {
        !          2513:        pte_t *pte;
        !          2514:        extern vm_offset_t virtual_avail;
        !          2515: 
        !          2516:        if (va < virtual_avail)
        !          2517:            return va;
        !          2518:        
        !          2519:        pte = find_or_allocate_pte(PPC_SID_KERNEL, trunc_page(va), FALSE);
        !          2520:        if (pte == NULL)
        !          2521:                return(0);
        !          2522:        return trunc_page(pte->pte1.word) | (va & page_mask);
        !          2523: }
        !          2524: 
        !          2525: 
        !          2526: 
        !          2527: /*
        !          2528:  * pmap_remove_all - Remove translations by physical address.
        !          2529:  *
        !          2530:  * This routine walks the pv_entry chain for the given va, and 
        !          2531:  * removes all translations.  It records modify bits.
        !          2532:  *
        !          2533:  * for 2.5vm <--> 3.0pmap this mapped to pmap_page_protect()
        !          2534:  */
        !          2535: void
        !          2536: pmap_remove_all(
        !          2537:        vm_offset_t phys)
        !          2538: {
        !          2539:        pmap_page_protect(phys, VM_PROT_NONE);
        !          2540: }
        !          2541: 
        !          2542: 
        !          2543: 
        !          2544: /*
        !          2545:  * pmap_copy_on_write - Ready region for copy-on-write handling.
        !          2546:  *
        !          2547:  * This routine removes write privleges from all physical maps
        !          2548:  * for a given physical page.
        !          2549:  *
        !          2550:  * for 2.5vm <--> 3.0pmap this mapped to pmap_page_protect()
        !          2551:  *
        !          2552:  */
        !          2553: void
        !          2554: pmap_copy_on_write(
        !          2555:        vm_offset_t phys)
        !          2556: {
        !          2557:        pmap_page_protect(phys, VM_PROT_READ);
        !          2558: }
        !          2559: 
        !          2560: 
        !          2561: 
        !          2562: /*
        !          2563:  * pmap_move_page - Move pages from one kernel vaddr to another.
        !          2564:  *
        !          2565:  * Parameters:
        !          2566:  *      from    - Original kernel va    (Mach. indep. page boundary)
        !          2567:  *      to      - New kernel va         (Mach. indep. page boundary)
        !          2568:  *      size    - Bytes to move.        (Mach. indep. pages)
        !          2569:  *
        !          2570:  * Returns:
        !          2571:  *      Nothing.
        !          2572:  */
        !          2573: static void
        !          2574: pmap_move_page(
        !          2575:        unsigned long from,
        !          2576:        unsigned long to,
        !          2577:        vm_size_t size)
        !          2578: {
        !          2579:         spl_t s;
        !          2580:         pte_t *pte;
        !          2581:                struct phys_entry *pp;
        !          2582:                struct mapping *mp;
        !          2583:         vm_prot_t prot;
        !          2584:                boolean_t wired;
        !          2585:                int ppn;
        !          2586: 
        !          2587:         s = splhigh();
        !          2588:         if ((size & page_mask) != 0) {
        !          2589:                 panic("pmap_move_page: partial mach indep page");
        !          2590:         }
        !          2591:         for (; size != (unsigned) 0; size -= PAGE_SIZE, from += PAGE_SIZE, to += PAGE_SIZE) {
        !          2592:                if ((pte = find_or_allocate_pte(kernel_pmap->space, trunc_page(from), FALSE)) == PTE_NULL) {
        !          2593:                         panic("pmap_move_page: from not mapped");
        !          2594:                 }
        !          2595:                /*
        !          2596:                 * Make sure the damn thing is valid.
        !          2597:                 */
        !          2598:                if (pte->pte0.bits.valid == FALSE) {
        !          2599:                        panic("pmap_move_page: invalid pte!");
        !          2600:                }
        !          2601:                 switch (pte->pte1.bits.protection) {
        !          2602:                 case 3:
        !          2603:                         prot = VM_PROT_READ;
        !          2604:                         break;
        !          2605:                 default:
        !          2606:                         prot = VM_PROT_READ|VM_PROT_WRITE;
        !          2607:                         break;
        !          2608:                 }
        !          2609: 
        !          2610:                ppn = pte->pte1.bits.phys_page;
        !          2611:                pp = pmap_find_physentry((vm_offset_t)(pte->pte1.word));
        !          2612:                queue_iterate(&pp->phys_link, mp,
        !          2613:                                struct mapping *, phys_link) {
        !          2614:                        if (mp->pte == pte)
        !          2615:                                break;
        !          2616:                }
        !          2617:                assert(mp != MAPPING_NULL);
        !          2618:                assert(mp->pte == pte);
        !          2619:                assert(((mp->vm_info.bits.page >> 10) & 0x3f) ==
        !          2620:                        mp->pte->pte0.bits.page_index);
        !          2621:                wired = mp->vm_info.bits.wired;
        !          2622: 
        !          2623:                /*
        !          2624:                 * Remove old mapping first (avoid aliasing)
        !          2625:                 */
        !          2626:                 pmap_remove(kernel_pmap, from, from + PAGE_SIZE);
        !          2627:                 pmap_enter(kernel_pmap, to, PMAP_PTOB(ppn), prot, wired);
        !          2628:         }
        !          2629:         splx(s);
        !          2630: }
        !          2631: 
        !          2632: 
        !          2633: 
        !          2634: /*
        !          2635:  * Move pages from one kernel virtual address to another.
        !          2636:  * Both addresses are assumed to reside in the Sysmap,
        !          2637:  * and size must be a multiple of the page size.
        !          2638:  */
        !          2639: void
        !          2640: pagemove(
        !          2641:        register caddr_t from,
        !          2642:        register caddr_t to,
        !          2643:        int size)
        !          2644: {
        !          2645:        pmap_move_page((unsigned long)from, (unsigned long)to, (vm_size_t)size);
        !          2646: }
        !          2647: 
        !          2648: 
        !          2649: 
        !          2650: #if DEBUG
        !          2651: /* This function is included to test compiler functionality, it does
        !          2652:  * a simple slide through the bitfields of a pte to check
        !          2653:  * minimally correct bitfield behaviour. We need bitfields to work!
        !          2654:  */
        !          2655: #define TEST(PTE0,PTE1)                                                        \
        !          2656:        if ((pte.pte0.word != (unsigned int)PTE0) ||                    \
        !          2657:            (pte.pte1.word != (unsigned int)PTE1)) {                    \
        !          2658:                DPRINTF(("BITFIELD TEST FAILED AT LINE %d\n",__LINE__));        \
        !          2659:                DPRINTF(("EXPECTING 0x%08x and 0x%08x\n",                       \
        !          2660:                       (unsigned int)PTE0,(unsigned int)PTE1));         \
        !          2661:                DPRINTF(("GOT       0x%08x and 0x%08x\n",                       \
        !          2662:                       pte.pte0.word,pte.pte1.word));                   \
        !          2663:                return 1;                                               \
        !          2664:        }
        !          2665: 
        !          2666: extern int pmap_test_bitfields(void); /* Prototyped also in go.c */
        !          2667: int pmap_test_bitfields(void)
        !          2668: {
        !          2669:        pte_t   pte;
        !          2670:        pte.pte0.word = 0;
        !          2671:        pte.pte1.word = 0;
        !          2672: 
        !          2673:        pte.pte0.bits.segment_id = 0x103; TEST(0x103 << 7, 0);
        !          2674:        pte.pte0.bits.valid = 1;
        !          2675:        TEST((unsigned int)(0x80000000U| 0x103<<7), 0);
        !          2676:        pte.pte0.bits.hash_id = 1;
        !          2677:        TEST((unsigned int)(0x80000000U | 0x103 << 7 | 1<<6), 0);
        !          2678:        pte.pte0.bits.page_index = 3;
        !          2679:        TEST((unsigned int)(0x80000000U|0x103<<7|1<<6|3), 0);
        !          2680: 
        !          2681:        pte.pte0.bits.segment_id = 0;
        !          2682:        TEST((unsigned int)(0x80000000U|1<<6|3), 0);
        !          2683:        pte.pte0.bits.page_index = 0;
        !          2684:        TEST((unsigned int)(0x80000000U|1<<6), 0);
        !          2685:        pte.pte0.bits.valid = 0;
        !          2686:        TEST(1<<6, 0);
        !          2687: 
        !          2688:        pte.pte1.bits.referenced = 1; TEST(1<<6, 1<<8);
        !          2689:        pte.pte1.bits.protection = 3; TEST(1<<6, (unsigned int)(1<<8 | 3));
        !          2690:        pte.pte1.bits.changed = 1; TEST(1<<6,(unsigned int)(1<<8|1<<7|3));
        !          2691:        pte.pte1.bits.phys_page = 0xfffff;
        !          2692:        TEST(1<<6,(unsigned int)(0xfffff000U|1<<8|1<<7|3));
        !          2693: 
        !          2694:        pte.pte1.bits.changed = 0;
        !          2695:        TEST(1<<6,(unsigned int)(0xfffff000U|1<<8|3));
        !          2696: 
        !          2697:        return 0;
        !          2698: }
        !          2699: #endif /* DEBUG */
        !          2700: 
        !          2701: 
        !          2702: 
        !          2703: void 
        !          2704: pmap_enter_cache_spec(
        !          2705:     pmap_t             pmap,
        !          2706:     vm_offset_t                va,
        !          2707:     vm_offset_t                pa,
        !          2708:     vm_prot_t          prot,
        !          2709:     boolean_t          wired,
        !          2710:     cache_spec_t       caching
        !          2711: )
        !          2712: {
        !          2713:     if (pmap == PMAP_NULL)
        !          2714:        return;
        !          2715: 
        !          2716:        pmap_enter(pmap, va, pa, prot, wired);
        !          2717:        
        !          2718: }
        !          2719: 
        !          2720: 
        !          2721: 
        !          2722: void
        !          2723: pmap_update( void )
        !          2724: {
        !          2725: #if DEBUG
        !          2726:        if (pmdebug & PDB_USER)
        !          2727:                DPRINTF(("()\n"));
        !          2728: #endif
        !          2729: }

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.