Return to vm_resident.c CVS log

Up to [GnuMach] / OSKit-Mach / vm

Annotation of OSKit-Mach/vm/vm_resident.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Mach Operating System
        !             3:  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
        !             4:  * Copyright (c) 1993,1994 The University of Utah and
        !             5:  * the Computer Systems Laboratory (CSL).
        !             6:  * All rights reserved.
        !             7:  *
        !             8:  * Permission to use, copy, modify and distribute this software and its
        !             9:  * documentation is hereby granted, provided that both the copyright
        !            10:  * notice and this permission notice appear in all copies of the
        !            11:  * software, derivative works or modified versions, and any portions
        !            12:  * thereof, and that both notices appear in supporting documentation.
        !            13:  *
        !            14:  * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
        !            15:  * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
        !            16:  * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
        !            17:  * THIS SOFTWARE.
        !            18:  *
        !            19:  * Carnegie Mellon requests users of this software to return to
        !            20:  *
        !            21:  *  Software Distribution Coordinator  or  [email protected]
        !            22:  *  School of Computer Science
        !            23:  *  Carnegie Mellon University
        !            24:  *  Pittsburgh PA 15213-3890
        !            25:  *
        !            26:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            27:  * the rights to redistribute these changes.
        !            28:  */
        !            29: /*
        !            30:  *     File:   vm/vm_page.c
        !            31:  *     Author: Avadis Tevanian, Jr., Michael Wayne Young
        !            32:  *
        !            33:  *     Resident memory management module.
        !            34:  */
        !            35: #include <cpus.h>
        !            36: 
        !            37: #include <mach/vm_prot.h>
        !            38: #include <kern/counters.h>
        !            39: #include <kern/sched_prim.h>
        !            40: #include <kern/task.h>
        !            41: #include <kern/thread.h>
        !            42: #include <mach/vm_statistics.h>
        !            43: #include "vm_param.h"
        !            44: #include <kern/zalloc.h>
        !            45: #include <vm/pmap.h>
        !            46: #include <vm/vm_map.h>
        !            47: #include <vm/vm_page.h>
        !            48: #include <vm/vm_pageout.h>
        !            49: #include <vm/vm_kern.h>
        !            50: 
        !            51: #include <mach_vm_debug.h>
        !            52: #if    MACH_VM_DEBUG
        !            53: #include <mach/kern_return.h>
        !            54: #include <mach_debug/hash_info.h>
        !            55: #include <vm/vm_user.h>
        !            56: #endif
        !            57: 
        !            58: /*
        !            59:  *     Associated with eacn page of user-allocatable memory is a
        !            60:  *     page structure.
        !            61:  */
        !            62: 
        !            63: /*
        !            64:  *     These variables record the values returned by vm_page_bootstrap,
        !            65:  *     for debugging purposes.  The implementation of pmap_steal_memory
        !            66:  *     and pmap_startup here also uses them internally.
        !            67:  */
        !            68: 
        !            69: vm_offset_t virtual_space_start;
        !            70: vm_offset_t virtual_space_end;
        !            71: 
        !            72: /*
        !            73:  *     The vm_page_lookup() routine, which provides for fast
        !            74:  *     (virtual memory object, offset) to page lookup, employs
        !            75:  *     the following hash table.  The vm_page_{insert,remove}
        !            76:  *     routines install and remove associations in the table.
        !            77:  *     [This table is often called the virtual-to-physical,
        !            78:  *     or VP, table.]
        !            79:  */
        !            80: typedef struct {
        !            81:        decl_simple_lock_data(,lock)
        !            82:        vm_page_t pages;
        !            83: } vm_page_bucket_t;
        !            84: 
        !            85: vm_page_bucket_t *vm_page_buckets;             /* Array of buckets */
        !            86: unsigned int   vm_page_bucket_count = 0;       /* How big is array? */
        !            87: unsigned int   vm_page_hash_mask;              /* Mask for hash function */
        !            88: 
        !            89: /*
        !            90:  *     Resident page structures are initialized from
        !            91:  *     a template (see vm_page_alloc).
        !            92:  *
        !            93:  *     When adding a new field to the virtual memory
        !            94:  *     object structure, be sure to add initialization
        !            95:  *     (see vm_page_bootstrap).
        !            96:  */
        !            97: struct vm_page vm_page_template;
        !            98: 
        !            99: /*
        !           100:  *     Resident pages that represent real memory
        !           101:  *     are allocated from a free list.
        !           102:  */
        !           103: vm_page_t      vm_page_queue_free;
        !           104: vm_page_t      vm_page_queue_fictitious;
        !           105: decl_simple_lock_data(,vm_page_queue_free_lock)
        !           106: unsigned int   vm_page_free_wanted;
        !           107: int            vm_page_queue_free_count;
        !           108: int            vm_page_unqueued_count;
        !           109: int            vm_page_fictitious_count;
        !           110: int            vm_page_external_count;
        !           111: 
        !           112: unsigned int   vm_page_free_count_minimum;     /* debugging */
        !           113: 
        !           114: /*
        !           115:  *     Occasionally, the virtual memory system uses
        !           116:  *     resident page structures that do not refer to
        !           117:  *     real pages, for example to leave a page with
        !           118:  *     important state information in the VP table.
        !           119:  *
        !           120:  *     These page structures are allocated the way
        !           121:  *     most other kernel structures are.
        !           122:  */
        !           123: zone_t vm_page_zone;
        !           124: 
        !           125: /*
        !           126:  *     Fictitious pages don't have a physical address,
        !           127:  *     but we must initialize phys_addr to something.
        !           128:  *     For debugging, this should be a strange value
        !           129:  *     that the pmap module can recognize in assertions.
        !           130:  */
        !           131: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
        !           132: 
        !           133: /*
        !           134:  *     Resident page structures are also chained on
        !           135:  *     queues that are used by the page replacement
        !           136:  *     system (pageout daemon).  These queues are
        !           137:  *     defined here, but are shared by the pageout
        !           138:  *     module.
        !           139:  */
        !           140: queue_head_t   vm_page_queue_active;
        !           141: queue_head_t   vm_page_queue_inactive;
        !           142: decl_simple_lock_data(,vm_page_queue_lock)
        !           143: int    vm_page_active_count;
        !           144: int    vm_page_inactive_count;
        !           145: int    vm_page_wire_count;
        !           146: 
        !           147: /*
        !           148:  *     Several page replacement parameters are also
        !           149:  *     shared with this module, so that page allocation
        !           150:  *     (done here in vm_page_alloc) can trigger the
        !           151:  *     pageout daemon.
        !           152:  */
        !           153: int    vm_page_free_target = 0;
        !           154: int    vm_page_free_min = 0;
        !           155: int    vm_page_inactive_target = 0;
        !           156: int    vm_page_free_reserved = 0;
        !           157: int    vm_page_laundry_count = 0;
        !           158: int    vm_page_external_limit = 0;
        !           159: 
        !           160: 
        !           161: /*
        !           162:  *     The VM system has a couple of heuristics for deciding
        !           163:  *     that pages are "uninteresting" and should be placed
        !           164:  *     on the inactive queue as likely candidates for replacement.
        !           165:  *     These variables let the heuristics be controlled at run-time
        !           166:  *     to make experimentation easier.
        !           167:  */
        !           168: 
        !           169: boolean_t vm_page_deactivate_behind = TRUE;
        !           170: boolean_t vm_page_deactivate_hint = TRUE;
        !           171: 
        !           172: /*
        !           173:  *     vm_page_bootstrap:
        !           174:  *
        !           175:  *     Initializes the resident memory module.
        !           176:  *
        !           177:  *     Allocates memory for the page cells, and
        !           178:  *     for the object/offset-to-page hash table headers.
        !           179:  *     Each page cell is initialized and placed on the free list.
        !           180:  *     Returns the range of available kernel virtual memory.
        !           181:  */
        !           182: 
        !           183: void vm_page_bootstrap(
        !           184:        vm_offset_t *startp,
        !           185:        vm_offset_t *endp)
        !           186: {
        !           187:        register vm_page_t m;
        !           188:        int i;
        !           189: 
        !           190:        /*
        !           191:         *      Initialize the vm_page template.
        !           192:         */
        !           193: 
        !           194:        m = &vm_page_template;
        !           195:        m->object = VM_OBJECT_NULL;     /* reset later */
        !           196:        m->offset = 0;                  /* reset later */
        !           197:        m->wire_count = 0;
        !           198: 
        !           199:        m->inactive = FALSE;
        !           200:        m->active = FALSE;
        !           201:        m->laundry = FALSE;
        !           202:        m->free = FALSE;
        !           203:        m->external = FALSE;
        !           204: 
        !           205:        m->busy = TRUE;
        !           206:        m->wanted = FALSE;
        !           207:        m->tabled = FALSE;
        !           208:        m->fictitious = FALSE;
        !           209:        m->private = FALSE;
        !           210:        m->absent = FALSE;
        !           211:        m->error = FALSE;
        !           212:        m->dirty = FALSE;
        !           213:        m->precious = FALSE;
        !           214:        m->reference = FALSE;
        !           215: 
        !           216:        m->phys_addr = 0;               /* reset later */
        !           217: 
        !           218:        m->page_lock = VM_PROT_NONE;
        !           219:        m->unlock_request = VM_PROT_NONE;
        !           220: 
        !           221:        /*
        !           222:         *      Initialize the page queues.
        !           223:         */
        !           224: 
        !           225:        simple_lock_init(&vm_page_queue_free_lock);
        !           226:        simple_lock_init(&vm_page_queue_lock);
        !           227: 
        !           228:        vm_page_queue_free = VM_PAGE_NULL;
        !           229:        vm_page_queue_fictitious = VM_PAGE_NULL;
        !           230:        queue_init(&vm_page_queue_active);
        !           231:        queue_init(&vm_page_queue_inactive);
        !           232: 
        !           233:        vm_page_free_wanted = 0;
        !           234: 
        !           235:        /*
        !           236:         *      Allocate (and initialize) the virtual-to-physical
        !           237:         *      table hash buckets.
        !           238:         *
        !           239:         *      The number of buckets should be a power of two to
        !           240:         *      get a good hash function.  The following computation
        !           241:         *      chooses the first power of two that is greater
        !           242:         *      than the number of physical pages in the system.
        !           243:         */
        !           244: 
        !           245:        if (vm_page_bucket_count == 0) {
        !           246:                unsigned int npages = pmap_free_pages();
        !           247: 
        !           248:                vm_page_bucket_count = 1;
        !           249:                while (vm_page_bucket_count < npages)
        !           250:                        vm_page_bucket_count <<= 1;
        !           251:        }
        !           252: 
        !           253:        vm_page_hash_mask = vm_page_bucket_count - 1;
        !           254: 
        !           255:        if (vm_page_hash_mask & vm_page_bucket_count)
        !           256:                printf("vm_page_bootstrap: WARNING -- strange page hash\n");
        !           257: 
        !           258:        vm_page_buckets = (vm_page_bucket_t *)
        !           259:                pmap_steal_memory(vm_page_bucket_count *
        !           260:                                  sizeof(vm_page_bucket_t));
        !           261: 
        !           262:        for (i = 0; i < vm_page_bucket_count; i++) {
        !           263:                register vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !           264: 
        !           265:                bucket->pages = VM_PAGE_NULL;
        !           266:                simple_lock_init(&bucket->lock);
        !           267:        }
        !           268: 
        !           269:        /*
        !           270:         *      Steal memory for the kentry zone.
        !           271:         */
        !           272: 
        !           273:        kentry_data_size = kentry_count * sizeof(struct vm_map_entry);
        !           274:        kentry_data = pmap_steal_memory(kentry_data_size);
        !           275: 
        !           276:        /*
        !           277:         *      Machine-dependent code allocates the resident page table.
        !           278:         *      It uses vm_page_init to initialize the page frames.
        !           279:         *      The code also returns to us the virtual space available
        !           280:         *      to the kernel.  We don't trust the pmap module
        !           281:         *      to get the alignment right.
        !           282:         */
        !           283: 
        !           284:        /* pmap_startup is responsible for stealing memory for
        !           285:           the zone system and putting it in zdata.  */
        !           286:        pmap_startup(&virtual_space_start, &virtual_space_end);
        !           287:        virtual_space_start = round_page(virtual_space_start);
        !           288:        virtual_space_end = trunc_page(virtual_space_end);
        !           289: 
        !           290:        *startp = virtual_space_start;
        !           291:        *endp = virtual_space_end;
        !           292: 
        !           293:        /*      printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
        !           294:        vm_page_free_count_minimum = vm_page_free_count;
        !           295: }
        !           296: 
        !           297: #ifndef        MACHINE_PAGES
        !           298: /*
        !           299:  *     We implement pmap_steal_memory and pmap_startup with the help
        !           300:  *     of two simpler functions, pmap_virtual_space and pmap_next_page.
        !           301:  */
        !           302: 
        !           303: vm_offset_t pmap_steal_memory(
        !           304:        vm_size_t size)
        !           305: {
        !           306:        vm_offset_t addr, vaddr, paddr;
        !           307: 
        !           308:        /*
        !           309:         *      We round the size to an integer multiple.
        !           310:         */
        !           311: 
        !           312:        size = (size + 3) &~ 3;
        !           313: 
        !           314:        /*
        !           315:         *      If this is the first call to pmap_steal_memory,
        !           316:         *      we have to initialize ourself.
        !           317:         */
        !           318: 
        !           319:        if (virtual_space_start == virtual_space_end) {
        !           320:                pmap_virtual_space(&virtual_space_start, &virtual_space_end);
        !           321: 
        !           322:                /*
        !           323:                 *      The initial values must be aligned properly, and
        !           324:                 *      we don't trust the pmap module to do it right.
        !           325:                 */
        !           326: 
        !           327:                virtual_space_start = round_page(virtual_space_start);
        !           328:                virtual_space_end = trunc_page(virtual_space_end);
        !           329:        }
        !           330: 
        !           331:        /*
        !           332:         *      Allocate virtual memory for this request.
        !           333:         */
        !           334: 
        !           335:        addr = virtual_space_start;
        !           336:        virtual_space_start += size;
        !           337: 
        !           338:        /*
        !           339:         *      Allocate and map physical pages to back new virtual pages.
        !           340:         */
        !           341: 
        !           342:        for (vaddr = round_page(addr);
        !           343:             vaddr < addr + size;
        !           344:             vaddr += PAGE_SIZE) {
        !           345:                paddr = pmap_grab_page();
        !           346: 
        !           347:                /*
        !           348:                 *      XXX Logically, these mappings should be wired,
        !           349:                 *      but some pmap modules barf if they are.
        !           350:                 */
        !           351: 
        !           352:                pmap_enter(kernel_pmap, vaddr, paddr,
        !           353:                           VM_PROT_READ|VM_PROT_WRITE, FALSE);
        !           354:        }
        !           355: 
        !           356:        return addr;
        !           357: }
        !           358: 
        !           359: #if 0
        !           360: void pmap_startup(
        !           361:        vm_offset_t *startp,
        !           362:        vm_offset_t *endp)
        !           363: {
        !           364:        unsigned int i, npages, pages_initialized;
        !           365:        vm_page_t pages;
        !           366:        vm_offset_t paddr;
        !           367: 
        !           368:        /*
        !           369:         *      We calculate how many page frames we will have
        !           370:         *      and then allocate the page structures in one chunk.
        !           371:         */
        !           372: 
        !           373:        npages = ((PAGE_SIZE * pmap_free_pages() +
        !           374:                   (round_page(virtual_space_start) - virtual_space_start)) /
        !           375:                  (PAGE_SIZE + sizeof *pages));
        !           376: 
        !           377:        pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
        !           378: 
        !           379:        /*
        !           380:         *      Initialize the page frames.
        !           381:         */
        !           382: 
        !           383:        for (i = 0, pages_initialized = 0; i < npages; i++) {
        !           384:                if (!pmap_next_page(&paddr))
        !           385:                        break;
        !           386: 
        !           387:                vm_page_init(&pages[i], paddr);
        !           388:                pages_initialized++;
        !           389:        }
        !           390: 
        !           391:        /*
        !           392:         * Release pages in reverse order so that physical pages
        !           393:         * initially get allocated in ascending addresses. This keeps
        !           394:         * the devices (which must address physical memory) happy if
        !           395:         * they require several consecutive pages.
        !           396:         */
        !           397: 
        !           398:        for (i = pages_initialized; i > 0; i--) {
        !           399:                vm_page_release(&pages[i - 1], FALSE);
        !           400:        }
        !           401: 
        !           402:        /*
        !           403:         *      We have to re-align virtual_space_start,
        !           404:         *      because pmap_steal_memory has been using it.
        !           405:         */
        !           406: 
        !           407:        virtual_space_start = round_page(virtual_space_start);
        !           408: 
        !           409:        *startp = virtual_space_start;
        !           410:        *endp = virtual_space_end;
        !           411: }
        !           412: #endif
        !           413: #endif /* MACHINE_PAGES */
        !           414: 
        !           415: /*
        !           416:  *     Routine:        vm_page_module_init
        !           417:  *     Purpose:
        !           418:  *             Second initialization pass, to be done after
        !           419:  *             the basic VM system is ready.
        !           420:  */
        !           421: void           vm_page_module_init(void)
        !           422: {
        !           423:        vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
        !           424:                             VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
        !           425:                             PAGE_SIZE,
        !           426:                             /*
        !           427:                              * This zone can be collectable, since we
        !           428:                              * dynamically add and remove pages from the
        !           429:                              * pool as the needs of drivers for contiguous
        !           430:                              * memory changes.
        !           431:                              */
        !           432:                             ZONE_COLLECTABLE, "vm pages");
        !           433: }
        !           434: 
        !           435: /*
        !           436:  *     Routine:        vm_page_create
        !           437:  *     Purpose:
        !           438:  *             After the VM system is up, machine-dependent code
        !           439:  *             may stumble across more physical memory.  For example,
        !           440:  *             memory that it was reserving for a frame buffer.
        !           441:  *             vm_page_create turns this memory into available pages.
        !           442:  */
        !           443: 
        !           444: void vm_page_create(
        !           445:        vm_offset_t     start,
        !           446:        vm_offset_t     end)
        !           447: {
        !           448:        vm_offset_t paddr;
        !           449:        vm_page_t m;
        !           450: 
        !           451:        for (paddr = round_page(start);
        !           452:             paddr < trunc_page(end);
        !           453:             paddr += PAGE_SIZE) {
        !           454:                m = (vm_page_t) zalloc(vm_page_zone);
        !           455:                if (m == VM_PAGE_NULL)
        !           456:                        panic("vm_page_create");
        !           457: 
        !           458:                vm_page_init(m, paddr);
        !           459:                vm_page_release(m, FALSE);
        !           460:        }
        !           461: }
        !           462: 
        !           463: /*
        !           464:  *     vm_page_hash:
        !           465:  *
        !           466:  *     Distributes the object/offset key pair among hash buckets.
        !           467:  *
        !           468:  *     NOTE:   To get a good hash function, the bucket count should
        !           469:  *             be a power of two.
        !           470:  */
        !           471: #define vm_page_hash(object, offset) \
        !           472:        (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
        !           473:                & vm_page_hash_mask)
        !           474: 
        !           475: /*
        !           476:  *     vm_page_insert:         [ internal use only ]
        !           477:  *
        !           478:  *     Inserts the given mem entry into the object/object-page
        !           479:  *     table and object list.
        !           480:  *
        !           481:  *     The object and page must be locked.
        !           482:  */
        !           483: 
        !           484: void vm_page_insert(
        !           485:        register vm_page_t      mem,
        !           486:        register vm_object_t    object,
        !           487:        register vm_offset_t    offset)
        !           488: {
        !           489:        register vm_page_bucket_t *bucket;
        !           490: 
        !           491:        VM_PAGE_CHECK(mem);
        !           492: 
        !           493:        if (mem->tabled)
        !           494:                panic("vm_page_insert");
        !           495: 
        !           496:        /*
        !           497:         *      Record the object/offset pair in this page
        !           498:         */
        !           499: 
        !           500:        mem->object = object;
        !           501:        mem->offset = offset;
        !           502: 
        !           503:        /*
        !           504:         *      Insert it into the object_object/offset hash table
        !           505:         */
        !           506: 
        !           507:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           508:        simple_lock(&bucket->lock);
        !           509:        mem->next = bucket->pages;
        !           510:        bucket->pages = mem;
        !           511:        simple_unlock(&bucket->lock);
        !           512: 
        !           513:        /*
        !           514:         *      Now link into the object's list of backed pages.
        !           515:         */
        !           516: 
        !           517:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           518:        mem->tabled = TRUE;
        !           519: 
        !           520:        /*
        !           521:         *      Show that the object has one more resident page.
        !           522:         */
        !           523: 
        !           524:        object->resident_page_count++;
        !           525: 
        !           526:        /*
        !           527:         *      Detect sequential access and inactivate previous page.
        !           528:         *      We ignore busy pages.
        !           529:         */
        !           530: 
        !           531:        if (vm_page_deactivate_behind &&
        !           532:            (offset == object->last_alloc + PAGE_SIZE)) {
        !           533:                vm_page_t       last_mem;
        !           534: 
        !           535:                last_mem = vm_page_lookup(object, object->last_alloc);
        !           536:                if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
        !           537:                        vm_page_deactivate(last_mem);
        !           538:        }
        !           539:        object->last_alloc = offset;
        !           540: }
        !           541: 
        !           542: /*
        !           543:  *     vm_page_replace:
        !           544:  *
        !           545:  *     Exactly like vm_page_insert, except that we first
        !           546:  *     remove any existing page at the given offset in object
        !           547:  *     and we don't do deactivate-behind.
        !           548:  *
        !           549:  *     The object and page must be locked.
        !           550:  */
        !           551: 
        !           552: void vm_page_replace(
        !           553:        register vm_page_t      mem,
        !           554:        register vm_object_t    object,
        !           555:        register vm_offset_t    offset)
        !           556: {
        !           557:        register vm_page_bucket_t *bucket;
        !           558: 
        !           559:        VM_PAGE_CHECK(mem);
        !           560: 
        !           561:        if (mem->tabled)
        !           562:                panic("vm_page_replace");
        !           563: 
        !           564:        /*
        !           565:         *      Record the object/offset pair in this page
        !           566:         */
        !           567: 
        !           568:        mem->object = object;
        !           569:        mem->offset = offset;
        !           570: 
        !           571:        /*
        !           572:         *      Insert it into the object_object/offset hash table,
        !           573:         *      replacing any page that might have been there.
        !           574:         */
        !           575: 
        !           576:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           577:        simple_lock(&bucket->lock);
        !           578:        if (bucket->pages) {
        !           579:                vm_page_t *mp = &bucket->pages;
        !           580:                register vm_page_t m = *mp;
        !           581:                do {
        !           582:                        if (m->object == object && m->offset == offset) {
        !           583:                                /*
        !           584:                                 * Remove page from bucket and from object,
        !           585:                                 * and return it to the free list.
        !           586:                                 */
        !           587:                                *mp = m->next;
        !           588:                                queue_remove(&object->memq, m, vm_page_t,
        !           589:                                             listq);
        !           590:                                m->tabled = FALSE;
        !           591:                                object->resident_page_count--;
        !           592: 
        !           593:                                /*
        !           594:                                 * Return page to the free list.
        !           595:                                 * Note the page is not tabled now, so this
        !           596:                                 * won't self-deadlock on the bucket lock.
        !           597:                                 */
        !           598: 
        !           599:                                vm_page_free(m);
        !           600:                                break;
        !           601:                        }
        !           602:                        mp = &m->next;
        !           603:                } while ((m = *mp) != 0);
        !           604:                mem->next = bucket->pages;
        !           605:        } else {
        !           606:                mem->next = VM_PAGE_NULL;
        !           607:        }
        !           608:        bucket->pages = mem;
        !           609:        simple_unlock(&bucket->lock);
        !           610: 
        !           611:        /*
        !           612:         *      Now link into the object's list of backed pages.
        !           613:         */
        !           614: 
        !           615:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           616:        mem->tabled = TRUE;
        !           617: 
        !           618:        /*
        !           619:         *      And show that the object has one more resident
        !           620:         *      page.
        !           621:         */
        !           622: 
        !           623:        object->resident_page_count++;
        !           624: }
        !           625: 
        !           626: /*
        !           627:  *     vm_page_remove:         [ internal use only ]
        !           628:  *
        !           629:  *     Removes the given mem entry from the object/offset-page
        !           630:  *     table and the object page list.
        !           631:  *
        !           632:  *     The object and page must be locked.
        !           633:  */
        !           634: 
        !           635: void vm_page_remove(
        !           636:        register vm_page_t      mem)
        !           637: {
        !           638:        register vm_page_bucket_t       *bucket;
        !           639:        register vm_page_t      this;
        !           640: 
        !           641:        assert(mem->tabled);
        !           642:        VM_PAGE_CHECK(mem);
        !           643: 
        !           644:        /*
        !           645:         *      Remove from the object_object/offset hash table
        !           646:         */
        !           647: 
        !           648:        bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
        !           649:        simple_lock(&bucket->lock);
        !           650:        if ((this = bucket->pages) == mem) {
        !           651:                /* optimize for common case */
        !           652: 
        !           653:                bucket->pages = mem->next;
        !           654:        } else {
        !           655:                register vm_page_t      *prev;
        !           656: 
        !           657:                for (prev = &this->next;
        !           658:                     (this = *prev) != mem;
        !           659:                     prev = &this->next)
        !           660:                        continue;
        !           661:                *prev = this->next;
        !           662:        }
        !           663:        simple_unlock(&bucket->lock);
        !           664: 
        !           665:        /*
        !           666:         *      Now remove from the object's list of backed pages.
        !           667:         */
        !           668: 
        !           669:        queue_remove(&mem->object->memq, mem, vm_page_t, listq);
        !           670: 
        !           671:        /*
        !           672:         *      And show that the object has one fewer resident
        !           673:         *      page.
        !           674:         */
        !           675: 
        !           676:        mem->object->resident_page_count--;
        !           677: 
        !           678:        mem->tabled = FALSE;
        !           679: }
        !           680: 
        !           681: /*
        !           682:  *     vm_page_lookup:
        !           683:  *
        !           684:  *     Returns the page associated with the object/offset
        !           685:  *     pair specified; if none is found, VM_PAGE_NULL is returned.
        !           686:  *
        !           687:  *     The object must be locked.  No side effects.
        !           688:  */
        !           689: 
        !           690: vm_page_t vm_page_lookup(
        !           691:        register vm_object_t    object,
        !           692:        register vm_offset_t    offset)
        !           693: {
        !           694:        register vm_page_t      mem;
        !           695:        register vm_page_bucket_t *bucket;
        !           696: 
        !           697:        /*
        !           698:         *      Search the hash table for this object/offset pair
        !           699:         */
        !           700: 
        !           701:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           702: 
        !           703:        simple_lock(&bucket->lock);
        !           704:        for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
        !           705:                VM_PAGE_CHECK(mem);
        !           706:                if ((mem->object == object) && (mem->offset == offset))
        !           707:                        break;
        !           708:        }
        !           709:        simple_unlock(&bucket->lock);
        !           710:        return mem;
        !           711: }
        !           712: 
        !           713: /*
        !           714:  *     vm_page_rename:
        !           715:  *
        !           716:  *     Move the given memory entry from its
        !           717:  *     current object to the specified target object/offset.
        !           718:  *
        !           719:  *     The object must be locked.
        !           720:  */
        !           721: void vm_page_rename(
        !           722:        register vm_page_t      mem,
        !           723:        register vm_object_t    new_object,
        !           724:        vm_offset_t             new_offset)
        !           725: {
        !           726:        /*
        !           727:         *      Changes to mem->object require the page lock because
        !           728:         *      the pageout daemon uses that lock to get the object.
        !           729:         */
        !           730: 
        !           731:        vm_page_lock_queues();
        !           732:        vm_page_remove(mem);
        !           733:        vm_page_insert(mem, new_object, new_offset);
        !           734:        vm_page_unlock_queues();
        !           735: }
        !           736: 
        !           737: /*
        !           738:  *     vm_page_init:
        !           739:  *
        !           740:  *     Initialize the fields in a new page.
        !           741:  *     This takes a structure with random values and initializes it
        !           742:  *     so that it can be given to vm_page_release or vm_page_insert.
        !           743:  */
        !           744: void vm_page_init(
        !           745:        vm_page_t       mem,
        !           746:        vm_offset_t     phys_addr)
        !           747: {
        !           748:        *mem = vm_page_template;
        !           749:        mem->phys_addr = phys_addr;
        !           750: }
        !           751: 
        !           752: /*
        !           753:  *     vm_page_grab_fictitious:
        !           754:  *
        !           755:  *     Remove a fictitious page from the free list.
        !           756:  *     Returns VM_PAGE_NULL if there are no free pages.
        !           757:  */
        !           758: 
        !           759: vm_page_t vm_page_grab_fictitious(void)
        !           760: {
        !           761:        register vm_page_t m;
        !           762: 
        !           763:        simple_lock(&vm_page_queue_free_lock);
        !           764:        m = vm_page_queue_fictitious;
        !           765:        if (m != VM_PAGE_NULL) {
        !           766:                vm_page_fictitious_count--;
        !           767:                vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
        !           768:                m->free = FALSE;
        !           769:        }
        !           770:        simple_unlock(&vm_page_queue_free_lock);
        !           771: 
        !           772:        return m;
        !           773: }
        !           774: 
        !           775: /*
        !           776:  *     vm_page_release_fictitious:
        !           777:  *
        !           778:  *     Release a fictitious page to the free list.
        !           779:  */
        !           780: 
        !           781: void vm_page_release_fictitious(
        !           782:        register vm_page_t m)
        !           783: {
        !           784:        simple_lock(&vm_page_queue_free_lock);
        !           785:        if (m->free)
        !           786:                panic("vm_page_release_fictitious");
        !           787:        m->free = TRUE;
        !           788:        m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
        !           789:        vm_page_queue_fictitious = m;
        !           790:        vm_page_fictitious_count++;
        !           791:        simple_unlock(&vm_page_queue_free_lock);
        !           792: }
        !           793: 
        !           794: /*
        !           795:  *     vm_page_more_fictitious:
        !           796:  *
        !           797:  *     Add more fictitious pages to the free list.
        !           798:  *     Allowed to block.
        !           799:  */
        !           800: 
        !           801: int vm_page_fictitious_quantum = 5;
        !           802: 
        !           803: void vm_page_more_fictitious(void)
        !           804: {
        !           805:        register vm_page_t m;
        !           806:        int i;
        !           807: 
        !           808:        for (i = 0; i < vm_page_fictitious_quantum; i++) {
        !           809:                m = (vm_page_t) zalloc(vm_page_zone);
        !           810:                if (m == VM_PAGE_NULL)
        !           811:                        panic("vm_page_more_fictitious");
        !           812: 
        !           813:                vm_page_init(m, vm_page_fictitious_addr);
        !           814:                m->fictitious = TRUE;
        !           815:                vm_page_release_fictitious(m);
        !           816:        }
        !           817: }
        !           818: 
        !           819: /*
        !           820:  *     vm_page_convert:
        !           821:  *
        !           822:  *     Attempt to convert a fictitious page into a real page.
        !           823:  */
        !           824: 
        !           825: boolean_t vm_page_convert(
        !           826:        register vm_page_t m,
        !           827:        boolean_t external)
        !           828: {
        !           829:        register vm_page_t real_m;
        !           830: 
        !           831:        real_m = vm_page_grab(external);
        !           832:        if (real_m == VM_PAGE_NULL)
        !           833:                return FALSE;
        !           834: 
        !           835:        m->phys_addr = real_m->phys_addr;
        !           836:        m->fictitious = FALSE;
        !           837: 
        !           838:        real_m->phys_addr = vm_page_fictitious_addr;
        !           839:        real_m->fictitious = TRUE;
        !           840: 
        !           841:        vm_page_release_fictitious(real_m);
        !           842:        return TRUE;
        !           843: }
        !           844: 
        !           845: /*
        !           846:  *     vm_page_grab:
        !           847:  *
        !           848:  *     Remove a page from the free list.
        !           849:  *     Returns VM_PAGE_NULL if the free list is too small.
        !           850:  */
        !           851: 
        !           852: vm_page_t vm_page_grab(
        !           853:        boolean_t external)
        !           854: {
        !           855:        register vm_page_t      mem;
        !           856: 
        !           857:        simple_lock(&vm_page_queue_free_lock);
        !           858: 
        !           859:        /*
        !           860:         *      Only let privileged threads (involved in pageout)
        !           861:         *      dip into the reserved pool or exceed the limit
        !           862:         *      for externally-managed pages.
        !           863:         */
        !           864: 
        !           865:        if (((vm_page_free_count < vm_page_free_reserved)
        !           866:             || (external
        !           867:                 && (vm_page_external_count > vm_page_external_limit)))
        !           868:            && !current_thread()->vm_privilege) {
        !           869:                simple_unlock(&vm_page_queue_free_lock);
        !           870:                return VM_PAGE_NULL;
        !           871:        }
        !           872: 
        !           873:        if (vm_page_queue_free == VM_PAGE_NULL) {
        !           874:                vm_page_grab_oskit_page();
        !           875:                assert (vm_page_queue_free != VM_PAGE_NULL);
        !           876:        }
        !           877: 
        !           878:        --vm_page_queue_free_count;
        !           879:        if (vm_page_free_count < vm_page_free_count_minimum)
        !           880:                vm_page_free_count_minimum = vm_page_free_count;
        !           881:        if (external)
        !           882:                vm_page_external_count++;
        !           883:        mem = vm_page_queue_free;
        !           884:        vm_page_queue_free = (vm_page_t) mem->pageq.next;
        !           885:        mem->free = FALSE;
        !           886:        mem->extcounted = mem->external = external;
        !           887:        simple_unlock(&vm_page_queue_free_lock);
        !           888: 
        !           889:        /*
        !           890:         *      Decide if we should poke the pageout daemon.
        !           891:         *      We do this if the free count is less than the low
        !           892:         *      water mark, or if the free count is less than the high
        !           893:         *      water mark (but above the low water mark) and the inactive
        !           894:         *      count is less than its target.
        !           895:         *
        !           896:         *      We don't have the counts locked ... if they change a little,
        !           897:         *      it doesn't really matter.
        !           898:         */
        !           899: 
        !           900:        if ((vm_page_free_count < vm_page_free_min) ||
        !           901:            ((vm_page_free_count < vm_page_free_target) &&
        !           902:             (vm_page_inactive_count < vm_page_inactive_target)) ||
        !           903:            vm_page_queue_free_count < vm_page_unqueued_count)
        !           904:                thread_wakeup((event_t) &vm_page_free_wanted);
        !           905: 
        !           906:        return mem;
        !           907: }
        !           908: 
        !           909: #if 0
        !           910: vm_offset_t vm_page_grab_phys_addr()
        !           911: {
        !           912:        vm_page_t p = vm_page_grab(FALSE);
        !           913:        if (p == VM_PAGE_NULL)
        !           914:                return -1;
        !           915:        else
        !           916:                return p->phys_addr;
        !           917: }
        !           918: 
        !           919: /*
        !           920:  *     vm_page_grab_contiguous_pages:
        !           921:  *
        !           922:  *     Take N pages off the free list, the pages should
        !           923:  *     cover a contiguous range of physical addresses.
        !           924:  *     [Used by device drivers to cope with DMA limitations]
        !           925:  *
        !           926:  *     Returns the page descriptors in ascending order, or
        !           927:  *     Returns KERN_RESOURCE_SHORTAGE if it could not.
        !           928:  */
        !           929: 
        !           930: /* Biggest phys page number for the pages we handle in VM */
        !           931: 
        !           932: vm_size_t      vm_page_big_pagenum = 0;        /* Set this before call! */
        !           933: 
        !           934: kern_return_t
        !           935: vm_page_grab_contiguous_pages(
        !           936:        int             npages,
        !           937:        vm_page_t       pages[],
        !           938:        natural_t       *bits,
        !           939:        boolean_t       external)
        !           940: {
        !           941:        register int    first_set;
        !           942:        int             size, alloc_size;
        !           943:        kern_return_t   ret;
        !           944:        vm_page_t       mem, prevmem;
        !           945: 
        !           946: #ifndef        NBBY
        !           947: #define        NBBY    8       /* size in bits of sizeof()`s unity */
        !           948: #endif
        !           949: 
        !           950: #define        NBPEL   (sizeof(natural_t)*NBBY)
        !           951: 
        !           952:        size = (vm_page_big_pagenum + NBPEL - 1)
        !           953:                & ~(NBPEL - 1);                         /* in bits */
        !           954: 
        !           955:        size = size / NBBY;                             /* in bytes */
        !           956: 
        !           957:        /*
        !           958:         * If we are called before the VM system is fully functional
        !           959:         * the invoker must provide us with the work space. [one bit
        !           960:         * per page starting at phys 0 and up to vm_page_big_pagenum]
        !           961:         */
        !           962:        if (bits == 0) {
        !           963:                alloc_size = round_page(size);
        !           964:                if (kmem_alloc_wired(kernel_map,
        !           965:                                     (vm_offset_t *)&bits,
        !           966:                                     alloc_size)
        !           967:                        != KERN_SUCCESS)
        !           968:                    return KERN_RESOURCE_SHORTAGE;
        !           969:        } else
        !           970:                alloc_size = 0;
        !           971: 
        !           972:        bzero(bits, size);
        !           973: 
        !           974:        /*
        !           975:         * A very large granularity call, its rare so that is ok
        !           976:         */
        !           977:        simple_lock(&vm_page_queue_free_lock);
        !           978: 
        !           979:        /*
        !           980:         *      Do not dip into the reserved pool.
        !           981:         */
        !           982: 
        !           983:        if ((vm_page_free_count < vm_page_free_reserved)
        !           984:            || (vm_page_external_count >= vm_page_external_limit)) {
        !           985:                simple_unlock(&vm_page_queue_free_lock);
        !           986:                return KERN_RESOURCE_SHORTAGE;
        !           987:        }
        !           988: 
        !           989:        /*
        !           990:         *      First pass through, build a big bit-array of
        !           991:         *      the pages that are free.  It is not going to
        !           992:         *      be too large anyways, in 4k we can fit info
        !           993:         *      for 32k pages.
        !           994:         */
        !           995:        mem = vm_page_queue_free;
        !           996:        while (mem) {
        !           997:                register int word_index, bit_index;
        !           998: 
        !           999:                bit_index = (mem->phys_addr >> PAGE_SHIFT);
        !          1000:                word_index = bit_index / NBPEL;
        !          1001:                bit_index = bit_index - (word_index * NBPEL);
        !          1002:                bits[word_index] |= 1 << bit_index;
        !          1003: 
        !          1004:                mem = (vm_page_t) mem->pageq.next;
        !          1005:        }
        !          1006: 
        !          1007:        /*
        !          1008:         *      Second loop. Scan the bit array for NPAGES
        !          1009:         *      contiguous bits.  That gives us, if any,
        !          1010:         *      the range of pages we will be grabbing off
        !          1011:         *      the free list.
        !          1012:         */
        !          1013:        {
        !          1014:            register int        bits_so_far = 0, i;
        !          1015: 
        !          1016:                first_set = 0;
        !          1017: 
        !          1018:                for (i = 0; i < size; i += sizeof(natural_t)) {
        !          1019: 
        !          1020:                    register natural_t  v = bits[i / sizeof(natural_t)];
        !          1021:                    register int        bitpos;
        !          1022: 
        !          1023:                    /*
        !          1024:                     * Bitscan this one word
        !          1025:                     */
        !          1026:                    if (v) {
        !          1027:                        /*
        !          1028:                         * keep counting them beans ?
        !          1029:                         */
        !          1030:                        bitpos = 0;
        !          1031: 
        !          1032:                        if (bits_so_far) {
        !          1033: count_ones:
        !          1034:                            while (v & 1) {
        !          1035:                                bitpos++;
        !          1036:                                /*
        !          1037:                                 * got enough beans ?
        !          1038:                                 */
        !          1039:                                if (++bits_so_far == npages)
        !          1040:                                    goto found_em;
        !          1041:                                v >>= 1;
        !          1042:                            }
        !          1043:                            /* if we are being lucky, roll again */
        !          1044:                            if (bitpos == NBPEL)
        !          1045:                                continue;
        !          1046:                        }
        !          1047: 
        !          1048:                        /*
        !          1049:                         * search for beans here
        !          1050:                         */
        !          1051:                        bits_so_far = 0;
        !          1052: count_zeroes:
        !          1053:                        while ((bitpos < NBPEL) && ((v & 1) == 0)) {
        !          1054:                            bitpos++;
        !          1055:                            v >>= 1;
        !          1056:                        }
        !          1057:                        if (v & 1) {
        !          1058:                            first_set = (i * NBBY) + bitpos;
        !          1059:                            goto count_ones;
        !          1060:                        }
        !          1061:                    }
        !          1062:                    /*
        !          1063:                     * No luck
        !          1064:                     */
        !          1065:                    bits_so_far = 0;
        !          1066:                }
        !          1067:        }
        !          1068: 
        !          1069:        /*
        !          1070:         *      We could not find enough contiguous pages.
        !          1071:         */
        !          1072: not_found_em:
        !          1073:        simple_unlock(&vm_page_queue_free_lock);
        !          1074: 
        !          1075:        ret = KERN_RESOURCE_SHORTAGE;
        !          1076:        goto out;
        !          1077: 
        !          1078:        /*
        !          1079:         *      Final pass. Now we know which pages we want.
        !          1080:         *      Scan the list until we find them all, grab
        !          1081:         *      pages as we go.  FIRST_SET tells us where
        !          1082:         *      in the bit-array our pages start.
        !          1083:         */
        !          1084: found_em:
        !          1085:        vm_page_free_count -= npages;
        !          1086:        if (vm_page_free_count < vm_page_free_count_minimum)
        !          1087:                vm_page_free_count_minimum = vm_page_free_count;
        !          1088:        if (external)
        !          1089:                vm_page_external_count += npages;
        !          1090:        {
        !          1091:            register vm_offset_t        first_phys, last_phys;
        !          1092: 
        !          1093:            /* cache values for compare */
        !          1094:            first_phys = first_set << PAGE_SHIFT;
        !          1095:            last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
        !          1096: 
        !          1097:            /* running pointers */
        !          1098:            mem = vm_page_queue_free;
        !          1099:            prevmem = VM_PAGE_NULL;
        !          1100: 
        !          1101:            while (mem) {
        !          1102: 
        !          1103:                register vm_offset_t    addr;
        !          1104: 
        !          1105:                addr = mem->phys_addr;
        !          1106: 
        !          1107:                if ((addr >= first_phys) &&
        !          1108:                    (addr <  last_phys)) {
        !          1109:                    if (prevmem)
        !          1110:                        prevmem->pageq.next = mem->pageq.next;
        !          1111:                    pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
        !          1112:                    mem->free = FALSE;
        !          1113:                    mem->extcounted = mem->external = external;
        !          1114:                    /*
        !          1115:                     * Got them all ?
        !          1116:                     */
        !          1117:                    if (--npages == 0) break;
        !          1118:                } else
        !          1119:                    prevmem = mem;
        !          1120: 
        !          1121:                mem = (vm_page_t) mem->pageq.next;
        !          1122:            }
        !          1123:        }
        !          1124: 
        !          1125:        simple_unlock(&vm_page_queue_free_lock);
        !          1126: 
        !          1127:        /*
        !          1128:         *      Decide if we should poke the pageout daemon.
        !          1129:         *      We do this if the free count is less than the low
        !          1130:         *      water mark, or if the free count is less than the high
        !          1131:         *      water mark (but above the low water mark) and the inactive
        !          1132:         *      count is less than its target.
        !          1133:         *
        !          1134:         *      We don't have the counts locked ... if they change a little,
        !          1135:         *      it doesn't really matter.
        !          1136:         */
        !          1137: 
        !          1138:        if ((vm_page_free_count < vm_page_free_min) ||
        !          1139:            ((vm_page_free_count < vm_page_free_target) &&
        !          1140:             (vm_page_inactive_count < vm_page_inactive_target)))
        !          1141:                thread_wakeup(&vm_page_free_wanted);
        !          1142: 
        !          1143:        ret = KERN_SUCCESS;
        !          1144: out:
        !          1145:        if (alloc_size)
        !          1146:                kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
        !          1147: 
        !          1148:        return ret;
        !          1149: }
        !          1150: #endif
        !          1151: 
        !          1152: /*
        !          1153:  *     vm_page_release:
        !          1154:  *
        !          1155:  *     Return a page to the free list.
        !          1156:  */
        !          1157: 
        !          1158: void vm_page_release(
        !          1159:        register vm_page_t      mem,
        !          1160:        boolean_t external)
        !          1161: {
        !          1162:        simple_lock(&vm_page_queue_free_lock);
        !          1163:        if (mem->free)
        !          1164:                panic("vm_page_release");
        !          1165:        mem->free = TRUE;
        !          1166:        mem->pageq.next = (queue_entry_t) vm_page_queue_free;
        !          1167:        vm_page_queue_free = mem;
        !          1168:        vm_page_queue_free_count++;
        !          1169:        if (external)
        !          1170:                vm_page_external_count--;
        !          1171: 
        !          1172:        /*
        !          1173:         *      Check if we should wake up someone waiting for page.
        !          1174:         *      But don't bother waking them unless they can allocate.
        !          1175:         *
        !          1176:         *      We wakeup only one thread, to prevent starvation.
        !          1177:         *      Because the scheduling system handles wait queues FIFO,
        !          1178:         *      if we wakeup all waiting threads, one greedy thread
        !          1179:         *      can starve multiple niceguy threads.  When the threads
        !          1180:         *      all wakeup, the greedy threads runs first, grabs the page,
        !          1181:         *      and waits for another page.  It will be the first to run
        !          1182:         *      when the next page is freed.
        !          1183:         *
        !          1184:         *      However, there is a slight danger here.
        !          1185:         *      The thread we wake might not use the free page.
        !          1186:         *      Then the other threads could wait indefinitely
        !          1187:         *      while the page goes unused.  To forestall this,
        !          1188:         *      the pageout daemon will keep making free pages
        !          1189:         *      as long as vm_page_free_wanted is non-zero.
        !          1190:         */
        !          1191: 
        !          1192:        if ((vm_page_free_wanted > 0) &&
        !          1193:            (vm_page_free_count >= vm_page_free_reserved)) {
        !          1194:                vm_page_free_wanted--;
        !          1195:                thread_wakeup_one((event_t) &vm_page_queue_free_count);
        !          1196:        }
        !          1197: 
        !          1198:        simple_unlock(&vm_page_queue_free_lock);
        !          1199: }
        !          1200: 
        !          1201: /*
        !          1202:  *     vm_page_wait:
        !          1203:  *
        !          1204:  *     Wait for a page to become available.
        !          1205:  *     If there are plenty of free pages, then we don't sleep.
        !          1206:  */
        !          1207: 
        !          1208: void vm_page_wait(
        !          1209:        void (*continuation)(void))
        !          1210: {
        !          1211: 
        !          1212: #ifndef CONTINUATIONS
        !          1213:        assert (continuation == 0);
        !          1214: #endif
        !          1215: 
        !          1216:        /*
        !          1217:         *      We can't use vm_page_free_reserved to make this
        !          1218:         *      determination.  Consider: some thread might
        !          1219:         *      need to allocate two pages.  The first allocation
        !          1220:         *      succeeds, the second fails.  After the first page is freed,
        !          1221:         *      a call to vm_page_wait must really block.
        !          1222:         */
        !          1223: 
        !          1224:        simple_lock(&vm_page_queue_free_lock);
        !          1225:        if ((vm_page_free_count < vm_page_free_target)
        !          1226:            || (vm_page_external_count > vm_page_external_limit)) {
        !          1227:                if (vm_page_free_wanted++ == 0)
        !          1228:                        thread_wakeup((event_t)&vm_page_free_wanted);
        !          1229:                assert_wait((event_t)&vm_page_queue_free_count, FALSE);
        !          1230:                simple_unlock(&vm_page_queue_free_lock);
        !          1231:                if (continuation != 0) {
        !          1232:                        counter(c_vm_page_wait_block_user++);
        !          1233:                        thread_block(continuation);
        !          1234:                } else {
        !          1235:                        counter(c_vm_page_wait_block_kernel++);
        !          1236:                        thread_block((void (*)(void)) 0);
        !          1237:                }
        !          1238:        } else
        !          1239:                simple_unlock(&vm_page_queue_free_lock);
        !          1240: }
        !          1241: 
        !          1242: /*
        !          1243:  *     vm_page_alloc:
        !          1244:  *
        !          1245:  *     Allocate and return a memory cell associated
        !          1246:  *     with this VM object/offset pair.
        !          1247:  *
        !          1248:  *     Object must be locked.
        !          1249:  */
        !          1250: 
        !          1251: vm_page_t vm_page_alloc(
        !          1252:        vm_object_t     object,
        !          1253:        vm_offset_t     offset)
        !          1254: {
        !          1255:        register vm_page_t      mem;
        !          1256: 
        !          1257:        mem = vm_page_grab(!object->internal);
        !          1258:        if (mem == VM_PAGE_NULL)
        !          1259:                return VM_PAGE_NULL;
        !          1260: 
        !          1261:        vm_page_lock_queues();
        !          1262:        vm_page_insert(mem, object, offset);
        !          1263:        vm_page_unlock_queues();
        !          1264: 
        !          1265:        return mem;
        !          1266: }
        !          1267: 
        !          1268: /*
        !          1269:  *     vm_page_free:
        !          1270:  *
        !          1271:  *     Returns the given page to the free list,
        !          1272:  *     disassociating it with any VM object.
        !          1273:  *
        !          1274:  *     Object and page queues must be locked prior to entry.
        !          1275:  */
        !          1276: void vm_page_free(
        !          1277:        register vm_page_t      mem)
        !          1278: {
        !          1279:        if (mem->free)
        !          1280:                panic("vm_page_free");
        !          1281: 
        !          1282:        if (mem->tabled)
        !          1283:                vm_page_remove(mem);
        !          1284:        VM_PAGE_QUEUES_REMOVE(mem);
        !          1285: 
        !          1286:        if (mem->wire_count != 0) {
        !          1287:                if (!mem->private && !mem->fictitious)
        !          1288:                        vm_page_wire_count--;
        !          1289:                mem->wire_count = 0;
        !          1290:        }
        !          1291: 
        !          1292:        if (mem->laundry) {
        !          1293:                vm_page_laundry_count--;
        !          1294:                mem->laundry = FALSE;
        !          1295:        }
        !          1296: 
        !          1297:        PAGE_WAKEUP_DONE(mem);
        !          1298: 
        !          1299:        if (mem->absent)
        !          1300:                vm_object_absent_release(mem->object);
        !          1301: 
        !          1302:        /*
        !          1303:         *      XXX The calls to vm_page_init here are
        !          1304:         *      really overkill.
        !          1305:         */
        !          1306: 
        !          1307:        if (mem->private || mem->fictitious) {
        !          1308:                vm_page_init(mem, vm_page_fictitious_addr);
        !          1309:                mem->fictitious = TRUE;
        !          1310:                vm_page_release_fictitious(mem);
        !          1311:        } else {
        !          1312:                int external = mem->external && mem->extcounted;
        !          1313:                vm_page_init(mem, mem->phys_addr);
        !          1314:                vm_page_release(mem, external);
        !          1315:        }
        !          1316: }
        !          1317: 
        !          1318: /*
        !          1319:  *     vm_page_wire:
        !          1320:  *
        !          1321:  *     Mark this page as wired down by yet
        !          1322:  *     another map, removing it from paging queues
        !          1323:  *     as necessary.
        !          1324:  *
        !          1325:  *     The page's object and the page queues must be locked.
        !          1326:  */
        !          1327: void vm_page_wire(
        !          1328:        register vm_page_t      mem)
        !          1329: {
        !          1330:        VM_PAGE_CHECK(mem);
        !          1331: 
        !          1332:        if (mem->wire_count == 0) {
        !          1333:                VM_PAGE_QUEUES_REMOVE(mem);
        !          1334:                if (!mem->private && !mem->fictitious)
        !          1335:                        vm_page_wire_count++;
        !          1336:        }
        !          1337:        mem->wire_count++;
        !          1338: }
        !          1339: 
        !          1340: /*
        !          1341:  *     vm_page_unwire:
        !          1342:  *
        !          1343:  *     Release one wiring of this page, potentially
        !          1344:  *     enabling it to be paged again.
        !          1345:  *
        !          1346:  *     The page's object and the page queues must be locked.
        !          1347:  */
        !          1348: void vm_page_unwire(
        !          1349:        register vm_page_t      mem)
        !          1350: {
        !          1351:        VM_PAGE_CHECK(mem);
        !          1352: 
        !          1353:        if (--mem->wire_count == 0) {
        !          1354:                queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
        !          1355:                vm_page_active_count++;
        !          1356:                mem->active = TRUE;
        !          1357:                if (!mem->private && !mem->fictitious)
        !          1358:                        vm_page_wire_count--;
        !          1359:        }
        !          1360: }
        !          1361: 
        !          1362: /*
        !          1363:  *     vm_page_deactivate:
        !          1364:  *
        !          1365:  *     Returns the given page to the inactive list,
        !          1366:  *     indicating that no physical maps have access
        !          1367:  *     to this page.  [Used by the physical mapping system.]
        !          1368:  *
        !          1369:  *     The page queues must be locked.
        !          1370:  */
        !          1371: void vm_page_deactivate(
        !          1372:        register vm_page_t      m)
        !          1373: {
        !          1374:        VM_PAGE_CHECK(m);
        !          1375: 
        !          1376:        /*
        !          1377:         *      This page is no longer very interesting.  If it was
        !          1378:         *      interesting (active or inactive/referenced), then we
        !          1379:         *      clear the reference bit and (re)enter it in the
        !          1380:         *      inactive queue.  Note wired pages should not have
        !          1381:         *      their reference bit cleared.
        !          1382:         */
        !          1383: 
        !          1384:        if (m->active || (m->inactive && m->reference)) {
        !          1385:                if (!m->fictitious && !m->absent)
        !          1386:                        pmap_clear_reference(m->phys_addr);
        !          1387:                m->reference = FALSE;
        !          1388:                VM_PAGE_QUEUES_REMOVE(m);
        !          1389:        }
        !          1390:        if (m->wire_count == 0 && !m->inactive) {
        !          1391:                queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
        !          1392:                m->inactive = TRUE;
        !          1393:                vm_page_inactive_count++;
        !          1394:        }
        !          1395: }
        !          1396: 
        !          1397: /*
        !          1398:  *     vm_page_activate:
        !          1399:  *
        !          1400:  *     Put the specified page on the active list (if appropriate).
        !          1401:  *
        !          1402:  *     The page queues must be locked.
        !          1403:  */
        !          1404: 
        !          1405: void vm_page_activate(
        !          1406:        register vm_page_t      m)
        !          1407: {
        !          1408:        VM_PAGE_CHECK(m);
        !          1409: 
        !          1410:        if (m->inactive) {
        !          1411:                queue_remove(&vm_page_queue_inactive, m, vm_page_t,
        !          1412:                                                pageq);
        !          1413:                vm_page_inactive_count--;
        !          1414:                m->inactive = FALSE;
        !          1415:        }
        !          1416:        if (m->wire_count == 0) {
        !          1417:                if (m->active)
        !          1418:                        panic("vm_page_activate: already active");
        !          1419: 
        !          1420:                queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
        !          1421:                m->active = TRUE;
        !          1422:                vm_page_active_count++;
        !          1423:        }
        !          1424: }
        !          1425: 
        !          1426: /*
        !          1427:  *     vm_page_zero_fill:
        !          1428:  *
        !          1429:  *     Zero-fill the specified page.
        !          1430:  */
        !          1431: void vm_page_zero_fill(
        !          1432:        vm_page_t       m)
        !          1433: {
        !          1434:        VM_PAGE_CHECK(m);
        !          1435: 
        !          1436:        pmap_zero_page(m->phys_addr);
        !          1437: }
        !          1438: 
        !          1439: /*
        !          1440:  *     vm_page_copy:
        !          1441:  *
        !          1442:  *     Copy one page to another
        !          1443:  */
        !          1444: 
        !          1445: void vm_page_copy(
        !          1446:        vm_page_t       src_m,
        !          1447:        vm_page_t       dest_m)
        !          1448: {
        !          1449:        VM_PAGE_CHECK(src_m);
        !          1450:        VM_PAGE_CHECK(dest_m);
        !          1451: 
        !          1452:        pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
        !          1453: }
        !          1454: 
        !          1455: #if    MACH_VM_DEBUG
        !          1456: /*
        !          1457:  *     Routine:        vm_page_info
        !          1458:  *     Purpose:
        !          1459:  *             Return information about the global VP table.
        !          1460:  *             Fills the buffer with as much information as possible
        !          1461:  *             and returns the desired size of the buffer.
        !          1462:  *     Conditions:
        !          1463:  *             Nothing locked.  The caller should provide
        !          1464:  *             possibly-pageable memory.
        !          1465:  */
        !          1466: 
        !          1467: unsigned int
        !          1468: vm_page_info(
        !          1469:        hash_info_bucket_t *info,
        !          1470:        unsigned int    count)
        !          1471: {
        !          1472:        int i;
        !          1473: 
        !          1474:        if (vm_page_bucket_count < count)
        !          1475:                count = vm_page_bucket_count;
        !          1476: 
        !          1477:        for (i = 0; i < count; i++) {
        !          1478:                vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !          1479:                unsigned int bucket_count = 0;
        !          1480:                vm_page_t m;
        !          1481: 
        !          1482:                simple_lock(&bucket->lock);
        !          1483:                for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
        !          1484:                        bucket_count++;
        !          1485:                simple_unlock(&bucket->lock);
        !          1486: 
        !          1487:                /* don't touch pageable memory while holding locks */
        !          1488:                info[i].hib_count = bucket_count;
        !          1489:        }
        !          1490: 
        !          1491:        return vm_page_bucket_count;
        !          1492: }
        !          1493: #endif /* MACH_VM_DEBUG */
        !          1494: 
        !          1495: #include <mach_kdb.h>
        !          1496: #if    MACH_KDB
        !          1497: #define        printf  kdbprintf
        !          1498: 
        !          1499: /*
        !          1500:  *     Routine:        vm_page_print [exported]
        !          1501:  */
        !          1502: void           vm_page_print(p)
        !          1503:        vm_page_t       p;
        !          1504: {
        !          1505:        iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
        !          1506:         printf(" offset 0x%X", (vm_offset_t) p->offset);
        !          1507:         printf("wire_count %d,", p->wire_count);
        !          1508:         printf(" %s",
        !          1509:                (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
        !          1510:         printf("%s",
        !          1511:                (p->free ? " free" : ""));
        !          1512:         printf("%s ",
        !          1513:                (p->laundry ? " laundry" : ""));
        !          1514:         printf("%s",
        !          1515:                (p->dirty ? "dirty" : "clean"));
        !          1516:         printf("%s",
        !          1517:                (p->busy ? " busy" : ""));
        !          1518:         printf("%s",
        !          1519:                (p->absent ? " absent" : ""));
        !          1520:         printf("%s",
        !          1521:                (p->error ? " error" : ""));
        !          1522:         printf("%s",
        !          1523:                (p->fictitious ? " fictitious" : ""));
        !          1524:         printf("%s",
        !          1525:                (p->private ? " private" : ""));
        !          1526:         printf("%s",
        !          1527:                (p->wanted ? " wanted" : ""));
        !          1528:         printf("%s,",
        !          1529:                (p->tabled ? "" : "not_tabled"));
        !          1530:         printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
        !          1531:                (vm_offset_t) p->phys_addr,
        !          1532:                (vm_offset_t) p->page_lock,
        !          1533:                (vm_offset_t) p->unlock_request);
        !          1534: }
        !          1535: #endif /* MACH_KDB */

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.