Return to vm_resident.c CVS log

Up to [Apple XNU] / XNU / osfmk / vm

Annotation of XNU/osfmk/vm/vm_resident.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
        !             3:  *
        !             4:  * @APPLE_LICENSE_HEADER_START@
        !             5:  * 
        !             6:  * The contents of this file constitute Original Code as defined in and
        !             7:  * are subject to the Apple Public Source License Version 1.1 (the
        !             8:  * "License").  You may not use this file except in compliance with the
        !             9:  * License.  Please obtain a copy of the License at
        !            10:  * http://www.apple.com/publicsource and read it before using this file.
        !            11:  * 
        !            12:  * This Original Code and all software distributed under the License are
        !            13:  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
        !            14:  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
        !            15:  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
        !            16:  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
        !            17:  * License for the specific language governing rights and limitations
        !            18:  * under the License.
        !            19:  * 
        !            20:  * @APPLE_LICENSE_HEADER_END@
        !            21:  */
        !            22: /*
        !            23:  * @OSF_COPYRIGHT@
        !            24:  */
        !            25: /* 
        !            26:  * Mach Operating System
        !            27:  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
        !            28:  * All Rights Reserved.
        !            29:  * 
        !            30:  * Permission to use, copy, modify and distribute this software and its
        !            31:  * documentation is hereby granted, provided that both the copyright
        !            32:  * notice and this permission notice appear in all copies of the
        !            33:  * software, derivative works or modified versions, and any portions
        !            34:  * thereof, and that both notices appear in supporting documentation.
        !            35:  * 
        !            36:  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
        !            37:  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
        !            38:  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
        !            39:  * 
        !            40:  * Carnegie Mellon requests users of this software to return to
        !            41:  * 
        !            42:  *  Software Distribution Coordinator  or  [email protected]
        !            43:  *  School of Computer Science
        !            44:  *  Carnegie Mellon University
        !            45:  *  Pittsburgh PA 15213-3890
        !            46:  * 
        !            47:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            48:  * the rights to redistribute these changes.
        !            49:  */
        !            50: /*
        !            51:  */
        !            52: /*
        !            53:  *     File:   vm/vm_page.c
        !            54:  *     Author: Avadis Tevanian, Jr., Michael Wayne Young
        !            55:  *
        !            56:  *     Resident memory management module.
        !            57:  */
        !            58: 
        !            59: #include <mach/vm_prot.h>
        !            60: #include <mach/vm_statistics.h>
        !            61: #include <kern/counters.h>
        !            62: #include <kern/sched_prim.h>
        !            63: #include <kern/task.h>
        !            64: #include <kern/thread.h>
        !            65: #include <kern/zalloc.h>
        !            66: #include <kern/xpr.h>
        !            67: #include <vm/pmap.h>
        !            68: #include <vm/vm_init.h>
        !            69: #include <vm/vm_map.h>
        !            70: #include <vm/vm_page.h>
        !            71: #include <vm/vm_pageout.h>
        !            72: #include <vm/vm_kern.h>                        /* kernel_memory_allocate() */
        !            73: #include <kern/misc_protos.h>
        !            74: #include <zone_debug.h>
        !            75: #include <vm/cpm.h>
        !            76: 
        !            77: /*
        !            78:  *     Associated with page of user-allocatable memory is a
        !            79:  *     page structure.
        !            80:  */
        !            81: 
        !            82: /*
        !            83:  *     These variables record the values returned by vm_page_bootstrap,
        !            84:  *     for debugging purposes.  The implementation of pmap_steal_memory
        !            85:  *     and pmap_startup here also uses them internally.
        !            86:  */
        !            87: 
        !            88: vm_offset_t virtual_space_start;
        !            89: vm_offset_t virtual_space_end;
        !            90: int    vm_page_pages;
        !            91: 
        !            92: /*
        !            93:  *     The vm_page_lookup() routine, which provides for fast
        !            94:  *     (virtual memory object, offset) to page lookup, employs
        !            95:  *     the following hash table.  The vm_page_{insert,remove}
        !            96:  *     routines install and remove associations in the table.
        !            97:  *     [This table is often called the virtual-to-physical,
        !            98:  *     or VP, table.]
        !            99:  */
        !           100: typedef struct {
        !           101:        vm_page_t       pages;
        !           102: #if    MACH_PAGE_HASH_STATS
        !           103:        int             cur_count;              /* current count */
        !           104:        int             hi_count;               /* high water mark */
        !           105: #endif /* MACH_PAGE_HASH_STATS */
        !           106: } vm_page_bucket_t;
        !           107: 
        !           108: vm_page_bucket_t *vm_page_buckets;             /* Array of buckets */
        !           109: unsigned int   vm_page_bucket_count = 0;       /* How big is array? */
        !           110: unsigned int   vm_page_hash_mask;              /* Mask for hash function */
        !           111: unsigned int   vm_page_hash_shift;             /* Shift for hash function */
        !           112: decl_simple_lock_data(,vm_page_bucket_lock)
        !           113: 
        !           114: #if    MACH_PAGE_HASH_STATS
        !           115: /* This routine is only for debug.  It is intended to be called by
        !           116:  * hand by a developer using a kernel debugger.  This routine prints
        !           117:  * out vm_page_hash table statistics to the kernel debug console.
        !           118:  */
        !           119: void
        !           120: hash_debug(void)
        !           121: {
        !           122:        int     i;
        !           123:        int     numbuckets = 0;
        !           124:        int     highsum = 0;
        !           125:        int     maxdepth = 0;
        !           126: 
        !           127:        for (i = 0; i < vm_page_bucket_count; i++) {
        !           128:                if (vm_page_buckets[i].hi_count) {
        !           129:                        numbuckets++;
        !           130:                        highsum += vm_page_buckets[i].hi_count;
        !           131:                        if (vm_page_buckets[i].hi_count > maxdepth)
        !           132:                                maxdepth = vm_page_buckets[i].hi_count;
        !           133:                }
        !           134:        }
        !           135:        printf("Total number of buckets: %d\n", vm_page_bucket_count);
        !           136:        printf("Number used buckets:     %d = %d%%\n",
        !           137:                numbuckets, 100*numbuckets/vm_page_bucket_count);
        !           138:        printf("Number unused buckets:   %d = %d%%\n",
        !           139:                vm_page_bucket_count - numbuckets,
        !           140:                100*(vm_page_bucket_count-numbuckets)/vm_page_bucket_count);
        !           141:        printf("Sum of bucket max depth: %d\n", highsum);
        !           142:        printf("Average bucket depth:    %d.%2d\n",
        !           143:                highsum/vm_page_bucket_count,
        !           144:                highsum%vm_page_bucket_count);
        !           145:        printf("Maximum bucket depth:    %d\n", maxdepth);
        !           146: }
        !           147: #endif /* MACH_PAGE_HASH_STATS */
        !           148: 
        !           149: /*
        !           150:  *     The virtual page size is currently implemented as a runtime
        !           151:  *     variable, but is constant once initialized using vm_set_page_size.
        !           152:  *     This initialization must be done in the machine-dependent
        !           153:  *     bootstrap sequence, before calling other machine-independent
        !           154:  *     initializations.
        !           155:  *
        !           156:  *     All references to the virtual page size outside this
        !           157:  *     module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT
        !           158:  *     constants.
        !           159:  */
        !           160: #ifndef PAGE_SIZE_FIXED
        !           161: vm_size_t      page_size  = 4096;
        !           162: vm_size_t      page_mask  = 4095;
        !           163: int            page_shift = 12;
        !           164: #endif /* PAGE_SIZE_FIXED */
        !           165: 
        !           166: /*
        !           167:  *     Resident page structures are initialized from
        !           168:  *     a template (see vm_page_alloc).
        !           169:  *
        !           170:  *     When adding a new field to the virtual memory
        !           171:  *     object structure, be sure to add initialization
        !           172:  *     (see vm_page_bootstrap).
        !           173:  */
        !           174: struct vm_page vm_page_template;
        !           175: 
        !           176: /*
        !           177:  *     Resident pages that represent real memory
        !           178:  *     are allocated from a free list.
        !           179:  */
        !           180: vm_page_t      vm_page_queue_free;
        !           181: vm_page_t       vm_page_queue_fictitious;
        !           182: decl_mutex_data(,vm_page_queue_free_lock)
        !           183: unsigned int   vm_page_free_wanted;
        !           184: int            vm_page_free_count;
        !           185: int            vm_page_fictitious_count;
        !           186: 
        !           187: unsigned int   vm_page_free_count_minimum;     /* debugging */
        !           188: 
        !           189: /*
        !           190:  *     Occasionally, the virtual memory system uses
        !           191:  *     resident page structures that do not refer to
        !           192:  *     real pages, for example to leave a page with
        !           193:  *     important state information in the VP table.
        !           194:  *
        !           195:  *     These page structures are allocated the way
        !           196:  *     most other kernel structures are.
        !           197:  */
        !           198: zone_t vm_page_zone;
        !           199: decl_mutex_data(,vm_page_alloc_lock)
        !           200: 
        !           201: /*
        !           202:  *     Fictitious pages don't have a physical address,
        !           203:  *     but we must initialize phys_addr to something.
        !           204:  *     For debugging, this should be a strange value
        !           205:  *     that the pmap module can recognize in assertions.
        !           206:  */
        !           207: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
        !           208: 
        !           209: /*
        !           210:  *     Resident page structures are also chained on
        !           211:  *     queues that are used by the page replacement
        !           212:  *     system (pageout daemon).  These queues are
        !           213:  *     defined here, but are shared by the pageout
        !           214:  *     module.
        !           215:  */
        !           216: queue_head_t   vm_page_queue_active;
        !           217: queue_head_t   vm_page_queue_inactive;
        !           218: decl_mutex_data(,vm_page_queue_lock)
        !           219: int            vm_page_active_count;
        !           220: int            vm_page_inactive_count;
        !           221: int            vm_page_wire_count;
        !           222: int            vm_page_gobble_count = 0;
        !           223: int            vm_page_wire_count_warning = 0;
        !           224: int            vm_page_gobble_count_warning = 0;
        !           225: 
        !           226: /* the following fields are protected by the vm_page_queue_lock */
        !           227: queue_head_t   vm_page_queue_limbo;
        !           228: int    vm_page_limbo_count = 0;                /* total pages in limbo */
        !           229: int    vm_page_limbo_real_count = 0;           /* real pages in limbo */
        !           230: int    vm_page_pin_count = 0;                  /* number of pinned pages */
        !           231: 
        !           232: decl_simple_lock_data(,vm_page_preppin_lock)
        !           233: 
        !           234: /*
        !           235:  *     Several page replacement parameters are also
        !           236:  *     shared with this module, so that page allocation
        !           237:  *     (done here in vm_page_alloc) can trigger the
        !           238:  *     pageout daemon.
        !           239:  */
        !           240: int    vm_page_free_target = 0;
        !           241: int    vm_page_free_min = 0;
        !           242: int    vm_page_inactive_target = 0;
        !           243: int    vm_page_free_reserved = 0;
        !           244: int    vm_page_laundry_count = 0;
        !           245: 
        !           246: /*
        !           247:  *     The VM system has a couple of heuristics for deciding
        !           248:  *     that pages are "uninteresting" and should be placed
        !           249:  *     on the inactive queue as likely candidates for replacement.
        !           250:  *     These variables let the heuristics be controlled at run-time
        !           251:  *     to make experimentation easier.
        !           252:  */
        !           253: 
        !           254: boolean_t vm_page_deactivate_hint = TRUE;
        !           255: 
        !           256: /* 
        !           257:  * Definition for automatic physical memory reservation.  Declared in
        !           258:  * vm_page.h, defined here.  See vm_page.h for details.  There's an
        !           259:  * extra zero entry to allow the code to compile if there are no requests
        !           260:  * for physical memory allocation; pmem_reserve_ctl_size is decremented
        !           261:  * by one to compensate.
        !           262:  */
        !           263: #include <flipc.h>
        !           264: #if FLIPC
        !           265: #include <flipc/flipc_usermsg.h>
        !           266: #endif
        !           267: 
        !           268: struct pmem_reserve pmem_reserve_ctl_array[] = {
        !           269: #if FLIPC
        !           270:     { &flipc_cb_length, (vm_offset_t *) &flipc_cb_base },
        !           271: #endif
        !           272:     { 0, (vm_offset_t *) 0 }
        !           273: };
        !           274: 
        !           275: struct pmem_reserve *pmem_reserve_ctl = &pmem_reserve_ctl_array[0];
        !           276: 
        !           277: int pmem_reserve_ctl_size =
        !           278:   (sizeof(pmem_reserve_ctl_array) / sizeof(struct pmem_reserve)) - 1;
        !           279: 
        !           280: /*
        !           281:  *     vm_set_page_size:
        !           282:  *
        !           283:  *     Sets the page size, perhaps based upon the memory
        !           284:  *     size.  Must be called before any use of page-size
        !           285:  *     dependent functions.
        !           286:  *
        !           287:  *     Sets page_shift and page_mask from page_size.
        !           288:  */
        !           289: void
        !           290: vm_set_page_size(void)
        !           291: {
        !           292: #ifndef PAGE_SIZE_FIXED
        !           293:        page_mask = page_size - 1;
        !           294: 
        !           295:        if ((page_mask & page_size) != 0)
        !           296:                panic("vm_set_page_size: page size not a power of two");
        !           297: 
        !           298:        for (page_shift = 0; ; page_shift++)
        !           299:                if ((1 << page_shift) == page_size)
        !           300:                        break;
        !           301: #endif /* PAGE_SIZE_FIXED */
        !           302: }
        !           303: 
        !           304: /*
        !           305:  *     vm_page_bootstrap:
        !           306:  *
        !           307:  *     Initializes the resident memory module.
        !           308:  *
        !           309:  *     Allocates memory for the page cells, and
        !           310:  *     for the object/offset-to-page hash table headers.
        !           311:  *     Each page cell is initialized and placed on the free list.
        !           312:  *     Returns the range of available kernel virtual memory.
        !           313:  */
        !           314: 
        !           315: void
        !           316: vm_page_bootstrap(
        !           317:        vm_offset_t             *startp,
        !           318:        vm_offset_t             *endp)
        !           319: {
        !           320:        register vm_page_t      m;
        !           321:        int                     i;
        !           322:        unsigned int            log1;
        !           323:        unsigned int            log2;
        !           324:        unsigned int            size;
        !           325: 
        !           326:        /*
        !           327:         *      Initialize the vm_page template.
        !           328:         */
        !           329: 
        !           330:        m = &vm_page_template;
        !           331:        m->object = VM_OBJECT_NULL;     /* reset later */
        !           332:        m->offset = 0;                  /* reset later */
        !           333:        m->wire_count = 0;
        !           334: 
        !           335:        m->inactive = FALSE;
        !           336:        m->active = FALSE;
        !           337:        m->laundry = FALSE;
        !           338:        m->free = FALSE;
        !           339:        m->reference = FALSE;
        !           340:        m->pageout = FALSE;
        !           341:        m->list_req_pending = FALSE;
        !           342: 
        !           343:        m->busy = TRUE;
        !           344:        m->wanted = FALSE;
        !           345:        m->tabled = FALSE;
        !           346:        m->fictitious = FALSE;
        !           347:        m->private = FALSE;
        !           348:        m->absent = FALSE;
        !           349:        m->error = FALSE;
        !           350:        m->dirty = FALSE;
        !           351:        m->cleaning = FALSE;
        !           352:        m->precious = FALSE;
        !           353:        m->clustered = FALSE;
        !           354:        m->lock_supplied = FALSE;
        !           355:        m->unusual = FALSE;
        !           356:        m->restart = FALSE;
        !           357:        m->limbo = FALSE;
        !           358: 
        !           359:        m->phys_addr = 0;               /* reset later */
        !           360: 
        !           361:        m->page_lock = VM_PROT_NONE;
        !           362:        m->unlock_request = VM_PROT_NONE;
        !           363:        m->page_error = KERN_SUCCESS;
        !           364: 
        !           365:        /*
        !           366:         *      Initialize the page queues.
        !           367:         */
        !           368: 
        !           369:        mutex_init(&vm_page_queue_free_lock, ETAP_VM_PAGEQ_FREE);
        !           370:        mutex_init(&vm_page_queue_lock, ETAP_VM_PAGEQ);
        !           371:        simple_lock_init(&vm_page_preppin_lock, ETAP_VM_PREPPIN);
        !           372: 
        !           373:        vm_page_queue_free = VM_PAGE_NULL;
        !           374:        vm_page_queue_fictitious = VM_PAGE_NULL;
        !           375:        queue_init(&vm_page_queue_active);
        !           376:        queue_init(&vm_page_queue_inactive);
        !           377:        queue_init(&vm_page_queue_limbo);
        !           378: 
        !           379:        vm_page_free_wanted = 0;
        !           380: 
        !           381:        /*
        !           382:         *      Steal memory for the map and zone subsystems.
        !           383:         */
        !           384: 
        !           385:        vm_map_steal_memory();
        !           386:        zone_steal_memory();
        !           387: 
        !           388:        /*
        !           389:         *      Allocate (and initialize) the virtual-to-physical
        !           390:         *      table hash buckets.
        !           391:         *
        !           392:         *      The number of buckets should be a power of two to
        !           393:         *      get a good hash function.  The following computation
        !           394:         *      chooses the first power of two that is greater
        !           395:         *      than the number of physical pages in the system.
        !           396:         */
        !           397: 
        !           398:        simple_lock_init(&vm_page_bucket_lock, ETAP_VM_BUCKET);
        !           399:        
        !           400:        if (vm_page_bucket_count == 0) {
        !           401:                unsigned int npages = pmap_free_pages();
        !           402: 
        !           403:                vm_page_bucket_count = 1;
        !           404:                while (vm_page_bucket_count < npages)
        !           405:                        vm_page_bucket_count <<= 1;
        !           406:        }
        !           407: 
        !           408:        vm_page_hash_mask = vm_page_bucket_count - 1;
        !           409: 
        !           410:        /*
        !           411:         *      Calculate object shift value for hashing algorithm:
        !           412:         *              O = log2(sizeof(struct vm_object))
        !           413:         *              B = log2(vm_page_bucket_count)
        !           414:         *              hash shifts the object left by
        !           415:         *              B/2 - O
        !           416:         */
        !           417:        size = vm_page_bucket_count;
        !           418:        for (log1 = 0; size > 1; log1++) 
        !           419:                size /= 2;
        !           420:        size = sizeof(struct vm_object);
        !           421:        for (log2 = 0; size > 1; log2++) 
        !           422:                size /= 2;
        !           423:        vm_page_hash_shift = log1/2 - log2 + 1;
        !           424: 
        !           425:        if (vm_page_hash_mask & vm_page_bucket_count)
        !           426:                printf("vm_page_bootstrap: WARNING -- strange page hash\n");
        !           427: 
        !           428:        vm_page_buckets = (vm_page_bucket_t *)
        !           429:                pmap_steal_memory(vm_page_bucket_count *
        !           430:                                  sizeof(vm_page_bucket_t));
        !           431: 
        !           432:        for (i = 0; i < vm_page_bucket_count; i++) {
        !           433:                register vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !           434: 
        !           435:                bucket->pages = VM_PAGE_NULL;
        !           436: #if     MACH_PAGE_HASH_STATS
        !           437:                bucket->cur_count = 0;
        !           438:                bucket->hi_count = 0;
        !           439: #endif /* MACH_PAGE_HASH_STATS */
        !           440:        }
        !           441: 
        !           442:        /*
        !           443:         *      Machine-dependent code allocates the resident page table.
        !           444:         *      It uses vm_page_init to initialize the page frames.
        !           445:         *      The code also returns to us the virtual space available
        !           446:         *      to the kernel.  We don't trust the pmap module
        !           447:         *      to get the alignment right.
        !           448:         */
        !           449: 
        !           450:        pmap_startup(&virtual_space_start, &virtual_space_end);
        !           451:        virtual_space_start = round_page(virtual_space_start);
        !           452:        virtual_space_end = trunc_page(virtual_space_end);
        !           453: 
        !           454:        *startp = virtual_space_start;
        !           455:        *endp = virtual_space_end;
        !           456: 
        !           457:        /*
        !           458:         *      Compute the initial "wire" count.
        !           459:         *      Up until now, the pages which have been set aside are not under 
        !           460:         *      the VM system's control, so although they aren't explicitly
        !           461:         *      wired, they nonetheless can't be moved. At this moment,
        !           462:         *      all VM managed pages are "free", courtesy of pmap_startup.
        !           463:         */
        !           464:        vm_page_wire_count = atop(mem_size) - vm_page_free_count;       /* initial value */
        !           465: 
        !           466:        printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);
        !           467:        vm_page_free_count_minimum = vm_page_free_count;
        !           468: }
        !           469: 
        !           470: #ifndef        MACHINE_PAGES
        !           471: /*
        !           472:  *     We implement pmap_steal_memory and pmap_startup with the help
        !           473:  *     of two simpler functions, pmap_virtual_space and pmap_next_page.
        !           474:  */
        !           475: 
        !           476: vm_offset_t
        !           477: pmap_steal_memory(
        !           478:        vm_size_t size)
        !           479: {
        !           480:        vm_offset_t addr, vaddr, paddr;
        !           481: 
        !           482:        /*
        !           483:         *      We round the size to a round multiple.
        !           484:         */
        !           485: 
        !           486:        size = (size + sizeof (void *) - 1) &~ (sizeof (void *) - 1);
        !           487: 
        !           488:        /*
        !           489:         *      If this is the first call to pmap_steal_memory,
        !           490:         *      we have to initialize ourself.
        !           491:         */
        !           492: 
        !           493:        if (virtual_space_start == virtual_space_end) {
        !           494:                pmap_virtual_space(&virtual_space_start, &virtual_space_end);
        !           495: 
        !           496:                /*
        !           497:                 *      The initial values must be aligned properly, and
        !           498:                 *      we don't trust the pmap module to do it right.
        !           499:                 */
        !           500: 
        !           501:                virtual_space_start = round_page(virtual_space_start);
        !           502:                virtual_space_end = trunc_page(virtual_space_end);
        !           503:        }
        !           504: 
        !           505:        /*
        !           506:         *      Allocate virtual memory for this request.
        !           507:         */
        !           508: 
        !           509:        addr = virtual_space_start;
        !           510:        virtual_space_start += size;
        !           511: 
        !           512:        kprintf("pmap_steal_memory: %08X - %08X; size=%08X\n", addr, virtual_space_start, size);        /* (TEST/DEBUG) */
        !           513: 
        !           514:        /*
        !           515:         *      Allocate and map physical pages to back new virtual pages.
        !           516:         */
        !           517: 
        !           518:        for (vaddr = round_page(addr);
        !           519:             vaddr < addr + size;
        !           520:             vaddr += PAGE_SIZE) {
        !           521:                if (!pmap_next_page(&paddr))
        !           522:                        panic("pmap_steal_memory");
        !           523: 
        !           524:                /*
        !           525:                 *      XXX Logically, these mappings should be wired,
        !           526:                 *      but some pmap modules barf if they are.
        !           527:                 */
        !           528: 
        !           529:                pmap_enter(kernel_pmap, vaddr, paddr,
        !           530:                           VM_PROT_READ|VM_PROT_WRITE, FALSE);
        !           531:                /*
        !           532:                 * Account for newly stolen memory
        !           533:                 */
        !           534:                vm_page_wire_count++;
        !           535: 
        !           536:        }
        !           537: 
        !           538:        return addr;
        !           539: }
        !           540: 
        !           541: void
        !           542: pmap_startup(
        !           543:        vm_offset_t *startp,
        !           544:        vm_offset_t *endp)
        !           545: {
        !           546:        unsigned int i, npages, pages_initialized;
        !           547:        vm_page_t pages;
        !           548:        vm_offset_t paddr;
        !           549: 
        !           550:        /*
        !           551:         *      We calculate how many page frames we will have
        !           552:         *      and then allocate the page structures in one chunk.
        !           553:         */
        !           554: 
        !           555:        npages = ((PAGE_SIZE * pmap_free_pages() +
        !           556:                   (round_page(virtual_space_start) - virtual_space_start)) /
        !           557:                  (PAGE_SIZE + sizeof *pages));
        !           558: 
        !           559:        pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
        !           560: 
        !           561:        /*
        !           562:         *      Initialize the page frames.
        !           563:         */
        !           564: 
        !           565:        for (i = 0, pages_initialized = 0; i < npages; i++) {
        !           566:                if (!pmap_next_page(&paddr))
        !           567:                        break;
        !           568: 
        !           569:                vm_page_init(&pages[i], paddr);
        !           570:                vm_page_pages++;
        !           571:                pages_initialized++;
        !           572:        }
        !           573: 
        !           574:        /*
        !           575:         * Release pages in reverse order so that physical pages
        !           576:         * initially get allocated in ascending addresses. This keeps
        !           577:         * the devices (which must address physical memory) happy if
        !           578:         * they require several consecutive pages.
        !           579:         */
        !           580: 
        !           581:        for (i = pages_initialized; i > 0; i--) {
        !           582:                vm_page_release(&pages[i - 1]);
        !           583:        }
        !           584: 
        !           585:        /*
        !           586:         *      We have to re-align virtual_space_start,
        !           587:         *      because pmap_steal_memory has been using it.
        !           588:         */
        !           589: 
        !           590:        virtual_space_start = round_page(virtual_space_start);
        !           591: 
        !           592:        *startp = virtual_space_start;
        !           593:        *endp = virtual_space_end;
        !           594: }
        !           595: #endif /* MACHINE_PAGES */
        !           596: 
        !           597: /*
        !           598:  *     Routine:        vm_page_module_init
        !           599:  *     Purpose:
        !           600:  *             Second initialization pass, to be done after
        !           601:  *             the basic VM system is ready.
        !           602:  */
        !           603: void
        !           604: vm_page_module_init(void)
        !           605: {
        !           606:        vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
        !           607:                             0, PAGE_SIZE, "vm pages");
        !           608: 
        !           609: #if    ZONE_DEBUG
        !           610:        zone_debug_disable(vm_page_zone);
        !           611: #endif /* ZONE_DEBUG */
        !           612: 
        !           613:        zone_change(vm_page_zone, Z_EXPAND, FALSE);
        !           614:        zone_change(vm_page_zone, Z_EXHAUST, TRUE);
        !           615:        zone_change(vm_page_zone, Z_FOREIGN, TRUE);
        !           616: 
        !           617:         /*
        !           618:          * Adjust zone statistics to account for the real pages allocated
        !           619:          * in vm_page_create(). [Q: is this really what we want?]
        !           620:          */
        !           621:         vm_page_zone->count += vm_page_pages;
        !           622:         vm_page_zone->cur_size += vm_page_pages * vm_page_zone->elem_size;
        !           623: 
        !           624:        mutex_init(&vm_page_alloc_lock, ETAP_VM_PAGE_ALLOC);
        !           625: }
        !           626: 
        !           627: /*
        !           628:  *     Routine:        vm_page_create
        !           629:  *     Purpose:
        !           630:  *             After the VM system is up, machine-dependent code
        !           631:  *             may stumble across more physical memory.  For example,
        !           632:  *             memory that it was reserving for a frame buffer.
        !           633:  *             vm_page_create turns this memory into available pages.
        !           634:  */
        !           635: 
        !           636: void
        !           637: vm_page_create(
        !           638:        vm_offset_t start,
        !           639:        vm_offset_t end)
        !           640: {
        !           641:        vm_offset_t paddr;
        !           642:        vm_page_t m;
        !           643: 
        !           644:        for (paddr = round_page(start);
        !           645:             paddr < trunc_page(end);
        !           646:             paddr += PAGE_SIZE) {
        !           647:                while ((m = (vm_page_t) vm_page_grab_fictitious())
        !           648:                        == VM_PAGE_NULL)
        !           649:                        vm_page_more_fictitious();
        !           650: 
        !           651:                vm_page_init(m, paddr);
        !           652:                vm_page_pages++;
        !           653:                vm_page_release(m);
        !           654:        }
        !           655: }
        !           656: 
        !           657: /*
        !           658:  *     vm_page_hash:
        !           659:  *
        !           660:  *     Distributes the object/offset key pair among hash buckets.
        !           661:  *
        !           662:  *     NOTE:   To get a good hash function, the bucket count should
        !           663:  *             be a power of two.
        !           664:  */
        !           665: #define vm_page_hash(object, offset) (\
        !           666:        ( ((natural_t)(vm_offset_t)object<<vm_page_hash_shift) + (natural_t)atop(offset))\
        !           667:         & vm_page_hash_mask)
        !           668: 
        !           669: /*
        !           670:  *     vm_page_insert:         [ internal use only ]
        !           671:  *
        !           672:  *     Inserts the given mem entry into the object/object-page
        !           673:  *     table and object list.
        !           674:  *
        !           675:  *     The object must be locked.
        !           676:  */
        !           677: 
        !           678: void
        !           679: vm_page_insert(
        !           680:        register vm_page_t      mem,
        !           681:        register vm_object_t    object,
        !           682:        register vm_offset_t    offset)
        !           683: {
        !           684:        register vm_page_bucket_t *bucket;
        !           685: 
        !           686:         XPR(XPR_VM_PAGE,
        !           687:                 "vm_page_insert, object 0x%X offset 0x%X page 0x%X\n",
        !           688:                 (integer_t)object, (integer_t)offset, (integer_t)mem, 0,0);
        !           689: 
        !           690:        VM_PAGE_CHECK(mem);
        !           691: 
        !           692:        if (mem->tabled)
        !           693:                panic("vm_page_insert");
        !           694: 
        !           695:        assert(!object->internal || offset < object->size);
        !           696: 
        !           697:        /* only insert "pageout" pages into "pageout" objects,
        !           698:         * and normal pages into normal objects */
        !           699:        assert(object->pageout == mem->pageout);
        !           700: 
        !           701:        /*
        !           702:         *      Record the object/offset pair in this page
        !           703:         */
        !           704: 
        !           705:        mem->object = object;
        !           706:        mem->offset = offset;
        !           707: 
        !           708:        /*
        !           709:         *      Insert it into the object_object/offset hash table
        !           710:         */
        !           711: 
        !           712:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           713:        simple_lock(&vm_page_bucket_lock);
        !           714:        mem->next = bucket->pages;
        !           715:        bucket->pages = mem;
        !           716: #if     MACH_PAGE_HASH_STATS
        !           717:        if (++bucket->cur_count > bucket->hi_count)
        !           718:                bucket->hi_count = bucket->cur_count;
        !           719: #endif /* MACH_PAGE_HASH_STATS */
        !           720:        simple_unlock(&vm_page_bucket_lock);
        !           721: 
        !           722:        /*
        !           723:         *      Now link into the object's list of backed pages.
        !           724:         */
        !           725: 
        !           726:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           727:        mem->tabled = TRUE;
        !           728: 
        !           729:        /*
        !           730:         *      Show that the object has one more resident page.
        !           731:         */
        !           732: 
        !           733:        object->resident_page_count++;
        !           734: }
        !           735: 
        !           736: /*
        !           737:  *     vm_page_replace:
        !           738:  *
        !           739:  *     Exactly like vm_page_insert, except that we first
        !           740:  *     remove any existing page at the given offset in object.
        !           741:  *
        !           742:  *     The object and page queues must be locked.
        !           743:  */
        !           744: 
        !           745: void
        !           746: vm_page_replace(
        !           747:        register vm_page_t      mem,
        !           748:        register vm_object_t    object,
        !           749:        register vm_offset_t    offset)
        !           750: {
        !           751:        register vm_page_bucket_t *bucket;
        !           752: 
        !           753:        VM_PAGE_CHECK(mem);
        !           754: 
        !           755:        if (mem->tabled)
        !           756:                panic("vm_page_replace");
        !           757: 
        !           758:        /*
        !           759:         *      Record the object/offset pair in this page
        !           760:         */
        !           761: 
        !           762:        mem->object = object;
        !           763:        mem->offset = offset;
        !           764: 
        !           765:        /*
        !           766:         *      Insert it into the object_object/offset hash table,
        !           767:         *      replacing any page that might have been there.
        !           768:         */
        !           769: 
        !           770:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           771:        simple_lock(&vm_page_bucket_lock);
        !           772:        if (bucket->pages) {
        !           773:                vm_page_t *mp = &bucket->pages;
        !           774:                register vm_page_t m = *mp;
        !           775:                do {
        !           776:                        if (m->object == object && m->offset == offset) {
        !           777:                                /*
        !           778:                                 * Remove page from bucket and from object,
        !           779:                                 * and return it to the free list.
        !           780:                                 */
        !           781:                                *mp = m->next;
        !           782:                                queue_remove(&object->memq, m, vm_page_t,
        !           783:                                             listq);
        !           784:                                m->tabled = FALSE;
        !           785:                                object->resident_page_count--;
        !           786: 
        !           787:                                /*
        !           788:                                 * Return page to the free list.
        !           789:                                 * Note the page is not tabled now, so this
        !           790:                                 * won't self-deadlock on the bucket lock.
        !           791:                                 */
        !           792: 
        !           793:                                vm_page_free(m);
        !           794:                                break;
        !           795:                        }
        !           796:                        mp = &m->next;
        !           797:                } while (m = *mp);
        !           798:                mem->next = bucket->pages;
        !           799:        } else {
        !           800:                mem->next = VM_PAGE_NULL;
        !           801:        }
        !           802:        bucket->pages = mem;
        !           803:        simple_unlock(&vm_page_bucket_lock);
        !           804: 
        !           805:        /*
        !           806:         *      Now link into the object's list of backed pages.
        !           807:         */
        !           808: 
        !           809:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           810:        mem->tabled = TRUE;
        !           811: 
        !           812:        /*
        !           813:         *      And show that the object has one more resident
        !           814:         *      page.
        !           815:         */
        !           816: 
        !           817:        object->resident_page_count++;
        !           818: }
        !           819: 
        !           820: /*
        !           821:  *     vm_page_remove:         [ internal use only ]
        !           822:  *
        !           823:  *     Removes the given mem entry from the object/offset-page
        !           824:  *     table and the object page list.
        !           825:  *
        !           826:  *     The object and page must be locked.
        !           827:  */
        !           828: 
        !           829: void
        !           830: vm_page_remove(
        !           831:        register vm_page_t      mem)
        !           832: {
        !           833:        register vm_page_bucket_t       *bucket;
        !           834:        register vm_page_t      this;
        !           835: 
        !           836:         XPR(XPR_VM_PAGE,
        !           837:                 "vm_page_remove, object 0x%X offset 0x%X page 0x%X\n",
        !           838:                 (integer_t)mem->object, (integer_t)mem->offset, 
        !           839:                (integer_t)mem, 0,0);
        !           840: 
        !           841:        assert(mem->tabled);
        !           842:        assert(!mem->cleaning);
        !           843:        VM_PAGE_CHECK(mem);
        !           844: 
        !           845:        /*
        !           846:         *      Remove from the object_object/offset hash table
        !           847:         */
        !           848: 
        !           849:        bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
        !           850:        simple_lock(&vm_page_bucket_lock);
        !           851:        if ((this = bucket->pages) == mem) {
        !           852:                /* optimize for common case */
        !           853: 
        !           854:                bucket->pages = mem->next;
        !           855:        } else {
        !           856:                register vm_page_t      *prev;
        !           857: 
        !           858:                for (prev = &this->next;
        !           859:                     (this = *prev) != mem;
        !           860:                     prev = &this->next)
        !           861:                        continue;
        !           862:                *prev = this->next;
        !           863:        }
        !           864: #if     MACH_PAGE_HASH_STATS
        !           865:        bucket->cur_count--;
        !           866: #endif /* MACH_PAGE_HASH_STATS */
        !           867:        simple_unlock(&vm_page_bucket_lock);
        !           868: 
        !           869:        /*
        !           870:         *      Now remove from the object's list of backed pages.
        !           871:         */
        !           872: 
        !           873:        queue_remove(&mem->object->memq, mem, vm_page_t, listq);
        !           874: 
        !           875:        /*
        !           876:         *      And show that the object has one fewer resident
        !           877:         *      page.
        !           878:         */
        !           879: 
        !           880:        mem->object->resident_page_count--;
        !           881: 
        !           882:        mem->tabled = FALSE;
        !           883:        mem->object = VM_OBJECT_NULL;
        !           884:        mem->offset = 0;
        !           885: }
        !           886: 
        !           887: /*
        !           888:  *     vm_page_lookup:
        !           889:  *
        !           890:  *     Returns the page associated with the object/offset
        !           891:  *     pair specified; if none is found, VM_PAGE_NULL is returned.
        !           892:  *
        !           893:  *     The object must be locked.  No side effects.
        !           894:  */
        !           895: 
        !           896: vm_page_t
        !           897: vm_page_lookup(
        !           898:        register vm_object_t    object,
        !           899:        register vm_offset_t    offset)
        !           900: {
        !           901:        register vm_page_t      mem;
        !           902:        register vm_page_bucket_t *bucket;
        !           903: 
        !           904:        /*
        !           905:         *      Search the hash table for this object/offset pair
        !           906:         */
        !           907: 
        !           908:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           909: 
        !           910:        simple_lock(&vm_page_bucket_lock);
        !           911:        for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
        !           912:                VM_PAGE_CHECK(mem);
        !           913:                if ((mem->object == object) && (mem->offset == offset))
        !           914:                        break;
        !           915:        }
        !           916:        simple_unlock(&vm_page_bucket_lock);
        !           917:        return(mem);
        !           918: }
        !           919: 
        !           920: /*
        !           921:  *     vm_page_rename:
        !           922:  *
        !           923:  *     Move the given memory entry from its
        !           924:  *     current object to the specified target object/offset.
        !           925:  *
        !           926:  *     The object must be locked.
        !           927:  */
        !           928: void
        !           929: vm_page_rename(
        !           930:        register vm_page_t      mem,
        !           931:        register vm_object_t    new_object,
        !           932:        vm_offset_t             new_offset)
        !           933: {
        !           934:        assert(mem->object != new_object);
        !           935:        /*
        !           936:         *      Changes to mem->object require the page lock because
        !           937:         *      the pageout daemon uses that lock to get the object.
        !           938:         */
        !           939: 
        !           940:         XPR(XPR_VM_PAGE,
        !           941:                 "vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n",
        !           942:                 (integer_t)new_object, (integer_t)new_offset, 
        !           943:                (integer_t)mem, 0,0);
        !           944: 
        !           945:        vm_page_lock_queues();
        !           946:        vm_page_remove(mem);
        !           947:        vm_page_insert(mem, new_object, new_offset);
        !           948:        vm_page_unlock_queues();
        !           949: }
        !           950: 
        !           951: /*
        !           952:  *     vm_page_init:
        !           953:  *
        !           954:  *     Initialize the fields in a new page.
        !           955:  *     This takes a structure with random values and initializes it
        !           956:  *     so that it can be given to vm_page_release or vm_page_insert.
        !           957:  */
        !           958: void
        !           959: vm_page_init(
        !           960:        vm_page_t       mem,
        !           961:        vm_offset_t     phys_addr)
        !           962: {
        !           963:        *mem = vm_page_template;
        !           964:        mem->phys_addr = phys_addr;
        !           965: }
        !           966: 
        !           967: /*
        !           968:  *     vm_page_grab_fictitious:
        !           969:  *
        !           970:  *     Remove a fictitious page from the free list.
        !           971:  *     Returns VM_PAGE_NULL if there are no free pages.
        !           972:  */
        !           973: int    c_vm_page_grab_fictitious = 0;
        !           974: int    c_vm_page_release_fictitious = 0;
        !           975: int    c_vm_page_more_fictitious = 0;
        !           976: 
        !           977: vm_page_t
        !           978: vm_page_grab_fictitious(void)
        !           979: {
        !           980:        register vm_page_t m;
        !           981: 
        !           982:        m = (vm_page_t)zget(vm_page_zone);
        !           983:        if (m) {
        !           984:                m->free = FALSE;
        !           985: #if    MACH_ASSERT || ZONE_DEBUG
        !           986:                vm_page_init(m, vm_page_fictitious_addr);
        !           987:                m->fictitious = TRUE;
        !           988: #endif /* MACH_ASSERT || ZONE_DEBUG */
        !           989:        }
        !           990: 
        !           991:        c_vm_page_grab_fictitious++;
        !           992:        return m;
        !           993: }
        !           994: 
        !           995: /*
        !           996:  *     vm_page_release_fictitious:
        !           997:  *
        !           998:  *     Release a fictitious page to the free list.
        !           999:  */
        !          1000: 
        !          1001: void
        !          1002: vm_page_release_fictitious(
        !          1003:        register vm_page_t m)
        !          1004: {
        !          1005:        assert(!m->free);
        !          1006:        assert(m->busy);
        !          1007:        assert(m->fictitious);
        !          1008:        assert(m->phys_addr == vm_page_fictitious_addr);
        !          1009: 
        !          1010:        c_vm_page_release_fictitious++;
        !          1011: 
        !          1012:        if (m->free)
        !          1013:                panic("vm_page_release_fictitious");
        !          1014:        m->free = TRUE;
        !          1015:        zfree(vm_page_zone, (vm_offset_t)m);
        !          1016: }
        !          1017: 
        !          1018: /*
        !          1019:  *     vm_page_more_fictitious:
        !          1020:  *
        !          1021:  *     Add more fictitious pages to the free list.
        !          1022:  *     Allowed to block. This routine is way intimate
        !          1023:  *     with the zones code, for several reasons:
        !          1024:  *     1. we need to carve some page structures out of physical
        !          1025:  *        memory before zones work, so they _cannot_ come from
        !          1026:  *        the zone_map.
        !          1027:  *     2. the zone needs to be collectable in order to prevent
        !          1028:  *        growth without bound. These structures are used by
        !          1029:  *        the device pager (by the hundreds and thousands), as
        !          1030:  *        private pages for pageout, and as blocking pages for
        !          1031:  *        pagein. Temporary bursts in demand should not result in
        !          1032:  *        permanent allocation of a resource.
        !          1033:  *     3. To smooth allocation humps, we allocate single pages
        !          1034:  *        with kernel_memory_allocate(), and cram them into the
        !          1035:  *        zone. This also allows us to initialize the vm_page_t's
        !          1036:  *        on the way into the zone, so that zget() always returns
        !          1037:  *        an initialized structure. The zone free element pointer
        !          1038:  *        and the free page pointer are both the first item in the
        !          1039:  *        vm_page_t.
        !          1040:  *     4. By having the pages in the zone pre-initialized, we need
        !          1041:  *        not keep 2 levels of lists. The garbage collector simply
        !          1042:  *        scans our list, and reduces physical memory usage as it
        !          1043:  *        sees fit.
        !          1044:  */
        !          1045: 
        !          1046: void vm_page_more_fictitious(void)
        !          1047: {
        !          1048:        extern vm_map_t zone_map;
        !          1049:        register vm_page_t m;
        !          1050:        vm_offset_t addr;
        !          1051:        kern_return_t retval;
        !          1052:        int i;
        !          1053: 
        !          1054:        c_vm_page_more_fictitious++;
        !          1055: 
        !          1056:        /* this may free up some fictitious pages */
        !          1057:        cleanup_limbo_queue();
        !          1058: 
        !          1059:        /*
        !          1060:         * Allocate a single page from the zone_map. Do not wait if no physical
        !          1061:         * pages are immediately available, and do not zero the space. We need
        !          1062:         * our own blocking lock here to prevent having multiple,
        !          1063:         * simultaneous requests from piling up on the zone_map lock. Exactly
        !          1064:         * one (of our) threads should be potentially waiting on the map lock.
        !          1065:         * If winner is not vm-privileged, then the page allocation will fail,
        !          1066:         * and it will temporarily block here in the vm_page_wait().
        !          1067:         */
        !          1068:        mutex_lock(&vm_page_alloc_lock);
        !          1069:        /*
        !          1070:         * If another thread allocated space, just bail out now.
        !          1071:         */
        !          1072:        if (zone_free_count(vm_page_zone) > 5) {
        !          1073:                /*
        !          1074:                 * The number "5" is a small number that is larger than the
        !          1075:                 * number of fictitious pages that any single caller will
        !          1076:                 * attempt to allocate. Otherwise, a thread will attempt to
        !          1077:                 * acquire a fictitious page (vm_page_grab_fictitious), fail,
        !          1078:                 * release all of the resources and locks already acquired,
        !          1079:                 * and then call this routine. This routine finds the pages
        !          1080:                 * that the caller released, so fails to allocate new space.
        !          1081:                 * The process repeats infinitely. The largest known number
        !          1082:                 * of fictitious pages required in this manner is 2. 5 is
        !          1083:                 * simply a somewhat larger number.
        !          1084:                 */
        !          1085:                mutex_unlock(&vm_page_alloc_lock);
        !          1086:                return;
        !          1087:        }
        !          1088: 
        !          1089:        if ((retval = kernel_memory_allocate(zone_map,
        !          1090:                        &addr, PAGE_SIZE, VM_PROT_ALL,
        !          1091:                        KMA_KOBJECT|KMA_NOPAGEWAIT)) != KERN_SUCCESS) { 
        !          1092:                /*
        !          1093:                 * No page was available. Tell the pageout daemon, drop the
        !          1094:                 * lock to give another thread a chance at it, and
        !          1095:                 * wait for the pageout daemon to make progress.
        !          1096:                 */
        !          1097:                mutex_unlock(&vm_page_alloc_lock);
        !          1098:                vm_page_wait();
        !          1099:                return;
        !          1100:        }
        !          1101:        /*
        !          1102:         * Initialize as many vm_page_t's as will fit on this page. This
        !          1103:         * depends on the zone code disturbing ONLY the first item of
        !          1104:         * each zone element.
        !          1105:         */
        !          1106:        m = (vm_page_t)addr;
        !          1107:        for (i = PAGE_SIZE/sizeof(struct vm_page); i > 0; i--) {
        !          1108:                vm_page_init(m, vm_page_fictitious_addr);
        !          1109:                m->fictitious = TRUE;
        !          1110:                m++;
        !          1111:        }
        !          1112:        zcram(vm_page_zone, addr, PAGE_SIZE);
        !          1113:        mutex_unlock(&vm_page_alloc_lock);
        !          1114: }
        !          1115: 
        !          1116: /*
        !          1117:  *     vm_page_convert:
        !          1118:  *
        !          1119:  *     Attempt to convert a fictitious page into a real page.
        !          1120:  */
        !          1121: 
        !          1122: boolean_t
        !          1123: vm_page_convert(
        !          1124:        register vm_page_t m)
        !          1125: {
        !          1126:        register vm_page_t real_m;
        !          1127: 
        !          1128:        assert(m->busy);
        !          1129:        assert(m->fictitious);
        !          1130:        assert(!m->dirty);
        !          1131: 
        !          1132:        real_m = vm_page_grab();
        !          1133:        if (real_m == VM_PAGE_NULL)
        !          1134:                return FALSE;
        !          1135: 
        !          1136:        m->phys_addr = real_m->phys_addr;
        !          1137:        m->fictitious = FALSE;
        !          1138: 
        !          1139:        vm_page_lock_queues();
        !          1140:        if (m->active)
        !          1141:                vm_page_active_count++;
        !          1142:        else if (m->inactive)
        !          1143:                vm_page_inactive_count++;
        !          1144:        vm_page_unlock_queues();
        !          1145: 
        !          1146:        real_m->phys_addr = vm_page_fictitious_addr;
        !          1147:        real_m->fictitious = TRUE;
        !          1148: 
        !          1149:        vm_page_release_fictitious(real_m);
        !          1150:        return TRUE;
        !          1151: }
        !          1152: 
        !          1153: /*
        !          1154:  *     vm_pool_low():
        !          1155:  *
        !          1156:  *     Return true if it is not likely that a non-vm_privileged thread
        !          1157:  *     can get memory without blocking.  Advisory only, since the
        !          1158:  *     situation may change under us.
        !          1159:  */
        !          1160: int
        !          1161: vm_pool_low(void)
        !          1162: {
        !          1163:        /* No locking, at worst we will fib. */
        !          1164:        return( vm_page_free_count < vm_page_free_reserved );
        !          1165: }
        !          1166: 
        !          1167: /*
        !          1168:  *     vm_page_grab:
        !          1169:  *
        !          1170:  *     Remove a page from the free list.
        !          1171:  *     Returns VM_PAGE_NULL if the free list is too small.
        !          1172:  */
        !          1173: 
        !          1174: unsigned long  vm_page_grab_count = 0; /* measure demand */
        !          1175: 
        !          1176: vm_page_t
        !          1177: vm_page_grab(void)
        !          1178: {
        !          1179:        register vm_page_t      mem;
        !          1180: 
        !          1181:        mutex_lock(&vm_page_queue_free_lock);
        !          1182:        vm_page_grab_count++;
        !          1183: 
        !          1184:        /*
        !          1185:         *      Optionally produce warnings if the wire or gobble
        !          1186:         *      counts exceed some threshold.
        !          1187:         */
        !          1188:        if (vm_page_wire_count_warning > 0
        !          1189:            && vm_page_wire_count >= vm_page_wire_count_warning) {
        !          1190:                printf("mk: vm_page_grab(): high wired page count of %d\n",
        !          1191:                        vm_page_wire_count);
        !          1192:                assert(vm_page_wire_count < vm_page_wire_count_warning);
        !          1193:        }
        !          1194:        if (vm_page_gobble_count_warning > 0
        !          1195:            && vm_page_gobble_count >= vm_page_gobble_count_warning) {
        !          1196:                printf("mk: vm_page_grab(): high gobbled page count of %d\n",
        !          1197:                        vm_page_gobble_count);
        !          1198:                assert(vm_page_gobble_count < vm_page_gobble_count_warning);
        !          1199:        }
        !          1200: 
        !          1201:        /*
        !          1202:         *      Only let privileged threads (involved in pageout)
        !          1203:         *      dip into the reserved pool.
        !          1204:         */
        !          1205: 
        !          1206:        if ((vm_page_free_count < vm_page_free_reserved) &&
        !          1207:            !current_thread()->vm_privilege) {
        !          1208:                mutex_unlock(&vm_page_queue_free_lock);
        !          1209:                mem = VM_PAGE_NULL;
        !          1210:                goto wakeup_pageout;
        !          1211:        }
        !          1212: 
        !          1213:        while (vm_page_queue_free == VM_PAGE_NULL) {
        !          1214:                printf("vm_page_grab: no free pages, trouble expected...\n");
        !          1215:                mutex_unlock(&vm_page_queue_free_lock);
        !          1216:                VM_PAGE_WAIT();
        !          1217:                mutex_lock(&vm_page_queue_free_lock);
        !          1218:        }
        !          1219: 
        !          1220:        if (--vm_page_free_count < vm_page_free_count_minimum)
        !          1221:                vm_page_free_count_minimum = vm_page_free_count;
        !          1222:        mem = vm_page_queue_free;
        !          1223:        vm_page_queue_free = (vm_page_t) mem->pageq.next;
        !          1224:        mem->free = FALSE;
        !          1225:        mutex_unlock(&vm_page_queue_free_lock);
        !          1226: 
        !          1227:        /*
        !          1228:         *      Decide if we should poke the pageout daemon.
        !          1229:         *      We do this if the free count is less than the low
        !          1230:         *      water mark, or if the free count is less than the high
        !          1231:         *      water mark (but above the low water mark) and the inactive
        !          1232:         *      count is less than its target.
        !          1233:         *
        !          1234:         *      We don't have the counts locked ... if they change a little,
        !          1235:         *      it doesn't really matter.
        !          1236:         */
        !          1237: 
        !          1238: wakeup_pageout:
        !          1239:        if ((vm_page_free_count < vm_page_free_min) ||
        !          1240:            ((vm_page_free_count < vm_page_free_target) &&
        !          1241:             (vm_page_inactive_count < vm_page_inactive_target)))
        !          1242:                thread_wakeup((event_t) &vm_page_free_wanted);
        !          1243: 
        !          1244: //     dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 4);      /* (TEST/DEBUG) */
        !          1245: 
        !          1246:        return mem;
        !          1247: }
        !          1248: 
        !          1249: /*
        !          1250:  *     vm_page_release:
        !          1251:  *
        !          1252:  *     Return a page to the free list.
        !          1253:  */
        !          1254: 
        !          1255: void
        !          1256: vm_page_release(
        !          1257:        register vm_page_t      mem)
        !          1258: {
        !          1259:        assert(!mem->private && !mem->fictitious);
        !          1260: 
        !          1261: //     dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 5);      /* (TEST/DEBUG) */
        !          1262: 
        !          1263:        mutex_lock(&vm_page_queue_free_lock);
        !          1264:        if (mem->free)
        !          1265:                panic("vm_page_release");
        !          1266:        mem->free = TRUE;
        !          1267:        mem->pageq.next = (queue_entry_t) vm_page_queue_free;
        !          1268:        vm_page_queue_free = mem;
        !          1269:        vm_page_free_count++;
        !          1270: 
        !          1271:        /*
        !          1272:         *      Check if we should wake up someone waiting for page.
        !          1273:         *      But don't bother waking them unless they can allocate.
        !          1274:         *
        !          1275:         *      We wakeup only one thread, to prevent starvation.
        !          1276:         *      Because the scheduling system handles wait queues FIFO,
        !          1277:         *      if we wakeup all waiting threads, one greedy thread
        !          1278:         *      can starve multiple niceguy threads.  When the threads
        !          1279:         *      all wakeup, the greedy threads runs first, grabs the page,
        !          1280:         *      and waits for another page.  It will be the first to run
        !          1281:         *      when the next page is freed.
        !          1282:         *
        !          1283:         *      However, there is a slight danger here.
        !          1284:         *      The thread we wake might not use the free page.
        !          1285:         *      Then the other threads could wait indefinitely
        !          1286:         *      while the page goes unused.  To forestall this,
        !          1287:         *      the pageout daemon will keep making free pages
        !          1288:         *      as long as vm_page_free_wanted is non-zero.
        !          1289:         */
        !          1290: 
        !          1291:        if ((vm_page_free_wanted > 0) &&
        !          1292:            (vm_page_free_count >= vm_page_free_reserved)) {
        !          1293:                vm_page_free_wanted--;
        !          1294:                thread_wakeup_one((event_t) &vm_page_free_count);
        !          1295:        }
        !          1296: 
        !          1297:        mutex_unlock(&vm_page_queue_free_lock);
        !          1298: }
        !          1299: 
        !          1300: /*
        !          1301:  * Release a page to the limbo list.
        !          1302:  * Put real pages at the head of the queue, fictitious at the tail.
        !          1303:  * Page queues must be locked.
        !          1304:  */
        !          1305: void
        !          1306: vm_page_release_limbo(
        !          1307:        register vm_page_t      m)
        !          1308: {
        !          1309:        assert(m->limbo);
        !          1310:        vm_page_limbo_count++;
        !          1311:        if (m->fictitious) {
        !          1312:                queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq);
        !          1313:        } else {
        !          1314:                vm_page_limbo_real_count++;
        !          1315:                queue_enter_first(&vm_page_queue_limbo, m, vm_page_t, pageq);
        !          1316:        }
        !          1317: }
        !          1318: 
        !          1319: /*
        !          1320:  * Exchange a real page in limbo (limbo_m) with a fictitious page (new_m).
        !          1321:  * The end result is that limbo_m is fictitious and still in limbo, and new_m
        !          1322:  * is the real page.  The prep and pin counts remain with the page in limbo
        !          1323:  * although they will be briefly cleared by vm_page_init.  This is OK since
        !          1324:  * there will be no interrupt-level interactions (the page is in limbo) and
        !          1325:  * vm_page_unprep must lock the page queues before changing the prep count.
        !          1326:  *
        !          1327:  * Page queues must be locked, and limbo_m must have been removed from its
        !          1328:  * object.
        !          1329:  */
        !          1330: void
        !          1331: vm_page_limbo_exchange(
        !          1332:        register vm_page_t      limbo_m,
        !          1333:        register vm_page_t      new_m)
        !          1334: {
        !          1335:        assert(limbo_m->limbo && !limbo_m->fictitious);
        !          1336:        assert(!limbo_m->tabled);
        !          1337:        assert(new_m->fictitious);
        !          1338: 
        !          1339:        *new_m = *limbo_m;
        !          1340:        vm_page_init(limbo_m, vm_page_fictitious_addr);
        !          1341: 
        !          1342:        limbo_m->fictitious = TRUE;
        !          1343:        limbo_m->limbo = TRUE;
        !          1344:        new_m->limbo = FALSE;
        !          1345: 
        !          1346:        limbo_m->prep_pin_count = new_m->prep_pin_count;
        !          1347:        new_m->prep_pin_count = 0;
        !          1348: }
        !          1349: 
        !          1350: /*
        !          1351:  *     vm_page_wait:
        !          1352:  *
        !          1353:  *     Wait for a page to become available.
        !          1354:  *     If there are plenty of free pages, then we don't sleep.
        !          1355:  */
        !          1356: 
        !          1357: void
        !          1358: vm_page_wait( void )
        !          1359: {
        !          1360:        /*
        !          1361:         *      We can't use vm_page_free_reserved to make this
        !          1362:         *      determination.  Consider: some thread might
        !          1363:         *      need to allocate two pages.  The first allocation
        !          1364:         *      succeeds, the second fails.  After the first page is freed,
        !          1365:         *      a call to vm_page_wait must really block.
        !          1366:         */
        !          1367: 
        !          1368:        mutex_lock(&vm_page_queue_free_lock);
        !          1369:        if (vm_page_free_count < vm_page_free_target) {
        !          1370:                if (vm_page_free_wanted++ == 0)
        !          1371:                        thread_wakeup((event_t)&vm_page_free_wanted);
        !          1372:                assert_wait((event_t)&vm_page_free_count, THREAD_UNINT);
        !          1373:                mutex_unlock(&vm_page_queue_free_lock);
        !          1374:                counter(c_vm_page_wait_block++);
        !          1375:                thread_block((void (*)(void))0);
        !          1376:        } else
        !          1377:                mutex_unlock(&vm_page_queue_free_lock);
        !          1378: }
        !          1379: 
        !          1380: /*
        !          1381:  *     vm_page_alloc:
        !          1382:  *
        !          1383:  *     Allocate and return a memory cell associated
        !          1384:  *     with this VM object/offset pair.
        !          1385:  *
        !          1386:  *     Object must be locked.
        !          1387:  */
        !          1388: 
        !          1389: vm_page_t
        !          1390: vm_page_alloc(
        !          1391:        vm_object_t     object,
        !          1392:        vm_offset_t     offset)
        !          1393: {
        !          1394:        register vm_page_t      mem;
        !          1395: 
        !          1396:        mem = vm_page_grab();
        !          1397:        if (mem == VM_PAGE_NULL)
        !          1398:                return VM_PAGE_NULL;
        !          1399: 
        !          1400:        vm_page_insert(mem, object, offset);
        !          1401: 
        !          1402:        return(mem);
        !          1403: }
        !          1404: 
        !          1405: int c_limbo_page_free = 0;     /* debugging */
        !          1406: int c_limbo_convert = 0;       /* debugging */
        !          1407: counter(unsigned int c_laundry_pages_freed = 0;)
        !          1408: 
        !          1409: int vm_pagein_cluster_unused = 0;
        !          1410: boolean_t      vm_page_free_verify = FALSE;
        !          1411: /*
        !          1412:  *     vm_page_free:
        !          1413:  *
        !          1414:  *     Returns the given page to the free list,
        !          1415:  *     disassociating it with any VM object.
        !          1416:  *
        !          1417:  *     Object and page queues must be locked prior to entry.
        !          1418:  */
        !          1419: void
        !          1420: vm_page_free(
        !          1421:        register vm_page_t      mem)
        !          1422: {
        !          1423:        vm_object_t     object = mem->object;
        !          1424: 
        !          1425:        assert(!mem->free);
        !          1426:        assert(!mem->cleaning);
        !          1427:        assert(!mem->pageout);
        !          1428:        assert(!vm_page_free_verify || pmap_verify_free(mem->phys_addr));
        !          1429: 
        !          1430:        if (mem->tabled)
        !          1431:                vm_page_remove(mem);    /* clears tabled, object, offset */
        !          1432:        VM_PAGE_QUEUES_REMOVE(mem);     /* clears active or inactive */
        !          1433: 
        !          1434:        if (mem->clustered) {
        !          1435:                mem->clustered = FALSE;
        !          1436:                vm_pagein_cluster_unused++;
        !          1437:        }
        !          1438: 
        !          1439:        if (mem->wire_count) {
        !          1440:                if (!mem->private && !mem->fictitious)
        !          1441:                        vm_page_wire_count--;
        !          1442:                mem->wire_count = 0;
        !          1443:                assert(!mem->gobbled);
        !          1444:        } else if (mem->gobbled) {
        !          1445:                if (!mem->private && !mem->fictitious)
        !          1446:                        vm_page_wire_count--;
        !          1447:                vm_page_gobble_count--;
        !          1448:        }
        !          1449:        mem->gobbled = FALSE;
        !          1450: 
        !          1451:        if (mem->laundry) {
        !          1452:                extern int vm_page_laundry_min;
        !          1453:                vm_page_laundry_count--;
        !          1454:                mem->laundry = FALSE;   /* laundry is now clear */
        !          1455:                counter(++c_laundry_pages_freed);
        !          1456:                if (vm_page_laundry_count < vm_page_laundry_min) {
        !          1457:                        vm_page_laundry_min = 0;
        !          1458:                        thread_wakeup((event_t) &vm_page_laundry_count);
        !          1459:                }
        !          1460:        }
        !          1461: 
        !          1462:        mem->discard_request = FALSE;
        !          1463: 
        !          1464:        PAGE_WAKEUP(mem);       /* clears wanted */
        !          1465: 
        !          1466:        if (mem->absent)
        !          1467:                vm_object_absent_release(object);
        !          1468: 
        !          1469:        if (mem->limbo) {
        !          1470:                /*
        !          1471:                 * The pageout daemon put this page into limbo and then freed
        !          1472:                 * it.  The page has already been removed from the object and
        !          1473:                 * queues, so any attempt to look it up will fail.  Put it
        !          1474:                 * on the limbo queue; the pageout daemon will convert it to a
        !          1475:                 * fictitious page and/or free the real one later.
        !          1476:                 */
        !          1477:                /* assert that it came from pageout daemon (how?) */
        !          1478:                assert(!mem->fictitious && !mem->absent);
        !          1479:                c_limbo_page_free++;
        !          1480:                vm_page_release_limbo(mem);
        !          1481:                return;
        !          1482:        }
        !          1483:        assert(mem->prep_pin_count == 0);
        !          1484: 
        !          1485:                /* Some of these may be unnecessary */
        !          1486:        mem->page_lock = 0;
        !          1487:        mem->unlock_request = 0;
        !          1488:        mem->busy = TRUE;
        !          1489:        mem->absent = FALSE;
        !          1490:        mem->error = FALSE;
        !          1491:        mem->dirty = FALSE;
        !          1492:        mem->precious = FALSE;
        !          1493:        mem->reference = FALSE;
        !          1494: 
        !          1495:        mem->page_error = KERN_SUCCESS;
        !          1496: 
        !          1497:        if (mem->private) {
        !          1498:                mem->private = FALSE;
        !          1499:                mem->fictitious = TRUE;
        !          1500:                mem->phys_addr = vm_page_fictitious_addr;
        !          1501:        }
        !          1502:        if (mem->fictitious) {
        !          1503:                vm_page_release_fictitious(mem);
        !          1504:        } else {
        !          1505:                vm_page_init(mem, mem->phys_addr);
        !          1506:                vm_page_release(mem);
        !          1507:        }
        !          1508: }
        !          1509: 
        !          1510: /*
        !          1511:  *     vm_page_wire:
        !          1512:  *
        !          1513:  *     Mark this page as wired down by yet
        !          1514:  *     another map, removing it from paging queues
        !          1515:  *     as necessary.
        !          1516:  *
        !          1517:  *     The page's object and the page queues must be locked.
        !          1518:  */
        !          1519: void
        !          1520: vm_page_wire(
        !          1521:        register vm_page_t      mem)
        !          1522: {
        !          1523: 
        !          1524: //     dbgLog(current_act(), mem->offset, mem->object, 1);     /* (TEST/DEBUG) */
        !          1525: 
        !          1526:        VM_PAGE_CHECK(mem);
        !          1527: 
        !          1528:        if (mem->wire_count == 0) {
        !          1529:                VM_PAGE_QUEUES_REMOVE(mem);
        !          1530:                if (!mem->private && !mem->fictitious && !mem->gobbled)
        !          1531:                        vm_page_wire_count++;
        !          1532:                if (mem->gobbled)
        !          1533:                        vm_page_gobble_count--;
        !          1534:                mem->gobbled = FALSE;
        !          1535:        }
        !          1536:        assert(!mem->gobbled);
        !          1537:        mem->wire_count++;
        !          1538: }
        !          1539: 
        !          1540: /*
        !          1541:  *      vm_page_gobble:
        !          1542:  *
        !          1543:  *      Mark this page as consumed by the vm/ipc/xmm subsystems.
        !          1544:  *
        !          1545:  *      Called only for freshly vm_page_grab()ed pages - w/ nothing locked.
        !          1546:  */
        !          1547: void
        !          1548: vm_page_gobble(
        !          1549:         register vm_page_t      mem)
        !          1550: {
        !          1551:         vm_page_lock_queues();
        !          1552:         VM_PAGE_CHECK(mem);
        !          1553: 
        !          1554:        assert(!mem->gobbled);
        !          1555:        assert(mem->wire_count == 0);
        !          1556: 
        !          1557:         if (!mem->gobbled && mem->wire_count == 0) {
        !          1558:                 if (!mem->private && !mem->fictitious)
        !          1559:                         vm_page_wire_count++;
        !          1560:         }
        !          1561:        vm_page_gobble_count++;
        !          1562:         mem->gobbled = TRUE;
        !          1563:         vm_page_unlock_queues();
        !          1564: }
        !          1565: 
        !          1566: /*
        !          1567:  *     vm_page_unwire:
        !          1568:  *
        !          1569:  *     Release one wiring of this page, potentially
        !          1570:  *     enabling it to be paged again.
        !          1571:  *
        !          1572:  *     The page's object and the page queues must be locked.
        !          1573:  */
        !          1574: void
        !          1575: vm_page_unwire(
        !          1576:        register vm_page_t      mem)
        !          1577: {
        !          1578: 
        !          1579: //     dbgLog(current_act(), mem->offset, mem->object, 0);     /* (TEST/DEBUG) */
        !          1580: 
        !          1581:        VM_PAGE_CHECK(mem);
        !          1582:        assert(mem->wire_count > 0);
        !          1583: 
        !          1584:        if (--mem->wire_count == 0) {
        !          1585:                assert(!mem->private && !mem->fictitious);
        !          1586:                vm_page_wire_count--;
        !          1587:                queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
        !          1588:                vm_page_active_count++;
        !          1589:                mem->active = TRUE;
        !          1590:                mem->reference = TRUE;
        !          1591:        }
        !          1592: }
        !          1593: 
        !          1594: /*
        !          1595:  *     vm_page_deactivate:
        !          1596:  *
        !          1597:  *     Returns the given page to the inactive list,
        !          1598:  *     indicating that no physical maps have access
        !          1599:  *     to this page.  [Used by the physical mapping system.]
        !          1600:  *
        !          1601:  *     The page queues must be locked.
        !          1602:  */
        !          1603: void
        !          1604: vm_page_deactivate(
        !          1605:        register vm_page_t      m)
        !          1606: {
        !          1607:        VM_PAGE_CHECK(m);
        !          1608: 
        !          1609: //     dbgLog(m->phys_addr, vm_page_free_count, vm_page_wire_count, 6);        /* (TEST/DEBUG) */
        !          1610: 
        !          1611:        /*
        !          1612:         *      This page is no longer very interesting.  If it was
        !          1613:         *      interesting (active or inactive/referenced), then we
        !          1614:         *      clear the reference bit and (re)enter it in the
        !          1615:         *      inactive queue.  Note wired pages should not have
        !          1616:         *      their reference bit cleared.
        !          1617:         */
        !          1618:        if (m->gobbled) {               /* can this happen? */
        !          1619:                assert(m->wire_count == 0);
        !          1620:                if (!m->private && !m->fictitious)
        !          1621:                        vm_page_wire_count--;
        !          1622:                vm_page_gobble_count--;
        !          1623:                m->gobbled = FALSE;
        !          1624:        }
        !          1625:        if (m->private || (m->wire_count != 0))
        !          1626:                return;
        !          1627:        if (m->active || (m->inactive && m->reference)) {
        !          1628:                if (!m->fictitious && !m->absent)
        !          1629:                        pmap_clear_reference(m->phys_addr);
        !          1630:                m->reference = FALSE;
        !          1631:                VM_PAGE_QUEUES_REMOVE(m);
        !          1632:        }
        !          1633:        if (m->wire_count == 0 && !m->inactive) {
        !          1634:                queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
        !          1635:                m->inactive = TRUE;
        !          1636:                if (!m->fictitious)
        !          1637:                        vm_page_inactive_count++;
        !          1638:        }
        !          1639: }
        !          1640: 
        !          1641: /*
        !          1642:  *     vm_page_activate:
        !          1643:  *
        !          1644:  *     Put the specified page on the active list (if appropriate).
        !          1645:  *
        !          1646:  *     The page queues must be locked.
        !          1647:  */
        !          1648: 
        !          1649: void
        !          1650: vm_page_activate(
        !          1651:        register vm_page_t      m)
        !          1652: {
        !          1653:        VM_PAGE_CHECK(m);
        !          1654: 
        !          1655:        if (m->gobbled) {
        !          1656:                assert(m->wire_count == 0);
        !          1657:                if (!m->private && !m->fictitious)
        !          1658:                        vm_page_wire_count--;
        !          1659:                vm_page_gobble_count--;
        !          1660:                m->gobbled = FALSE;
        !          1661:        }
        !          1662:        if (m->private)
        !          1663:                return;
        !          1664: 
        !          1665:        if (m->inactive) {
        !          1666:                queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
        !          1667:                if (!m->fictitious)
        !          1668:                        vm_page_inactive_count--;
        !          1669:                m->inactive = FALSE;
        !          1670:        }
        !          1671:        if (m->wire_count == 0) {
        !          1672:                if (m->active)
        !          1673:                        panic("vm_page_activate: already active");
        !          1674: 
        !          1675:                queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
        !          1676:                m->active = TRUE;
        !          1677:                m->reference = TRUE;
        !          1678:                if (!m->fictitious)
        !          1679:                        vm_page_active_count++;
        !          1680:        }
        !          1681: }
        !          1682: 
        !          1683: /*
        !          1684:  *     vm_page_part_zero_fill:
        !          1685:  *
        !          1686:  *     Zero-fill a part of the page.
        !          1687:  */
        !          1688: void
        !          1689: vm_page_part_zero_fill(
        !          1690:        vm_page_t       m,
        !          1691:        vm_offset_t     m_pa,
        !          1692:        vm_size_t       len)
        !          1693: {
        !          1694:        VM_PAGE_CHECK(m);
        !          1695: 
        !          1696:        pmap_zero_part_page(m->phys_addr, m_pa, len);
        !          1697: }
        !          1698: 
        !          1699: /*
        !          1700:  *     vm_page_zero_fill:
        !          1701:  *
        !          1702:  *     Zero-fill the specified page.
        !          1703:  */
        !          1704: void
        !          1705: vm_page_zero_fill(
        !          1706:        vm_page_t       m)
        !          1707: {
        !          1708:         XPR(XPR_VM_PAGE,
        !          1709:                 "vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n",
        !          1710:                 (integer_t)m->object, (integer_t)m->offset, (integer_t)m, 0,0);
        !          1711: 
        !          1712:        VM_PAGE_CHECK(m);
        !          1713: 
        !          1714:        pmap_zero_page(m->phys_addr);
        !          1715: }
        !          1716: 
        !          1717: /*
        !          1718:  *     vm_page_part_copy:
        !          1719:  *
        !          1720:  *     copy part of one page to another
        !          1721:  */
        !          1722: 
        !          1723: void
        !          1724: vm_page_part_copy(
        !          1725:        vm_page_t       src_m,
        !          1726:        vm_offset_t     src_pa,
        !          1727:        vm_page_t       dst_m,
        !          1728:        vm_offset_t     dst_pa,
        !          1729:        vm_size_t       len)
        !          1730: {
        !          1731:        VM_PAGE_CHECK(src_m);
        !          1732:        VM_PAGE_CHECK(dst_m);
        !          1733: 
        !          1734:        pmap_copy_part_page(src_m->phys_addr, src_pa,
        !          1735:                        dst_m->phys_addr, dst_pa, len);
        !          1736: }
        !          1737: 
        !          1738: /*
        !          1739:  *     vm_page_copy:
        !          1740:  *
        !          1741:  *     Copy one page to another
        !          1742:  */
        !          1743: 
        !          1744: void
        !          1745: vm_page_copy(
        !          1746:        vm_page_t       src_m,
        !          1747:        vm_page_t       dest_m)
        !          1748: {
        !          1749:         XPR(XPR_VM_PAGE,
        !          1750:         "vm_page_copy, object 0x%X offset 0x%X to object 0x%X offset 0x%X\n",
        !          1751:         (integer_t)src_m->object, src_m->offset, 
        !          1752:        (integer_t)dest_m->object, dest_m->offset,
        !          1753:        0);
        !          1754: 
        !          1755:        VM_PAGE_CHECK(src_m);
        !          1756:        VM_PAGE_CHECK(dest_m);
        !          1757: 
        !          1758:        pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
        !          1759: }
        !          1760: 
        !          1761: /*
        !          1762:  * Limbo pages are placed on the limbo queue to await their prep count
        !          1763:  * going to zero.  A page is put into limbo by the pageout daemon.  If the
        !          1764:  * page is real, then the pageout daemon did not need to page out the page,
        !          1765:  * it just freed it.  When the prep_pin_count is zero the page can be freed.
        !          1766:  * Real pages with a non-zero prep count are converted to fictitious pages
        !          1767:  * so that the memory can be reclaimed; the fictitious page will remain on
        !          1768:  * the limbo queue until its prep count reaches zero.
        !          1769:  *
        !          1770:  * cleanup_limbo_queue is called by vm_page_more_fictitious and the pageout
        !          1771:  * daemon since it can free both real and fictitious pages.
        !          1772:  * It returns the number of fictitious pages freed.
        !          1773:  */
        !          1774: void
        !          1775: cleanup_limbo_queue(void)
        !          1776: {
        !          1777:        register vm_page_t free_m, m;
        !          1778:        vm_offset_t phys_addr;
        !          1779: 
        !          1780:        vm_page_lock_queues();
        !          1781:        assert(vm_page_limbo_count >= vm_page_limbo_real_count);
        !          1782: 
        !          1783:        /*
        !          1784:         * first free up all pages with prep/pin counts of zero.  This
        !          1785:         * may free both real and fictitious pages, which may be needed
        !          1786:         * later to convert real ones.
        !          1787:         */
        !          1788:        m = (vm_page_t)queue_first(&vm_page_queue_limbo);
        !          1789:        while (!queue_end(&vm_page_queue_limbo, (queue_entry_t)m)) {
        !          1790:                if (m->prep_pin_count == 0) {
        !          1791:                        free_m = m;
        !          1792:                        m = (vm_page_t)queue_next(&m->pageq);
        !          1793:                        queue_remove(&vm_page_queue_limbo, free_m, vm_page_t,
        !          1794:                                                                        pageq);
        !          1795:                        vm_page_limbo_count--;
        !          1796:                        if (!free_m->fictitious)
        !          1797:                                vm_page_limbo_real_count--;
        !          1798:                        free_m->limbo = FALSE;
        !          1799:                        vm_page_free(free_m);
        !          1800:                        assert(vm_page_limbo_count >= 0);
        !          1801:                        assert(vm_page_limbo_real_count >= 0);
        !          1802:                } else {
        !          1803:                        m = (vm_page_t)queue_next(&m->pageq);
        !          1804:                }
        !          1805:        }
        !          1806: 
        !          1807:        /*
        !          1808:         * now convert any remaining real pages to fictitious and free the
        !          1809:         * real ones.
        !          1810:         */
        !          1811:        while (vm_page_limbo_real_count > 0) {
        !          1812:                queue_remove_first(&vm_page_queue_limbo, m, vm_page_t, pageq);
        !          1813:                assert(!m->fictitious);
        !          1814:                assert(m->limbo);
        !          1815: 
        !          1816:                /*
        !          1817:                 * Try to get a fictitious page.  If impossible,
        !          1818:                 * requeue the real one and give up.
        !          1819:                 */
        !          1820:                free_m = vm_page_grab_fictitious();
        !          1821:                if (free_m == VM_PAGE_NULL) {
        !          1822:                        queue_enter_first(&vm_page_queue_limbo, m, vm_page_t,
        !          1823:                                                                        pageq);
        !          1824:                        break;
        !          1825:                }
        !          1826:                c_limbo_convert++;
        !          1827:                vm_page_limbo_exchange(m, free_m);
        !          1828:                assert(m->limbo && m->fictitious);
        !          1829:                assert(!free_m->limbo && !free_m->fictitious);
        !          1830:                queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq);
        !          1831:                vm_page_free(free_m);
        !          1832:                vm_page_limbo_real_count--;
        !          1833:        }
        !          1834: 
        !          1835:        vm_page_unlock_queues();
        !          1836: }
        !          1837: 
        !          1838: /*
        !          1839:  * Increment prep_count on a page.
        !          1840:  * Must be called in thread context.  Page must not disappear: object
        !          1841:  * must be locked.
        !          1842:  */
        !          1843: kern_return_t
        !          1844: vm_page_prep(
        !          1845:        register vm_page_t      m)
        !          1846: {
        !          1847:        kern_return_t retval = KERN_SUCCESS;
        !          1848: 
        !          1849:        assert(m != VM_PAGE_NULL);
        !          1850:        vm_page_lock_queues();
        !          1851:        if (!m->busy && !m->error && !m->fictitious && !m->absent) {
        !          1852:                if (m->prep_pin_count != 0) {
        !          1853:                        vm_page_pin_lock();
        !          1854:                        m->prep_count++;
        !          1855:                        vm_page_pin_unlock();
        !          1856:                } else {
        !          1857:                        m->prep_count++;
        !          1858:                }
        !          1859:                assert(m->prep_count != 0);     /* check for wraparound */
        !          1860:        } else {
        !          1861:                retval = KERN_FAILURE;
        !          1862:        }
        !          1863:        vm_page_unlock_queues();
        !          1864:        return retval;
        !          1865: }
        !          1866: 
        !          1867: 
        !          1868: /*
        !          1869:  * Pin a page (increment pin count).
        !          1870:  * Must have been previously prepped.
        !          1871:  *
        !          1872:  * MUST BE CALLED AT SPLVM.
        !          1873:  *
        !          1874:  * May be called from thread or interrupt context.
        !          1875:  * If page is in "limbo" it cannot be pinned.
        !          1876:  */
        !          1877: kern_return_t
        !          1878: vm_page_pin(
        !          1879:        register vm_page_t      m)
        !          1880: {
        !          1881:        kern_return_t retval = KERN_SUCCESS;
        !          1882: 
        !          1883:        assert(m != VM_PAGE_NULL);
        !          1884:        vm_page_pin_lock();
        !          1885:        if (m->limbo || m->prep_count == 0) {
        !          1886:                retval = KERN_FAILURE;
        !          1887:        } else {
        !          1888:                assert(!m->fictitious);
        !          1889:                if (m->pin_count == 0)
        !          1890:                        vm_page_pin_count++;
        !          1891:                m->pin_count++;
        !          1892:        }
        !          1893:        vm_page_pin_unlock();
        !          1894:        return retval;
        !          1895: }
        !          1896: 
        !          1897: 
        !          1898: /*
        !          1899:  * Unprep a page (decrement prep count).
        !          1900:  * Must have been previously prepped.
        !          1901:  * Called to decrement prep count after an attempt to pin failed.
        !          1902:  * Must be called from thread context.
        !          1903:  */
        !          1904: kern_return_t
        !          1905: vm_page_unprep(
        !          1906:        register vm_page_t      m)
        !          1907: {
        !          1908:        kern_return_t retval = KERN_SUCCESS;
        !          1909: 
        !          1910:        assert(m != VM_PAGE_NULL);
        !          1911:        vm_page_lock_queues();
        !          1912:        vm_page_pin_lock();
        !          1913:        assert(m->prep_count != 0);
        !          1914:        if (m->prep_count == 0)
        !          1915:                retval = KERN_FAILURE;          /* shouldn't happen */
        !          1916:        else
        !          1917:                m->prep_count--;
        !          1918:        vm_page_pin_unlock();
        !          1919:        vm_page_unlock_queues();
        !          1920:        return retval;
        !          1921: }
        !          1922: 
        !          1923: 
        !          1924: /*
        !          1925:  * Unpin a page: decrement pin AND prep counts.
        !          1926:  * Must have been previously prepped AND pinned.
        !          1927:  *
        !          1928:  * MUST BE CALLED AT SPLVM.
        !          1929:  *
        !          1930:  * May be called from thread or interrupt context.
        !          1931:  */
        !          1932: kern_return_t
        !          1933: vm_page_unpin(
        !          1934:        register vm_page_t      m)
        !          1935: {
        !          1936:        kern_return_t retval = KERN_SUCCESS;
        !          1937: 
        !          1938:        assert(m != VM_PAGE_NULL);
        !          1939:        vm_page_pin_lock();
        !          1940:        assert(m->prep_count != 0 && m->pin_count != 0);
        !          1941:        assert(m->prep_count >= m->pin_count);
        !          1942:        assert(!m->limbo && !m->fictitious);
        !          1943:        if (m->prep_count != 0 && m->pin_count != 0) {
        !          1944:                m->prep_count--;
        !          1945:                m->pin_count--;
        !          1946:                if (m->pin_count == 0)
        !          1947:                        vm_page_pin_count--;
        !          1948:        } else {
        !          1949:                retval = KERN_FAILURE;          /* shouldn't happen */
        !          1950:        }
        !          1951:        vm_page_pin_unlock();
        !          1952:        return retval;
        !          1953: }
        !          1954: 
        !          1955: /*
        !          1956:  *     Currently, this is a primitive allocator that grabs
        !          1957:  *     free pages from the system, sorts them by physical
        !          1958:  *     address, then searches for a region large enough to
        !          1959:  *     satisfy the user's request.
        !          1960:  *
        !          1961:  *     Additional levels of effort:
        !          1962:  *             + steal clean active/inactive pages
        !          1963:  *             + force pageouts of dirty pages
        !          1964:  *             + maintain a map of available physical
        !          1965:  *             memory
        !          1966:  */
        !          1967: 
        !          1968: #define        SET_NEXT_PAGE(m,n)      ((m)->pageq.next = (struct queue_entry *) (n))
        !          1969: 
        !          1970: #if    MACH_ASSERT
        !          1971: int    vm_page_verify_contiguous(
        !          1972:                vm_page_t       pages,
        !          1973:                unsigned int    npages);
        !          1974: #endif /* MACH_ASSERT */
        !          1975: 
        !          1976: cpm_counter(unsigned int       vpfls_pages_handled = 0;)
        !          1977: cpm_counter(unsigned int       vpfls_head_insertions = 0;)
        !          1978: cpm_counter(unsigned int       vpfls_tail_insertions = 0;)
        !          1979: cpm_counter(unsigned int       vpfls_general_insertions = 0;)
        !          1980: cpm_counter(unsigned int       vpfc_failed = 0;)
        !          1981: cpm_counter(unsigned int       vpfc_satisfied = 0;)
        !          1982: 
        !          1983: /*
        !          1984:  *     Sort free list by ascending physical address,
        !          1985:  *     using a not-particularly-bright sort algorithm.
        !          1986:  *     Caller holds vm_page_queue_free_lock.
        !          1987:  */
        !          1988: static void
        !          1989: vm_page_free_list_sort(void)
        !          1990: {
        !          1991:        vm_page_t       sort_list;
        !          1992:        vm_page_t       sort_list_end;
        !          1993:        vm_page_t       m, m1, *prev, next_m;
        !          1994:        vm_offset_t     addr;
        !          1995: #if    MACH_ASSERT
        !          1996:        unsigned int    npages;
        !          1997:        int             old_free_count;
        !          1998: #endif /* MACH_ASSERT */
        !          1999: 
        !          2000: #if    MACH_ASSERT
        !          2001:        /*
        !          2002:         *      Verify pages in the free list..
        !          2003:         */
        !          2004:        npages = 0;
        !          2005:        for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m))
        !          2006:                ++npages;
        !          2007:        if (npages != vm_page_free_count)
        !          2008:                panic("vm_sort_free_list:  prelim:  npages %d free_count %d",
        !          2009:                      npages, vm_page_free_count);
        !          2010:        old_free_count = vm_page_free_count;
        !          2011: #endif /* MACH_ASSERT */
        !          2012: 
        !          2013:        sort_list = sort_list_end = vm_page_queue_free;
        !          2014:        m = NEXT_PAGE(vm_page_queue_free);
        !          2015:        SET_NEXT_PAGE(vm_page_queue_free, VM_PAGE_NULL);
        !          2016:        cpm_counter(vpfls_pages_handled = 0);
        !          2017:        while (m != VM_PAGE_NULL) {
        !          2018:                cpm_counter(++vpfls_pages_handled);
        !          2019:                next_m = NEXT_PAGE(m);
        !          2020:                if (m->phys_addr < sort_list->phys_addr) {
        !          2021:                        cpm_counter(++vpfls_head_insertions);
        !          2022:                        SET_NEXT_PAGE(m, sort_list);
        !          2023:                        sort_list = m;
        !          2024:                } else if (m->phys_addr > sort_list_end->phys_addr) {
        !          2025:                        cpm_counter(++vpfls_tail_insertions);
        !          2026:                        SET_NEXT_PAGE(sort_list_end, m);
        !          2027:                        SET_NEXT_PAGE(m, VM_PAGE_NULL);
        !          2028:                        sort_list_end = m;
        !          2029:                } else {
        !          2030:                        cpm_counter(++vpfls_general_insertions);
        !          2031:                        /* general sorted list insertion */
        !          2032:                        prev = &sort_list;
        !          2033:                        for (m1=sort_list; m1!=VM_PAGE_NULL; m1=NEXT_PAGE(m1)) {
        !          2034:                                if (m1->phys_addr > m->phys_addr) {
        !          2035:                                        if (*prev != m1)
        !          2036:                                                panic("vm_sort_free_list: ugh");
        !          2037:                                        SET_NEXT_PAGE(m, *prev);
        !          2038:                                        *prev = m;
        !          2039:                                        break;
        !          2040:                                }
        !          2041:                                prev = (vm_page_t *) &m1->pageq.next;
        !          2042:                        }
        !          2043:                }
        !          2044:                m = next_m;
        !          2045:        }
        !          2046: 
        !          2047: #if    MACH_ASSERT
        !          2048:        /*
        !          2049:         *      Verify that pages are sorted into ascending order.
        !          2050:         */
        !          2051:        for (m = sort_list, npages = 0; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
        !          2052:                if (m != sort_list &&
        !          2053:                    m->phys_addr <= addr) {
        !          2054:                        printf("m 0x%x addr 0x%x\n", m, addr);
        !          2055:                        panic("vm_sort_free_list");
        !          2056:                }
        !          2057:                addr = m->phys_addr;
        !          2058:                ++npages;
        !          2059:        }
        !          2060:        if (old_free_count != vm_page_free_count)
        !          2061:                panic("vm_sort_free_list:  old_free %d free_count %d",
        !          2062:                      old_free_count, vm_page_free_count);
        !          2063:        if (npages != vm_page_free_count)
        !          2064:                panic("vm_sort_free_list:  npages %d free_count %d",
        !          2065:                      npages, vm_page_free_count);
        !          2066: #endif /* MACH_ASSERT */
        !          2067: 
        !          2068:        vm_page_queue_free = sort_list;
        !          2069: }
        !          2070: 
        !          2071: 
        !          2072: #if    MACH_ASSERT
        !          2073: /*
        !          2074:  *     Check that the list of pages is ordered by
        !          2075:  *     ascending physical address and has no holes.
        !          2076:  */
        !          2077: int
        !          2078: vm_page_verify_contiguous(
        !          2079:        vm_page_t       pages,
        !          2080:        unsigned int    npages)
        !          2081: {
        !          2082:        register vm_page_t      m;
        !          2083:        unsigned int            page_count;
        !          2084:        vm_offset_t             prev_addr;
        !          2085: 
        !          2086:        prev_addr = pages->phys_addr;
        !          2087:        page_count = 1;
        !          2088:        for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
        !          2089:                if (m->phys_addr != prev_addr + page_size) {
        !          2090:                        printf("m 0x%x prev_addr 0x%x, current addr 0x%x\n",
        !          2091:                               m, prev_addr, m->phys_addr);
        !          2092:                        printf("pages 0x%x page_count %d\n", pages, page_count);
        !          2093:                        panic("vm_page_verify_contiguous:  not contiguous!");
        !          2094:                }
        !          2095:                prev_addr = m->phys_addr;
        !          2096:                ++page_count;
        !          2097:        }
        !          2098:        if (page_count != npages) {
        !          2099:                printf("pages 0x%x actual count 0x%x but requested 0x%x\n",
        !          2100:                       pages, page_count, npages);
        !          2101:                panic("vm_page_verify_contiguous:  count error");
        !          2102:        }
        !          2103:        return 1;
        !          2104: }
        !          2105: #endif /* MACH_ASSERT */
        !          2106: 
        !          2107: 
        !          2108: /*
        !          2109:  *     Find a region large enough to contain at least npages
        !          2110:  *     of contiguous physical memory.
        !          2111:  *
        !          2112:  *     Requirements:
        !          2113:  *             - Called while holding vm_page_queue_free_lock.
        !          2114:  *             - Doesn't respect vm_page_free_reserved; caller
        !          2115:  *             must not ask for more pages than are legal to grab.
        !          2116:  *
        !          2117:  *     Returns a pointer to a list of gobbled pages or VM_PAGE_NULL.
        !          2118:  *
        !          2119:  */
        !          2120: static vm_page_t
        !          2121: vm_page_find_contiguous(
        !          2122:        int             npages)
        !          2123: {
        !          2124:        vm_page_t       m, *contig_prev, *prev_ptr;
        !          2125:        vm_offset_t     prev_addr;
        !          2126:        unsigned int    contig_npages;
        !          2127:        vm_page_t       list;
        !          2128: 
        !          2129:        if (npages < 1)
        !          2130:                return VM_PAGE_NULL;
        !          2131: 
        !          2132:        prev_addr = vm_page_queue_free->phys_addr - (page_size + 1);
        !          2133:        prev_ptr = &vm_page_queue_free;
        !          2134:        for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
        !          2135: 
        !          2136:                if (m->phys_addr != prev_addr + page_size) {
        !          2137:                        /*
        !          2138:                         *      Whoops!  Pages aren't contiguous.  Start over.
        !          2139:                         */
        !          2140:                        contig_npages = 0;
        !          2141:                        contig_prev = prev_ptr;
        !          2142:                }
        !          2143: 
        !          2144:                if (++contig_npages == npages) {
        !          2145:                        /*
        !          2146:                         *      Chop these pages out of the free list.
        !          2147:                         *      Mark them all as gobbled.
        !          2148:                         */
        !          2149:                        list = *contig_prev;
        !          2150:                        *contig_prev = NEXT_PAGE(m);
        !          2151:                        SET_NEXT_PAGE(m, VM_PAGE_NULL);
        !          2152:                        for (m = list; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
        !          2153:                                assert(m->free);
        !          2154:                                assert(!m->wanted);
        !          2155:                                m->free = FALSE;
        !          2156:                                m->gobbled = TRUE;
        !          2157:                        }
        !          2158:                        vm_page_free_count -= npages;
        !          2159:                        if (vm_page_free_count < vm_page_free_count_minimum)
        !          2160:                                vm_page_free_count_minimum = vm_page_free_count;
        !          2161:                        vm_page_gobble_count += npages;
        !          2162:                        cpm_counter(++vpfc_satisfied);
        !          2163:                        assert(vm_page_verify_contiguous(list, contig_npages));
        !          2164:                        return list;
        !          2165:                }
        !          2166: 
        !          2167:                assert(contig_npages < npages);
        !          2168:                prev_ptr = (vm_page_t *) &m->pageq.next;
        !          2169:                prev_addr = m->phys_addr;
        !          2170:        }
        !          2171:        cpm_counter(++vpfc_failed);
        !          2172:        return VM_PAGE_NULL;
        !          2173: }
        !          2174: 
        !          2175: /*
        !          2176:  *     Allocate a list of contiguous, wired pages.
        !          2177:  */
        !          2178: kern_return_t
        !          2179: cpm_allocate(
        !          2180:        vm_size_t       size,
        !          2181:        vm_page_t       *list,
        !          2182:        boolean_t       wire)
        !          2183: {
        !          2184:        register vm_page_t      m;
        !          2185:        vm_page_t               *first_contig;
        !          2186:        vm_page_t               free_list, pages;
        !          2187:        unsigned int            npages, n1pages;
        !          2188:        int                     vm_pages_available;
        !          2189: 
        !          2190:        if (size % page_size != 0)
        !          2191:                return KERN_INVALID_ARGUMENT;
        !          2192: 
        !          2193:        vm_page_lock_queues();
        !          2194:        mutex_lock(&vm_page_queue_free_lock);
        !          2195: 
        !          2196:        /*
        !          2197:         *      Should also take active and inactive pages
        !          2198:         *      into account...  One day...
        !          2199:         */
        !          2200:        vm_pages_available = vm_page_free_count - vm_page_free_reserved;
        !          2201: 
        !          2202:        if (size > vm_pages_available * page_size) {
        !          2203:                mutex_unlock(&vm_page_queue_free_lock);
        !          2204:                return KERN_RESOURCE_SHORTAGE;
        !          2205:        }
        !          2206: 
        !          2207:        vm_page_free_list_sort();
        !          2208: 
        !          2209:        npages = size / page_size;
        !          2210: 
        !          2211:        /*
        !          2212:         *      Obtain a pointer to a subset of the free
        !          2213:         *      list large enough to satisfy the request;
        !          2214:         *      the region will be physically contiguous.
        !          2215:         */
        !          2216:        pages = vm_page_find_contiguous(npages);
        !          2217:        if (pages == VM_PAGE_NULL) {
        !          2218:                mutex_unlock(&vm_page_queue_free_lock);
        !          2219:                return KERN_NO_SPACE;
        !          2220:        }
        !          2221: 
        !          2222:        mutex_unlock(&vm_page_queue_free_lock);
        !          2223: 
        !          2224:        /*
        !          2225:         *      Walk the returned list, wiring the pages.
        !          2226:         */
        !          2227:        if (wire == TRUE)
        !          2228:                for (m = pages; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
        !          2229:                        /*
        !          2230:                         *      Essentially inlined vm_page_wire.
        !          2231:                         */
        !          2232:                        assert(!m->active);
        !          2233:                        assert(!m->inactive);
        !          2234:                        assert(!m->private);
        !          2235:                        assert(!m->fictitious);
        !          2236:                        assert(m->wire_count == 0);
        !          2237:                        assert(m->gobbled);
        !          2238:                        m->gobbled = FALSE;
        !          2239:                        m->wire_count++;
        !          2240:                        ++vm_page_wire_count;
        !          2241:                        --vm_page_gobble_count;
        !          2242:                }
        !          2243:        vm_page_unlock_queues();
        !          2244: 
        !          2245:        /*
        !          2246:         *      The CPM pages should now be available and
        !          2247:         *      ordered by ascending physical address.
        !          2248:         */
        !          2249:        assert(vm_page_verify_contiguous(pages, npages));
        !          2250: 
        !          2251:        *list = pages;
        !          2252:        return KERN_SUCCESS;
        !          2253: }
        !          2254: 
        !          2255: 
        !          2256: #include <mach_vm_debug.h>
        !          2257: #if    MACH_VM_DEBUG
        !          2258: 
        !          2259: #include <mach_debug/hash_info.h>
        !          2260: #include <vm/vm_debug.h>
        !          2261: 
        !          2262: /*
        !          2263:  *     Routine:        vm_page_info
        !          2264:  *     Purpose:
        !          2265:  *             Return information about the global VP table.
        !          2266:  *             Fills the buffer with as much information as possible
        !          2267:  *             and returns the desired size of the buffer.
        !          2268:  *     Conditions:
        !          2269:  *             Nothing locked.  The caller should provide
        !          2270:  *             possibly-pageable memory.
        !          2271:  */
        !          2272: 
        !          2273: unsigned int
        !          2274: vm_page_info(
        !          2275:        hash_info_bucket_t *info,
        !          2276:        unsigned int count)
        !          2277: {
        !          2278:        int i;
        !          2279: 
        !          2280:        if (vm_page_bucket_count < count)
        !          2281:                count = vm_page_bucket_count;
        !          2282: 
        !          2283:        for (i = 0; i < count; i++) {
        !          2284:                vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !          2285:                unsigned int bucket_count = 0;
        !          2286:                vm_page_t m;
        !          2287: 
        !          2288:                simple_lock(&vm_page_bucket_lock);
        !          2289:                for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
        !          2290:                        bucket_count++;
        !          2291:                simple_unlock(&vm_page_bucket_lock);
        !          2292: 
        !          2293:                /* don't touch pageable memory while holding locks */
        !          2294:                info[i].hib_count = bucket_count;
        !          2295:        }
        !          2296: 
        !          2297:        return vm_page_bucket_count;
        !          2298: }
        !          2299: #endif /* MACH_VM_DEBUG */
        !          2300: 
        !          2301: #include <mach_kdb.h>
        !          2302: #if    MACH_KDB
        !          2303: 
        !          2304: #include <ddb/db_output.h>
        !          2305: #include <vm/vm_print.h>
        !          2306: #define        printf  kdbprintf
        !          2307: 
        !          2308: /*
        !          2309:  *     Routine:        vm_page_print [exported]
        !          2310:  */
        !          2311: void
        !          2312: vm_page_print(
        !          2313:        vm_page_t       p)
        !          2314: {
        !          2315:        extern db_indent;
        !          2316: 
        !          2317:        iprintf("page 0x%x\n", p);
        !          2318: 
        !          2319:        db_indent += 2;
        !          2320: 
        !          2321:        iprintf("object=0x%x", p->object);
        !          2322:        printf(", offset=0x%x", p->offset);
        !          2323:        printf(", wire_count=%d", p->wire_count);
        !          2324:        printf(", prep_count=%d", p->prep_count);
        !          2325:        printf(", pin_count=%d\n", p->pin_count);
        !          2326: 
        !          2327:        iprintf("%sinactive, %sactive, %sgobbled, %slaundry, %sfree, %sref, %sdiscard\n",
        !          2328:                (p->inactive ? "" : "!"),
        !          2329:                (p->active ? "" : "!"),
        !          2330:                (p->gobbled ? "" : "!"),
        !          2331:                (p->laundry ? "" : "!"),
        !          2332:                (p->free ? "" : "!"),
        !          2333:                (p->reference ? "" : "!"),
        !          2334:                (p->discard_request ? "" : "!"));
        !          2335:        iprintf("%sbusy, %swanted, %stabled, %sfictitious, %sprivate, %sprecious\n",
        !          2336:                (p->busy ? "" : "!"),
        !          2337:                (p->wanted ? "" : "!"),
        !          2338:                (p->tabled ? "" : "!"),
        !          2339:                (p->fictitious ? "" : "!"),
        !          2340:                (p->private ? "" : "!"),
        !          2341:                (p->precious ? "" : "!"));
        !          2342:        iprintf("%sabsent, %serror, %sdirty, %scleaning, %spageout, %sclustered\n",
        !          2343:                (p->absent ? "" : "!"),
        !          2344:                (p->error ? "" : "!"),
        !          2345:                (p->dirty ? "" : "!"),
        !          2346:                (p->cleaning ? "" : "!"),
        !          2347:                (p->pageout ? "" : "!"),
        !          2348:                (p->clustered ? "" : "!"));
        !          2349:        iprintf("%slock_supplied, %soverwriting, %srestart, %sunusual, %slimbo\n",
        !          2350:                (p->lock_supplied ? "" : "!"),
        !          2351:                (p->overwriting ? "" : "!"),
        !          2352:                (p->restart ? "" : "!"),
        !          2353:                (p->unusual ? "" : "!"),
        !          2354:                (p->limbo ? "" : "!"));
        !          2355: 
        !          2356:        iprintf("phys_addr=0x%x", p->phys_addr);
        !          2357:        printf(", page_error=0x%x", p->page_error);
        !          2358:        printf(", page_lock=0x%x", p->page_lock);
        !          2359:        printf(", unlock_request=%d\n", p->unlock_request);
        !          2360: 
        !          2361:        db_indent -= 2;
        !          2362: }
        !          2363: #endif /* MACH_KDB */

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.