Return to vm_resident.c CVS log

Up to [GnuMach] / Gnu-Mach / vm

Annotation of Gnu-Mach/vm/vm_resident.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Mach Operating System
        !             3:  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
        !             4:  * Copyright (c) 1993,1994 The University of Utah and
        !             5:  * the Computer Systems Laboratory (CSL).
        !             6:  * All rights reserved.
        !             7:  *
        !             8:  * Permission to use, copy, modify and distribute this software and its
        !             9:  * documentation is hereby granted, provided that both the copyright
        !            10:  * notice and this permission notice appear in all copies of the
        !            11:  * software, derivative works or modified versions, and any portions
        !            12:  * thereof, and that both notices appear in supporting documentation.
        !            13:  *
        !            14:  * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
        !            15:  * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
        !            16:  * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
        !            17:  * THIS SOFTWARE.
        !            18:  *
        !            19:  * Carnegie Mellon requests users of this software to return to
        !            20:  *
        !            21:  *  Software Distribution Coordinator  or  [email protected]
        !            22:  *  School of Computer Science
        !            23:  *  Carnegie Mellon University
        !            24:  *  Pittsburgh PA 15213-3890
        !            25:  *
        !            26:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            27:  * the rights to redistribute these changes.
        !            28:  */
        !            29: /*
        !            30:  *     File:   vm/vm_page.c
        !            31:  *     Author: Avadis Tevanian, Jr., Michael Wayne Young
        !            32:  *
        !            33:  *     Resident memory management module.
        !            34:  */
        !            35: #include <cpus.h>
        !            36: 
        !            37: #include <mach/vm_prot.h>
        !            38: #include <kern/counters.h>
        !            39: #include <kern/sched_prim.h>
        !            40: #include <kern/task.h>
        !            41: #include <kern/thread.h>
        !            42: #include <mach/vm_statistics.h>
        !            43: #include "vm_param.h"
        !            44: #include <kern/xpr.h>
        !            45: #include <kern/zalloc.h>
        !            46: #include <vm/pmap.h>
        !            47: #include <vm/vm_map.h>
        !            48: #include <vm/vm_page.h>
        !            49: #include <vm/vm_pageout.h>
        !            50: #include <vm/vm_kern.h>
        !            51: 
        !            52: #include <mach_vm_debug.h>
        !            53: #if    MACH_VM_DEBUG
        !            54: #include <mach/kern_return.h>
        !            55: #include <mach_debug/hash_info.h>
        !            56: #include <vm/vm_user.h>
        !            57: #endif
        !            58: 
        !            59: /* in zalloc.c XXX */
        !            60: extern vm_offset_t     zdata;
        !            61: extern vm_size_t       zdata_size;
        !            62: 
        !            63: /*
        !            64:  *     Associated with eacn page of user-allocatable memory is a
        !            65:  *     page structure.
        !            66:  */
        !            67: 
        !            68: /*
        !            69:  *     These variables record the values returned by vm_page_bootstrap,
        !            70:  *     for debugging purposes.  The implementation of pmap_steal_memory
        !            71:  *     and pmap_startup here also uses them internally.
        !            72:  */
        !            73: 
        !            74: vm_offset_t virtual_space_start;
        !            75: vm_offset_t virtual_space_end;
        !            76: 
        !            77: /*
        !            78:  *     The vm_page_lookup() routine, which provides for fast
        !            79:  *     (virtual memory object, offset) to page lookup, employs
        !            80:  *     the following hash table.  The vm_page_{insert,remove}
        !            81:  *     routines install and remove associations in the table.
        !            82:  *     [This table is often called the virtual-to-physical,
        !            83:  *     or VP, table.]
        !            84:  */
        !            85: typedef struct {
        !            86:        decl_simple_lock_data(,lock)
        !            87:        vm_page_t pages;
        !            88: } vm_page_bucket_t;
        !            89: 
        !            90: vm_page_bucket_t *vm_page_buckets;             /* Array of buckets */
        !            91: unsigned int   vm_page_bucket_count = 0;       /* How big is array? */
        !            92: unsigned int   vm_page_hash_mask;              /* Mask for hash function */
        !            93: 
        !            94: /*
        !            95:  *     Resident page structures are initialized from
        !            96:  *     a template (see vm_page_alloc).
        !            97:  *
        !            98:  *     When adding a new field to the virtual memory
        !            99:  *     object structure, be sure to add initialization
        !           100:  *     (see vm_page_bootstrap).
        !           101:  */
        !           102: struct vm_page vm_page_template;
        !           103: 
        !           104: /*
        !           105:  *     Resident pages that represent real memory
        !           106:  *     are allocated from a free list.
        !           107:  */
        !           108: vm_page_t      vm_page_queue_free;
        !           109: vm_page_t      vm_page_queue_fictitious;
        !           110: decl_simple_lock_data(,vm_page_queue_free_lock)
        !           111: unsigned int   vm_page_free_wanted;
        !           112: int            vm_page_free_count;
        !           113: int            vm_page_fictitious_count;
        !           114: 
        !           115: unsigned int   vm_page_free_count_minimum;     /* debugging */
        !           116: 
        !           117: /*
        !           118:  *     Occasionally, the virtual memory system uses
        !           119:  *     resident page structures that do not refer to
        !           120:  *     real pages, for example to leave a page with
        !           121:  *     important state information in the VP table.
        !           122:  *
        !           123:  *     These page structures are allocated the way
        !           124:  *     most other kernel structures are.
        !           125:  */
        !           126: zone_t vm_page_zone;
        !           127: 
        !           128: /*
        !           129:  *     Fictitious pages don't have a physical address,
        !           130:  *     but we must initialize phys_addr to something.
        !           131:  *     For debugging, this should be a strange value
        !           132:  *     that the pmap module can recognize in assertions.
        !           133:  */
        !           134: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
        !           135: 
        !           136: /*
        !           137:  *     Resident page structures are also chained on
        !           138:  *     queues that are used by the page replacement
        !           139:  *     system (pageout daemon).  These queues are
        !           140:  *     defined here, but are shared by the pageout
        !           141:  *     module.
        !           142:  */
        !           143: queue_head_t   vm_page_queue_active;
        !           144: queue_head_t   vm_page_queue_inactive;
        !           145: decl_simple_lock_data(,vm_page_queue_lock)
        !           146: int    vm_page_active_count;
        !           147: int    vm_page_inactive_count;
        !           148: int    vm_page_wire_count;
        !           149: 
        !           150: /*
        !           151:  *     Several page replacement parameters are also
        !           152:  *     shared with this module, so that page allocation
        !           153:  *     (done here in vm_page_alloc) can trigger the
        !           154:  *     pageout daemon.
        !           155:  */
        !           156: int    vm_page_free_target = 0;
        !           157: int    vm_page_free_min = 0;
        !           158: int    vm_page_inactive_target = 0;
        !           159: int    vm_page_free_reserved = 0;
        !           160: int    vm_page_laundry_count = 0;
        !           161: 
        !           162: /*
        !           163:  *     The VM system has a couple of heuristics for deciding
        !           164:  *     that pages are "uninteresting" and should be placed
        !           165:  *     on the inactive queue as likely candidates for replacement.
        !           166:  *     These variables let the heuristics be controlled at run-time
        !           167:  *     to make experimentation easier.
        !           168:  */
        !           169: 
        !           170: boolean_t vm_page_deactivate_behind = TRUE;
        !           171: boolean_t vm_page_deactivate_hint = TRUE;
        !           172: 
        !           173: /*
        !           174:  *     vm_page_bootstrap:
        !           175:  *
        !           176:  *     Initializes the resident memory module.
        !           177:  *
        !           178:  *     Allocates memory for the page cells, and
        !           179:  *     for the object/offset-to-page hash table headers.
        !           180:  *     Each page cell is initialized and placed on the free list.
        !           181:  *     Returns the range of available kernel virtual memory.
        !           182:  */
        !           183: 
        !           184: void vm_page_bootstrap(
        !           185:        vm_offset_t *startp,
        !           186:        vm_offset_t *endp)
        !           187: {
        !           188:        register vm_page_t m;
        !           189:        int i;
        !           190: 
        !           191:        /*
        !           192:         *      Initialize the vm_page template.
        !           193:         */
        !           194: 
        !           195:        m = &vm_page_template;
        !           196:        m->object = VM_OBJECT_NULL;     /* reset later */
        !           197:        m->offset = 0;                  /* reset later */
        !           198:        m->wire_count = 0;
        !           199: 
        !           200:        m->inactive = FALSE;
        !           201:        m->active = FALSE;
        !           202:        m->laundry = FALSE;
        !           203:        m->free = FALSE;
        !           204: 
        !           205:        m->busy = TRUE;
        !           206:        m->wanted = FALSE;
        !           207:        m->tabled = FALSE;
        !           208:        m->fictitious = FALSE;
        !           209:        m->private = FALSE;
        !           210:        m->absent = FALSE;
        !           211:        m->error = FALSE;
        !           212:        m->dirty = FALSE;
        !           213:        m->precious = FALSE;
        !           214:        m->reference = FALSE;
        !           215: 
        !           216:        m->phys_addr = 0;               /* reset later */
        !           217: 
        !           218:        m->page_lock = VM_PROT_NONE;
        !           219:        m->unlock_request = VM_PROT_NONE;
        !           220: 
        !           221:        /*
        !           222:         *      Initialize the page queues.
        !           223:         */
        !           224: 
        !           225:        simple_lock_init(&vm_page_queue_free_lock);
        !           226:        simple_lock_init(&vm_page_queue_lock);
        !           227: 
        !           228:        vm_page_queue_free = VM_PAGE_NULL;
        !           229:        vm_page_queue_fictitious = VM_PAGE_NULL;
        !           230:        queue_init(&vm_page_queue_active);
        !           231:        queue_init(&vm_page_queue_inactive);
        !           232: 
        !           233:        vm_page_free_wanted = 0;
        !           234: 
        !           235:        /*
        !           236:         *      Steal memory for the zone system.
        !           237:         */
        !           238: 
        !           239:        kentry_data_size = kentry_count * sizeof(struct vm_map_entry);
        !           240:        kentry_data = pmap_steal_memory(kentry_data_size);
        !           241: 
        !           242:        zdata = pmap_steal_memory(zdata_size);
        !           243: 
        !           244:        /*
        !           245:         *      Allocate (and initialize) the virtual-to-physical
        !           246:         *      table hash buckets.
        !           247:         *
        !           248:         *      The number of buckets should be a power of two to
        !           249:         *      get a good hash function.  The following computation
        !           250:         *      chooses the first power of two that is greater
        !           251:         *      than the number of physical pages in the system.
        !           252:         */
        !           253: 
        !           254:        if (vm_page_bucket_count == 0) {
        !           255:                unsigned int npages = pmap_free_pages();
        !           256: 
        !           257:                vm_page_bucket_count = 1;
        !           258:                while (vm_page_bucket_count < npages)
        !           259:                        vm_page_bucket_count <<= 1;
        !           260:        }
        !           261: 
        !           262:        vm_page_hash_mask = vm_page_bucket_count - 1;
        !           263: 
        !           264:        if (vm_page_hash_mask & vm_page_bucket_count)
        !           265:                printf("vm_page_bootstrap: WARNING -- strange page hash\n");
        !           266: 
        !           267:        vm_page_buckets = (vm_page_bucket_t *)
        !           268:                pmap_steal_memory(vm_page_bucket_count *
        !           269:                                  sizeof(vm_page_bucket_t));
        !           270: 
        !           271:        for (i = 0; i < vm_page_bucket_count; i++) {
        !           272:                register vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !           273: 
        !           274:                bucket->pages = VM_PAGE_NULL;
        !           275:                simple_lock_init(&bucket->lock);
        !           276:        }
        !           277: 
        !           278:        /*
        !           279:         *      Machine-dependent code allocates the resident page table.
        !           280:         *      It uses vm_page_init to initialize the page frames.
        !           281:         *      The code also returns to us the virtual space available
        !           282:         *      to the kernel.  We don't trust the pmap module
        !           283:         *      to get the alignment right.
        !           284:         */
        !           285: 
        !           286:        pmap_startup(&virtual_space_start, &virtual_space_end);
        !           287:        virtual_space_start = round_page(virtual_space_start);
        !           288:        virtual_space_end = trunc_page(virtual_space_end);
        !           289: 
        !           290:        *startp = virtual_space_start;
        !           291:        *endp = virtual_space_end;
        !           292: 
        !           293:        /*      printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
        !           294:        vm_page_free_count_minimum = vm_page_free_count;
        !           295: }
        !           296: 
        !           297: #ifndef        MACHINE_PAGES
        !           298: /*
        !           299:  *     We implement pmap_steal_memory and pmap_startup with the help
        !           300:  *     of two simpler functions, pmap_virtual_space and pmap_next_page.
        !           301:  */
        !           302: 
        !           303: vm_offset_t pmap_steal_memory(
        !           304:        vm_size_t size)
        !           305: {
        !           306:        vm_offset_t addr, vaddr, paddr;
        !           307: 
        !           308:        /*
        !           309:         *      We round the size to an integer multiple.
        !           310:         */
        !           311: 
        !           312:        size = (size + 3) &~ 3;
        !           313: 
        !           314:        /*
        !           315:         *      If this is the first call to pmap_steal_memory,
        !           316:         *      we have to initialize ourself.
        !           317:         */
        !           318: 
        !           319:        if (virtual_space_start == virtual_space_end) {
        !           320:                pmap_virtual_space(&virtual_space_start, &virtual_space_end);
        !           321: 
        !           322:                /*
        !           323:                 *      The initial values must be aligned properly, and
        !           324:                 *      we don't trust the pmap module to do it right.
        !           325:                 */
        !           326: 
        !           327:                virtual_space_start = round_page(virtual_space_start);
        !           328:                virtual_space_end = trunc_page(virtual_space_end);
        !           329:        }
        !           330: 
        !           331:        /*
        !           332:         *      Allocate virtual memory for this request.
        !           333:         */
        !           334: 
        !           335:        addr = virtual_space_start;
        !           336:        virtual_space_start += size;
        !           337: 
        !           338:        /*
        !           339:         *      Allocate and map physical pages to back new virtual pages.
        !           340:         */
        !           341: 
        !           342:        for (vaddr = round_page(addr);
        !           343:             vaddr < addr + size;
        !           344:             vaddr += PAGE_SIZE) {
        !           345:                if (!pmap_next_page(&paddr))
        !           346:                        panic("pmap_steal_memory");
        !           347: 
        !           348:                /*
        !           349:                 *      XXX Logically, these mappings should be wired,
        !           350:                 *      but some pmap modules barf if they are.
        !           351:                 */
        !           352: 
        !           353:                pmap_enter(kernel_pmap, vaddr, paddr,
        !           354:                           VM_PROT_READ|VM_PROT_WRITE, FALSE);
        !           355:        }
        !           356: 
        !           357:        return addr;
        !           358: }
        !           359: 
        !           360: void pmap_startup(
        !           361:        vm_offset_t *startp,
        !           362:        vm_offset_t *endp)
        !           363: {
        !           364:        unsigned int i, npages, pages_initialized;
        !           365:        vm_page_t pages;
        !           366:        vm_offset_t paddr;
        !           367: 
        !           368:        /*
        !           369:         *      We calculate how many page frames we will have
        !           370:         *      and then allocate the page structures in one chunk.
        !           371:         */
        !           372: 
        !           373:        npages = ((PAGE_SIZE * pmap_free_pages() +
        !           374:                   (round_page(virtual_space_start) - virtual_space_start)) /
        !           375:                  (PAGE_SIZE + sizeof *pages));
        !           376: 
        !           377:        pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
        !           378: 
        !           379:        /*
        !           380:         *      Initialize the page frames.
        !           381:         */
        !           382: 
        !           383:        for (i = 0, pages_initialized = 0; i < npages; i++) {
        !           384:                if (!pmap_next_page(&paddr))
        !           385:                        break;
        !           386: 
        !           387:                vm_page_init(&pages[i], paddr);
        !           388:                pages_initialized++;
        !           389:        }
        !           390: 
        !           391:        /*
        !           392:         * Release pages in reverse order so that physical pages
        !           393:         * initially get allocated in ascending addresses. This keeps
        !           394:         * the devices (which must address physical memory) happy if
        !           395:         * they require several consecutive pages.
        !           396:         */
        !           397: 
        !           398:        for (i = pages_initialized; i > 0; i--) {
        !           399:                vm_page_release(&pages[i - 1]);
        !           400:        }
        !           401: 
        !           402:        /*
        !           403:         *      We have to re-align virtual_space_start,
        !           404:         *      because pmap_steal_memory has been using it.
        !           405:         */
        !           406: 
        !           407:        virtual_space_start = round_page(virtual_space_start);
        !           408: 
        !           409:        *startp = virtual_space_start;
        !           410:        *endp = virtual_space_end;
        !           411: }
        !           412: #endif /* MACHINE_PAGES */
        !           413: 
        !           414: /*
        !           415:  *     Routine:        vm_page_module_init
        !           416:  *     Purpose:
        !           417:  *             Second initialization pass, to be done after
        !           418:  *             the basic VM system is ready.
        !           419:  */
        !           420: void           vm_page_module_init(void)
        !           421: {
        !           422:        vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
        !           423:                             VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
        !           424:                             PAGE_SIZE,
        !           425:                             0, "vm pages");
        !           426: }
        !           427: 
        !           428: /*
        !           429:  *     Routine:        vm_page_create
        !           430:  *     Purpose:
        !           431:  *             After the VM system is up, machine-dependent code
        !           432:  *             may stumble across more physical memory.  For example,
        !           433:  *             memory that it was reserving for a frame buffer.
        !           434:  *             vm_page_create turns this memory into available pages.
        !           435:  */
        !           436: 
        !           437: void vm_page_create(
        !           438:        vm_offset_t     start,
        !           439:        vm_offset_t     end)
        !           440: {
        !           441:        vm_offset_t paddr;
        !           442:        vm_page_t m;
        !           443: 
        !           444:        for (paddr = round_page(start);
        !           445:             paddr < trunc_page(end);
        !           446:             paddr += PAGE_SIZE) {
        !           447:                m = (vm_page_t) zalloc(vm_page_zone);
        !           448:                if (m == VM_PAGE_NULL)
        !           449:                        panic("vm_page_create");
        !           450: 
        !           451:                vm_page_init(m, paddr);
        !           452:                vm_page_release(m);
        !           453:        }
        !           454: }
        !           455: 
        !           456: /*
        !           457:  *     vm_page_hash:
        !           458:  *
        !           459:  *     Distributes the object/offset key pair among hash buckets.
        !           460:  *
        !           461:  *     NOTE:   To get a good hash function, the bucket count should
        !           462:  *             be a power of two.
        !           463:  */
        !           464: #define vm_page_hash(object, offset) \
        !           465:        (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
        !           466:                & vm_page_hash_mask)
        !           467: 
        !           468: /*
        !           469:  *     vm_page_insert:         [ internal use only ]
        !           470:  *
        !           471:  *     Inserts the given mem entry into the object/object-page
        !           472:  *     table and object list.
        !           473:  *
        !           474:  *     The object and page must be locked.
        !           475:  */
        !           476: 
        !           477: void vm_page_insert(
        !           478:        register vm_page_t      mem,
        !           479:        register vm_object_t    object,
        !           480:        register vm_offset_t    offset)
        !           481: {
        !           482:        register vm_page_bucket_t *bucket;
        !           483: 
        !           484:        VM_PAGE_CHECK(mem);
        !           485: 
        !           486:        if (mem->tabled)
        !           487:                panic("vm_page_insert");
        !           488: 
        !           489:        /*
        !           490:         *      Record the object/offset pair in this page
        !           491:         */
        !           492: 
        !           493:        mem->object = object;
        !           494:        mem->offset = offset;
        !           495: 
        !           496:        /*
        !           497:         *      Insert it into the object_object/offset hash table
        !           498:         */
        !           499: 
        !           500:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           501:        simple_lock(&bucket->lock);
        !           502:        mem->next = bucket->pages;
        !           503:        bucket->pages = mem;
        !           504:        simple_unlock(&bucket->lock);
        !           505: 
        !           506:        /*
        !           507:         *      Now link into the object's list of backed pages.
        !           508:         */
        !           509: 
        !           510:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           511:        mem->tabled = TRUE;
        !           512: 
        !           513:        /*
        !           514:         *      Show that the object has one more resident page.
        !           515:         */
        !           516: 
        !           517:        object->resident_page_count++;
        !           518: 
        !           519:        /*
        !           520:         *      Detect sequential access and inactivate previous page.
        !           521:         *      We ignore busy pages.
        !           522:         */
        !           523: 
        !           524:        if (vm_page_deactivate_behind &&
        !           525:            (offset == object->last_alloc + PAGE_SIZE)) {
        !           526:                vm_page_t       last_mem;
        !           527: 
        !           528:                last_mem = vm_page_lookup(object, object->last_alloc);
        !           529:                if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
        !           530:                        vm_page_deactivate(last_mem);
        !           531:        }
        !           532:        object->last_alloc = offset;
        !           533: }
        !           534: 
        !           535: /*
        !           536:  *     vm_page_replace:
        !           537:  *
        !           538:  *     Exactly like vm_page_insert, except that we first
        !           539:  *     remove any existing page at the given offset in object
        !           540:  *     and we don't do deactivate-behind.
        !           541:  *
        !           542:  *     The object and page must be locked.
        !           543:  */
        !           544: 
        !           545: void vm_page_replace(
        !           546:        register vm_page_t      mem,
        !           547:        register vm_object_t    object,
        !           548:        register vm_offset_t    offset)
        !           549: {
        !           550:        register vm_page_bucket_t *bucket;
        !           551: 
        !           552:        VM_PAGE_CHECK(mem);
        !           553: 
        !           554:        if (mem->tabled)
        !           555:                panic("vm_page_replace");
        !           556: 
        !           557:        /*
        !           558:         *      Record the object/offset pair in this page
        !           559:         */
        !           560: 
        !           561:        mem->object = object;
        !           562:        mem->offset = offset;
        !           563: 
        !           564:        /*
        !           565:         *      Insert it into the object_object/offset hash table,
        !           566:         *      replacing any page that might have been there.
        !           567:         */
        !           568: 
        !           569:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           570:        simple_lock(&bucket->lock);
        !           571:        if (bucket->pages) {
        !           572:                vm_page_t *mp = &bucket->pages;
        !           573:                register vm_page_t m = *mp;
        !           574:                do {
        !           575:                        if (m->object == object && m->offset == offset) {
        !           576:                                /*
        !           577:                                 * Remove page from bucket and from object,
        !           578:                                 * and return it to the free list.
        !           579:                                 */
        !           580:                                *mp = m->next;
        !           581:                                queue_remove(&object->memq, m, vm_page_t,
        !           582:                                             listq);
        !           583:                                m->tabled = FALSE;
        !           584:                                object->resident_page_count--;
        !           585: 
        !           586:                                /*
        !           587:                                 * Return page to the free list.
        !           588:                                 * Note the page is not tabled now, so this
        !           589:                                 * won't self-deadlock on the bucket lock.
        !           590:                                 */
        !           591: 
        !           592:                                vm_page_free(m);
        !           593:                                break;
        !           594:                        }
        !           595:                        mp = &m->next;
        !           596:                } while ((m = *mp) != 0);
        !           597:                mem->next = bucket->pages;
        !           598:        } else {
        !           599:                mem->next = VM_PAGE_NULL;
        !           600:        }
        !           601:        bucket->pages = mem;
        !           602:        simple_unlock(&bucket->lock);
        !           603: 
        !           604:        /*
        !           605:         *      Now link into the object's list of backed pages.
        !           606:         */
        !           607: 
        !           608:        queue_enter(&object->memq, mem, vm_page_t, listq);
        !           609:        mem->tabled = TRUE;
        !           610: 
        !           611:        /*
        !           612:         *      And show that the object has one more resident
        !           613:         *      page.
        !           614:         */
        !           615: 
        !           616:        object->resident_page_count++;
        !           617: }
        !           618: 
        !           619: /*
        !           620:  *     vm_page_remove:         [ internal use only ]
        !           621:  *
        !           622:  *     Removes the given mem entry from the object/offset-page
        !           623:  *     table and the object page list.
        !           624:  *
        !           625:  *     The object and page must be locked.
        !           626:  */
        !           627: 
        !           628: void vm_page_remove(
        !           629:        register vm_page_t      mem)
        !           630: {
        !           631:        register vm_page_bucket_t       *bucket;
        !           632:        register vm_page_t      this;
        !           633: 
        !           634:        assert(mem->tabled);
        !           635:        VM_PAGE_CHECK(mem);
        !           636: 
        !           637:        /*
        !           638:         *      Remove from the object_object/offset hash table
        !           639:         */
        !           640: 
        !           641:        bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
        !           642:        simple_lock(&bucket->lock);
        !           643:        if ((this = bucket->pages) == mem) {
        !           644:                /* optimize for common case */
        !           645: 
        !           646:                bucket->pages = mem->next;
        !           647:        } else {
        !           648:                register vm_page_t      *prev;
        !           649: 
        !           650:                for (prev = &this->next;
        !           651:                     (this = *prev) != mem;
        !           652:                     prev = &this->next)
        !           653:                        continue;
        !           654:                *prev = this->next;
        !           655:        }
        !           656:        simple_unlock(&bucket->lock);
        !           657: 
        !           658:        /*
        !           659:         *      Now remove from the object's list of backed pages.
        !           660:         */
        !           661: 
        !           662:        queue_remove(&mem->object->memq, mem, vm_page_t, listq);
        !           663: 
        !           664:        /*
        !           665:         *      And show that the object has one fewer resident
        !           666:         *      page.
        !           667:         */
        !           668: 
        !           669:        mem->object->resident_page_count--;
        !           670: 
        !           671:        mem->tabled = FALSE;
        !           672: }
        !           673: 
        !           674: /*
        !           675:  *     vm_page_lookup:
        !           676:  *
        !           677:  *     Returns the page associated with the object/offset
        !           678:  *     pair specified; if none is found, VM_PAGE_NULL is returned.
        !           679:  *
        !           680:  *     The object must be locked.  No side effects.
        !           681:  */
        !           682: 
        !           683: vm_page_t vm_page_lookup(
        !           684:        register vm_object_t    object,
        !           685:        register vm_offset_t    offset)
        !           686: {
        !           687:        register vm_page_t      mem;
        !           688:        register vm_page_bucket_t *bucket;
        !           689: 
        !           690:        /*
        !           691:         *      Search the hash table for this object/offset pair
        !           692:         */
        !           693: 
        !           694:        bucket = &vm_page_buckets[vm_page_hash(object, offset)];
        !           695: 
        !           696:        simple_lock(&bucket->lock);
        !           697:        for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
        !           698:                VM_PAGE_CHECK(mem);
        !           699:                if ((mem->object == object) && (mem->offset == offset))
        !           700:                        break;
        !           701:        }
        !           702:        simple_unlock(&bucket->lock);
        !           703:        return mem;
        !           704: }
        !           705: 
        !           706: /*
        !           707:  *     vm_page_rename:
        !           708:  *
        !           709:  *     Move the given memory entry from its
        !           710:  *     current object to the specified target object/offset.
        !           711:  *
        !           712:  *     The object must be locked.
        !           713:  */
        !           714: void vm_page_rename(
        !           715:        register vm_page_t      mem,
        !           716:        register vm_object_t    new_object,
        !           717:        vm_offset_t             new_offset)
        !           718: {
        !           719:        /*
        !           720:         *      Changes to mem->object require the page lock because
        !           721:         *      the pageout daemon uses that lock to get the object.
        !           722:         */
        !           723: 
        !           724:        vm_page_lock_queues();
        !           725:        vm_page_remove(mem);
        !           726:        vm_page_insert(mem, new_object, new_offset);
        !           727:        vm_page_unlock_queues();
        !           728: }
        !           729: 
        !           730: /*
        !           731:  *     vm_page_init:
        !           732:  *
        !           733:  *     Initialize the fields in a new page.
        !           734:  *     This takes a structure with random values and initializes it
        !           735:  *     so that it can be given to vm_page_release or vm_page_insert.
        !           736:  */
        !           737: void vm_page_init(
        !           738:        vm_page_t       mem,
        !           739:        vm_offset_t     phys_addr)
        !           740: {
        !           741:        *mem = vm_page_template;
        !           742:        mem->phys_addr = phys_addr;
        !           743: }
        !           744: 
        !           745: /*
        !           746:  *     vm_page_grab_fictitious:
        !           747:  *
        !           748:  *     Remove a fictitious page from the free list.
        !           749:  *     Returns VM_PAGE_NULL if there are no free pages.
        !           750:  */
        !           751: 
        !           752: vm_page_t vm_page_grab_fictitious(void)
        !           753: {
        !           754:        register vm_page_t m;
        !           755: 
        !           756:        simple_lock(&vm_page_queue_free_lock);
        !           757:        m = vm_page_queue_fictitious;
        !           758:        if (m != VM_PAGE_NULL) {
        !           759:                vm_page_fictitious_count--;
        !           760:                vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
        !           761:                m->free = FALSE;
        !           762:        }
        !           763:        simple_unlock(&vm_page_queue_free_lock);
        !           764: 
        !           765:        return m;
        !           766: }
        !           767: 
        !           768: /*
        !           769:  *     vm_page_release_fictitious:
        !           770:  *
        !           771:  *     Release a fictitious page to the free list.
        !           772:  */
        !           773: 
        !           774: void vm_page_release_fictitious(
        !           775:        register vm_page_t m)
        !           776: {
        !           777:        simple_lock(&vm_page_queue_free_lock);
        !           778:        if (m->free)
        !           779:                panic("vm_page_release_fictitious");
        !           780:        m->free = TRUE;
        !           781:        m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
        !           782:        vm_page_queue_fictitious = m;
        !           783:        vm_page_fictitious_count++;
        !           784:        simple_unlock(&vm_page_queue_free_lock);
        !           785: }
        !           786: 
        !           787: /*
        !           788:  *     vm_page_more_fictitious:
        !           789:  *
        !           790:  *     Add more fictitious pages to the free list.
        !           791:  *     Allowed to block.
        !           792:  */
        !           793: 
        !           794: int vm_page_fictitious_quantum = 5;
        !           795: 
        !           796: void vm_page_more_fictitious(void)
        !           797: {
        !           798:        register vm_page_t m;
        !           799:        int i;
        !           800: 
        !           801:        for (i = 0; i < vm_page_fictitious_quantum; i++) {
        !           802:                m = (vm_page_t) zalloc(vm_page_zone);
        !           803:                if (m == VM_PAGE_NULL)
        !           804:                        panic("vm_page_more_fictitious");
        !           805: 
        !           806:                vm_page_init(m, vm_page_fictitious_addr);
        !           807:                m->fictitious = TRUE;
        !           808:                vm_page_release_fictitious(m);
        !           809:        }
        !           810: }
        !           811: 
        !           812: /*
        !           813:  *     vm_page_convert:
        !           814:  *
        !           815:  *     Attempt to convert a fictitious page into a real page.
        !           816:  */
        !           817: 
        !           818: boolean_t vm_page_convert(
        !           819:        register vm_page_t m)
        !           820: {
        !           821:        register vm_page_t real_m;
        !           822: 
        !           823:        real_m = vm_page_grab();
        !           824:        if (real_m == VM_PAGE_NULL)
        !           825:                return FALSE;
        !           826: 
        !           827:        m->phys_addr = real_m->phys_addr;
        !           828:        m->fictitious = FALSE;
        !           829: 
        !           830:        real_m->phys_addr = vm_page_fictitious_addr;
        !           831:        real_m->fictitious = TRUE;
        !           832: 
        !           833:        vm_page_release_fictitious(real_m);
        !           834:        return TRUE;
        !           835: }
        !           836: 
        !           837: /*
        !           838:  *     vm_page_grab:
        !           839:  *
        !           840:  *     Remove a page from the free list.
        !           841:  *     Returns VM_PAGE_NULL if the free list is too small.
        !           842:  */
        !           843: 
        !           844: vm_page_t vm_page_grab(void)
        !           845: {
        !           846:        register vm_page_t      mem;
        !           847: 
        !           848:        simple_lock(&vm_page_queue_free_lock);
        !           849: 
        !           850:        /*
        !           851:         *      Only let privileged threads (involved in pageout)
        !           852:         *      dip into the reserved pool.
        !           853:         */
        !           854: 
        !           855:        if ((vm_page_free_count < vm_page_free_reserved) &&
        !           856:            !current_thread()->vm_privilege) {
        !           857:                simple_unlock(&vm_page_queue_free_lock);
        !           858:                return VM_PAGE_NULL;
        !           859:        }
        !           860: 
        !           861:        if (vm_page_queue_free == VM_PAGE_NULL)
        !           862:                panic("vm_page_grab");
        !           863: 
        !           864:        if (--vm_page_free_count < vm_page_free_count_minimum)
        !           865:                vm_page_free_count_minimum = vm_page_free_count;
        !           866:        mem = vm_page_queue_free;
        !           867:        vm_page_queue_free = (vm_page_t) mem->pageq.next;
        !           868:        mem->free = FALSE;
        !           869:        simple_unlock(&vm_page_queue_free_lock);
        !           870: 
        !           871:        /*
        !           872:         *      Decide if we should poke the pageout daemon.
        !           873:         *      We do this if the free count is less than the low
        !           874:         *      water mark, or if the free count is less than the high
        !           875:         *      water mark (but above the low water mark) and the inactive
        !           876:         *      count is less than its target.
        !           877:         *
        !           878:         *      We don't have the counts locked ... if they change a little,
        !           879:         *      it doesn't really matter.
        !           880:         */
        !           881: 
        !           882:        if ((vm_page_free_count < vm_page_free_min) ||
        !           883:            ((vm_page_free_count < vm_page_free_target) &&
        !           884:             (vm_page_inactive_count < vm_page_inactive_target)))
        !           885:                thread_wakeup((event_t) &vm_page_free_wanted);
        !           886: 
        !           887:        return mem;
        !           888: }
        !           889: 
        !           890: vm_offset_t vm_page_grab_phys_addr(void)
        !           891: {
        !           892:        vm_page_t p = vm_page_grab();
        !           893:        if (p == VM_PAGE_NULL)
        !           894:                return -1;
        !           895:        else
        !           896:                return p->phys_addr;
        !           897: }
        !           898: 
        !           899: /*
        !           900:  *     vm_page_grab_contiguous_pages:
        !           901:  *
        !           902:  *     Take N pages off the free list, the pages should
        !           903:  *     cover a contiguous range of physical addresses.
        !           904:  *     [Used by device drivers to cope with DMA limitations]
        !           905:  *
        !           906:  *     Returns the page descriptors in ascending order, or
        !           907:  *     Returns KERN_RESOURCE_SHORTAGE if it could not.
        !           908:  */
        !           909: 
        !           910: /* Biggest phys page number for the pages we handle in VM */
        !           911: 
        !           912: vm_size_t      vm_page_big_pagenum = 0;        /* Set this before call! */
        !           913: 
        !           914: kern_return_t
        !           915: vm_page_grab_contiguous_pages(
        !           916:        int             npages,
        !           917:        vm_page_t       pages[],
        !           918:        natural_t       *bits)
        !           919: {
        !           920:        register int    first_set;
        !           921:        int             size, alloc_size;
        !           922:        kern_return_t   ret;
        !           923:        vm_page_t       mem, prevmem;
        !           924: 
        !           925: #ifndef        NBBY
        !           926: #define        NBBY    8       /* size in bits of sizeof()`s unity */
        !           927: #endif
        !           928: 
        !           929: #define        NBPEL   (sizeof(natural_t)*NBBY)
        !           930: 
        !           931:        size = (vm_page_big_pagenum + NBPEL - 1)
        !           932:                & ~(NBPEL - 1);                         /* in bits */
        !           933: 
        !           934:        size = size / NBBY;                             /* in bytes */
        !           935: 
        !           936:        /*
        !           937:         * If we are called before the VM system is fully functional
        !           938:         * the invoker must provide us with the work space. [one bit
        !           939:         * per page starting at phys 0 and up to vm_page_big_pagenum]
        !           940:         */
        !           941:        if (bits == 0) {
        !           942:                alloc_size = round_page(size);
        !           943:                if (kmem_alloc_wired(kernel_map,
        !           944:                                     (vm_offset_t *)&bits,
        !           945:                                     alloc_size)
        !           946:                        != KERN_SUCCESS)
        !           947:                    return KERN_RESOURCE_SHORTAGE;
        !           948:        } else
        !           949:                alloc_size = 0;
        !           950: 
        !           951:        bzero(bits, size);
        !           952: 
        !           953:        /*
        !           954:         * A very large granularity call, its rare so that is ok
        !           955:         */
        !           956:        simple_lock(&vm_page_queue_free_lock);
        !           957: 
        !           958:        /*
        !           959:         *      Do not dip into the reserved pool.
        !           960:         */
        !           961: 
        !           962:        if (vm_page_free_count < vm_page_free_reserved) {
        !           963:                simple_unlock(&vm_page_queue_free_lock);
        !           964:                return KERN_RESOURCE_SHORTAGE;
        !           965:        }
        !           966: 
        !           967:        /*
        !           968:         *      First pass through, build a big bit-array of
        !           969:         *      the pages that are free.  It is not going to
        !           970:         *      be too large anyways, in 4k we can fit info
        !           971:         *      for 32k pages.
        !           972:         */
        !           973:        mem = vm_page_queue_free;
        !           974:        while (mem) {
        !           975:                register int word_index, bit_index;
        !           976: 
        !           977:                bit_index = (mem->phys_addr >> PAGE_SHIFT);
        !           978:                word_index = bit_index / NBPEL;
        !           979:                bit_index = bit_index - (word_index * NBPEL);
        !           980:                bits[word_index] |= 1 << bit_index;
        !           981: 
        !           982:                mem = (vm_page_t) mem->pageq.next;
        !           983:        }
        !           984: 
        !           985:        /*
        !           986:         *      Second loop. Scan the bit array for NPAGES
        !           987:         *      contiguous bits.  That gives us, if any,
        !           988:         *      the range of pages we will be grabbing off
        !           989:         *      the free list.
        !           990:         */
        !           991:        {
        !           992:            register int        bits_so_far = 0, i;
        !           993: 
        !           994:                first_set = 0;
        !           995: 
        !           996:                for (i = 0; i < size; i += sizeof(natural_t)) {
        !           997: 
        !           998:                    register natural_t  v = bits[i / sizeof(natural_t)];
        !           999:                    register int        bitpos;
        !          1000: 
        !          1001:                    /*
        !          1002:                     * Bitscan this one word
        !          1003:                     */
        !          1004:                    if (v) {
        !          1005:                        /*
        !          1006:                         * keep counting them beans ?
        !          1007:                         */
        !          1008:                        bitpos = 0;
        !          1009: 
        !          1010:                        if (bits_so_far) {
        !          1011: count_ones:
        !          1012:                            while (v & 1) {
        !          1013:                                bitpos++;
        !          1014:                                /*
        !          1015:                                 * got enough beans ?
        !          1016:                                 */
        !          1017:                                if (++bits_so_far == npages)
        !          1018:                                    goto found_em;
        !          1019:                                v >>= 1;
        !          1020:                            }
        !          1021:                            /* if we are being lucky, roll again */
        !          1022:                            if (bitpos == NBPEL)
        !          1023:                                continue;
        !          1024:                        }
        !          1025: 
        !          1026:                        /*
        !          1027:                         * search for beans here
        !          1028:                         */
        !          1029:                        bits_so_far = 0;
        !          1030: count_zeroes:
        !          1031:                        while ((bitpos < NBPEL) && ((v & 1) == 0)) {
        !          1032:                            bitpos++;
        !          1033:                            v >>= 1;
        !          1034:                        }
        !          1035:                        if (v & 1) {
        !          1036:                            first_set = (i * NBBY) + bitpos;
        !          1037:                            goto count_ones;
        !          1038:                        }
        !          1039:                    }
        !          1040:                    /*
        !          1041:                     * No luck
        !          1042:                     */
        !          1043:                    bits_so_far = 0;
        !          1044:                }
        !          1045:        }
        !          1046: 
        !          1047:        /*
        !          1048:         *      We could not find enough contiguous pages.
        !          1049:         */
        !          1050: not_found_em:
        !          1051:        simple_unlock(&vm_page_queue_free_lock);
        !          1052: 
        !          1053:        ret = KERN_RESOURCE_SHORTAGE;
        !          1054:        goto out;
        !          1055: 
        !          1056:        /*
        !          1057:         *      Final pass. Now we know which pages we want.
        !          1058:         *      Scan the list until we find them all, grab
        !          1059:         *      pages as we go.  FIRST_SET tells us where
        !          1060:         *      in the bit-array our pages start.
        !          1061:         */
        !          1062: found_em:
        !          1063:        vm_page_free_count -= npages;
        !          1064:        if (vm_page_free_count < vm_page_free_count_minimum)
        !          1065:                vm_page_free_count_minimum = vm_page_free_count;
        !          1066: 
        !          1067:        {
        !          1068:            register vm_offset_t        first_phys, last_phys;
        !          1069: 
        !          1070:            /* cache values for compare */
        !          1071:            first_phys = first_set << PAGE_SHIFT;
        !          1072:            last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
        !          1073: 
        !          1074:            /* running pointers */
        !          1075:            mem = vm_page_queue_free;
        !          1076:            prevmem = VM_PAGE_NULL;
        !          1077: 
        !          1078:            while (mem) {
        !          1079: 
        !          1080:                register vm_offset_t    addr;
        !          1081: 
        !          1082:                addr = mem->phys_addr;
        !          1083: 
        !          1084:                if ((addr >= first_phys) &&
        !          1085:                    (addr <  last_phys)) {
        !          1086:                    if (prevmem)
        !          1087:                        prevmem->pageq.next = mem->pageq.next;
        !          1088:                    pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
        !          1089:                    mem->free = FALSE;
        !          1090:                    /*
        !          1091:                     * Got them all ?
        !          1092:                     */
        !          1093:                    if (--npages == 0) break;
        !          1094:                } else
        !          1095:                    prevmem = mem;
        !          1096: 
        !          1097:                mem = (vm_page_t) mem->pageq.next;
        !          1098:            }
        !          1099:        }
        !          1100: 
        !          1101:        simple_unlock(&vm_page_queue_free_lock);
        !          1102: 
        !          1103:        /*
        !          1104:         *      Decide if we should poke the pageout daemon.
        !          1105:         *      We do this if the free count is less than the low
        !          1106:         *      water mark, or if the free count is less than the high
        !          1107:         *      water mark (but above the low water mark) and the inactive
        !          1108:         *      count is less than its target.
        !          1109:         *
        !          1110:         *      We don't have the counts locked ... if they change a little,
        !          1111:         *      it doesn't really matter.
        !          1112:         */
        !          1113: 
        !          1114:        if ((vm_page_free_count < vm_page_free_min) ||
        !          1115:            ((vm_page_free_count < vm_page_free_target) &&
        !          1116:             (vm_page_inactive_count < vm_page_inactive_target)))
        !          1117:                thread_wakeup(&vm_page_free_wanted);
        !          1118: 
        !          1119:        ret = KERN_SUCCESS;
        !          1120: out:
        !          1121:        if (alloc_size)
        !          1122:                kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
        !          1123: 
        !          1124:        return ret;
        !          1125: }
        !          1126: 
        !          1127: /*
        !          1128:  *     vm_page_release:
        !          1129:  *
        !          1130:  *     Return a page to the free list.
        !          1131:  */
        !          1132: 
        !          1133: void vm_page_release(
        !          1134:        register vm_page_t      mem)
        !          1135: {
        !          1136:        simple_lock(&vm_page_queue_free_lock);
        !          1137:        if (mem->free)
        !          1138:                panic("vm_page_release");
        !          1139:        mem->free = TRUE;
        !          1140:        mem->pageq.next = (queue_entry_t) vm_page_queue_free;
        !          1141:        vm_page_queue_free = mem;
        !          1142:        vm_page_free_count++;
        !          1143: 
        !          1144:        /*
        !          1145:         *      Check if we should wake up someone waiting for page.
        !          1146:         *      But don't bother waking them unless they can allocate.
        !          1147:         *
        !          1148:         *      We wakeup only one thread, to prevent starvation.
        !          1149:         *      Because the scheduling system handles wait queues FIFO,
        !          1150:         *      if we wakeup all waiting threads, one greedy thread
        !          1151:         *      can starve multiple niceguy threads.  When the threads
        !          1152:         *      all wakeup, the greedy threads runs first, grabs the page,
        !          1153:         *      and waits for another page.  It will be the first to run
        !          1154:         *      when the next page is freed.
        !          1155:         *
        !          1156:         *      However, there is a slight danger here.
        !          1157:         *      The thread we wake might not use the free page.
        !          1158:         *      Then the other threads could wait indefinitely
        !          1159:         *      while the page goes unused.  To forestall this,
        !          1160:         *      the pageout daemon will keep making free pages
        !          1161:         *      as long as vm_page_free_wanted is non-zero.
        !          1162:         */
        !          1163: 
        !          1164:        if ((vm_page_free_wanted > 0) &&
        !          1165:            (vm_page_free_count >= vm_page_free_reserved)) {
        !          1166:                vm_page_free_wanted--;
        !          1167:                thread_wakeup_one((event_t) &vm_page_free_count);
        !          1168:        }
        !          1169: 
        !          1170:        simple_unlock(&vm_page_queue_free_lock);
        !          1171: }
        !          1172: 
        !          1173: /*
        !          1174:  *     vm_page_wait:
        !          1175:  *
        !          1176:  *     Wait for a page to become available.
        !          1177:  *     If there are plenty of free pages, then we don't sleep.
        !          1178:  */
        !          1179: 
        !          1180: void vm_page_wait(
        !          1181:        void (*continuation)(void))
        !          1182: {
        !          1183: 
        !          1184: #ifndef CONTINUATIONS
        !          1185:        assert (continuation == 0);
        !          1186: #endif
        !          1187: 
        !          1188:        /*
        !          1189:         *      We can't use vm_page_free_reserved to make this
        !          1190:         *      determination.  Consider: some thread might
        !          1191:         *      need to allocate two pages.  The first allocation
        !          1192:         *      succeeds, the second fails.  After the first page is freed,
        !          1193:         *      a call to vm_page_wait must really block.
        !          1194:         */
        !          1195: 
        !          1196:        simple_lock(&vm_page_queue_free_lock);
        !          1197:        if (vm_page_free_count < vm_page_free_target) {
        !          1198:                if (vm_page_free_wanted++ == 0)
        !          1199:                        thread_wakeup((event_t)&vm_page_free_wanted);
        !          1200:                assert_wait((event_t)&vm_page_free_count, FALSE);
        !          1201:                simple_unlock(&vm_page_queue_free_lock);
        !          1202:                if (continuation != 0) {
        !          1203:                        counter(c_vm_page_wait_block_user++);
        !          1204:                        thread_block(continuation);
        !          1205:                } else {
        !          1206:                        counter(c_vm_page_wait_block_kernel++);
        !          1207:                        thread_block((void (*)(void)) 0);
        !          1208:                }
        !          1209:        } else
        !          1210:                simple_unlock(&vm_page_queue_free_lock);
        !          1211: }
        !          1212: 
        !          1213: /*
        !          1214:  *     vm_page_alloc:
        !          1215:  *
        !          1216:  *     Allocate and return a memory cell associated
        !          1217:  *     with this VM object/offset pair.
        !          1218:  *
        !          1219:  *     Object must be locked.
        !          1220:  */
        !          1221: 
        !          1222: vm_page_t vm_page_alloc(
        !          1223:        vm_object_t     object,
        !          1224:        vm_offset_t     offset)
        !          1225: {
        !          1226:        register vm_page_t      mem;
        !          1227: 
        !          1228:        mem = vm_page_grab();
        !          1229:        if (mem == VM_PAGE_NULL)
        !          1230:                return VM_PAGE_NULL;
        !          1231: 
        !          1232:        vm_page_lock_queues();
        !          1233:        vm_page_insert(mem, object, offset);
        !          1234:        vm_page_unlock_queues();
        !          1235: 
        !          1236:        return mem;
        !          1237: }
        !          1238: 
        !          1239: /*
        !          1240:  *     vm_page_free:
        !          1241:  *
        !          1242:  *     Returns the given page to the free list,
        !          1243:  *     disassociating it with any VM object.
        !          1244:  *
        !          1245:  *     Object and page queues must be locked prior to entry.
        !          1246:  */
        !          1247: void vm_page_free(
        !          1248:        register vm_page_t      mem)
        !          1249: {
        !          1250:        if (mem->free)
        !          1251:                panic("vm_page_free");
        !          1252: 
        !          1253:        if (mem->tabled)
        !          1254:                vm_page_remove(mem);
        !          1255:        VM_PAGE_QUEUES_REMOVE(mem);
        !          1256: 
        !          1257:        if (mem->wire_count != 0) {
        !          1258:                if (!mem->private && !mem->fictitious)
        !          1259:                        vm_page_wire_count--;
        !          1260:                mem->wire_count = 0;
        !          1261:        }
        !          1262: 
        !          1263:        if (mem->laundry) {
        !          1264:                vm_page_laundry_count--;
        !          1265:                mem->laundry = FALSE;
        !          1266:        }
        !          1267: 
        !          1268:        PAGE_WAKEUP_DONE(mem);
        !          1269: 
        !          1270:        if (mem->absent)
        !          1271:                vm_object_absent_release(mem->object);
        !          1272: 
        !          1273:        /*
        !          1274:         *      XXX The calls to vm_page_init here are
        !          1275:         *      really overkill.
        !          1276:         */
        !          1277: 
        !          1278:        if (mem->private || mem->fictitious) {
        !          1279:                vm_page_init(mem, vm_page_fictitious_addr);
        !          1280:                mem->fictitious = TRUE;
        !          1281:                vm_page_release_fictitious(mem);
        !          1282:        } else {
        !          1283:                vm_page_init(mem, mem->phys_addr);
        !          1284:                vm_page_release(mem);
        !          1285:        }
        !          1286: }
        !          1287: 
        !          1288: /*
        !          1289:  *     vm_page_wire:
        !          1290:  *
        !          1291:  *     Mark this page as wired down by yet
        !          1292:  *     another map, removing it from paging queues
        !          1293:  *     as necessary.
        !          1294:  *
        !          1295:  *     The page's object and the page queues must be locked.
        !          1296:  */
        !          1297: void vm_page_wire(
        !          1298:        register vm_page_t      mem)
        !          1299: {
        !          1300:        VM_PAGE_CHECK(mem);
        !          1301: 
        !          1302:        if (mem->wire_count == 0) {
        !          1303:                VM_PAGE_QUEUES_REMOVE(mem);
        !          1304:                if (!mem->private && !mem->fictitious)
        !          1305:                        vm_page_wire_count++;
        !          1306:        }
        !          1307:        mem->wire_count++;
        !          1308: }
        !          1309: 
        !          1310: /*
        !          1311:  *     vm_page_unwire:
        !          1312:  *
        !          1313:  *     Release one wiring of this page, potentially
        !          1314:  *     enabling it to be paged again.
        !          1315:  *
        !          1316:  *     The page's object and the page queues must be locked.
        !          1317:  */
        !          1318: void vm_page_unwire(
        !          1319:        register vm_page_t      mem)
        !          1320: {
        !          1321:        VM_PAGE_CHECK(mem);
        !          1322: 
        !          1323:        if (--mem->wire_count == 0) {
        !          1324:                queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
        !          1325:                vm_page_active_count++;
        !          1326:                mem->active = TRUE;
        !          1327:                if (!mem->private && !mem->fictitious)
        !          1328:                        vm_page_wire_count--;
        !          1329:        }
        !          1330: }
        !          1331: 
        !          1332: /*
        !          1333:  *     vm_page_deactivate:
        !          1334:  *
        !          1335:  *     Returns the given page to the inactive list,
        !          1336:  *     indicating that no physical maps have access
        !          1337:  *     to this page.  [Used by the physical mapping system.]
        !          1338:  *
        !          1339:  *     The page queues must be locked.
        !          1340:  */
        !          1341: void vm_page_deactivate(
        !          1342:        register vm_page_t      m)
        !          1343: {
        !          1344:        VM_PAGE_CHECK(m);
        !          1345: 
        !          1346:        /*
        !          1347:         *      This page is no longer very interesting.  If it was
        !          1348:         *      interesting (active or inactive/referenced), then we
        !          1349:         *      clear the reference bit and (re)enter it in the
        !          1350:         *      inactive queue.  Note wired pages should not have
        !          1351:         *      their reference bit cleared.
        !          1352:         */
        !          1353: 
        !          1354:        if (m->active || (m->inactive && m->reference)) {
        !          1355:                if (!m->fictitious && !m->absent)
        !          1356:                        pmap_clear_reference(m->phys_addr);
        !          1357:                m->reference = FALSE;
        !          1358:                VM_PAGE_QUEUES_REMOVE(m);
        !          1359:        }
        !          1360:        if (m->wire_count == 0 && !m->inactive) {
        !          1361:                queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
        !          1362:                m->inactive = TRUE;
        !          1363:                vm_page_inactive_count++;
        !          1364:        }
        !          1365: }
        !          1366: 
        !          1367: /*
        !          1368:  *     vm_page_activate:
        !          1369:  *
        !          1370:  *     Put the specified page on the active list (if appropriate).
        !          1371:  *
        !          1372:  *     The page queues must be locked.
        !          1373:  */
        !          1374: 
        !          1375: void vm_page_activate(
        !          1376:        register vm_page_t      m)
        !          1377: {
        !          1378:        VM_PAGE_CHECK(m);
        !          1379: 
        !          1380:        if (m->inactive) {
        !          1381:                queue_remove(&vm_page_queue_inactive, m, vm_page_t,
        !          1382:                                                pageq);
        !          1383:                vm_page_inactive_count--;
        !          1384:                m->inactive = FALSE;
        !          1385:        }
        !          1386:        if (m->wire_count == 0) {
        !          1387:                if (m->active)
        !          1388:                        panic("vm_page_activate: already active");
        !          1389: 
        !          1390:                queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
        !          1391:                m->active = TRUE;
        !          1392:                vm_page_active_count++;
        !          1393:        }
        !          1394: }
        !          1395: 
        !          1396: /*
        !          1397:  *     vm_page_zero_fill:
        !          1398:  *
        !          1399:  *     Zero-fill the specified page.
        !          1400:  */
        !          1401: void vm_page_zero_fill(
        !          1402:        vm_page_t       m)
        !          1403: {
        !          1404:        VM_PAGE_CHECK(m);
        !          1405: 
        !          1406:        pmap_zero_page(m->phys_addr);
        !          1407: }
        !          1408: 
        !          1409: /*
        !          1410:  *     vm_page_copy:
        !          1411:  *
        !          1412:  *     Copy one page to another
        !          1413:  */
        !          1414: 
        !          1415: void vm_page_copy(
        !          1416:        vm_page_t       src_m,
        !          1417:        vm_page_t       dest_m)
        !          1418: {
        !          1419:        VM_PAGE_CHECK(src_m);
        !          1420:        VM_PAGE_CHECK(dest_m);
        !          1421: 
        !          1422:        pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
        !          1423: }
        !          1424: 
        !          1425: #if    MACH_VM_DEBUG
        !          1426: /*
        !          1427:  *     Routine:        vm_page_info
        !          1428:  *     Purpose:
        !          1429:  *             Return information about the global VP table.
        !          1430:  *             Fills the buffer with as much information as possible
        !          1431:  *             and returns the desired size of the buffer.
        !          1432:  *     Conditions:
        !          1433:  *             Nothing locked.  The caller should provide
        !          1434:  *             possibly-pageable memory.
        !          1435:  */
        !          1436: 
        !          1437: unsigned int
        !          1438: vm_page_info(
        !          1439:        hash_info_bucket_t *info,
        !          1440:        unsigned int    count)
        !          1441: {
        !          1442:        int i;
        !          1443: 
        !          1444:        if (vm_page_bucket_count < count)
        !          1445:                count = vm_page_bucket_count;
        !          1446: 
        !          1447:        for (i = 0; i < count; i++) {
        !          1448:                vm_page_bucket_t *bucket = &vm_page_buckets[i];
        !          1449:                unsigned int bucket_count = 0;
        !          1450:                vm_page_t m;
        !          1451: 
        !          1452:                simple_lock(&bucket->lock);
        !          1453:                for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
        !          1454:                        bucket_count++;
        !          1455:                simple_unlock(&bucket->lock);
        !          1456: 
        !          1457:                /* don't touch pageable memory while holding locks */
        !          1458:                info[i].hib_count = bucket_count;
        !          1459:        }
        !          1460: 
        !          1461:        return vm_page_bucket_count;
        !          1462: }
        !          1463: #endif /* MACH_VM_DEBUG */
        !          1464: 
        !          1465: #include <mach_kdb.h>
        !          1466: #if    MACH_KDB
        !          1467: #define        printf  kdbprintf
        !          1468: 
        !          1469: /*
        !          1470:  *     Routine:        vm_page_print [exported]
        !          1471:  */
        !          1472: void           vm_page_print(p)
        !          1473:        vm_page_t       p;
        !          1474: {
        !          1475:        iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
        !          1476:         printf(" offset 0x%X", (vm_offset_t) p->offset);
        !          1477:         printf("wire_count %d,", p->wire_count);
        !          1478:         printf(" %s",
        !          1479:                (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
        !          1480:         printf("%s",
        !          1481:                (p->free ? " free" : ""));
        !          1482:         printf("%s ",
        !          1483:                (p->laundry ? " laundry" : ""));
        !          1484:         printf("%s",
        !          1485:                (p->dirty ? "dirty" : "clean"));
        !          1486:         printf("%s",
        !          1487:                (p->busy ? " busy" : ""));
        !          1488:         printf("%s",
        !          1489:                (p->absent ? " absent" : ""));
        !          1490:         printf("%s",
        !          1491:                (p->error ? " error" : ""));
        !          1492:         printf("%s",
        !          1493:                (p->fictitious ? " fictitious" : ""));
        !          1494:         printf("%s",
        !          1495:                (p->private ? " private" : ""));
        !          1496:         printf("%s",
        !          1497:                (p->wanted ? " wanted" : ""));
        !          1498:         printf("%s,",
        !          1499:                (p->tabled ? "" : "not_tabled"));
        !          1500:         printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
        !          1501:                (vm_offset_t) p->phys_addr,
        !          1502:                (vm_offset_t) p->page_lock,
        !          1503:                (vm_offset_t) p->unlock_request);
        !          1504: }
        !          1505: #endif /* MACH_KDB */

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.