Return to vm_object.c CVS log

Up to [GnuMach] / OSKit-Mach / vm

Annotation of OSKit-Mach/vm/vm_object.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Mach Operating System
        !             3:  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
        !             4:  * Copyright (c) 1993,1994 The University of Utah and
        !             5:  * the Computer Systems Laboratory (CSL).
        !             6:  * All rights reserved.
        !             7:  *
        !             8:  * Permission to use, copy, modify and distribute this software and its
        !             9:  * documentation is hereby granted, provided that both the copyright
        !            10:  * notice and this permission notice appear in all copies of the
        !            11:  * software, derivative works or modified versions, and any portions
        !            12:  * thereof, and that both notices appear in supporting documentation.
        !            13:  *
        !            14:  * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
        !            15:  * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
        !            16:  * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
        !            17:  * THIS SOFTWARE.
        !            18:  *
        !            19:  * Carnegie Mellon requests users of this software to return to
        !            20:  *
        !            21:  *  Software Distribution Coordinator  or  [email protected]
        !            22:  *  School of Computer Science
        !            23:  *  Carnegie Mellon University
        !            24:  *  Pittsburgh PA 15213-3890
        !            25:  *
        !            26:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            27:  * the rights to redistribute these changes.
        !            28:  */
        !            29: /*
        !            30:  *     File:   vm/vm_object.c
        !            31:  *     Author: Avadis Tevanian, Jr., Michael Wayne Young
        !            32:  *
        !            33:  *     Virtual memory object module.
        !            34:  */
        !            35: 
        !            36: #include <norma_vm.h>
        !            37: #include <mach_pagemap.h>
        !            38: 
        !            39: #if    NORMA_VM
        !            40: #include <norma/xmm_server_rename.h>
        !            41: #endif /* NORMA_VM */
        !            42: 
        !            43: #include <mach/memory_object.h>
        !            44: #include "memory_object_default.h"
        !            45: #include "memory_object_user.h"
        !            46: #include "vm_param.h"
        !            47: #include <ipc/ipc_port.h>
        !            48: #include <ipc/ipc_space.h>
        !            49: #include <kern/assert.h>
        !            50: #include <kern/lock.h>
        !            51: #include <kern/queue.h>
        !            52: #include <kern/zalloc.h>
        !            53: #include <kern/debug.h>
        !            54: #include <vm/memory_object.h>
        !            55: #include <vm/vm_fault.h>
        !            56: #include <vm/vm_map.h>
        !            57: #include <vm/vm_object.h>
        !            58: #include <vm/vm_page.h>
        !            59: #include <vm/vm_pageout.h>
        !            60: 
        !            61: 
        !            62: void memory_object_release(
        !            63:        ipc_port_t      pager,
        !            64:        pager_request_t pager_request,
        !            65:        ipc_port_t      pager_name); /* forward */
        !            66: 
        !            67: void vm_object_deactivate_pages(vm_object_t);
        !            68: 
        !            69: /*
        !            70:  *     Virtual memory objects maintain the actual data
        !            71:  *     associated with allocated virtual memory.  A given
        !            72:  *     page of memory exists within exactly one object.
        !            73:  *
        !            74:  *     An object is only deallocated when all "references"
        !            75:  *     are given up.  Only one "reference" to a given
        !            76:  *     region of an object should be writeable.
        !            77:  *
        !            78:  *     Associated with each object is a list of all resident
        !            79:  *     memory pages belonging to that object; this list is
        !            80:  *     maintained by the "vm_page" module, but locked by the object's
        !            81:  *     lock.
        !            82:  *
        !            83:  *     Each object also records the memory object port
        !            84:  *     that is used by the kernel to request and write
        !            85:  *     back data (the memory object port, field "pager"),
        !            86:  *     and the ports provided to the memory manager, the server that
        !            87:  *     manages that data, to return data and control its
        !            88:  *     use (the memory object control port, field "pager_request")
        !            89:  *     and for naming (the memory object name port, field "pager_name").
        !            90:  *
        !            91:  *     Virtual memory objects are allocated to provide
        !            92:  *     zero-filled memory (vm_allocate) or map a user-defined
        !            93:  *     memory object into a virtual address space (vm_map).
        !            94:  *
        !            95:  *     Virtual memory objects that refer to a user-defined
        !            96:  *     memory object are called "permanent", because all changes
        !            97:  *     made in virtual memory are reflected back to the
        !            98:  *     memory manager, which may then store it permanently.
        !            99:  *     Other virtual memory objects are called "temporary",
        !           100:  *     meaning that changes need be written back only when
        !           101:  *     necessary to reclaim pages, and that storage associated
        !           102:  *     with the object can be discarded once it is no longer
        !           103:  *     mapped.
        !           104:  *
        !           105:  *     A permanent memory object may be mapped into more
        !           106:  *     than one virtual address space.  Moreover, two threads
        !           107:  *     may attempt to make the first mapping of a memory
        !           108:  *     object concurrently.  Only one thread is allowed to
        !           109:  *     complete this mapping; all others wait for the
        !           110:  *     "pager_initialized" field is asserted, indicating
        !           111:  *     that the first thread has initialized all of the
        !           112:  *     necessary fields in the virtual memory object structure.
        !           113:  *
        !           114:  *     The kernel relies on a *default memory manager* to
        !           115:  *     provide backing storage for the zero-filled virtual
        !           116:  *     memory objects.  The memory object ports associated
        !           117:  *     with these temporary virtual memory objects are only
        !           118:  *     generated and passed to the default memory manager
        !           119:  *     when it becomes necessary.  Virtual memory objects
        !           120:  *     that depend on the default memory manager are called
        !           121:  *     "internal".  The "pager_created" field is provided to
        !           122:  *     indicate whether these ports have ever been allocated.
        !           123:  *
        !           124:  *     The kernel may also create virtual memory objects to
        !           125:  *     hold changed pages after a copy-on-write operation.
        !           126:  *     In this case, the virtual memory object (and its
        !           127:  *     backing storage -- its memory object) only contain
        !           128:  *     those pages that have been changed.  The "shadow"
        !           129:  *     field refers to the virtual memory object that contains
        !           130:  *     the remainder of the contents.  The "shadow_offset"
        !           131:  *     field indicates where in the "shadow" these contents begin.
        !           132:  *     The "copy" field refers to a virtual memory object
        !           133:  *     to which changed pages must be copied before changing
        !           134:  *     this object, in order to implement another form
        !           135:  *     of copy-on-write optimization.
        !           136:  *
        !           137:  *     The virtual memory object structure also records
        !           138:  *     the attributes associated with its memory object.
        !           139:  *     The "pager_ready", "can_persist" and "copy_strategy"
        !           140:  *     fields represent those attributes.  The "cached_list"
        !           141:  *     field is used in the implementation of the persistence
        !           142:  *     attribute.
        !           143:  *
        !           144:  * ZZZ Continue this comment.
        !           145:  */
        !           146: 
        !           147: zone_t         vm_object_zone;         /* vm backing store zone */
        !           148: 
        !           149: /*
        !           150:  *     All wired-down kernel memory belongs to a single virtual
        !           151:  *     memory object (kernel_object) to avoid wasting data structures.
        !           152:  */
        !           153: vm_object_t    kernel_object;
        !           154: 
        !           155: /*
        !           156:  *     Virtual memory objects that are not referenced by
        !           157:  *     any address maps, but that are allowed to persist
        !           158:  *     (an attribute specified by the associated memory manager),
        !           159:  *     are kept in a queue (vm_object_cached_list).
        !           160:  *
        !           161:  *     When an object from this queue is referenced again,
        !           162:  *     for example to make another address space mapping,
        !           163:  *     it must be removed from the queue.  That is, the
        !           164:  *     queue contains *only* objects with zero references.
        !           165:  *
        !           166:  *     The kernel may choose to terminate objects from this
        !           167:  *     queue in order to reclaim storage.  The current policy
        !           168:  *     is to permit a fixed maximum number of unreferenced
        !           169:  *     objects (vm_object_cached_max).
        !           170:  *
        !           171:  *     A simple lock (accessed by routines
        !           172:  *     vm_object_cache_{lock,lock_try,unlock}) governs the
        !           173:  *     object cache.  It must be held when objects are
        !           174:  *     added to or removed from the cache (in vm_object_terminate).
        !           175:  *     The routines that acquire a reference to a virtual
        !           176:  *     memory object based on one of the memory object ports
        !           177:  *     must also lock the cache.
        !           178:  *
        !           179:  *     Ideally, the object cache should be more isolated
        !           180:  *     from the reference mechanism, so that the lock need
        !           181:  *     not be held to make simple references.
        !           182:  */
        !           183: queue_head_t   vm_object_cached_list;
        !           184: int            vm_object_cached_count;
        !           185: int            vm_object_cached_max = 200;     /* may be patched*/
        !           186: 
        !           187: decl_simple_lock_data(,vm_object_cached_lock_data)
        !           188: 
        !           189: #define vm_object_cache_lock()         \
        !           190:                simple_lock(&vm_object_cached_lock_data)
        !           191: #define vm_object_cache_lock_try()     \
        !           192:                simple_lock_try(&vm_object_cached_lock_data)
        !           193: #define vm_object_cache_unlock()       \
        !           194:                simple_unlock(&vm_object_cached_lock_data)
        !           195: 
        !           196: /*
        !           197:  *     Virtual memory objects are initialized from
        !           198:  *     a template (see vm_object_allocate).
        !           199:  *
        !           200:  *     When adding a new field to the virtual memory
        !           201:  *     object structure, be sure to add initialization
        !           202:  *     (see vm_object_init).
        !           203:  */
        !           204: vm_object_t    vm_object_template;
        !           205: 
        !           206: /*
        !           207:  *     vm_object_allocate:
        !           208:  *
        !           209:  *     Returns a new object with the given size.
        !           210:  */
        !           211: 
        !           212: vm_object_t _vm_object_allocate(
        !           213:        vm_size_t               size)
        !           214: {
        !           215:        register vm_object_t object;
        !           216: 
        !           217:        object = (vm_object_t) zalloc(vm_object_zone);
        !           218: 
        !           219:        *object = *vm_object_template;
        !           220:        queue_init(&object->memq);
        !           221:        vm_object_lock_init(object);
        !           222:        object->size = size;
        !           223: 
        !           224:        return object;
        !           225: }
        !           226: 
        !           227: vm_object_t vm_object_allocate(
        !           228:        vm_size_t       size)
        !           229: {
        !           230:        register vm_object_t object;
        !           231:        register ipc_port_t port;
        !           232: 
        !           233:        object = _vm_object_allocate(size);
        !           234: #if    !NORMA_VM
        !           235:        port = ipc_port_alloc_kernel();
        !           236:        if (port == IP_NULL)
        !           237:                panic("vm_object_allocate");
        !           238:        object->pager_name = port;
        !           239:        ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
        !           240: #endif /* !NORMA_VM */
        !           241: 
        !           242:        return object;
        !           243: }
        !           244: 
        !           245: /*
        !           246:  *     vm_object_bootstrap:
        !           247:  *
        !           248:  *     Initialize the VM objects module.
        !           249:  */
        !           250: void vm_object_bootstrap(void)
        !           251: {
        !           252:        vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
        !           253:                                round_page(512*1024),
        !           254:                                round_page(12*1024),
        !           255:                                0, "objects");
        !           256: 
        !           257:        queue_init(&vm_object_cached_list);
        !           258:        simple_lock_init(&vm_object_cached_lock_data);
        !           259: 
        !           260:        /*
        !           261:         *      Fill in a template object, for quick initialization
        !           262:         */
        !           263: 
        !           264:        vm_object_template = (vm_object_t) zalloc(vm_object_zone);
        !           265:        bzero((char *) vm_object_template, sizeof *vm_object_template);
        !           266: 
        !           267:        vm_object_template->ref_count = 1;
        !           268:        vm_object_template->size = 0;
        !           269:        vm_object_template->resident_page_count = 0;
        !           270:        vm_object_template->copy = VM_OBJECT_NULL;
        !           271:        vm_object_template->shadow = VM_OBJECT_NULL;
        !           272:        vm_object_template->shadow_offset = (vm_offset_t) 0;
        !           273: 
        !           274:        vm_object_template->pager = IP_NULL;
        !           275:        vm_object_template->paging_offset = 0;
        !           276:        vm_object_template->pager_request = PAGER_REQUEST_NULL;
        !           277:        vm_object_template->pager_name = IP_NULL;
        !           278: 
        !           279:        vm_object_template->pager_created = FALSE;
        !           280:        vm_object_template->pager_initialized = FALSE;
        !           281:        vm_object_template->pager_ready = FALSE;
        !           282: 
        !           283:        vm_object_template->copy_strategy = MEMORY_OBJECT_COPY_NONE;
        !           284:                /* ignored if temporary, will be reset before
        !           285:                 * permanent object becomes ready */
        !           286:        vm_object_template->use_shared_copy = FALSE;
        !           287:        vm_object_template->shadowed = FALSE;
        !           288: 
        !           289:        vm_object_template->absent_count = 0;
        !           290:        vm_object_template->all_wanted = 0; /* all bits FALSE */
        !           291: 
        !           292:        vm_object_template->paging_in_progress = 0;
        !           293:        vm_object_template->can_persist = FALSE;
        !           294:        vm_object_template->internal = TRUE;
        !           295:        vm_object_template->temporary = TRUE;
        !           296:        vm_object_template->alive = TRUE;
        !           297:        vm_object_template->lock_in_progress = FALSE;
        !           298:        vm_object_template->lock_restart = FALSE;
        !           299:        vm_object_template->use_old_pageout = TRUE; /* XXX change later */
        !           300:        vm_object_template->last_alloc = (vm_offset_t) 0;
        !           301: 
        !           302: #if    MACH_PAGEMAP
        !           303:        vm_object_template->existence_info = VM_EXTERNAL_NULL;
        !           304: #endif /* MACH_PAGEMAP */
        !           305: 
        !           306:                /*
        !           307:         *      Initialize the "kernel object"
        !           308:         */
        !           309: 
        !           310:        kernel_object = _vm_object_allocate(
        !           311:                VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
        !           312: 
        !           313:        /*
        !           314:         *      Initialize the "submap object".  Make it as large as the
        !           315:         *      kernel object so that no limit is imposed on submap sizes.
        !           316:         */
        !           317: 
        !           318:        vm_submap_object = _vm_object_allocate(
        !           319:                VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
        !           320: 
        !           321: #if    MACH_PAGEMAP
        !           322:        vm_external_module_initialize();
        !           323: #endif /* MACH_PAGEMAP */
        !           324: }
        !           325: 
        !           326: void vm_object_init(void)
        !           327: {
        !           328: #if    !NORMA_VM
        !           329:        /*
        !           330:         *      Finish initializing the kernel object.
        !           331:         *      The submap object doesn't need a name port.
        !           332:         */
        !           333: 
        !           334:        kernel_object->pager_name = ipc_port_alloc_kernel();
        !           335:        ipc_kobject_set(kernel_object->pager_name,
        !           336:                        (ipc_kobject_t) kernel_object,
        !           337:                        IKOT_PAGING_NAME);
        !           338: #endif /* !NORMA_VM */
        !           339: }
        !           340: 
        !           341: /*
        !           342:  *     vm_object_reference:
        !           343:  *
        !           344:  *     Gets another reference to the given object.
        !           345:  */
        !           346: void vm_object_reference(
        !           347:        register vm_object_t    object)
        !           348: {
        !           349:        if (object == VM_OBJECT_NULL)
        !           350:                return;
        !           351: 
        !           352:        vm_object_lock(object);
        !           353:        assert(object->ref_count > 0);
        !           354:        object->ref_count++;
        !           355:        vm_object_unlock(object);
        !           356: }
        !           357: 
        !           358: /*
        !           359:  *     vm_object_deallocate:
        !           360:  *
        !           361:  *     Release a reference to the specified object,
        !           362:  *     gained either through a vm_object_allocate
        !           363:  *     or a vm_object_reference call.  When all references
        !           364:  *     are gone, storage associated with this object
        !           365:  *     may be relinquished.
        !           366:  *
        !           367:  *     No object may be locked.
        !           368:  */
        !           369: void vm_object_deallocate(
        !           370:        register vm_object_t    object)
        !           371: {
        !           372:        vm_object_t     temp;
        !           373: 
        !           374:        while (object != VM_OBJECT_NULL) {
        !           375: 
        !           376:                /*
        !           377:                 *      The cache holds a reference (uncounted) to
        !           378:                 *      the object; we must lock it before removing
        !           379:                 *      the object.
        !           380:                 */
        !           381: 
        !           382:                vm_object_cache_lock();
        !           383: 
        !           384:                /*
        !           385:                 *      Lose the reference
        !           386:                 */
        !           387:                vm_object_lock(object);
        !           388:                if (--(object->ref_count) > 0) {
        !           389: 
        !           390:                        /*
        !           391:                         *      If there are still references, then
        !           392:                         *      we are done.
        !           393:                         */
        !           394:                        vm_object_unlock(object);
        !           395:                        vm_object_cache_unlock();
        !           396:                        return;
        !           397:                }
        !           398: 
        !           399:                /*
        !           400:                 *      See whether this object can persist.  If so, enter
        !           401:                 *      it in the cache, then deactivate all of its
        !           402:                 *      pages.
        !           403:                 */
        !           404:                if (object->can_persist) {
        !           405:                        boolean_t       overflow;
        !           406: 
        !           407:                        /*
        !           408:                         *      Enter the object onto the queue
        !           409:                         *      of "cached" objects.  Remember whether
        !           410:                         *      we've caused the queue to overflow,
        !           411:                         *      as a hint.
        !           412:                         */
        !           413: 
        !           414:                        queue_enter(&vm_object_cached_list, object,
        !           415:                                vm_object_t, cached_list);
        !           416:                        overflow = (++vm_object_cached_count > vm_object_cached_max);
        !           417:                        vm_object_cache_unlock();
        !           418: 
        !           419:                        vm_object_deactivate_pages(object);
        !           420:                        vm_object_unlock(object);
        !           421: 
        !           422:                        /*
        !           423:                         *      If we didn't overflow, or if the queue has
        !           424:                         *      been reduced back to below the specified
        !           425:                         *      minimum, then quit.
        !           426:                         */
        !           427:                        if (!overflow)
        !           428:                                return;
        !           429: 
        !           430:                        while (TRUE) {
        !           431:                                vm_object_cache_lock();
        !           432:                                if (vm_object_cached_count <=
        !           433:                                    vm_object_cached_max) {
        !           434:                                        vm_object_cache_unlock();
        !           435:                                        return;
        !           436:                                }
        !           437: 
        !           438:                                /*
        !           439:                                 *      If we must trim down the queue, take
        !           440:                                 *      the first object, and proceed to
        !           441:                                 *      terminate it instead of the original
        !           442:                                 *      object.  Have to wait for pager init.
        !           443:                                 *  if it's in progress.
        !           444:                                 */
        !           445:                                object= (vm_object_t)
        !           446:                                    queue_first(&vm_object_cached_list);
        !           447:                                vm_object_lock(object);
        !           448: 
        !           449:                                if (!(object->pager_created &&
        !           450:                                    !object->pager_initialized)) {
        !           451: 
        !           452:                                        /*
        !           453:                                         *  Ok to terminate, hang on to lock.
        !           454:                                         */
        !           455:                                        break;
        !           456:                                }
        !           457: 
        !           458:                                vm_object_assert_wait(object,
        !           459:                                        VM_OBJECT_EVENT_INITIALIZED, FALSE);
        !           460:                                vm_object_unlock(object);
        !           461:                                vm_object_cache_unlock();
        !           462:                                thread_block((void (*)()) 0);
        !           463: 
        !           464:                                /*
        !           465:                                 *  Continue loop to check if cache still
        !           466:                                 *  needs to be trimmed.
        !           467:                                 */
        !           468:                        }
        !           469: 
        !           470:                        /*
        !           471:                         *      Actually remove object from cache.
        !           472:                         */
        !           473: 
        !           474:                        queue_remove(&vm_object_cached_list, object,
        !           475:                                        vm_object_t, cached_list);
        !           476:                        vm_object_cached_count--;
        !           477: 
        !           478:                        assert(object->ref_count == 0);
        !           479:                }
        !           480:                else {
        !           481:                        if (object->pager_created &&
        !           482:                            !object->pager_initialized) {
        !           483: 
        !           484:                                /*
        !           485:                                 *      Have to wait for initialization.
        !           486:                                 *      Put reference back and retry
        !           487:                                 *      when it's initialized.
        !           488:                                 */
        !           489:                                object->ref_count++;
        !           490:                                vm_object_assert_wait(object,
        !           491:                                        VM_OBJECT_EVENT_INITIALIZED, FALSE);
        !           492:                                vm_object_unlock(object);
        !           493:                                vm_object_cache_unlock();
        !           494:                                thread_block((void (*)()) 0);
        !           495:                                continue;
        !           496:                          }
        !           497:                }
        !           498: 
        !           499:                /*
        !           500:                 *      Take the reference to the shadow object
        !           501:                 *      out of the object to be destroyed.
        !           502:                 */
        !           503: 
        !           504:                temp = object->shadow;
        !           505: 
        !           506:                /*
        !           507:                 *      Destroy the object; the cache lock will
        !           508:                 *      be released in the process.
        !           509:                 */
        !           510: 
        !           511:                vm_object_terminate(object);
        !           512: 
        !           513:                /*
        !           514:                 *      Deallocate the reference to the shadow
        !           515:                 *      by continuing the loop with that object
        !           516:                 *      in place of the original.
        !           517:                 */
        !           518: 
        !           519:                object = temp;
        !           520:        }
        !           521: }
        !           522: 
        !           523: boolean_t      vm_object_terminate_remove_all = FALSE;
        !           524: 
        !           525: /*
        !           526:  *     Routine:        vm_object_terminate
        !           527:  *     Purpose:
        !           528:  *             Free all resources associated with a vm_object.
        !           529:  *     In/out conditions:
        !           530:  *             Upon entry, the object and the cache must be locked,
        !           531:  *             and the object must have no references.
        !           532:  *
        !           533:  *             The shadow object reference is left alone.
        !           534:  *
        !           535:  *             Upon exit, the cache will be unlocked, and the
        !           536:  *             object will cease to exist.
        !           537:  */
        !           538: void vm_object_terminate(
        !           539:        register vm_object_t    object)
        !           540: {
        !           541:        register vm_page_t      p;
        !           542:        vm_object_t             shadow_object;
        !           543: 
        !           544:        /*
        !           545:         *      Make sure the object isn't already being terminated
        !           546:         */
        !           547: 
        !           548:        assert(object->alive);
        !           549:        object->alive = FALSE;
        !           550: 
        !           551:        /*
        !           552:         *      Make sure no one can look us up now.
        !           553:         */
        !           554: 
        !           555:        vm_object_remove(object);
        !           556:        vm_object_cache_unlock();
        !           557: 
        !           558:        /*
        !           559:         *      Detach the object from its shadow if we are the shadow's
        !           560:         *      copy.
        !           561:         */
        !           562:        if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
        !           563:                vm_object_lock(shadow_object);
        !           564:                assert((shadow_object->copy == object) ||
        !           565:                       (shadow_object->copy == VM_OBJECT_NULL));
        !           566:                shadow_object->copy = VM_OBJECT_NULL;
        !           567:                vm_object_unlock(shadow_object);
        !           568:        }
        !           569: 
        !           570:        /*
        !           571:         *      The pageout daemon might be playing with our pages.
        !           572:         *      Now that the object is dead, it won't touch any more
        !           573:         *      pages, but some pages might already be on their way out.
        !           574:         *      Hence, we wait until the active paging activities have ceased.
        !           575:         */
        !           576: 
        !           577:        vm_object_paging_wait(object, FALSE);
        !           578: 
        !           579:        /*
        !           580:         *      Clean or free the pages, as appropriate.
        !           581:         *      It is possible for us to find busy/absent pages,
        !           582:         *      if some faults on this object were aborted.
        !           583:         */
        !           584: 
        !           585:        if ((object->temporary) || (object->pager == IP_NULL)) {
        !           586:                while (!queue_empty(&object->memq)) {
        !           587:                        p = (vm_page_t) queue_first(&object->memq);
        !           588: 
        !           589:                        VM_PAGE_CHECK(p);
        !           590: 
        !           591:                        if (p->busy && !p->absent)
        !           592:                                panic("vm_object_terminate.2 0x%x 0x%x",
        !           593:                                      object, p);
        !           594: 
        !           595:                        VM_PAGE_FREE(p);
        !           596:                }
        !           597:        } else while (!queue_empty(&object->memq)) {
        !           598:                p = (vm_page_t) queue_first(&object->memq);
        !           599: 
        !           600:                VM_PAGE_CHECK(p);
        !           601: 
        !           602:                if (p->busy && !p->absent)
        !           603:                        panic("vm_object_terminate.3 0x%x 0x%x", object, p);
        !           604: 
        !           605:                vm_page_lock_queues();
        !           606:                VM_PAGE_QUEUES_REMOVE(p);
        !           607:                vm_page_unlock_queues();
        !           608: 
        !           609:                if (p->absent || p->private) {
        !           610: 
        !           611:                        /*
        !           612:                         *      For private pages, VM_PAGE_FREE just
        !           613:                         *      leaves the page structure around for
        !           614:                         *      its owner to clean up.  For absent
        !           615:                         *      pages, the structure is returned to
        !           616:                         *      the appropriate pool.
        !           617:                         */
        !           618: 
        !           619:                        goto free_page;
        !           620:                }
        !           621: 
        !           622:                if (p->fictitious)
        !           623:                        panic("vm_object_terminate.4 0x%x 0x%x", object, p);
        !           624: 
        !           625:                if (!p->dirty)
        !           626:                        p->dirty = pmap_is_modified(p->phys_addr);
        !           627: 
        !           628:                if (p->dirty || p->precious) {
        !           629:                        p->busy = TRUE;
        !           630:                        vm_pageout_page(p, FALSE, TRUE); /* flush page */
        !           631:                } else {
        !           632:                    free_page:
        !           633:                        VM_PAGE_FREE(p);
        !           634:                }
        !           635:        }
        !           636: 
        !           637:        assert(object->ref_count == 0);
        !           638:        assert(object->paging_in_progress == 0);
        !           639: 
        !           640:        /*
        !           641:         *      Throw away port rights... note that they may
        !           642:         *      already have been thrown away (by vm_object_destroy
        !           643:         *      or memory_object_destroy).
        !           644:         *
        !           645:         *      Instead of destroying the control and name ports,
        !           646:         *      we send all rights off to the memory manager instead,
        !           647:         *      using memory_object_terminate.
        !           648:         */
        !           649: 
        !           650:        vm_object_unlock(object);
        !           651: 
        !           652:        if (object->pager != IP_NULL) {
        !           653:                /* consumes our rights for pager, pager_request, pager_name */
        !           654:                memory_object_release(object->pager,
        !           655:                                             object->pager_request,
        !           656:                                             object->pager_name);
        !           657:        } else if (object->pager_name != IP_NULL) {
        !           658:                /* consumes our right for pager_name */
        !           659: #if    NORMA_VM
        !           660:                ipc_port_release_send(object->pager_name);
        !           661: #else  /* NORMA_VM */
        !           662:                ipc_port_dealloc_kernel(object->pager_name);
        !           663: #endif /* NORMA_VM */
        !           664:        }
        !           665: 
        !           666: #if    MACH_PAGEMAP
        !           667:        vm_external_destroy(object->existence_info);
        !           668: #endif /* MACH_PAGEMAP */
        !           669: 
        !           670:        /*
        !           671:         *      Free the space for the object.
        !           672:         */
        !           673: 
        !           674:        zfree(vm_object_zone, (vm_offset_t) object);
        !           675: }
        !           676: 
        !           677: /*
        !           678:  *     Routine:        vm_object_pager_wakeup
        !           679:  *     Purpose:        Wake up anyone waiting for IKOT_PAGER_TERMINATING
        !           680:  */
        !           681: 
        !           682: void
        !           683: vm_object_pager_wakeup(
        !           684:        ipc_port_t      pager)
        !           685: {
        !           686:        boolean_t someone_waiting;
        !           687: 
        !           688:        /*
        !           689:         *      If anyone was waiting for the memory_object_terminate
        !           690:         *      to be queued, wake them up now.
        !           691:         */
        !           692:        vm_object_cache_lock();
        !           693:        assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
        !           694:        someone_waiting = (pager->ip_kobject != IKO_NULL);
        !           695:        if (ip_active(pager))
        !           696:                ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
        !           697:        vm_object_cache_unlock();
        !           698:        if (someone_waiting) {
        !           699:                thread_wakeup((event_t) pager);
        !           700:        }
        !           701: }
        !           702: 
        !           703: /*
        !           704:  *     Routine:        memory_object_release
        !           705:  *     Purpose:        Terminate the pager and release port rights,
        !           706:  *                     just like memory_object_terminate, except
        !           707:  *                     that we wake up anyone blocked in vm_object_enter
        !           708:  *                     waiting for termination message to be queued
        !           709:  *                     before calling memory_object_init.
        !           710:  */
        !           711: void memory_object_release(
        !           712:        ipc_port_t      pager,
        !           713:        pager_request_t pager_request,
        !           714:        ipc_port_t      pager_name)
        !           715: {
        !           716: 
        !           717:        /*
        !           718:         *      Keep a reference to pager port;
        !           719:         *      the terminate might otherwise release all references.
        !           720:         */
        !           721:        ip_reference(pager);
        !           722: 
        !           723:        /*
        !           724:         *      Terminate the pager.
        !           725:         */
        !           726:        (void) memory_object_terminate(pager, pager_request, pager_name);
        !           727: 
        !           728:        /*
        !           729:         *      Wakeup anyone waiting for this terminate
        !           730:         */
        !           731:        vm_object_pager_wakeup(pager);
        !           732: 
        !           733:        /*
        !           734:         *      Release reference to pager port.
        !           735:         */
        !           736:        ip_release(pager);
        !           737: }
        !           738: 
        !           739: /*
        !           740:  *     Routine:        vm_object_abort_activity [internal use only]
        !           741:  *     Purpose:
        !           742:  *             Abort paging requests pending on this object.
        !           743:  *     In/out conditions:
        !           744:  *             The object is locked on entry and exit.
        !           745:  */
        !           746: void vm_object_abort_activity(
        !           747:        vm_object_t     object)
        !           748: {
        !           749:        register
        !           750:        vm_page_t       p;
        !           751:        vm_page_t       next;
        !           752: 
        !           753:        /*
        !           754:         *      Abort all activity that would be waiting
        !           755:         *      for a result on this memory object.
        !           756:         *
        !           757:         *      We could also choose to destroy all pages
        !           758:         *      that we have in memory for this object, but
        !           759:         *      we don't.
        !           760:         */
        !           761: 
        !           762:        p = (vm_page_t) queue_first(&object->memq);
        !           763:        while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !           764:                next = (vm_page_t) queue_next(&p->listq);
        !           765: 
        !           766:                /*
        !           767:                 *      If it's being paged in, destroy it.
        !           768:                 *      If an unlock has been requested, start it again.
        !           769:                 */
        !           770: 
        !           771:                if (p->busy && p->absent) {
        !           772:                        VM_PAGE_FREE(p);
        !           773:                }
        !           774:                 else {
        !           775:                        if (p->unlock_request != VM_PROT_NONE)
        !           776:                                p->unlock_request = VM_PROT_NONE;
        !           777:                        PAGE_WAKEUP(p);
        !           778:                }
        !           779: 
        !           780:                p = next;
        !           781:        }
        !           782: 
        !           783:        /*
        !           784:         *      Wake up threads waiting for the memory object to
        !           785:         *      become ready.
        !           786:         */
        !           787: 
        !           788:        object->pager_ready = TRUE;
        !           789:        vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
        !           790: }
        !           791: 
        !           792: /*
        !           793:  *     Routine:        memory_object_destroy [user interface]
        !           794:  *     Purpose:
        !           795:  *             Shut down a memory object, despite the
        !           796:  *             presence of address map (or other) references
        !           797:  *             to the vm_object.
        !           798:  *     Note:
        !           799:  *             This routine may be called either from the user interface,
        !           800:  *             or from port destruction handling (via vm_object_destroy).
        !           801:  */
        !           802: kern_return_t memory_object_destroy(
        !           803:        register
        !           804:        vm_object_t     object,
        !           805:        kern_return_t   reason)
        !           806: {
        !           807:        ipc_port_t      old_object,  old_name;
        !           808:        pager_request_t old_control;
        !           809: 
        !           810: #ifdef lint
        !           811:        reason++;
        !           812: #endif /* lint */
        !           813: 
        !           814:        if (object == VM_OBJECT_NULL)
        !           815:                return KERN_SUCCESS;
        !           816: 
        !           817:        /*
        !           818:         *      Remove the port associations immediately.
        !           819:         *
        !           820:         *      This will prevent the memory manager from further
        !           821:         *      meddling.  [If it wanted to flush data or make
        !           822:         *      other changes, it should have done so before performing
        !           823:         *      the destroy call.]
        !           824:         */
        !           825: 
        !           826:        vm_object_cache_lock();
        !           827:        vm_object_lock(object);
        !           828:        vm_object_remove(object);
        !           829:        object->can_persist = FALSE;
        !           830:        vm_object_cache_unlock();
        !           831: 
        !           832:        /*
        !           833:         *      Rip out the ports from the vm_object now... this
        !           834:         *      will prevent new memory_object calls from succeeding.
        !           835:         */
        !           836: 
        !           837:        old_object = object->pager;
        !           838:        object->pager = IP_NULL;
        !           839: 
        !           840:        old_control = object->pager_request;
        !           841:        object->pager_request = PAGER_REQUEST_NULL;
        !           842: 
        !           843:        old_name = object->pager_name;
        !           844:        object->pager_name = IP_NULL;
        !           845: 
        !           846: 
        !           847:        /*
        !           848:         *      Wait for existing paging activity (that might
        !           849:         *      have the old ports) to subside.
        !           850:         */
        !           851: 
        !           852:        vm_object_paging_wait(object, FALSE);
        !           853:        vm_object_unlock(object);
        !           854: 
        !           855:        /*
        !           856:         *      Shut down the ports now.
        !           857:         *
        !           858:         *      [Paging operations may be proceeding concurrently --
        !           859:         *      they'll get the null values established above.]
        !           860:         */
        !           861: 
        !           862:        if (old_object != IP_NULL) {
        !           863:                /* consumes our rights for object, control, name */
        !           864:                memory_object_release(old_object, old_control,
        !           865:                                             old_name);
        !           866:        } else if (old_name != IP_NULL) {
        !           867:                /* consumes our right for name */
        !           868: #if    NORMA_VM
        !           869:                ipc_port_release_send(object->pager_name);
        !           870: #else  /* NORMA_VM */
        !           871:                ipc_port_dealloc_kernel(object->pager_name);
        !           872: #endif /* NORMA_VM */
        !           873:        }
        !           874: 
        !           875:        /*
        !           876:         *      Lose the reference that was donated for this routine
        !           877:         */
        !           878: 
        !           879:        vm_object_deallocate(object);
        !           880: 
        !           881:        return KERN_SUCCESS;
        !           882: }
        !           883: 
        !           884: /*
        !           885:  *     vm_object_deactivate_pages
        !           886:  *
        !           887:  *     Deactivate all pages in the specified object.  (Keep its pages
        !           888:  *     in memory even though it is no longer referenced.)
        !           889:  *
        !           890:  *     The object must be locked.
        !           891:  */
        !           892: void vm_object_deactivate_pages(
        !           893:        register vm_object_t    object)
        !           894: {
        !           895:        register vm_page_t      p;
        !           896: 
        !           897:        queue_iterate(&object->memq, p, vm_page_t, listq) {
        !           898:                vm_page_lock_queues();
        !           899:                if (!p->busy)
        !           900:                        vm_page_deactivate(p);
        !           901:                vm_page_unlock_queues();
        !           902:        }
        !           903: }
        !           904: 
        !           905: 
        !           906: /*
        !           907:  *     Routine:        vm_object_pmap_protect
        !           908:  *
        !           909:  *     Purpose:
        !           910:  *             Reduces the permission for all physical
        !           911:  *             pages in the specified object range.
        !           912:  *
        !           913:  *             If removing write permission only, it is
        !           914:  *             sufficient to protect only the pages in
        !           915:  *             the top-level object; only those pages may
        !           916:  *             have write permission.
        !           917:  *
        !           918:  *             If removing all access, we must follow the
        !           919:  *             shadow chain from the top-level object to
        !           920:  *             remove access to all pages in shadowed objects.
        !           921:  *
        !           922:  *             The object must *not* be locked.  The object must
        !           923:  *             be temporary/internal.
        !           924:  *
        !           925:  *              If pmap is not NULL, this routine assumes that
        !           926:  *              the only mappings for the pages are in that
        !           927:  *              pmap.
        !           928:  */
        !           929: boolean_t vm_object_pmap_protect_by_page = FALSE;
        !           930: 
        !           931: void vm_object_pmap_protect(
        !           932:        register vm_object_t    object,
        !           933:        register vm_offset_t    offset,
        !           934:        vm_offset_t             size,
        !           935:        pmap_t                  pmap,
        !           936:        vm_offset_t             pmap_start,
        !           937:        vm_prot_t               prot)
        !           938: {
        !           939:        if (object == VM_OBJECT_NULL)
        !           940:            return;
        !           941: 
        !           942:        vm_object_lock(object);
        !           943: 
        !           944:        assert(object->temporary && object->internal);
        !           945: 
        !           946:        while (TRUE) {
        !           947:            if (object->resident_page_count > atop(size) / 2 &&
        !           948:                    pmap != PMAP_NULL) {
        !           949:                vm_object_unlock(object);
        !           950:                pmap_protect(pmap, pmap_start, pmap_start + size, prot);
        !           951:                return;
        !           952:            }
        !           953: 
        !           954:            {
        !           955:                register vm_page_t      p;
        !           956:                register vm_offset_t    end;
        !           957: 
        !           958:                end = offset + size;
        !           959: 
        !           960:                queue_iterate(&object->memq, p, vm_page_t, listq) {
        !           961:                    if (!p->fictitious &&
        !           962:                        (offset <= p->offset) &&
        !           963:                        (p->offset < end)) {
        !           964:                        if ((pmap == PMAP_NULL) ||
        !           965:                            vm_object_pmap_protect_by_page) {
        !           966:                            pmap_page_protect(p->phys_addr,
        !           967:                                              prot & ~p->page_lock);
        !           968:                        } else {
        !           969:                            vm_offset_t start =
        !           970:                                        pmap_start +
        !           971:                                        (p->offset - offset);
        !           972: 
        !           973:                            pmap_protect(pmap,
        !           974:                                         start,
        !           975:                                         start + PAGE_SIZE,
        !           976:                                         prot);
        !           977:                        }
        !           978:                    }
        !           979:                }
        !           980:            }
        !           981: 
        !           982:            if (prot == VM_PROT_NONE) {
        !           983:                /*
        !           984:                 * Must follow shadow chain to remove access
        !           985:                 * to pages in shadowed objects.
        !           986:                 */
        !           987:                register vm_object_t    next_object;
        !           988: 
        !           989:                next_object = object->shadow;
        !           990:                if (next_object != VM_OBJECT_NULL) {
        !           991:                    offset += object->shadow_offset;
        !           992:                    vm_object_lock(next_object);
        !           993:                    vm_object_unlock(object);
        !           994:                    object = next_object;
        !           995:                }
        !           996:                else {
        !           997:                    /*
        !           998:                     * End of chain - we are done.
        !           999:                     */
        !          1000:                    break;
        !          1001:                }
        !          1002:            }
        !          1003:            else {
        !          1004:                /*
        !          1005:                 * Pages in shadowed objects may never have
        !          1006:                 * write permission - we may stop here.
        !          1007:                 */
        !          1008:                break;
        !          1009:            }
        !          1010:        }
        !          1011: 
        !          1012:        vm_object_unlock(object);
        !          1013: }
        !          1014: 
        !          1015: /*
        !          1016:  *     vm_object_pmap_remove:
        !          1017:  *
        !          1018:  *     Removes all physical pages in the specified
        !          1019:  *     object range from all physical maps.
        !          1020:  *
        !          1021:  *     The object must *not* be locked.
        !          1022:  */
        !          1023: void vm_object_pmap_remove(
        !          1024:        register vm_object_t    object,
        !          1025:        register vm_offset_t    start,
        !          1026:        register vm_offset_t    end)
        !          1027: {
        !          1028:        register vm_page_t      p;
        !          1029: 
        !          1030:        if (object == VM_OBJECT_NULL)
        !          1031:                return;
        !          1032: 
        !          1033:        vm_object_lock(object);
        !          1034:        queue_iterate(&object->memq, p, vm_page_t, listq) {
        !          1035:                if (!p->fictitious &&
        !          1036:                    (start <= p->offset) &&
        !          1037:                    (p->offset < end))
        !          1038:                        pmap_page_protect(p->phys_addr, VM_PROT_NONE);
        !          1039:        }
        !          1040:        vm_object_unlock(object);
        !          1041: }
        !          1042: 
        !          1043: /*
        !          1044:  *     Routine:        vm_object_copy_slowly
        !          1045:  *
        !          1046:  *     Description:
        !          1047:  *             Copy the specified range of the source
        !          1048:  *             virtual memory object without using
        !          1049:  *             protection-based optimizations (such
        !          1050:  *             as copy-on-write).  The pages in the
        !          1051:  *             region are actually copied.
        !          1052:  *
        !          1053:  *     In/out conditions:
        !          1054:  *             The caller must hold a reference and a lock
        !          1055:  *             for the source virtual memory object.  The source
        !          1056:  *             object will be returned *unlocked*.
        !          1057:  *
        !          1058:  *     Results:
        !          1059:  *             If the copy is completed successfully, KERN_SUCCESS is
        !          1060:  *             returned.  If the caller asserted the interruptible
        !          1061:  *             argument, and an interruption occurred while waiting
        !          1062:  *             for a user-generated event, MACH_SEND_INTERRUPTED is
        !          1063:  *             returned.  Other values may be returned to indicate
        !          1064:  *             hard errors during the copy operation.
        !          1065:  *
        !          1066:  *             A new virtual memory object is returned in a
        !          1067:  *             parameter (_result_object).  The contents of this
        !          1068:  *             new object, starting at a zero offset, are a copy
        !          1069:  *             of the source memory region.  In the event of
        !          1070:  *             an error, this parameter will contain the value
        !          1071:  *             VM_OBJECT_NULL.
        !          1072:  */
        !          1073: kern_return_t vm_object_copy_slowly(
        !          1074:        register
        !          1075:        vm_object_t     src_object,
        !          1076:        vm_offset_t     src_offset,
        !          1077:        vm_size_t       size,
        !          1078:        boolean_t       interruptible,
        !          1079:        vm_object_t     *_result_object)        /* OUT */
        !          1080: {
        !          1081:        vm_object_t     new_object;
        !          1082:        vm_offset_t     new_offset;
        !          1083: 
        !          1084:        if (size == 0) {
        !          1085:                vm_object_unlock(src_object);
        !          1086:                *_result_object = VM_OBJECT_NULL;
        !          1087:                return KERN_INVALID_ARGUMENT;
        !          1088:        }
        !          1089: 
        !          1090:        /*
        !          1091:         *      Prevent destruction of the source object while we copy.
        !          1092:         */
        !          1093: 
        !          1094:        assert(src_object->ref_count > 0);
        !          1095:        src_object->ref_count++;
        !          1096:        vm_object_unlock(src_object);
        !          1097: 
        !          1098:        /*
        !          1099:         *      Create a new object to hold the copied pages.
        !          1100:         *      A few notes:
        !          1101:         *              We fill the new object starting at offset 0,
        !          1102:         *               regardless of the input offset.
        !          1103:         *              We don't bother to lock the new object within
        !          1104:         *               this routine, since we have the only reference.
        !          1105:         */
        !          1106: 
        !          1107:        new_object = vm_object_allocate(size);
        !          1108:        new_offset = 0;
        !          1109: 
        !          1110:        assert(size == trunc_page(size));       /* Will the loop terminate? */
        !          1111: 
        !          1112:        for ( ;
        !          1113:            size != 0 ;
        !          1114:            src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
        !          1115:            ) {
        !          1116:                vm_page_t       new_page;
        !          1117:                vm_fault_return_t result;
        !          1118: 
        !          1119:                while ((new_page = vm_page_alloc(new_object, new_offset))
        !          1120:                                == VM_PAGE_NULL) {
        !          1121:                        VM_PAGE_WAIT((void (*)()) 0);
        !          1122:                }
        !          1123: 
        !          1124:                do {
        !          1125:                        vm_prot_t       prot = VM_PROT_READ;
        !          1126:                        vm_page_t       _result_page;
        !          1127:                        vm_page_t       top_page;
        !          1128:                        register
        !          1129:                        vm_page_t       result_page;
        !          1130: 
        !          1131:                        vm_object_lock(src_object);
        !          1132:                        src_object->paging_in_progress++;
        !          1133: 
        !          1134:                        result = vm_fault_page(src_object, src_offset,
        !          1135:                                VM_PROT_READ, FALSE, interruptible,
        !          1136:                                &prot, &_result_page, &top_page,
        !          1137:                                FALSE, (void (*)()) 0);
        !          1138: 
        !          1139:                        switch(result) {
        !          1140:                                case VM_FAULT_SUCCESS:
        !          1141:                                        result_page = _result_page;
        !          1142: 
        !          1143:                                        /*
        !          1144:                                         *      We don't need to hold the object
        !          1145:                                         *      lock -- the busy page will be enough.
        !          1146:                                         *      [We don't care about picking up any
        !          1147:                                         *      new modifications.]
        !          1148:                                         *
        !          1149:                                         *      Copy the page to the new object.
        !          1150:                                         *
        !          1151:                                         *      POLICY DECISION:
        !          1152:                                         *              If result_page is clean,
        !          1153:                                         *              we could steal it instead
        !          1154:                                         *              of copying.
        !          1155:                                         */
        !          1156: 
        !          1157:                                        vm_object_unlock(result_page->object);
        !          1158:                                        vm_page_copy(result_page, new_page);
        !          1159: 
        !          1160:                                        /*
        !          1161:                                         *      Let go of both pages (make them
        !          1162:                                         *      not busy, perform wakeup, activate).
        !          1163:                                         */
        !          1164: 
        !          1165:                                        new_page->busy = FALSE;
        !          1166:                                        new_page->dirty = TRUE;
        !          1167:                                        vm_object_lock(result_page->object);
        !          1168:                                        PAGE_WAKEUP_DONE(result_page);
        !          1169: 
        !          1170:                                        vm_page_lock_queues();
        !          1171:                                        if (!result_page->active &&
        !          1172:                                            !result_page->inactive)
        !          1173:                                                vm_page_activate(result_page);
        !          1174:                                        vm_page_activate(new_page);
        !          1175:                                        vm_page_unlock_queues();
        !          1176: 
        !          1177:                                        /*
        !          1178:                                         *      Release paging references and
        !          1179:                                         *      top-level placeholder page, if any.
        !          1180:                                         */
        !          1181: 
        !          1182:                                        vm_fault_cleanup(result_page->object,
        !          1183:                                                        top_page);
        !          1184: 
        !          1185:                                        break;
        !          1186: 
        !          1187:                                case VM_FAULT_RETRY:
        !          1188:                                        break;
        !          1189: 
        !          1190:                                case VM_FAULT_MEMORY_SHORTAGE:
        !          1191:                                        VM_PAGE_WAIT((void (*)()) 0);
        !          1192:                                        break;
        !          1193: 
        !          1194:                                case VM_FAULT_FICTITIOUS_SHORTAGE:
        !          1195:                                        vm_page_more_fictitious();
        !          1196:                                        break;
        !          1197: 
        !          1198:                                case VM_FAULT_INTERRUPTED:
        !          1199:                                        vm_page_free(new_page);
        !          1200:                                        vm_object_deallocate(new_object);
        !          1201:                                        vm_object_deallocate(src_object);
        !          1202:                                        *_result_object = VM_OBJECT_NULL;
        !          1203:                                        return MACH_SEND_INTERRUPTED;
        !          1204: 
        !          1205:                                case VM_FAULT_MEMORY_ERROR:
        !          1206:                                        /*
        !          1207:                                         * A policy choice:
        !          1208:                                         *      (a) ignore pages that we can't
        !          1209:                                         *          copy
        !          1210:                                         *      (b) return the null object if
        !          1211:                                         *          any page fails [chosen]
        !          1212:                                         */
        !          1213: 
        !          1214:                                        vm_page_free(new_page);
        !          1215:                                        vm_object_deallocate(new_object);
        !          1216:                                        vm_object_deallocate(src_object);
        !          1217:                                        *_result_object = VM_OBJECT_NULL;
        !          1218:                                        return KERN_MEMORY_ERROR;
        !          1219:                        }
        !          1220:                } while (result != VM_FAULT_SUCCESS);
        !          1221:        }
        !          1222: 
        !          1223:        /*
        !          1224:         *      Lose the extra reference, and return our object.
        !          1225:         */
        !          1226: 
        !          1227:        vm_object_deallocate(src_object);
        !          1228:        *_result_object = new_object;
        !          1229:        return KERN_SUCCESS;
        !          1230: }
        !          1231: 
        !          1232: /*
        !          1233:  *     Routine:        vm_object_copy_temporary
        !          1234:  *
        !          1235:  *     Purpose:
        !          1236:  *             Copy the specified range of the source virtual
        !          1237:  *             memory object, if it can be done without blocking.
        !          1238:  *
        !          1239:  *     Results:
        !          1240:  *             If the copy is successful, the copy is returned in
        !          1241:  *             the arguments; otherwise, the arguments are not
        !          1242:  *             affected.
        !          1243:  *
        !          1244:  *     In/out conditions:
        !          1245:  *             The object should be unlocked on entry and exit.
        !          1246:  */
        !          1247: 
        !          1248: vm_object_t    vm_object_copy_delayed();       /* forward declaration */
        !          1249: 
        !          1250: boolean_t vm_object_copy_temporary(
        !          1251:        vm_object_t     *_object,               /* INOUT */
        !          1252:        vm_offset_t     *_offset,               /* INOUT */
        !          1253:        boolean_t       *_src_needs_copy,       /* OUT */
        !          1254:        boolean_t       *_dst_needs_copy)       /* OUT */
        !          1255: {
        !          1256:        vm_object_t     object = *_object;
        !          1257: 
        !          1258: #ifdef lint
        !          1259:        ++*_offset;
        !          1260: #endif /* lint */
        !          1261: 
        !          1262:        if (object == VM_OBJECT_NULL) {
        !          1263:                *_src_needs_copy = FALSE;
        !          1264:                *_dst_needs_copy = FALSE;
        !          1265:                return TRUE;
        !          1266:        }
        !          1267: 
        !          1268:        /*
        !          1269:         *      If the object is temporary, we can perform
        !          1270:         *      a symmetric copy-on-write without asking.
        !          1271:         */
        !          1272: 
        !          1273:        vm_object_lock(object);
        !          1274:        if (object->temporary) {
        !          1275: 
        !          1276:                /*
        !          1277:                 *      Shared objects use delayed copy
        !          1278:                 */
        !          1279:                if (object->use_shared_copy) {
        !          1280: 
        !          1281:                        /*
        !          1282:                         *      Asymmetric copy strategy.  Destination
        !          1283:                         *      must be copied (to allow copy object reuse).
        !          1284:                         *      Source is unaffected.
        !          1285:                         */
        !          1286:                        vm_object_unlock(object);
        !          1287:                        object = vm_object_copy_delayed(object);
        !          1288:                        *_object = object;
        !          1289:                        *_src_needs_copy = FALSE;
        !          1290:                        *_dst_needs_copy = TRUE;
        !          1291:                        return TRUE;
        !          1292:                }
        !          1293: 
        !          1294:                /*
        !          1295:                 *      Make another reference to the object.
        !          1296:                 *
        !          1297:                 *      Leave object/offset unchanged.
        !          1298:                 */
        !          1299: 
        !          1300:                assert(object->ref_count > 0);
        !          1301:                object->ref_count++;
        !          1302:                object->shadowed = TRUE;
        !          1303:                vm_object_unlock(object);
        !          1304: 
        !          1305:                /*
        !          1306:                 *      Both source and destination must make
        !          1307:                 *      shadows, and the source must be made
        !          1308:                 *      read-only if not already.
        !          1309:                 */
        !          1310: 
        !          1311:                *_src_needs_copy = TRUE;
        !          1312:                *_dst_needs_copy = TRUE;
        !          1313:                return TRUE;
        !          1314:        }
        !          1315: 
        !          1316:        if (object->pager_ready &&
        !          1317:            (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
        !          1318:                /* XXX Do something intelligent (see temporary code above) */
        !          1319:        }
        !          1320:        vm_object_unlock(object);
        !          1321: 
        !          1322:        return FALSE;
        !          1323: }
        !          1324: 
        !          1325: /*
        !          1326:  *     Routine:        vm_object_copy_call [internal]
        !          1327:  *
        !          1328:  *     Description:
        !          1329:  *             Copy the specified (src_offset, size) portion
        !          1330:  *             of the source object (src_object), using the
        !          1331:  *             user-managed copy algorithm.
        !          1332:  *
        !          1333:  *     In/out conditions:
        !          1334:  *             The source object must be locked on entry.  It
        !          1335:  *             will be *unlocked* on exit.
        !          1336:  *
        !          1337:  *     Results:
        !          1338:  *             If the copy is successful, KERN_SUCCESS is returned.
        !          1339:  *             This routine is interruptible; if a wait for
        !          1340:  *             a user-generated event is interrupted, MACH_SEND_INTERRUPTED
        !          1341:  *             is returned.  Other return values indicate hard errors
        !          1342:  *             in creating the user-managed memory object for the copy.
        !          1343:  *
        !          1344:  *             A new object that represents the copied virtual
        !          1345:  *             memory is returned in a parameter (*_result_object).
        !          1346:  *             If the return value indicates an error, this parameter
        !          1347:  *             is not valid.
        !          1348:  */
        !          1349: kern_return_t vm_object_copy_call(
        !          1350:        vm_object_t     src_object,
        !          1351:        vm_offset_t     src_offset,
        !          1352:        vm_size_t       size,
        !          1353:        vm_object_t     *_result_object)        /* OUT */
        !          1354: {
        !          1355:        vm_offset_t     src_end = src_offset + size;
        !          1356:        ipc_port_t      new_memory_object;
        !          1357:        vm_object_t     new_object;
        !          1358:        vm_page_t       p;
        !          1359: 
        !          1360:        /*
        !          1361:         *      Set the backing object for the new
        !          1362:         *      temporary object.
        !          1363:         */
        !          1364: 
        !          1365:        assert(src_object->ref_count > 0);
        !          1366:        src_object->ref_count++;
        !          1367:        vm_object_paging_begin(src_object);
        !          1368:        vm_object_unlock(src_object);
        !          1369: 
        !          1370:        /*
        !          1371:         *      Create a memory object port to be associated
        !          1372:         *      with this new vm_object.
        !          1373:         *
        !          1374:         *      Since the kernel has the only rights to this
        !          1375:         *      port, we need not hold the cache lock.
        !          1376:         *
        !          1377:         *      Since we have the only object reference, we
        !          1378:         *      need not be worried about collapse operations.
        !          1379:         *
        !          1380:         */
        !          1381: 
        !          1382:        new_memory_object = ipc_port_alloc_kernel();
        !          1383:        if (new_memory_object == IP_NULL) {
        !          1384:                panic("vm_object_copy_call: allocate memory object port");
        !          1385:                /* XXX Shouldn't panic here. */
        !          1386:        }
        !          1387: 
        !          1388:        /* we hold a naked receive right for new_memory_object */
        !          1389:        (void) ipc_port_make_send(new_memory_object);
        !          1390:        /* now we also hold a naked send right for new_memory_object */
        !          1391: 
        !          1392:        /*
        !          1393:         *      Let the memory manager know that a copy operation
        !          1394:         *      is in progress.  Note that we're using the old
        !          1395:         *      memory object's ports (for which we're holding
        !          1396:         *      a paging reference)... the memory manager cannot
        !          1397:         *      yet affect the new memory object.
        !          1398:         */
        !          1399: 
        !          1400:        (void) memory_object_copy(src_object->pager,
        !          1401:                                src_object->pager_request,
        !          1402:                                src_offset, size,
        !          1403:                                new_memory_object);
        !          1404:        /* no longer hold the naked receive right for new_memory_object */
        !          1405: 
        !          1406:        vm_object_lock(src_object);
        !          1407:        vm_object_paging_end(src_object);
        !          1408: 
        !          1409:        /*
        !          1410:         *      Remove write access from all of the pages of
        !          1411:         *      the old memory object that we can.
        !          1412:         */
        !          1413: 
        !          1414:        queue_iterate(&src_object->memq, p, vm_page_t, listq) {
        !          1415:            if (!p->fictitious &&
        !          1416:                (src_offset <= p->offset) &&
        !          1417:                (p->offset < src_end) &&
        !          1418:                !(p->page_lock & VM_PROT_WRITE)) {
        !          1419:                p->page_lock |= VM_PROT_WRITE;
        !          1420:                pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
        !          1421:            }
        !          1422:        }
        !          1423: 
        !          1424:        vm_object_unlock(src_object);
        !          1425: 
        !          1426:        /*
        !          1427:         *      Initialize the rest of the paging stuff
        !          1428:         */
        !          1429: 
        !          1430:        new_object = vm_object_enter(new_memory_object, size, FALSE);
        !          1431:        new_object->shadow = src_object;
        !          1432:        new_object->shadow_offset = src_offset;
        !          1433: 
        !          1434:        /*
        !          1435:         *      Drop the reference for new_memory_object taken above.
        !          1436:         */
        !          1437: 
        !          1438:        ipc_port_release_send(new_memory_object);
        !          1439:        /* no longer hold the naked send right for new_memory_object */
        !          1440: 
        !          1441:        *_result_object = new_object;
        !          1442:        return KERN_SUCCESS;
        !          1443: }
        !          1444: 
        !          1445: /*
        !          1446:  *     Routine:        vm_object_copy_delayed [internal]
        !          1447:  *
        !          1448:  *     Description:
        !          1449:  *             Copy the specified virtual memory object, using
        !          1450:  *             the asymmetric copy-on-write algorithm.
        !          1451:  *
        !          1452:  *     In/out conditions:
        !          1453:  *             The object must be unlocked on entry.
        !          1454:  *
        !          1455:  *             This routine will not block waiting for user-generated
        !          1456:  *             events.  It is not interruptible.
        !          1457:  */
        !          1458: vm_object_t vm_object_copy_delayed(
        !          1459:        vm_object_t     src_object)
        !          1460: {
        !          1461:        vm_object_t     new_copy;
        !          1462:        vm_object_t     old_copy;
        !          1463:        vm_page_t       p;
        !          1464: 
        !          1465:        /*
        !          1466:         *      The user-level memory manager wants to see
        !          1467:         *      all of the changes to this object, but it
        !          1468:         *      has promised not to make any changes on its own.
        !          1469:         *
        !          1470:         *      Perform an asymmetric copy-on-write, as follows:
        !          1471:         *              Create a new object, called a "copy object"
        !          1472:         *               to hold pages modified by the new mapping
        !          1473:         *               (i.e., the copy, not the original mapping).
        !          1474:         *              Record the original object as the backing
        !          1475:         *               object for the copy object.  If the
        !          1476:         *               original mapping does not change a page,
        !          1477:         *               it may be used read-only by the copy.
        !          1478:         *              Record the copy object in the original
        !          1479:         *               object.  When the original mapping causes
        !          1480:         *               a page to be modified, it must be copied
        !          1481:         *               to a new page that is "pushed" to the
        !          1482:         *               copy object.
        !          1483:         *              Mark the new mapping (the copy object)
        !          1484:         *               copy-on-write.  This makes the copy
        !          1485:         *               object itself read-only, allowing it
        !          1486:         *               to be reused if the original mapping
        !          1487:         *               makes no changes, and simplifying the
        !          1488:         *               synchronization required in the "push"
        !          1489:         *               operation described above.
        !          1490:         *
        !          1491:         *      The copy-on-write is said to be assymetric because
        !          1492:         *      the original object is *not* marked copy-on-write.
        !          1493:         *      A copied page is pushed to the copy object, regardless
        !          1494:         *      which party attempted to modify the page.
        !          1495:         *
        !          1496:         *      Repeated asymmetric copy operations may be done.
        !          1497:         *      If the original object has not been changed since
        !          1498:         *      the last copy, its copy object can be reused.
        !          1499:         *      Otherwise, a new copy object can be inserted
        !          1500:         *      between the original object and its previous
        !          1501:         *      copy object.  Since any copy object is read-only,
        !          1502:         *      this cannot affect the contents of the previous copy
        !          1503:         *      object.
        !          1504:         *
        !          1505:         *      Note that a copy object is higher in the object
        !          1506:         *      tree than the original object; therefore, use of
        !          1507:         *      the copy object recorded in the original object
        !          1508:         *      must be done carefully, to avoid deadlock.
        !          1509:         */
        !          1510: 
        !          1511:        /*
        !          1512:         *      Allocate a new copy object before locking, even
        !          1513:         *      though we may not need it later.
        !          1514:         */
        !          1515: 
        !          1516:        new_copy = vm_object_allocate(src_object->size);
        !          1517: 
        !          1518:        vm_object_lock(src_object);
        !          1519: 
        !          1520:        /*
        !          1521:         *      See whether we can reuse the result of a previous
        !          1522:         *      copy operation.
        !          1523:         */
        !          1524:  Retry:
        !          1525:        old_copy = src_object->copy;
        !          1526:        if (old_copy != VM_OBJECT_NULL) {
        !          1527:                /*
        !          1528:                 *      Try to get the locks (out of order)
        !          1529:                 */
        !          1530:                if (!vm_object_lock_try(old_copy)) {
        !          1531:                        vm_object_unlock(src_object);
        !          1532: 
        !          1533:                        simple_lock_pause();    /* wait a bit */
        !          1534: 
        !          1535:                        vm_object_lock(src_object);
        !          1536:                        goto Retry;
        !          1537:                }
        !          1538: 
        !          1539:                /*
        !          1540:                 *      Determine whether the old copy object has
        !          1541:                 *      been modified.
        !          1542:                 */
        !          1543: 
        !          1544:                if (old_copy->resident_page_count == 0 &&
        !          1545:                    !old_copy->pager_created) {
        !          1546:                        /*
        !          1547:                         *      It has not been modified.
        !          1548:                         *
        !          1549:                         *      Return another reference to
        !          1550:                         *      the existing copy-object.
        !          1551:                         */
        !          1552:                        assert(old_copy->ref_count > 0);
        !          1553:                        old_copy->ref_count++;
        !          1554:                        vm_object_unlock(old_copy);
        !          1555:                        vm_object_unlock(src_object);
        !          1556: 
        !          1557:                        vm_object_deallocate(new_copy);
        !          1558: 
        !          1559:                        return old_copy;
        !          1560:                }
        !          1561: 
        !          1562:                /*
        !          1563:                 *      The copy-object is always made large enough to
        !          1564:                 *      completely shadow the original object, since
        !          1565:                 *      it may have several users who want to shadow
        !          1566:                 *      the original object at different points.
        !          1567:                 */
        !          1568: 
        !          1569:                assert((old_copy->shadow == src_object) &&
        !          1570:                    (old_copy->shadow_offset == (vm_offset_t) 0));
        !          1571: 
        !          1572:                /*
        !          1573:                 *      Make the old copy-object shadow the new one.
        !          1574:                 *      It will receive no more pages from the original
        !          1575:                 *      object.
        !          1576:                 */
        !          1577: 
        !          1578:                src_object->ref_count--;        /* remove ref. from old_copy */
        !          1579:                assert(src_object->ref_count > 0);
        !          1580:                old_copy->shadow = new_copy;
        !          1581:                assert(new_copy->ref_count > 0);
        !          1582:                new_copy->ref_count++;
        !          1583:                vm_object_unlock(old_copy);     /* done with old_copy */
        !          1584:        }
        !          1585: 
        !          1586:        /*
        !          1587:         *      Point the new copy at the existing object.
        !          1588:         */
        !          1589: 
        !          1590:        new_copy->shadow = src_object;
        !          1591:        new_copy->shadow_offset = 0;
        !          1592:        new_copy->shadowed = TRUE;      /* caller must set needs_copy */
        !          1593:        assert(src_object->ref_count > 0);
        !          1594:        src_object->ref_count++;
        !          1595:        src_object->copy = new_copy;
        !          1596: 
        !          1597:        /*
        !          1598:         *      Mark all pages of the existing object copy-on-write.
        !          1599:         *      This object may have a shadow chain below it, but
        !          1600:         *      those pages will already be marked copy-on-write.
        !          1601:         */
        !          1602: 
        !          1603:        queue_iterate(&src_object->memq, p, vm_page_t, listq) {
        !          1604:            if (!p->fictitious)
        !          1605:                pmap_page_protect(p->phys_addr,
        !          1606:                                  (VM_PROT_ALL & ~VM_PROT_WRITE &
        !          1607:                                   ~p->page_lock));
        !          1608:        }
        !          1609: 
        !          1610:        vm_object_unlock(src_object);
        !          1611: 
        !          1612:        return new_copy;
        !          1613: }
        !          1614: 
        !          1615: /*
        !          1616:  *     Routine:        vm_object_copy_strategically
        !          1617:  *
        !          1618:  *     Purpose:
        !          1619:  *             Perform a copy according to the source object's
        !          1620:  *             declared strategy.  This operation may block,
        !          1621:  *             and may be interrupted.
        !          1622:  */
        !          1623: kern_return_t  vm_object_copy_strategically(
        !          1624:        register
        !          1625:        vm_object_t     src_object,
        !          1626:        vm_offset_t     src_offset,
        !          1627:        vm_size_t       size,
        !          1628:        vm_object_t     *dst_object,    /* OUT */
        !          1629:        vm_offset_t     *dst_offset,    /* OUT */
        !          1630:        boolean_t       *dst_needs_copy) /* OUT */
        !          1631: {
        !          1632:        kern_return_t   result = KERN_SUCCESS;  /* to quiet gcc warnings */
        !          1633:        boolean_t       interruptible = TRUE; /* XXX */
        !          1634: 
        !          1635:        assert(src_object != VM_OBJECT_NULL);
        !          1636: 
        !          1637:        vm_object_lock(src_object);
        !          1638: 
        !          1639:        /* XXX assert(!src_object->temporary);  JSB FIXME */
        !          1640: 
        !          1641:        /*
        !          1642:         *      The copy strategy is only valid if the memory manager
        !          1643:         *      is "ready".
        !          1644:         */
        !          1645: 
        !          1646:        while (!src_object->pager_ready) {
        !          1647:                vm_object_wait( src_object,
        !          1648:                                VM_OBJECT_EVENT_PAGER_READY,
        !          1649:                                interruptible);
        !          1650:                if (interruptible &&
        !          1651:                    (current_thread()->wait_result != THREAD_AWAKENED)) {
        !          1652:                        *dst_object = VM_OBJECT_NULL;
        !          1653:                        *dst_offset = 0;
        !          1654:                        *dst_needs_copy = FALSE;
        !          1655:                        return MACH_SEND_INTERRUPTED;
        !          1656:                }
        !          1657:                vm_object_lock(src_object);
        !          1658:        }
        !          1659: 
        !          1660:        /*
        !          1661:         *      The object may be temporary (even though it is external).
        !          1662:         *      If so, do a symmetric copy.
        !          1663:         */
        !          1664: 
        !          1665:        if (src_object->temporary) {
        !          1666:                /*
        !          1667:                 *      XXX
        !          1668:                 *      This does not count as intelligent!
        !          1669:                 *      This buys us the object->temporary optimizations,
        !          1670:                 *      but we aren't using a symmetric copy,
        !          1671:                 *      which may confuse the vm code. The correct thing
        !          1672:                 *      to do here is to figure out what to call to get
        !          1673:                 *      a temporary shadowing set up.
        !          1674:                 */
        !          1675:                src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
        !          1676:        }
        !          1677: 
        !          1678:        /*
        !          1679:         *      The object is permanent. Use the appropriate copy strategy.
        !          1680:         */
        !          1681: 
        !          1682:        switch (src_object->copy_strategy) {
        !          1683:            case MEMORY_OBJECT_COPY_NONE:
        !          1684:                if ((result = vm_object_copy_slowly(
        !          1685:                                        src_object,
        !          1686:                                        src_offset,
        !          1687:                                        size,
        !          1688:                                        interruptible,
        !          1689:                                        dst_object))
        !          1690:                    == KERN_SUCCESS) {
        !          1691:                        *dst_offset = 0;
        !          1692:                        *dst_needs_copy = FALSE;
        !          1693:                }
        !          1694:                break;
        !          1695: 
        !          1696:            case MEMORY_OBJECT_COPY_CALL:
        !          1697:                if ((result = vm_object_copy_call(
        !          1698:                                src_object,
        !          1699:                                src_offset,
        !          1700:                                size,
        !          1701:                                dst_object))
        !          1702:                    == KERN_SUCCESS) {
        !          1703:                        *dst_offset = 0;
        !          1704:                        *dst_needs_copy = FALSE;
        !          1705:                }
        !          1706:                break;
        !          1707: 
        !          1708:            case MEMORY_OBJECT_COPY_DELAY:
        !          1709:                vm_object_unlock(src_object);
        !          1710:                *dst_object = vm_object_copy_delayed(src_object);
        !          1711:                *dst_offset = src_offset;
        !          1712:                *dst_needs_copy = TRUE;
        !          1713: 
        !          1714:                result = KERN_SUCCESS;
        !          1715:                break;
        !          1716:        }
        !          1717: 
        !          1718:        return result;
        !          1719: }
        !          1720: 
        !          1721: /*
        !          1722:  *     vm_object_shadow:
        !          1723:  *
        !          1724:  *     Create a new object which is backed by the
        !          1725:  *     specified existing object range.  The source
        !          1726:  *     object reference is deallocated.
        !          1727:  *
        !          1728:  *     The new object and offset into that object
        !          1729:  *     are returned in the source parameters.
        !          1730:  */
        !          1731: 
        !          1732: void vm_object_shadow(
        !          1733:        vm_object_t     *object,        /* IN/OUT */
        !          1734:        vm_offset_t     *offset,        /* IN/OUT */
        !          1735:        vm_size_t       length)
        !          1736: {
        !          1737:        register vm_object_t    source;
        !          1738:        register vm_object_t    result;
        !          1739: 
        !          1740:        source = *object;
        !          1741: 
        !          1742:        /*
        !          1743:         *      Allocate a new object with the given length
        !          1744:         */
        !          1745: 
        !          1746:        if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
        !          1747:                panic("vm_object_shadow: no object for shadowing");
        !          1748: 
        !          1749:        /*
        !          1750:         *      The new object shadows the source object, adding
        !          1751:         *      a reference to it.  Our caller changes his reference
        !          1752:         *      to point to the new object, removing a reference to
        !          1753:         *      the source object.  Net result: no change of reference
        !          1754:         *      count.
        !          1755:         */
        !          1756:        result->shadow = source;
        !          1757: 
        !          1758:        /*
        !          1759:         *      Store the offset into the source object,
        !          1760:         *      and fix up the offset into the new object.
        !          1761:         */
        !          1762: 
        !          1763:        result->shadow_offset = *offset;
        !          1764: 
        !          1765:        /*
        !          1766:         *      Return the new things
        !          1767:         */
        !          1768: 
        !          1769:        *offset = 0;
        !          1770:        *object = result;
        !          1771: }
        !          1772: 
        !          1773: /*
        !          1774:  *     The relationship between vm_object structures and
        !          1775:  *     the memory_object ports requires careful synchronization.
        !          1776:  *
        !          1777:  *     All associations are created by vm_object_enter.  All three
        !          1778:  *     port fields are filled in, as follows:
        !          1779:  *             pager:  the memory_object port itself, supplied by
        !          1780:  *                     the user requesting a mapping (or the kernel,
        !          1781:  *                     when initializing internal objects); the
        !          1782:  *                     kernel simulates holding send rights by keeping
        !          1783:  *                     a port reference;
        !          1784:  *             pager_request:
        !          1785:  *             pager_name:
        !          1786:  *                     the memory object control and name ports,
        !          1787:  *                     created by the kernel; the kernel holds
        !          1788:  *                     receive (and ownership) rights to these
        !          1789:  *                     ports, but no other references.
        !          1790:  *     All of the ports are referenced by their global names.
        !          1791:  *
        !          1792:  *     When initialization is complete, the "initialized" field
        !          1793:  *     is asserted.  Other mappings using a particular memory object,
        !          1794:  *     and any references to the vm_object gained through the
        !          1795:  *     port association must wait for this initialization to occur.
        !          1796:  *
        !          1797:  *     In order to allow the memory manager to set attributes before
        !          1798:  *     requests (notably virtual copy operations, but also data or
        !          1799:  *     unlock requests) are made, a "ready" attribute is made available.
        !          1800:  *     Only the memory manager may affect the value of this attribute.
        !          1801:  *     Its value does not affect critical kernel functions, such as
        !          1802:  *     internal object initialization or destruction.  [Furthermore,
        !          1803:  *     memory objects created by the kernel are assumed to be ready
        !          1804:  *     immediately; the default memory manager need not explicitly
        !          1805:  *     set the "ready" attribute.]
        !          1806:  *
        !          1807:  *     [Both the "initialized" and "ready" attribute wait conditions
        !          1808:  *     use the "pager" field as the wait event.]
        !          1809:  *
        !          1810:  *     The port associations can be broken down by any of the
        !          1811:  *     following routines:
        !          1812:  *             vm_object_terminate:
        !          1813:  *                     No references to the vm_object remain, and
        !          1814:  *                     the object cannot (or will not) be cached.
        !          1815:  *                     This is the normal case, and is done even
        !          1816:  *                     though one of the other cases has already been
        !          1817:  *                     done.
        !          1818:  *             vm_object_destroy:
        !          1819:  *                     The memory_object port has been destroyed,
        !          1820:  *                     meaning that the kernel cannot flush dirty
        !          1821:  *                     pages or request new data or unlock existing
        !          1822:  *                     data.
        !          1823:  *             memory_object_destroy:
        !          1824:  *                     The memory manager has requested that the
        !          1825:  *                     kernel relinquish rights to the memory object
        !          1826:  *                     port.  [The memory manager may not want to
        !          1827:  *                     destroy the port, but may wish to refuse or
        !          1828:  *                     tear down existing memory mappings.]
        !          1829:  *     Each routine that breaks an association must break all of
        !          1830:  *     them at once.  At some later time, that routine must clear
        !          1831:  *     the vm_object port fields and release the port rights.
        !          1832:  *     [Furthermore, each routine must cope with the simultaneous
        !          1833:  *     or previous operations of the others.]
        !          1834:  *
        !          1835:  *     In addition to the lock on the object, the vm_object_cache_lock
        !          1836:  *     governs the port associations.  References gained through the
        !          1837:  *     port association require use of the cache lock.
        !          1838:  *
        !          1839:  *     Because the port fields may be cleared spontaneously, they
        !          1840:  *     cannot be used to determine whether a memory object has
        !          1841:  *     ever been associated with a particular vm_object.  [This
        !          1842:  *     knowledge is important to the shadow object mechanism.]
        !          1843:  *     For this reason, an additional "created" attribute is
        !          1844:  *     provided.
        !          1845:  *
        !          1846:  *     During various paging operations, the port values found in the
        !          1847:  *     vm_object must be valid.  To prevent these port rights from being
        !          1848:  *     released, and to prevent the port associations from changing
        !          1849:  *     (other than being removed, i.e., made null), routines may use
        !          1850:  *     the vm_object_paging_begin/end routines [actually, macros].
        !          1851:  *     The implementation uses the "paging_in_progress" and "wanted" fields.
        !          1852:  *     [Operations that alter the validity of the port values include the
        !          1853:  *     termination routines and vm_object_collapse.]
        !          1854:  */
        !          1855: 
        !          1856: vm_object_t vm_object_lookup(
        !          1857:        ipc_port_t      port)
        !          1858: {
        !          1859:        vm_object_t     object = VM_OBJECT_NULL;
        !          1860: 
        !          1861:        if (IP_VALID(port)) {
        !          1862:                ip_lock(port);
        !          1863:                if (ip_active(port) &&
        !          1864: #if    NORMA_VM
        !          1865:                    (ip_kotype(port) == IKOT_PAGER)) {
        !          1866: #else  /* NORMA_VM */
        !          1867:                    (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
        !          1868: #endif /* NORMA_VM */
        !          1869:                        vm_object_cache_lock();
        !          1870:                        object = (vm_object_t) port->ip_kobject;
        !          1871:                        vm_object_lock(object);
        !          1872: 
        !          1873:                        assert(object->alive);
        !          1874: 
        !          1875:                        if (object->ref_count == 0) {
        !          1876:                                queue_remove(&vm_object_cached_list, object,
        !          1877:                                             vm_object_t, cached_list);
        !          1878:                                vm_object_cached_count--;
        !          1879:                        }
        !          1880: 
        !          1881:                        object->ref_count++;
        !          1882:                        vm_object_unlock(object);
        !          1883:                        vm_object_cache_unlock();
        !          1884:                }
        !          1885:                ip_unlock(port);
        !          1886:        }
        !          1887: 
        !          1888:        return object;
        !          1889: }
        !          1890: 
        !          1891: vm_object_t vm_object_lookup_name(
        !          1892:        ipc_port_t      port)
        !          1893: {
        !          1894:        vm_object_t     object = VM_OBJECT_NULL;
        !          1895: 
        !          1896:        if (IP_VALID(port)) {
        !          1897:                ip_lock(port);
        !          1898:                if (ip_active(port) &&
        !          1899:                    (ip_kotype(port) == IKOT_PAGING_NAME)) {
        !          1900:                        vm_object_cache_lock();
        !          1901:                        object = (vm_object_t) port->ip_kobject;
        !          1902:                        vm_object_lock(object);
        !          1903: 
        !          1904:                        assert(object->alive);
        !          1905: 
        !          1906:                        if (object->ref_count == 0) {
        !          1907:                                queue_remove(&vm_object_cached_list, object,
        !          1908:                                             vm_object_t, cached_list);
        !          1909:                                vm_object_cached_count--;
        !          1910:                        }
        !          1911: 
        !          1912:                        object->ref_count++;
        !          1913:                        vm_object_unlock(object);
        !          1914:                        vm_object_cache_unlock();
        !          1915:                }
        !          1916:                ip_unlock(port);
        !          1917:        }
        !          1918: 
        !          1919:        return object;
        !          1920: }
        !          1921: 
        !          1922: void vm_object_destroy(
        !          1923:        ipc_port_t      pager)
        !          1924: {
        !          1925:        vm_object_t     object;
        !          1926:        pager_request_t old_request;
        !          1927:        ipc_port_t      old_name;
        !          1928: 
        !          1929:        /*
        !          1930:         *      Perform essentially the same operations as in vm_object_lookup,
        !          1931:         *      except that this time we look up based on the memory_object
        !          1932:         *      port, not the control port.
        !          1933:         */
        !          1934:        vm_object_cache_lock();
        !          1935:        if (ip_kotype(pager) != IKOT_PAGER) {
        !          1936:                vm_object_cache_unlock();
        !          1937:                return;
        !          1938:        }
        !          1939: 
        !          1940:        object = (vm_object_t) pager->ip_kobject;
        !          1941:        vm_object_lock(object);
        !          1942:        if (object->ref_count == 0) {
        !          1943:                queue_remove(&vm_object_cached_list, object,
        !          1944:                                vm_object_t, cached_list);
        !          1945:                vm_object_cached_count--;
        !          1946:        }
        !          1947:        object->ref_count++;
        !          1948: 
        !          1949:        object->can_persist = FALSE;
        !          1950: 
        !          1951:        assert(object->pager == pager);
        !          1952: 
        !          1953:        /*
        !          1954:         *      Remove the port associations.
        !          1955:         *
        !          1956:         *      Note that the memory_object itself is dead, so
        !          1957:         *      we don't bother with it.
        !          1958:         */
        !          1959: 
        !          1960:        object->pager = IP_NULL;
        !          1961:        vm_object_remove(object);
        !          1962: 
        !          1963:        old_request = object->pager_request;
        !          1964:        object->pager_request = PAGER_REQUEST_NULL;
        !          1965: 
        !          1966:        old_name = object->pager_name;
        !          1967:        object->pager_name = IP_NULL;
        !          1968: 
        !          1969:        vm_object_unlock(object);
        !          1970:        vm_object_cache_unlock();
        !          1971: 
        !          1972:        /*
        !          1973:         *      Clean up the port references.  Note that there's no
        !          1974:         *      point in trying the memory_object_terminate call
        !          1975:         *      because the memory_object itself is dead.
        !          1976:         */
        !          1977: 
        !          1978:        ipc_port_release_send(pager);
        !          1979: #if    !NORMA_VM
        !          1980:        if (old_request != IP_NULL)
        !          1981:                ipc_port_dealloc_kernel(old_request);
        !          1982: #endif /* !NORMA_VM */
        !          1983:        if (old_name != IP_NULL)
        !          1984: #if    NORMA_VM
        !          1985:                ipc_port_release_send(old_name);
        !          1986: #else  /* NORMA_VM */
        !          1987:                ipc_port_dealloc_kernel(old_name);
        !          1988: #endif /* NORMA_VM */
        !          1989: 
        !          1990:        /*
        !          1991:         *      Restart pending page requests
        !          1992:         */
        !          1993: 
        !          1994:        vm_object_abort_activity(object);
        !          1995: 
        !          1996:        /*
        !          1997:         *      Lose the object reference.
        !          1998:         */
        !          1999: 
        !          2000:        vm_object_deallocate(object);
        !          2001: }
        !          2002: 
        !          2003: boolean_t      vm_object_accept_old_init_protocol = FALSE;
        !          2004: 
        !          2005: /*
        !          2006:  *     Routine:        vm_object_enter
        !          2007:  *     Purpose:
        !          2008:  *             Find a VM object corresponding to the given
        !          2009:  *             pager; if no such object exists, create one,
        !          2010:  *             and initialize the pager.
        !          2011:  */
        !          2012: vm_object_t vm_object_enter(
        !          2013:        ipc_port_t      pager,
        !          2014:        vm_size_t       size,
        !          2015:        boolean_t       internal)
        !          2016: {
        !          2017:        register
        !          2018:        vm_object_t     object;
        !          2019:        vm_object_t     new_object;
        !          2020:        boolean_t       must_init;
        !          2021:        ipc_kobject_type_t po;
        !          2022: 
        !          2023: restart:
        !          2024:        if (!IP_VALID(pager))
        !          2025:                return vm_object_allocate(size);
        !          2026: 
        !          2027:        new_object = VM_OBJECT_NULL;
        !          2028:        must_init = FALSE;
        !          2029: 
        !          2030:        /*
        !          2031:         *      Look for an object associated with this port.
        !          2032:         */
        !          2033: 
        !          2034:        vm_object_cache_lock();
        !          2035:        for (;;) {
        !          2036:                po = ip_kotype(pager);
        !          2037: 
        !          2038:                /*
        !          2039:                 *      If a previous object is being terminated,
        !          2040:                 *      we must wait for the termination message
        !          2041:                 *      to be queued.
        !          2042:                 *
        !          2043:                 *      We set kobject to a non-null value to let the
        !          2044:                 *      terminator know that someone is waiting.
        !          2045:                 *      Among the possibilities is that the port
        !          2046:                 *      could die while we're waiting.  Must restart
        !          2047:                 *      instead of continuing the loop.
        !          2048:                 */
        !          2049: 
        !          2050:                if (po == IKOT_PAGER_TERMINATING) {
        !          2051:                        pager->ip_kobject = (ipc_kobject_t) pager;
        !          2052:                        assert_wait((event_t) pager, FALSE);
        !          2053:                        vm_object_cache_unlock();
        !          2054:                        thread_block((void (*)()) 0);
        !          2055:                        goto restart;
        !          2056:                }
        !          2057: 
        !          2058:                /*
        !          2059:                 *      Bail if there is already a kobject associated
        !          2060:                 *      with the pager port.
        !          2061:                 */
        !          2062:                if (po != IKOT_NONE) {
        !          2063:                        break;
        !          2064:                }
        !          2065: 
        !          2066:                /*
        !          2067:                 *      We must unlock to create a new object;
        !          2068:                 *      if we do so, we must try the lookup again.
        !          2069:                 */
        !          2070: 
        !          2071:                if (new_object == VM_OBJECT_NULL) {
        !          2072:                        vm_object_cache_unlock();
        !          2073:                        new_object = vm_object_allocate(size);
        !          2074:                        vm_object_cache_lock();
        !          2075:                } else {
        !          2076:                        /*
        !          2077:                         *      Lookup failed twice, and we have something
        !          2078:                         *      to insert; set the object.
        !          2079:                         */
        !          2080: 
        !          2081:                        ipc_kobject_set(pager,
        !          2082:                                        (ipc_kobject_t) new_object,
        !          2083:                                        IKOT_PAGER);
        !          2084:                        new_object = VM_OBJECT_NULL;
        !          2085:                        must_init = TRUE;
        !          2086:                }
        !          2087:        }
        !          2088: 
        !          2089:        if (internal)
        !          2090:                must_init = TRUE;
        !          2091: 
        !          2092:        /*
        !          2093:         *      It's only good if it's a VM object!
        !          2094:         */
        !          2095: 
        !          2096:        object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
        !          2097:                                    : VM_OBJECT_NULL;
        !          2098: 
        !          2099:        if ((object != VM_OBJECT_NULL) && !must_init) {
        !          2100:                vm_object_lock(object);
        !          2101:                if (object->ref_count == 0) {
        !          2102:                        queue_remove(&vm_object_cached_list, object,
        !          2103:                                        vm_object_t, cached_list);
        !          2104:                        vm_object_cached_count--;
        !          2105:                }
        !          2106:                object->ref_count++;
        !          2107:                vm_object_unlock(object);
        !          2108: 
        !          2109:                vm_stat.hits++;
        !          2110:        }
        !          2111:        assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
        !          2112:                ((object->paging_in_progress != 0) && internal));
        !          2113: 
        !          2114:        vm_stat.lookups++;
        !          2115: 
        !          2116:        vm_object_cache_unlock();
        !          2117: 
        !          2118:        /*
        !          2119:         *      If we raced to create a vm_object but lost, let's
        !          2120:         *      throw away ours.
        !          2121:         */
        !          2122: 
        !          2123:        if (new_object != VM_OBJECT_NULL)
        !          2124:                vm_object_deallocate(new_object);
        !          2125: 
        !          2126:        if (object == VM_OBJECT_NULL)
        !          2127:                return(object);
        !          2128: 
        !          2129:        if (must_init) {
        !          2130:                /*
        !          2131:                 *      Copy the naked send right we were given.
        !          2132:                 */
        !          2133: 
        !          2134:                pager = ipc_port_copy_send(pager);
        !          2135:                if (!IP_VALID(pager))
        !          2136:                        panic("vm_object_enter: port died"); /* XXX */
        !          2137: 
        !          2138:                object->pager_created = TRUE;
        !          2139:                object->pager = pager;
        !          2140: 
        !          2141: #if    NORMA_VM
        !          2142: 
        !          2143:                /*
        !          2144:                 *      Let the xmm system know that we want to use the pager.
        !          2145:                 *
        !          2146:                 *      Name port will be provided by the xmm system
        !          2147:                 *      when set_attributes_common is called.
        !          2148:                 */
        !          2149: 
        !          2150:                object->internal = internal;
        !          2151:                object->pager_ready = internal;
        !          2152:                if (internal) {
        !          2153:                        assert(object->temporary);
        !          2154:                } else {
        !          2155:                        object->temporary = FALSE;
        !          2156:                }
        !          2157:                object->pager_name = IP_NULL;
        !          2158: 
        !          2159:                (void) xmm_memory_object_init(object);
        !          2160: #else  /* NORMA_VM */
        !          2161: 
        !          2162:                /*
        !          2163:                 *      Allocate request port.
        !          2164:                 */
        !          2165: 
        !          2166:                object->pager_request = ipc_port_alloc_kernel();
        !          2167:                if (object->pager_request == IP_NULL)
        !          2168:                        panic("vm_object_enter: pager request alloc");
        !          2169: 
        !          2170:                ipc_kobject_set(object->pager_request,
        !          2171:                                (ipc_kobject_t) object,
        !          2172:                                IKOT_PAGING_REQUEST);
        !          2173: 
        !          2174:                /*
        !          2175:                 *      Let the pager know we're using it.
        !          2176:                 */
        !          2177: 
        !          2178:                if (internal) {
        !          2179:                        /* acquire a naked send right for the DMM */
        !          2180:                        ipc_port_t DMM = memory_manager_default_reference();
        !          2181: 
        !          2182:                        /* mark the object internal */
        !          2183:                        object->internal = TRUE;
        !          2184:                        assert(object->temporary);
        !          2185: 
        !          2186:                        /* default-pager objects are ready immediately */
        !          2187:                        object->pager_ready = TRUE;
        !          2188: 
        !          2189:                        /* consumes the naked send right for DMM */
        !          2190:                        (void) memory_object_create(DMM,
        !          2191:                                pager,
        !          2192:                                object->size,
        !          2193:                                object->pager_request,
        !          2194:                                object->pager_name,
        !          2195:                                PAGE_SIZE);
        !          2196:                } else {
        !          2197:                        /* the object is external and not temporary */
        !          2198:                        object->internal = FALSE;
        !          2199:                        object->temporary = FALSE;
        !          2200: 
        !          2201:                        /* user pager objects are not ready until marked so */
        !          2202:                        object->pager_ready = FALSE;
        !          2203: 
        !          2204:                        (void) memory_object_init(pager,
        !          2205:                                object->pager_request,
        !          2206:                                object->pager_name,
        !          2207:                                PAGE_SIZE);
        !          2208: 
        !          2209:                }
        !          2210: #endif /* NORMA_VM */
        !          2211: 
        !          2212:                vm_object_lock(object);
        !          2213:                object->pager_initialized = TRUE;
        !          2214: 
        !          2215:                if (vm_object_accept_old_init_protocol)
        !          2216:                        object->pager_ready = TRUE;
        !          2217: 
        !          2218:                vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
        !          2219:        } else {
        !          2220:                vm_object_lock(object);
        !          2221:        }
        !          2222:        /*
        !          2223:         *      [At this point, the object must be locked]
        !          2224:         */
        !          2225: 
        !          2226:        /*
        !          2227:         *      Wait for the work above to be done by the first
        !          2228:         *      thread to map this object.
        !          2229:         */
        !          2230: 
        !          2231:        while (!object->pager_initialized) {
        !          2232:                vm_object_wait( object,
        !          2233:                                VM_OBJECT_EVENT_INITIALIZED,
        !          2234:                                FALSE);
        !          2235:                vm_object_lock(object);
        !          2236:        }
        !          2237:        vm_object_unlock(object);
        !          2238: 
        !          2239:        return object;
        !          2240: }
        !          2241: 
        !          2242: /*
        !          2243:  *     Routine:        vm_object_pager_create
        !          2244:  *     Purpose:
        !          2245:  *             Create a memory object for an internal object.
        !          2246:  *     In/out conditions:
        !          2247:  *             The object is locked on entry and exit;
        !          2248:  *             it may be unlocked within this call.
        !          2249:  *     Limitations:
        !          2250:  *             Only one thread may be performing a
        !          2251:  *             vm_object_pager_create on an object at
        !          2252:  *             a time.  Presumably, only the pageout
        !          2253:  *             daemon will be using this routine.
        !          2254:  */
        !          2255: void vm_object_pager_create(
        !          2256:        register
        !          2257:        vm_object_t     object)
        !          2258: {
        !          2259:        ipc_port_t      pager;
        !          2260: 
        !          2261:        if (object->pager_created) {
        !          2262:                /*
        !          2263:                 *      Someone else got to it first...
        !          2264:                 *      wait for them to finish initializing
        !          2265:                 */
        !          2266: 
        !          2267:                while (!object->pager_initialized) {
        !          2268:                        vm_object_wait( object,
        !          2269:                                        VM_OBJECT_EVENT_PAGER_READY,
        !          2270:                                        FALSE);
        !          2271:                        vm_object_lock(object);
        !          2272:                }
        !          2273:                return;
        !          2274:        }
        !          2275: 
        !          2276:        /*
        !          2277:         *      Indicate that a memory object has been assigned
        !          2278:         *      before dropping the lock, to prevent a race.
        !          2279:         */
        !          2280: 
        !          2281:        object->pager_created = TRUE;
        !          2282: 
        !          2283:        /*
        !          2284:         *      Prevent collapse or termination by
        !          2285:         *      holding a paging reference
        !          2286:         */
        !          2287: 
        !          2288:        vm_object_paging_begin(object);
        !          2289:        vm_object_unlock(object);
        !          2290: 
        !          2291: #if    MACH_PAGEMAP
        !          2292:        object->existence_info = vm_external_create(
        !          2293:                                        object->size +
        !          2294:                                        object->paging_offset);
        !          2295:        assert((object->size + object->paging_offset) >=
        !          2296:                object->size);
        !          2297: #endif /* MACH_PAGEMAP */
        !          2298: 
        !          2299:        /*
        !          2300:         *      Create the pager, and associate with it
        !          2301:         *      this object.
        !          2302:         *
        !          2303:         *      Note that we only make the port association
        !          2304:         *      so that vm_object_enter can properly look up
        !          2305:         *      the object to complete the initialization...
        !          2306:         *      we do not expect any user to ever map this
        !          2307:         *      object.
        !          2308:         *
        !          2309:         *      Since the kernel has the only rights to the
        !          2310:         *      port, it's safe to install the association
        !          2311:         *      without holding the cache lock.
        !          2312:         */
        !          2313: 
        !          2314:        pager = ipc_port_alloc_kernel();
        !          2315:        if (pager == IP_NULL)
        !          2316:                panic("vm_object_pager_create: allocate pager port");
        !          2317: 
        !          2318:        (void) ipc_port_make_send(pager);
        !          2319:        ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
        !          2320: 
        !          2321:        /*
        !          2322:         *      Initialize the rest of the paging stuff
        !          2323:         */
        !          2324: 
        !          2325:        if (vm_object_enter(pager, object->size, TRUE) != object)
        !          2326:                panic("vm_object_pager_create: mismatch");
        !          2327: 
        !          2328:        /*
        !          2329:         *      Drop the naked send right taken above.
        !          2330:         */
        !          2331: 
        !          2332:        ipc_port_release_send(pager);
        !          2333: 
        !          2334:        /*
        !          2335:         *      Release the paging reference
        !          2336:         */
        !          2337: 
        !          2338:        vm_object_lock(object);
        !          2339:        vm_object_paging_end(object);
        !          2340: }
        !          2341: 
        !          2342: /*
        !          2343:  *     Routine:        vm_object_remove
        !          2344:  *     Purpose:
        !          2345:  *             Eliminate the pager/object association
        !          2346:  *             for this pager.
        !          2347:  *     Conditions:
        !          2348:  *             The object cache must be locked.
        !          2349:  */
        !          2350: void vm_object_remove(
        !          2351:        vm_object_t     object)
        !          2352: {
        !          2353:        ipc_port_t port;
        !          2354: 
        !          2355:        if ((port = object->pager) != IP_NULL) {
        !          2356:                if (ip_kotype(port) == IKOT_PAGER)
        !          2357:                        ipc_kobject_set(port, IKO_NULL,
        !          2358:                                        IKOT_PAGER_TERMINATING);
        !          2359:                 else if (ip_kotype(port) != IKOT_NONE)
        !          2360:                        panic("vm_object_remove: bad object port");
        !          2361:        }
        !          2362: #if    !NORMA_VM
        !          2363:        if ((port = object->pager_request) != IP_NULL) {
        !          2364:                if (ip_kotype(port) == IKOT_PAGING_REQUEST)
        !          2365:                        ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
        !          2366:                 else if (ip_kotype(port) != IKOT_NONE)
        !          2367:                        panic("vm_object_remove: bad request port");
        !          2368:        }
        !          2369:        if ((port = object->pager_name) != IP_NULL) {
        !          2370:                if (ip_kotype(port) == IKOT_PAGING_NAME)
        !          2371:                        ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
        !          2372:                 else if (ip_kotype(port) != IKOT_NONE)
        !          2373:                        panic("vm_object_remove: bad name port");
        !          2374:        }
        !          2375: #endif /* !NORMA_VM */
        !          2376: }
        !          2377: 
        !          2378: /*
        !          2379:  *     Global variables for vm_object_collapse():
        !          2380:  *
        !          2381:  *             Counts for normal collapses and bypasses.
        !          2382:  *             Debugging variables, to watch or disable collapse.
        !          2383:  */
        !          2384: long   object_collapses = 0;
        !          2385: long   object_bypasses  = 0;
        !          2386: 
        !          2387: int            vm_object_collapse_debug = 0;
        !          2388: boolean_t      vm_object_collapse_allowed = TRUE;
        !          2389: boolean_t      vm_object_collapse_bypass_allowed = TRUE;
        !          2390: 
        !          2391: /*
        !          2392:  *     vm_object_collapse:
        !          2393:  *
        !          2394:  *     Collapse an object with the object backing it.
        !          2395:  *     Pages in the backing object are moved into the
        !          2396:  *     parent, and the backing object is deallocated.
        !          2397:  *
        !          2398:  *     Requires that the object be locked and the page
        !          2399:  *     queues be unlocked.  May unlock/relock the object,
        !          2400:  *     so the caller should hold a reference for the object.
        !          2401:  */
        !          2402: void vm_object_collapse(
        !          2403:        register vm_object_t    object)
        !          2404: {
        !          2405:        register vm_object_t    backing_object;
        !          2406:        register vm_offset_t    backing_offset;
        !          2407:        register vm_size_t      size;
        !          2408:        register vm_offset_t    new_offset;
        !          2409:        register vm_page_t      p, pp;
        !          2410:        ipc_port_t old_name_port;
        !          2411: 
        !          2412:        if (!vm_object_collapse_allowed)
        !          2413:                return;
        !          2414: 
        !          2415:        while (TRUE) {
        !          2416:                /*
        !          2417:                 *      Verify that the conditions are right for collapse:
        !          2418:                 *
        !          2419:                 *      The object exists and no pages in it are currently
        !          2420:                 *      being paged out (or have ever been paged out).
        !          2421:                 *
        !          2422:                 *      This check is probably overkill -- if a memory
        !          2423:                 *      object has not been created, the fault handler
        !          2424:                 *      shouldn't release the object lock while paging
        !          2425:                 *      is in progress or absent pages exist.
        !          2426:                 */
        !          2427:                if (object == VM_OBJECT_NULL ||
        !          2428:                    object->pager_created ||
        !          2429:                    object->paging_in_progress != 0 ||
        !          2430:                    object->absent_count != 0)
        !          2431:                        return;
        !          2432: 
        !          2433:                /*
        !          2434:                 *              There is a backing object, and
        !          2435:                 */
        !          2436: 
        !          2437:                if ((backing_object = object->shadow) == VM_OBJECT_NULL)
        !          2438:                        return;
        !          2439: 
        !          2440:                vm_object_lock(backing_object);
        !          2441:                /*
        !          2442:                 *      ...
        !          2443:                 *              The backing object is not read_only,
        !          2444:                 *              and no pages in the backing object are
        !          2445:                 *              currently being paged out.
        !          2446:                 *              The backing object is internal.
        !          2447:                 *
        !          2448:                 *      XXX It may be sufficient for the backing
        !          2449:                 *      XXX object to be temporary.
        !          2450:                 */
        !          2451: 
        !          2452:                if (!backing_object->internal ||
        !          2453:                    backing_object->paging_in_progress != 0) {
        !          2454:                        vm_object_unlock(backing_object);
        !          2455:                        return;
        !          2456:                }
        !          2457: 
        !          2458:                /*
        !          2459:                 *      The backing object can't be a copy-object:
        !          2460:                 *      the shadow_offset for the copy-object must stay
        !          2461:                 *      as 0.  Furthermore (for the 'we have all the
        !          2462:                 *      pages' case), if we bypass backing_object and
        !          2463:                 *      just shadow the next object in the chain, old
        !          2464:                 *      pages from that object would then have to be copied
        !          2465:                 *      BOTH into the (former) backing_object and into the
        !          2466:                 *      parent object.
        !          2467:                 */
        !          2468:                if (backing_object->shadow != VM_OBJECT_NULL &&
        !          2469:                    backing_object->shadow->copy != VM_OBJECT_NULL) {
        !          2470:                        vm_object_unlock(backing_object);
        !          2471:                        return;
        !          2472:                }
        !          2473: 
        !          2474:                /*
        !          2475:                 *      We know that we can either collapse the backing
        !          2476:                 *      object (if the parent is the only reference to
        !          2477:                 *      it) or (perhaps) remove the parent's reference
        !          2478:                 *      to it.
        !          2479:                 */
        !          2480: 
        !          2481:                backing_offset = object->shadow_offset;
        !          2482:                size = object->size;
        !          2483: 
        !          2484:                /*
        !          2485:                 *      If there is exactly one reference to the backing
        !          2486:                 *      object, we can collapse it into the parent.
        !          2487:                 */
        !          2488: 
        !          2489:                if (backing_object->ref_count == 1) {
        !          2490:                        if (!vm_object_cache_lock_try()) {
        !          2491:                                vm_object_unlock(backing_object);
        !          2492:                                return;
        !          2493:                        }
        !          2494: 
        !          2495:                        /*
        !          2496:                         *      We can collapse the backing object.
        !          2497:                         *
        !          2498:                         *      Move all in-memory pages from backing_object
        !          2499:                         *      to the parent.  Pages that have been paged out
        !          2500:                         *      will be overwritten by any of the parent's
        !          2501:                         *      pages that shadow them.
        !          2502:                         */
        !          2503: 
        !          2504:                        while (!queue_empty(&backing_object->memq)) {
        !          2505: 
        !          2506:                                p = (vm_page_t)
        !          2507:                                        queue_first(&backing_object->memq);
        !          2508: 
        !          2509:                                new_offset = (p->offset - backing_offset);
        !          2510: 
        !          2511:                                assert(!p->busy || p->absent);
        !          2512: 
        !          2513:                                /*
        !          2514:                                 *      If the parent has a page here, or if
        !          2515:                                 *      this page falls outside the parent,
        !          2516:                                 *      dispose of it.
        !          2517:                                 *
        !          2518:                                 *      Otherwise, move it as planned.
        !          2519:                                 */
        !          2520: 
        !          2521:                                if (p->offset < backing_offset ||
        !          2522:                                    new_offset >= size) {
        !          2523:                                        vm_page_lock_queues();
        !          2524:                                        vm_page_free(p);
        !          2525:                                        vm_page_unlock_queues();
        !          2526:                                } else {
        !          2527:                                    pp = vm_page_lookup(object, new_offset);
        !          2528:                                    if (pp != VM_PAGE_NULL && !pp->absent) {
        !          2529:                                        /*
        !          2530:                                         *      Parent object has a real page.
        !          2531:                                         *      Throw away the backing object's
        !          2532:                                         *      page.
        !          2533:                                         */
        !          2534:                                        vm_page_lock_queues();
        !          2535:                                        vm_page_free(p);
        !          2536:                                        vm_page_unlock_queues();
        !          2537:                                    }
        !          2538:                                    else {
        !          2539:                                        if (pp != VM_PAGE_NULL) {
        !          2540:                                            /*
        !          2541:                                             *  Parent has an absent page...
        !          2542:                                             *  it's not being paged in, so
        !          2543:                                             *  it must really be missing from
        !          2544:                                             *  the parent.
        !          2545:                                             *
        !          2546:                                             *  Throw out the absent page...
        !          2547:                                             *  any faults looking for that
        !          2548:                                             *  page will restart with the new
        !          2549:                                             *  one.
        !          2550:                                             */
        !          2551: 
        !          2552:                                            /*
        !          2553:                                             *  This should never happen -- the
        !          2554:                                             *  parent cannot have ever had an
        !          2555:                                             *  external memory object, and thus
        !          2556:                                             *  cannot have absent pages.
        !          2557:                                             */
        !          2558:                                            panic("vm_object_collapse: bad case");
        !          2559: 
        !          2560:                                            vm_page_lock_queues();
        !          2561:                                            vm_page_free(pp);
        !          2562:                                            vm_page_unlock_queues();
        !          2563: 
        !          2564:                                            /*
        !          2565:                                             *  Fall through to move the backing
        !          2566:                                             *  object's page up.
        !          2567:                                             */
        !          2568:                                        }
        !          2569:                                        /*
        !          2570:                                         *      Parent now has no page.
        !          2571:                                         *      Move the backing object's page up.
        !          2572:                                         */
        !          2573:                                        vm_page_rename(p, object, new_offset);
        !          2574:                                    }
        !          2575:                                }
        !          2576:                        }
        !          2577: 
        !          2578:                        /*
        !          2579:                         *      Move the pager from backing_object to object.
        !          2580:                         *
        !          2581:                         *      XXX We're only using part of the paging space
        !          2582:                         *      for keeps now... we ought to discard the
        !          2583:                         *      unused portion.
        !          2584:                         */
        !          2585: 
        !          2586:                        switch (vm_object_collapse_debug) {
        !          2587:                            case 0:
        !          2588:                                break;
        !          2589:                            case 1:
        !          2590:                                if ((backing_object->pager == IP_NULL) &&
        !          2591:                                    (backing_object->pager_request ==
        !          2592:                                     PAGER_REQUEST_NULL))
        !          2593:                                    break;
        !          2594:                                /* Fall through to... */
        !          2595: 
        !          2596:                            default:
        !          2597:                                printf("vm_object_collapse: %#x (pager %#x, request %#x) up to %#x\n",
        !          2598:                                        backing_object, backing_object->pager, backing_object->pager_request,
        !          2599:                                        object);
        !          2600:                                if (vm_object_collapse_debug > 2)
        !          2601:                                    Debugger("vm_object_collapse");
        !          2602:                        }
        !          2603: 
        !          2604:                        object->pager = backing_object->pager;
        !          2605:                        if (object->pager != IP_NULL)
        !          2606:                                ipc_kobject_set(object->pager,
        !          2607:                                                (ipc_kobject_t) object,
        !          2608:                                                IKOT_PAGER);
        !          2609:                        object->pager_initialized = backing_object->pager_initialized;
        !          2610:                        object->pager_ready = backing_object->pager_ready;
        !          2611:                        object->pager_created = backing_object->pager_created;
        !          2612: 
        !          2613:                        object->pager_request = backing_object->pager_request;
        !          2614: #if    NORMA_VM
        !          2615:                        old_name_port = object->pager_name;
        !          2616:                        object->pager_name = backing_object->pager_name;
        !          2617: #else  /* NORMA_VM */
        !          2618:                        if (object->pager_request != IP_NULL)
        !          2619:                                ipc_kobject_set(object->pager_request,
        !          2620:                                                (ipc_kobject_t) object,
        !          2621:                                                IKOT_PAGING_REQUEST);
        !          2622:                        old_name_port = object->pager_name;
        !          2623:                        if (old_name_port != IP_NULL)
        !          2624:                                ipc_kobject_set(old_name_port,
        !          2625:                                                IKO_NULL, IKOT_NONE);
        !          2626:                        object->pager_name = backing_object->pager_name;
        !          2627:                        if (object->pager_name != IP_NULL)
        !          2628:                                ipc_kobject_set(object->pager_name,
        !          2629:                                                (ipc_kobject_t) object,
        !          2630:                                                IKOT_PAGING_NAME);
        !          2631: #endif /* NORMA_VM */
        !          2632: 
        !          2633:                        vm_object_cache_unlock();
        !          2634: 
        !          2635:                        /*
        !          2636:                         * If there is no pager, leave paging-offset alone.
        !          2637:                         */
        !          2638:                        if (object->pager != IP_NULL)
        !          2639:                                object->paging_offset =
        !          2640:                                        backing_object->paging_offset +
        !          2641:                                                backing_offset;
        !          2642: 
        !          2643: #if    MACH_PAGEMAP
        !          2644:                        assert(object->existence_info == VM_EXTERNAL_NULL);
        !          2645:                        object->existence_info = backing_object->existence_info;
        !          2646: #endif /* MACH_PAGEMAP */
        !          2647: 
        !          2648:                        /*
        !          2649:                         *      Object now shadows whatever backing_object did.
        !          2650:                         *      Note that the reference to backing_object->shadow
        !          2651:                         *      moves from within backing_object to within object.
        !          2652:                         */
        !          2653: 
        !          2654:                        object->shadow = backing_object->shadow;
        !          2655:                        object->shadow_offset += backing_object->shadow_offset;
        !          2656:                        if (object->shadow != VM_OBJECT_NULL &&
        !          2657:                            object->shadow->copy != VM_OBJECT_NULL) {
        !          2658:                                panic("vm_object_collapse: we collapsed a copy-object!");
        !          2659:                        }
        !          2660:                        /*
        !          2661:                         *      Discard backing_object.
        !          2662:                         *
        !          2663:                         *      Since the backing object has no pages, no
        !          2664:                         *      pager left, and no object references within it,
        !          2665:                         *      all that is necessary is to dispose of it.
        !          2666:                         */
        !          2667: 
        !          2668:                        assert(
        !          2669:                                (backing_object->ref_count == 1) &&
        !          2670:                                (backing_object->resident_page_count == 0) &&
        !          2671:                                (backing_object->paging_in_progress == 0)
        !          2672:                        );
        !          2673: 
        !          2674:                        assert(backing_object->alive);
        !          2675:                        backing_object->alive = FALSE;
        !          2676:                        vm_object_unlock(backing_object);
        !          2677: 
        !          2678:                        vm_object_unlock(object);
        !          2679:                        if (old_name_port != IP_NULL)
        !          2680: #if    NORMA_VM
        !          2681:                                ipc_port_release_send(old_name_port);
        !          2682: #else  /* NORMA_VM */
        !          2683:                                ipc_port_dealloc_kernel(old_name_port);
        !          2684: #endif /* NORMA_VM */
        !          2685:                        zfree(vm_object_zone, (vm_offset_t) backing_object);
        !          2686:                        vm_object_lock(object);
        !          2687: 
        !          2688:                        object_collapses++;
        !          2689:                }
        !          2690:                else {
        !          2691:                        if (!vm_object_collapse_bypass_allowed) {
        !          2692:                                vm_object_unlock(backing_object);
        !          2693:                                return;
        !          2694:                        }
        !          2695: 
        !          2696:                        /*
        !          2697:                         *      If all of the pages in the backing object are
        !          2698:                         *      shadowed by the parent object, the parent
        !          2699:                         *      object no longer has to shadow the backing
        !          2700:                         *      object; it can shadow the next one in the
        !          2701:                         *      chain.
        !          2702:                         *
        !          2703:                         *      The backing object must not be paged out - we'd
        !          2704:                         *      have to check all of the paged-out pages, as
        !          2705:                         *      well.
        !          2706:                         */
        !          2707: 
        !          2708:                        if (backing_object->pager_created) {
        !          2709:                                vm_object_unlock(backing_object);
        !          2710:                                return;
        !          2711:                        }
        !          2712: 
        !          2713:                        /*
        !          2714:                         *      Should have a check for a 'small' number
        !          2715:                         *      of pages here.
        !          2716:                         */
        !          2717: 
        !          2718:                        queue_iterate(&backing_object->memq, p,
        !          2719:                                      vm_page_t, listq)
        !          2720:                        {
        !          2721:                                new_offset = (p->offset - backing_offset);
        !          2722: 
        !          2723:                                /*
        !          2724:                                 *      If the parent has a page here, or if
        !          2725:                                 *      this page falls outside the parent,
        !          2726:                                 *      keep going.
        !          2727:                                 *
        !          2728:                                 *      Otherwise, the backing_object must be
        !          2729:                                 *      left in the chain.
        !          2730:                                 */
        !          2731: 
        !          2732:                                if (p->offset >= backing_offset &&
        !          2733:                                    new_offset <= size &&
        !          2734:                                    (pp = vm_page_lookup(object, new_offset))
        !          2735:                                      == VM_PAGE_NULL) {
        !          2736:                                        /*
        !          2737:                                         *      Page still needed.
        !          2738:                                         *      Can't go any further.
        !          2739:                                         */
        !          2740:                                        vm_object_unlock(backing_object);
        !          2741:                                        return;
        !          2742:                                }
        !          2743:                        }
        !          2744: 
        !          2745:                        /*
        !          2746:                         *      Make the parent shadow the next object
        !          2747:                         *      in the chain.  Deallocating backing_object
        !          2748:                         *      will not remove it, since its reference
        !          2749:                         *      count is at least 2.
        !          2750:                         */
        !          2751: 
        !          2752:                        vm_object_reference(object->shadow = backing_object->shadow);
        !          2753:                        object->shadow_offset += backing_object->shadow_offset;
        !          2754: 
        !          2755:                        /*
        !          2756:                         *      Backing object might have had a copy pointer
        !          2757:                         *      to us.  If it did, clear it.
        !          2758:                         */
        !          2759:                        if (backing_object->copy == object)
        !          2760:                                backing_object->copy = VM_OBJECT_NULL;
        !          2761: 
        !          2762:                        /*
        !          2763:                         *      Drop the reference count on backing_object.
        !          2764:                         *      Since its ref_count was at least 2, it
        !          2765:                         *      will not vanish; so we don't need to call
        !          2766:                         *      vm_object_deallocate.
        !          2767:                         */
        !          2768:                        backing_object->ref_count--;
        !          2769:                        assert(backing_object->ref_count > 0);
        !          2770:                        vm_object_unlock(backing_object);
        !          2771: 
        !          2772:                        object_bypasses ++;
        !          2773: 
        !          2774:                }
        !          2775: 
        !          2776:                /*
        !          2777:                 *      Try again with this object's new backing object.
        !          2778:                 */
        !          2779:        }
        !          2780: }
        !          2781: 
        !          2782: /*
        !          2783:  *     Routine:        vm_object_page_remove: [internal]
        !          2784:  *     Purpose:
        !          2785:  *             Removes all physical pages in the specified
        !          2786:  *             object range from the object's list of pages.
        !          2787:  *
        !          2788:  *     In/out conditions:
        !          2789:  *             The object must be locked.
        !          2790:  */
        !          2791: unsigned int vm_object_page_remove_lookup = 0;
        !          2792: unsigned int vm_object_page_remove_iterate = 0;
        !          2793: 
        !          2794: void vm_object_page_remove(
        !          2795:        register vm_object_t    object,
        !          2796:        register vm_offset_t    start,
        !          2797:        register vm_offset_t    end)
        !          2798: {
        !          2799:        register vm_page_t      p, next;
        !          2800: 
        !          2801:        /*
        !          2802:         *      One and two page removals are most popular.
        !          2803:         *      The factor of 16 here is somewhat arbitrary.
        !          2804:         *      It balances vm_object_lookup vs iteration.
        !          2805:         */
        !          2806: 
        !          2807:        if (atop(end - start) < (unsigned)object->resident_page_count/16) {
        !          2808:                vm_object_page_remove_lookup++;
        !          2809: 
        !          2810:                for (; start < end; start += PAGE_SIZE) {
        !          2811:                        p = vm_page_lookup(object, start);
        !          2812:                        if (p != VM_PAGE_NULL) {
        !          2813:                                if (!p->fictitious)
        !          2814:                                        pmap_page_protect(p->phys_addr,
        !          2815:                                                          VM_PROT_NONE);
        !          2816:                                vm_page_lock_queues();
        !          2817:                                vm_page_free(p);
        !          2818:                                vm_page_unlock_queues();
        !          2819:                        }
        !          2820:                }
        !          2821:        } else {
        !          2822:                vm_object_page_remove_iterate++;
        !          2823: 
        !          2824:                p = (vm_page_t) queue_first(&object->memq);
        !          2825:                while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !          2826:                        next = (vm_page_t) queue_next(&p->listq);
        !          2827:                        if ((start <= p->offset) && (p->offset < end)) {
        !          2828:                                if (!p->fictitious)
        !          2829:                                    pmap_page_protect(p->phys_addr,
        !          2830:                                                      VM_PROT_NONE);
        !          2831:                                vm_page_lock_queues();
        !          2832:                                vm_page_free(p);
        !          2833:                                vm_page_unlock_queues();
        !          2834:                        }
        !          2835:                        p = next;
        !          2836:                }
        !          2837:        }
        !          2838: }
        !          2839: 
        !          2840: /*
        !          2841:  *     Routine:        vm_object_coalesce
        !          2842:  *     Function:       Coalesces two objects backing up adjoining
        !          2843:  *                     regions of memory into a single object.
        !          2844:  *
        !          2845:  *     returns TRUE if objects were combined.
        !          2846:  *
        !          2847:  *     NOTE:   Only works at the moment if the second object is NULL -
        !          2848:  *             if it's not, which object do we lock first?
        !          2849:  *
        !          2850:  *     Parameters:
        !          2851:  *             prev_object     First object to coalesce
        !          2852:  *             prev_offset     Offset into prev_object
        !          2853:  *             next_object     Second object into coalesce
        !          2854:  *             next_offset     Offset into next_object
        !          2855:  *
        !          2856:  *             prev_size       Size of reference to prev_object
        !          2857:  *             next_size       Size of reference to next_object
        !          2858:  *
        !          2859:  *     Conditions:
        !          2860:  *     The object must *not* be locked.
        !          2861:  */
        !          2862: 
        !          2863: boolean_t vm_object_coalesce(
        !          2864:        register vm_object_t prev_object,
        !          2865:        vm_object_t     next_object,
        !          2866:        vm_offset_t     prev_offset,
        !          2867:        vm_offset_t     next_offset,
        !          2868:        vm_size_t       prev_size,
        !          2869:        vm_size_t       next_size)
        !          2870: {
        !          2871:        vm_size_t       newsize;
        !          2872: 
        !          2873: #ifdef lint
        !          2874:        next_offset++;
        !          2875: #endif /* lint */
        !          2876: 
        !          2877:        if (next_object != VM_OBJECT_NULL) {
        !          2878:                return FALSE;
        !          2879:        }
        !          2880: 
        !          2881:        if (prev_object == VM_OBJECT_NULL) {
        !          2882:                return TRUE;
        !          2883:        }
        !          2884: 
        !          2885:        vm_object_lock(prev_object);
        !          2886: 
        !          2887:        /*
        !          2888:         *      Try to collapse the object first
        !          2889:         */
        !          2890:        vm_object_collapse(prev_object);
        !          2891: 
        !          2892:        /*
        !          2893:         *      Can't coalesce if pages not mapped to
        !          2894:         *      prev_entry may be in use anyway:
        !          2895:         *      . more than one reference
        !          2896:         *      . paged out
        !          2897:         *      . shadows another object
        !          2898:         *      . has a copy elsewhere
        !          2899:         *      . paging references (pages might be in page-list)
        !          2900:         */
        !          2901: 
        !          2902:        if ((prev_object->ref_count > 1) ||
        !          2903:            prev_object->pager_created ||
        !          2904:            (prev_object->shadow != VM_OBJECT_NULL) ||
        !          2905:            (prev_object->copy != VM_OBJECT_NULL) ||
        !          2906:            (prev_object->paging_in_progress != 0)) {
        !          2907:                vm_object_unlock(prev_object);
        !          2908:                return FALSE;
        !          2909:        }
        !          2910: 
        !          2911:        /*
        !          2912:         *      Remove any pages that may still be in the object from
        !          2913:         *      a previous deallocation.
        !          2914:         */
        !          2915: 
        !          2916:        vm_object_page_remove(prev_object,
        !          2917:                        prev_offset + prev_size,
        !          2918:                        prev_offset + prev_size + next_size);
        !          2919: 
        !          2920:        /*
        !          2921:         *      Extend the object if necessary.
        !          2922:         */
        !          2923:        newsize = prev_offset + prev_size + next_size;
        !          2924:        if (newsize > prev_object->size)
        !          2925:                prev_object->size = newsize;
        !          2926: 
        !          2927:        vm_object_unlock(prev_object);
        !          2928:        return TRUE;
        !          2929: }
        !          2930: 
        !          2931: vm_object_t    vm_object_request_object(
        !          2932:        ipc_port_t      p)
        !          2933: {
        !          2934:        return vm_object_lookup(p);
        !          2935: }
        !          2936: 
        !          2937: /*
        !          2938:  *     Routine:        vm_object_name
        !          2939:  *     Purpose:
        !          2940:  *             Returns a naked send right to the "name" port associated
        !          2941:  *             with this object.
        !          2942:  */
        !          2943: ipc_port_t     vm_object_name(
        !          2944:        vm_object_t     object)
        !          2945: {
        !          2946:        ipc_port_t      p;
        !          2947: 
        !          2948:        if (object == VM_OBJECT_NULL)
        !          2949:                return IP_NULL;
        !          2950: 
        !          2951:        vm_object_lock(object);
        !          2952: 
        !          2953:        while (object->shadow != VM_OBJECT_NULL) {
        !          2954:                vm_object_t     new_object = object->shadow;
        !          2955:                vm_object_lock(new_object);
        !          2956:                vm_object_unlock(object);
        !          2957:                object = new_object;
        !          2958:        }
        !          2959: 
        !          2960:        p = object->pager_name;
        !          2961:        if (p != IP_NULL)
        !          2962: #if    NORMA_VM
        !          2963:                p = ipc_port_copy_send(p);
        !          2964: #else  /* NORMA_VM */
        !          2965:                p = ipc_port_make_send(p);
        !          2966: #endif /* NORMA_VM */
        !          2967:        vm_object_unlock(object);
        !          2968: 
        !          2969:        return p;
        !          2970: }
        !          2971: 
        !          2972: /*
        !          2973:  *     Attach a set of physical pages to an object, so that they can
        !          2974:  *     be mapped by mapping the object.  Typically used to map IO memory.
        !          2975:  *
        !          2976:  *     The mapping function and its private data are used to obtain the
        !          2977:  *     physical addresses for each page to be mapped.
        !          2978:  */
        !          2979: void
        !          2980: vm_object_page_map(
        !          2981:        vm_object_t     object,
        !          2982:        vm_offset_t     offset,
        !          2983:        vm_size_t       size,
        !          2984:        vm_offset_t     (*map_fn)(void *, vm_offset_t),
        !          2985:        void *          map_fn_data)    /* private to map_fn */
        !          2986: {
        !          2987:        int     num_pages;
        !          2988:        int     i;
        !          2989:        vm_page_t       m;
        !          2990:        vm_page_t       old_page;
        !          2991:        vm_offset_t     addr;
        !          2992: 
        !          2993:        num_pages = atop(size);
        !          2994: 
        !          2995:        for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
        !          2996: 
        !          2997:            addr = (*map_fn)(map_fn_data, offset);
        !          2998: 
        !          2999:            while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
        !          3000:                vm_page_more_fictitious();
        !          3001: 
        !          3002:            vm_object_lock(object);
        !          3003:            if ((old_page = vm_page_lookup(object, offset))
        !          3004:                        != VM_PAGE_NULL)
        !          3005:            {
        !          3006:                vm_page_lock_queues();
        !          3007:                vm_page_free(old_page);
        !          3008:                vm_page_unlock_queues();
        !          3009:            }
        !          3010: 
        !          3011:            vm_page_init(m, addr);
        !          3012:            m->private = TRUE;          /* don`t free page */
        !          3013:            m->wire_count = 1;
        !          3014:            vm_page_lock_queues();
        !          3015:            vm_page_insert(m, object, offset);
        !          3016:            vm_page_unlock_queues();
        !          3017: 
        !          3018:            PAGE_WAKEUP_DONE(m);
        !          3019:            vm_object_unlock(object);
        !          3020:        }
        !          3021: }
        !          3022: 
        !          3023: #include <mach_kdb.h>
        !          3024: 
        !          3025: 
        !          3026: #if    MACH_KDB
        !          3027: #define printf kdbprintf
        !          3028: 
        !          3029: boolean_t      vm_object_print_pages = FALSE;
        !          3030: 
        !          3031: /*
        !          3032:  *     vm_object_print:        [ debug ]
        !          3033:  */
        !          3034: void vm_object_print(
        !          3035:        vm_object_t     object)
        !          3036: {
        !          3037:        register vm_page_t      p;
        !          3038:        extern indent;
        !          3039: 
        !          3040:        register int count;
        !          3041: 
        !          3042:        if (object == VM_OBJECT_NULL)
        !          3043:                return;
        !          3044: 
        !          3045:        iprintf("Object 0x%X: size=0x%X",
        !          3046:                (vm_offset_t) object, (vm_offset_t) object->size);
        !          3047:         printf(", %d references, %d resident pages,", object->ref_count,
        !          3048:                object->resident_page_count);
        !          3049:         printf(" %d absent pages,", object->absent_count);
        !          3050:         printf(" %d paging ops\n", object->paging_in_progress);
        !          3051:        indent += 2;
        !          3052:        iprintf("memory object=0x%X (offset=0x%X),",
        !          3053:                 (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
        !          3054:         printf("control=0x%X, name=0x%X\n",
        !          3055:                (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
        !          3056:        iprintf("%s%s",
        !          3057:                object->pager_ready ? " ready" : "",
        !          3058:                object->pager_created ? " created" : "");
        !          3059:         printf("%s,%s ",
        !          3060:                object->pager_initialized ? "" : "uninitialized",
        !          3061:                object->temporary ? "temporary" : "permanent");
        !          3062:         printf("%s%s,",
        !          3063:                object->internal ? "internal" : "external",
        !          3064:                object->can_persist ? " cacheable" : "");
        !          3065:         printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
        !          3066:        iprintf("shadow=0x%X (offset=0x%X),",
        !          3067:                (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
        !          3068:         printf("copy=0x%X\n", (vm_offset_t) object->copy);
        !          3069: 
        !          3070:        indent += 2;
        !          3071: 
        !          3072:        if (vm_object_print_pages) {
        !          3073:                count = 0;
        !          3074:                p = (vm_page_t) queue_first(&object->memq);
        !          3075:                while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !          3076:                        if (count == 0) iprintf("memory:=");
        !          3077:                        else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
        !          3078:                        else printf(",");
        !          3079:                        count++;
        !          3080: 
        !          3081:                        printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
        !          3082:                        p = (vm_page_t) queue_next(&p->listq);
        !          3083:                }
        !          3084:                if (count != 0)
        !          3085:                        printf("\n");
        !          3086:        }
        !          3087:        indent -= 4;
        !          3088: }
        !          3089: 
        !          3090: #endif /* MACH_KDB */

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.