Return to vm_object.c CVS log

Up to [Apple XNU] / XNU / osfmk / vm

Annotation of XNU/osfmk/vm/vm_object.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
        !             3:  *
        !             4:  * @APPLE_LICENSE_HEADER_START@
        !             5:  * 
        !             6:  * The contents of this file constitute Original Code as defined in and
        !             7:  * are subject to the Apple Public Source License Version 1.1 (the
        !             8:  * "License").  You may not use this file except in compliance with the
        !             9:  * License.  Please obtain a copy of the License at
        !            10:  * http://www.apple.com/publicsource and read it before using this file.
        !            11:  * 
        !            12:  * This Original Code and all software distributed under the License are
        !            13:  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
        !            14:  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
        !            15:  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
        !            16:  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
        !            17:  * License for the specific language governing rights and limitations
        !            18:  * under the License.
        !            19:  * 
        !            20:  * @APPLE_LICENSE_HEADER_END@
        !            21:  */
        !            22: /*
        !            23:  * @OSF_COPYRIGHT@
        !            24:  */
        !            25: /* 
        !            26:  * Mach Operating System
        !            27:  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
        !            28:  * All Rights Reserved.
        !            29:  * 
        !            30:  * Permission to use, copy, modify and distribute this software and its
        !            31:  * documentation is hereby granted, provided that both the copyright
        !            32:  * notice and this permission notice appear in all copies of the
        !            33:  * software, derivative works or modified versions, and any portions
        !            34:  * thereof, and that both notices appear in supporting documentation.
        !            35:  * 
        !            36:  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
        !            37:  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
        !            38:  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
        !            39:  * 
        !            40:  * Carnegie Mellon requests users of this software to return to
        !            41:  * 
        !            42:  *  Software Distribution Coordinator  or  [email protected]
        !            43:  *  School of Computer Science
        !            44:  *  Carnegie Mellon University
        !            45:  *  Pittsburgh PA 15213-3890
        !            46:  * 
        !            47:  * any improvements or extensions that they make and grant Carnegie Mellon
        !            48:  * the rights to redistribute these changes.
        !            49:  */
        !            50: /*
        !            51:  */
        !            52: /*
        !            53:  *     File:   vm/vm_object.c
        !            54:  *     Author: Avadis Tevanian, Jr., Michael Wayne Young
        !            55:  *
        !            56:  *     Virtual memory object module.
        !            57:  */
        !            58: 
        !            59: #ifdef MACH_BSD
        !            60: /* remove as part of compoenent support merge */
        !            61: extern int     vnode_pager_workaround;
        !            62: #endif
        !            63: 
        !            64: #include <mach_pagemap.h>
        !            65: #include <task_swapper.h>
        !            66: 
        !            67: #include <mach/memory_object.h>
        !            68: #include <mach/memory_object_default.h>
        !            69: #include <mach/memory_object_control_server.h>
        !            70: #include <mach/vm_param.h>
        !            71: #include <ipc/ipc_port.h>
        !            72: #include <ipc/ipc_space.h>
        !            73: #include <kern/assert.h>
        !            74: #include <kern/lock.h>
        !            75: #include <kern/queue.h>
        !            76: #include <kern/xpr.h>
        !            77: #include <kern/zalloc.h>
        !            78: #include <kern/host.h>
        !            79: #include <kern/host_statistics.h>
        !            80: #include <vm/memory_object.h>
        !            81: #include <vm/vm_fault.h>
        !            82: #include <vm/vm_map.h>
        !            83: #include <vm/vm_object.h>
        !            84: #include <vm/vm_page.h>
        !            85: #include <vm/vm_pageout.h>
        !            86: #include <kern/misc_protos.h>
        !            87: 
        !            88: /*
        !            89:  *     Virtual memory objects maintain the actual data
        !            90:  *     associated with allocated virtual memory.  A given
        !            91:  *     page of memory exists within exactly one object.
        !            92:  *
        !            93:  *     An object is only deallocated when all "references"
        !            94:  *     are given up.  Only one "reference" to a given
        !            95:  *     region of an object should be writeable.
        !            96:  *
        !            97:  *     Associated with each object is a list of all resident
        !            98:  *     memory pages belonging to that object; this list is
        !            99:  *     maintained by the "vm_page" module, but locked by the object's
        !           100:  *     lock.
        !           101:  *
        !           102:  *     Each object also records the memory object port
        !           103:  *     that is used by the kernel to request and write
        !           104:  *     back data (the memory object port, field "pager"),
        !           105:  *     and the ports provided to the memory manager, the server that
        !           106:  *     manages that data, to return data and control its
        !           107:  *     use (the memory object control port, field "pager_request")
        !           108:  *     and for naming (the memory object name port, field "pager_name").
        !           109:  *
        !           110:  *     Virtual memory objects are allocated to provide
        !           111:  *     zero-filled memory (vm_allocate) or map a user-defined
        !           112:  *     memory object into a virtual address space (vm_map).
        !           113:  *
        !           114:  *     Virtual memory objects that refer to a user-defined
        !           115:  *     memory object are called "permanent", because all changes
        !           116:  *     made in virtual memory are reflected back to the
        !           117:  *     memory manager, which may then store it permanently.
        !           118:  *     Other virtual memory objects are called "temporary",
        !           119:  *     meaning that changes need be written back only when
        !           120:  *     necessary to reclaim pages, and that storage associated
        !           121:  *     with the object can be discarded once it is no longer
        !           122:  *     mapped.
        !           123:  *
        !           124:  *     A permanent memory object may be mapped into more
        !           125:  *     than one virtual address space.  Moreover, two threads
        !           126:  *     may attempt to make the first mapping of a memory
        !           127:  *     object concurrently.  Only one thread is allowed to
        !           128:  *     complete this mapping; all others wait for the
        !           129:  *     "pager_initialized" field is asserted, indicating
        !           130:  *     that the first thread has initialized all of the
        !           131:  *     necessary fields in the virtual memory object structure.
        !           132:  *
        !           133:  *     The kernel relies on a *default memory manager* to
        !           134:  *     provide backing storage for the zero-filled virtual
        !           135:  *     memory objects.  The memory object ports associated
        !           136:  *     with these temporary virtual memory objects are only
        !           137:  *     generated and passed to the default memory manager
        !           138:  *     when it becomes necessary.  Virtual memory objects
        !           139:  *     that depend on the default memory manager are called
        !           140:  *     "internal".  The "pager_created" field is provided to
        !           141:  *     indicate whether these ports have ever been allocated.
        !           142:  *     
        !           143:  *     The kernel may also create virtual memory objects to
        !           144:  *     hold changed pages after a copy-on-write operation.
        !           145:  *     In this case, the virtual memory object (and its
        !           146:  *     backing storage -- its memory object) only contain
        !           147:  *     those pages that have been changed.  The "shadow"
        !           148:  *     field refers to the virtual memory object that contains
        !           149:  *     the remainder of the contents.  The "shadow_offset"
        !           150:  *     field indicates where in the "shadow" these contents begin.
        !           151:  *     The "copy" field refers to a virtual memory object
        !           152:  *     to which changed pages must be copied before changing
        !           153:  *     this object, in order to implement another form
        !           154:  *     of copy-on-write optimization.
        !           155:  *
        !           156:  *     The virtual memory object structure also records
        !           157:  *     the attributes associated with its memory object.
        !           158:  *     The "pager_ready", "can_persist" and "copy_strategy"
        !           159:  *     fields represent those attributes.  The "cached_list"
        !           160:  *     field is used in the implementation of the persistence
        !           161:  *     attribute.
        !           162:  *
        !           163:  * ZZZ Continue this comment.
        !           164:  */
        !           165: 
        !           166: /* Forward declarations for internal functions. */
        !           167: extern void            _vm_object_allocate(
        !           168:                                vm_size_t       size,
        !           169:                                vm_object_t     object);
        !           170: 
        !           171: extern void            vm_object_terminate(
        !           172:                                vm_object_t     object);
        !           173: 
        !           174: extern void            vm_object_remove(
        !           175:                                vm_object_t     object);
        !           176: 
        !           177: extern vm_object_t     vm_object_cache_trim(
        !           178:                                boolean_t called_from_vm_object_deallocate);
        !           179: 
        !           180: extern void            vm_object_deactivate_pages(
        !           181:                                vm_object_t     object);
        !           182: 
        !           183: extern void            vm_object_abort_activity(
        !           184:                                vm_object_t     object);
        !           185: 
        !           186: extern kern_return_t   vm_object_copy_call(
        !           187:                                vm_object_t     src_object,
        !           188:                                vm_offset_t     src_offset,
        !           189:                                vm_size_t       size,
        !           190:                                vm_object_t     *_result_object);
        !           191: 
        !           192: extern void            vm_object_do_collapse(
        !           193:                                vm_object_t     object,
        !           194:                                vm_object_t     backing_object);
        !           195: 
        !           196: extern void            vm_object_do_bypass(
        !           197:                                vm_object_t     object,
        !           198:                                vm_object_t     backing_object);
        !           199: 
        !           200: extern void            memory_object_release(
        !           201:                                ipc_port_t      pager,
        !           202:                                pager_request_t pager_request);
        !           203: 
        !           204: zone_t         vm_object_zone;         /* vm backing store zone */
        !           205: 
        !           206: /*
        !           207:  *     All wired-down kernel memory belongs to a single virtual
        !           208:  *     memory object (kernel_object) to avoid wasting data structures.
        !           209:  */
        !           210: struct vm_object       kernel_object_store;
        !           211: vm_object_t            kernel_object = &kernel_object_store;
        !           212: 
        !           213: /*
        !           214:  *     The submap object is used as a placeholder for vm_map_submap
        !           215:  *     operations.  The object is declared in vm_map.c because it
        !           216:  *     is exported by the vm_map module.  The storage is declared
        !           217:  *     here because it must be initialized here.
        !           218:  */
        !           219: struct vm_object       vm_submap_object_store;
        !           220: 
        !           221: /*
        !           222:  *     Virtual memory objects are initialized from
        !           223:  *     a template (see vm_object_allocate).
        !           224:  *
        !           225:  *     When adding a new field to the virtual memory
        !           226:  *     object structure, be sure to add initialization
        !           227:  *     (see vm_object_init).
        !           228:  */
        !           229: struct vm_object       vm_object_template;
        !           230: 
        !           231: /*
        !           232:  *     Virtual memory objects that are not referenced by
        !           233:  *     any address maps, but that are allowed to persist
        !           234:  *     (an attribute specified by the associated memory manager),
        !           235:  *     are kept in a queue (vm_object_cached_list).
        !           236:  *
        !           237:  *     When an object from this queue is referenced again,
        !           238:  *     for example to make another address space mapping,
        !           239:  *     it must be removed from the queue.  That is, the
        !           240:  *     queue contains *only* objects with zero references.
        !           241:  *
        !           242:  *     The kernel may choose to terminate objects from this
        !           243:  *     queue in order to reclaim storage.  The current policy
        !           244:  *     is to permit a fixed maximum number of unreferenced
        !           245:  *     objects (vm_object_cached_max).
        !           246:  *
        !           247:  *     A spin lock (accessed by routines
        !           248:  *     vm_object_cache_{lock,lock_try,unlock}) governs the
        !           249:  *     object cache.  It must be held when objects are
        !           250:  *     added to or removed from the cache (in vm_object_terminate).
        !           251:  *     The routines that acquire a reference to a virtual
        !           252:  *     memory object based on one of the memory object ports
        !           253:  *     must also lock the cache.
        !           254:  *
        !           255:  *     Ideally, the object cache should be more isolated
        !           256:  *     from the reference mechanism, so that the lock need
        !           257:  *     not be held to make simple references.
        !           258:  */
        !           259: queue_head_t   vm_object_cached_list;
        !           260: int            vm_object_cached_count;
        !           261: int            vm_object_cached_high;  /* highest # of cached objects */
        !           262: int            vm_object_cached_max = 500;     /* may be patched*/
        !           263: 
        !           264: decl_mutex_data(,vm_object_cached_lock_data)
        !           265: 
        !           266: #define vm_object_cache_lock()         \
        !           267:                mutex_lock(&vm_object_cached_lock_data)
        !           268: #define vm_object_cache_lock_try()     \
        !           269:                mutex_try(&vm_object_cached_lock_data)
        !           270: #define vm_object_cache_unlock()       \
        !           271:                mutex_unlock(&vm_object_cached_lock_data)
        !           272: 
        !           273: #define        VM_OBJECT_HASH_COUNT            1024
        !           274: queue_head_t   vm_object_hashtable[VM_OBJECT_HASH_COUNT];
        !           275: struct zone    *vm_object_hash_zone;
        !           276: 
        !           277: struct vm_object_hash_entry {
        !           278:        queue_chain_t           hash_link;      /* hash chain link */
        !           279:        ipc_port_t              pager;          /* pager we represent */
        !           280:        vm_object_t             object;         /* corresponding object */
        !           281:        boolean_t               waiting;        /* someone waiting for
        !           282:                                                 * termination */
        !           283: };
        !           284: 
        !           285: typedef struct vm_object_hash_entry    *vm_object_hash_entry_t;
        !           286: #define VM_OBJECT_HASH_ENTRY_NULL      ((vm_object_hash_entry_t) 0)
        !           287: 
        !           288: #define VM_OBJECT_HASH_SHIFT   8
        !           289: #define vm_object_hash(pager) \
        !           290:        ((((unsigned)pager) >> VM_OBJECT_HASH_SHIFT) % VM_OBJECT_HASH_COUNT)
        !           291: 
        !           292: /*
        !           293:  *     vm_object_hash_lookup looks up a pager in the hashtable
        !           294:  *     and returns the corresponding entry, with optional removal.
        !           295:  */
        !           296: 
        !           297: vm_object_hash_entry_t
        !           298: vm_object_hash_lookup(
        !           299:        ipc_port_t      pager,
        !           300:        boolean_t       remove_entry)
        !           301: {
        !           302:        register queue_t                bucket;
        !           303:        register vm_object_hash_entry_t entry;
        !           304: 
        !           305:        bucket = &vm_object_hashtable[vm_object_hash(pager)];
        !           306: 
        !           307:        entry = (vm_object_hash_entry_t)queue_first(bucket);
        !           308:        while (!queue_end(bucket, (queue_entry_t)entry)) {
        !           309:                if (entry->pager == pager && !remove_entry)
        !           310:                        return(entry);
        !           311:                else if (entry->pager == pager) {
        !           312:                        queue_remove(bucket, entry,
        !           313:                                        vm_object_hash_entry_t, hash_link);
        !           314:                        return(entry);
        !           315:                }
        !           316: 
        !           317:                entry = (vm_object_hash_entry_t)queue_next(&entry->hash_link);
        !           318:        }
        !           319: 
        !           320:        return(VM_OBJECT_HASH_ENTRY_NULL);
        !           321: }
        !           322: 
        !           323: /*
        !           324:  *     vm_object_hash_enter enters the specified
        !           325:  *     pager / cache object association in the hashtable.
        !           326:  */
        !           327: 
        !           328: void
        !           329: vm_object_hash_insert(
        !           330:        vm_object_hash_entry_t  entry)
        !           331: {
        !           332:        register queue_t                bucket;
        !           333: 
        !           334:        bucket = &vm_object_hashtable[vm_object_hash(entry->pager)];
        !           335: 
        !           336:        queue_enter(bucket, entry, vm_object_hash_entry_t, hash_link);
        !           337: }
        !           338: 
        !           339: vm_object_hash_entry_t
        !           340: vm_object_hash_entry_alloc(
        !           341:        ipc_port_t      pager)
        !           342: {
        !           343:        vm_object_hash_entry_t  entry;
        !           344: 
        !           345:        entry = (vm_object_hash_entry_t)zalloc(vm_object_hash_zone);
        !           346:        entry->pager = pager;
        !           347:        entry->object = VM_OBJECT_NULL;
        !           348:        entry->waiting = FALSE;
        !           349: 
        !           350:        return(entry);
        !           351: }
        !           352: 
        !           353: void
        !           354: vm_object_hash_entry_free(
        !           355:        vm_object_hash_entry_t  entry)
        !           356: {
        !           357:        zfree(vm_object_hash_zone, (vm_offset_t)entry);
        !           358: }
        !           359: 
        !           360: /*
        !           361:  *     vm_object_allocate:
        !           362:  *
        !           363:  *     Returns a new object with the given size.
        !           364:  */
        !           365: 
        !           366: void
        !           367: _vm_object_allocate(
        !           368:        vm_size_t       size,
        !           369:        vm_object_t     object)
        !           370: {
        !           371:        XPR(XPR_VM_OBJECT,
        !           372:                "vm_object_allocate, object 0x%X size 0x%X\n",
        !           373:                (integer_t)object, size, 0,0,0);
        !           374: 
        !           375:        *object = vm_object_template;
        !           376:        queue_init(&object->memq);
        !           377:        queue_init(&object->msr_q);
        !           378:        vm_object_lock_init(object);
        !           379:        object->size = size;
        !           380: }
        !           381: 
        !           382: vm_object_t
        !           383: vm_object_allocate(
        !           384:        vm_size_t       size)
        !           385: {
        !           386:        register vm_object_t object;
        !           387:        register ipc_port_t port;
        !           388: 
        !           389:        object = (vm_object_t) zalloc(vm_object_zone);
        !           390:        
        !           391: //     dbgLog(object, size, 0, 2);                                             /* (TEST/DEBUG) */
        !           392:        
        !           393:        _vm_object_allocate(size, object);
        !           394: 
        !           395:        return object;
        !           396: }
        !           397: 
        !           398: /*
        !           399:  *     vm_object_bootstrap:
        !           400:  *
        !           401:  *     Initialize the VM objects module.
        !           402:  */
        !           403: void
        !           404: vm_object_bootstrap(void)
        !           405: {
        !           406:        register        i;
        !           407: 
        !           408:        vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
        !           409:                                round_page(512*1024),
        !           410:                                round_page(12*1024),
        !           411:                                "vm objects");
        !           412: 
        !           413:        queue_init(&vm_object_cached_list);
        !           414:        mutex_init(&vm_object_cached_lock_data, ETAP_VM_OBJ_CACHE);
        !           415: 
        !           416:        vm_object_hash_zone =
        !           417:                        zinit((vm_size_t) sizeof (struct vm_object_hash_entry),
        !           418:                              round_page(512*1024),
        !           419:                              round_page(12*1024),
        !           420:                              "vm object hash entries");
        !           421: 
        !           422:        for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
        !           423:                queue_init(&vm_object_hashtable[i]);
        !           424: 
        !           425:        /*
        !           426:         *      Fill in a template object, for quick initialization
        !           427:         */
        !           428: 
        !           429:        /* memq; Lock; init after allocation */
        !           430:        vm_object_template.size = 0;
        !           431:        vm_object_template.frozen_size = 0;
        !           432:        vm_object_template.ref_count = 1;
        !           433: #if    TASK_SWAPPER
        !           434:        vm_object_template.res_count = 1;
        !           435: #endif /* TASK_SWAPPER */
        !           436:        vm_object_template.resident_page_count = 0;
        !           437:        vm_object_template.copy = VM_OBJECT_NULL;
        !           438:        vm_object_template.shadow = VM_OBJECT_NULL;
        !           439:        vm_object_template.shadow_offset = (vm_offset_t) 0;
        !           440:        vm_object_template.true_share = FALSE;
        !           441: 
        !           442:        vm_object_template.pager = IP_NULL;
        !           443:        vm_object_template.paging_offset = 0;
        !           444:        vm_object_template.pager_request = PAGER_REQUEST_NULL;
        !           445:        /* msr_q; init after allocation */
        !           446: 
        !           447:        vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_SYMMETRIC;
        !           448:        vm_object_template.absent_count = 0;
        !           449:        vm_object_template.paging_in_progress = 0;
        !           450: 
        !           451:        /* Begin bitfields */
        !           452:        vm_object_template.all_wanted = 0; /* all bits FALSE */
        !           453:        vm_object_template.pager_created = FALSE;
        !           454:        vm_object_template.pager_initialized = FALSE;
        !           455:        vm_object_template.pager_ready = FALSE;
        !           456:        vm_object_template.pager_trusted = FALSE;
        !           457:        vm_object_template.can_persist = FALSE;
        !           458:        vm_object_template.internal = TRUE;
        !           459:        vm_object_template.temporary = TRUE;
        !           460:        vm_object_template.private = FALSE;
        !           461:        vm_object_template.pageout = FALSE;
        !           462:        vm_object_template.alive = TRUE;
        !           463:        vm_object_template.lock_in_progress = FALSE;
        !           464:        vm_object_template.lock_restart = FALSE;
        !           465:        vm_object_template.silent_overwrite = FALSE;
        !           466:        vm_object_template.advisory_pageout = FALSE;
        !           467:        vm_object_template.shadowed = FALSE;
        !           468:        /* End bitfields */
        !           469: 
        !           470:        /* cached_list; init after allocation */
        !           471:        vm_object_template.last_alloc = (vm_offset_t) 0;
        !           472:        vm_object_template.cluster_size = 0;
        !           473: #if    MACH_PAGEMAP
        !           474:        vm_object_template.existence_map = VM_EXTERNAL_NULL;
        !           475: #endif /* MACH_PAGEMAP */
        !           476: #if    MACH_ASSERT
        !           477:        vm_object_template.paging_object = VM_OBJECT_NULL;
        !           478: #endif /* MACH_ASSERT */
        !           479: 
        !           480:        /*
        !           481:         *      Initialize the "kernel object"
        !           482:         */
        !           483: 
        !           484:        kernel_object = &kernel_object_store;
        !           485: 
        !           486: /*
        !           487:  *     Note that in the following size specifications, we need to add 1 because 
        !           488:  *     VM_MAX_KERNEL_ADDRESS is a maximum address, not a size.
        !           489:  */
        !           490:        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
        !           491:                        kernel_object);
        !           492: 
        !           493:        /*
        !           494:         *      Initialize the "submap object".  Make it as large as the
        !           495:         *      kernel object so that no limit is imposed on submap sizes.
        !           496:         */
        !           497: 
        !           498:        vm_submap_object = &vm_submap_object_store;
        !           499:        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
        !           500:                        vm_submap_object);
        !           501:        /*
        !           502:         * Create an "extra" reference to this object so that we never
        !           503:         * try to deallocate it; zfree doesn't like to be called with
        !           504:         * non-zone memory.
        !           505:         */
        !           506:        vm_object_reference(vm_submap_object);
        !           507: 
        !           508: #if    MACH_PAGEMAP
        !           509:        vm_external_module_initialize();
        !           510: #endif /* MACH_PAGEMAP */
        !           511: }
        !           512: 
        !           513: void
        !           514: vm_object_init(void)
        !           515: {
        !           516:        /*
        !           517:         *      Finish initializing the kernel object.
        !           518:         */
        !           519: }
        !           520: 
        !           521: #if    TASK_SWAPPER
        !           522: /*
        !           523:  * vm_object_res_deallocate
        !           524:  *
        !           525:  * (recursively) decrement residence counts on vm objects and their shadows.
        !           526:  * Called from vm_object_deallocate and when swapping out an object.
        !           527:  *
        !           528:  * The object is locked, and remains locked throughout the function,
        !           529:  * even as we iterate down the shadow chain.  Locks on intermediate objects
        !           530:  * will be dropped, but not the original object.
        !           531:  *
        !           532:  * NOTE: this function used to use recursion, rather than iteration.
        !           533:  */
        !           534: 
        !           535: void
        !           536: vm_object_res_deallocate(
        !           537:        vm_object_t     object)
        !           538: {
        !           539:        vm_object_t orig_object = object;
        !           540:        /*
        !           541:         * Object is locked so it can be called directly
        !           542:         * from vm_object_deallocate.  Original object is never
        !           543:         * unlocked.
        !           544:         */
        !           545:        assert(object->res_count > 0);
        !           546:        while  (--object->res_count == 0) {
        !           547:                assert(object->ref_count >= object->res_count);
        !           548:                vm_object_deactivate_pages(object);
        !           549:                /* iterate on shadow, if present */
        !           550:                if (object->shadow != VM_OBJECT_NULL) {
        !           551:                        vm_object_t tmp_object = object->shadow;
        !           552:                        vm_object_lock(tmp_object);
        !           553:                        if (object != orig_object)
        !           554:                                vm_object_unlock(object);
        !           555:                        object = tmp_object;
        !           556:                        assert(object->res_count > 0);
        !           557:                } else
        !           558:                        break;
        !           559:        }
        !           560:        if (object != orig_object)
        !           561:                vm_object_unlock(object);
        !           562: }
        !           563: 
        !           564: /*
        !           565:  * vm_object_res_reference
        !           566:  *
        !           567:  * Internal function to increment residence count on a vm object
        !           568:  * and its shadows.  It is called only from vm_object_reference, and
        !           569:  * when swapping in a vm object, via vm_map_swap.
        !           570:  *
        !           571:  * The object is locked, and remains locked throughout the function,
        !           572:  * even as we iterate down the shadow chain.  Locks on intermediate objects
        !           573:  * will be dropped, but not the original object.
        !           574:  *
        !           575:  * NOTE: this function used to use recursion, rather than iteration.
        !           576:  */
        !           577: 
        !           578: void
        !           579: vm_object_res_reference(
        !           580:        vm_object_t     object)
        !           581: {
        !           582:        vm_object_t orig_object = object;
        !           583:        /* 
        !           584:         * Object is locked, so this can be called directly
        !           585:         * from vm_object_reference.  This lock is never released.
        !           586:         */
        !           587:        while  ((++object->res_count == 1)  && 
        !           588:                (object->shadow != VM_OBJECT_NULL)) {
        !           589:                vm_object_t tmp_object = object->shadow;
        !           590: 
        !           591:                assert(object->ref_count >= object->res_count);
        !           592:                vm_object_lock(tmp_object);
        !           593:                if (object != orig_object)
        !           594:                        vm_object_unlock(object);
        !           595:                object = tmp_object;
        !           596:        }
        !           597:        if (object != orig_object)
        !           598:                vm_object_unlock(object);
        !           599:        assert(orig_object->ref_count >= orig_object->res_count);
        !           600: }
        !           601: #endif /* TASK_SWAPPER */
        !           602: 
        !           603: #if    MACH_ASSERT
        !           604: /*
        !           605:  *     vm_object_reference:
        !           606:  *
        !           607:  *     Gets another reference to the given object.
        !           608:  */
        !           609: void
        !           610: vm_object_reference(
        !           611:        register vm_object_t    object)
        !           612: {
        !           613:        if (object == VM_OBJECT_NULL)
        !           614:                return;
        !           615: 
        !           616:        vm_object_lock(object);
        !           617:        assert(object->ref_count > 0);
        !           618:        object->ref_count++;
        !           619:        vm_object_res_reference(object);
        !           620:        vm_object_unlock(object);
        !           621: }
        !           622: #endif /* MACH_ASSERT */
        !           623: 
        !           624: #define        MIGHT_NOT_CACHE_SHADOWS         1
        !           625: #if    MIGHT_NOT_CACHE_SHADOWS
        !           626: int cache_shadows = TRUE;
        !           627: #endif /* MIGHT_NOT_CACHE_SHADOWS */
        !           628: 
        !           629: /*
        !           630:  *     vm_object_deallocate:
        !           631:  *
        !           632:  *     Release a reference to the specified object,
        !           633:  *     gained either through a vm_object_allocate
        !           634:  *     or a vm_object_reference call.  When all references
        !           635:  *     are gone, storage associated with this object
        !           636:  *     may be relinquished.
        !           637:  *
        !           638:  *     No object may be locked.
        !           639:  */
        !           640: void
        !           641: vm_object_deallocate(
        !           642:        register vm_object_t    object)
        !           643: {
        !           644:        boolean_t retry_cache_trim = FALSE;
        !           645:        vm_object_t shadow;
        !           646:        
        !           647: //     if(object)dbgLog(object, object->ref_count, object->can_persist, 3);    /* (TEST/DEBUG) */
        !           648: //     else dbgLog(object, 0, 0, 3);   /* (TEST/DEBUG) */
        !           649: 
        !           650: 
        !           651:        while (object != VM_OBJECT_NULL) {
        !           652: 
        !           653:                /*
        !           654:                 *      The cache holds a reference (uncounted) to
        !           655:                 *      the object; we must lock it before removing
        !           656:                 *      the object.
        !           657:                 */
        !           658: 
        !           659:                vm_object_cache_lock();
        !           660:                vm_object_lock(object);
        !           661:                assert(object->alive);
        !           662: 
        !           663:                /*
        !           664:                 *      Lose the reference. If other references
        !           665:                 *      remain, then we are done, unless we need
        !           666:                 *      to retry a cache trim.
        !           667:                 *      If it is the last reference, then keep it
        !           668:                 *      until any pending initialization is completed.
        !           669:                 */
        !           670: 
        !           671:                assert(object->ref_count > 0);
        !           672:                if (object->ref_count > 1) {
        !           673:                        object->ref_count--;
        !           674:                        vm_object_res_deallocate(object);
        !           675:                        vm_object_unlock(object);
        !           676:                        vm_object_cache_unlock();
        !           677:                        if (retry_cache_trim &&
        !           678:                            ((object = vm_object_cache_trim(TRUE)) !=
        !           679:                             VM_OBJECT_NULL)) {
        !           680:                                continue;
        !           681:                        }
        !           682:                        return;
        !           683:                }
        !           684: 
        !           685:                /*
        !           686:                 *      We have to wait for initialization
        !           687:                 *      before destroying or caching the object.
        !           688:                 */
        !           689:                
        !           690:                if (object->pager_created && ! object->pager_initialized) {
        !           691:                        assert(! object->can_persist);
        !           692:                        vm_object_assert_wait(object,
        !           693:                                              VM_OBJECT_EVENT_INITIALIZED,
        !           694:                                              THREAD_UNINT);
        !           695:                        vm_object_unlock(object);
        !           696:                        vm_object_cache_unlock();
        !           697:                        thread_block((void (*)(void))0);
        !           698:                        continue;
        !           699:                }
        !           700: 
        !           701:                /*
        !           702:                 *      If this object can persist, then enter it in
        !           703:                 *      the cache. Otherwise, terminate it.
        !           704:                 *
        !           705:                 *      NOTE:  Only permanent objects are cached, and
        !           706:                 *      permanent objects cannot have shadows.  This
        !           707:                 *      affects the residence counting logic in a minor
        !           708:                 *      way (can do it in-line, mostly).
        !           709:                 */
        !           710: 
        !           711:                if (object->can_persist) {
        !           712:                        /*
        !           713:                         *      Now it is safe to decrement reference count,
        !           714:                         *      and to return if reference count is > 0.
        !           715:                         */
        !           716:                        if (--object->ref_count > 0) {
        !           717:                                vm_object_res_deallocate(object);
        !           718:                                vm_object_unlock(object);
        !           719:                                vm_object_cache_unlock();
        !           720:                                if (retry_cache_trim &&
        !           721:                                    ((object = vm_object_cache_trim(TRUE)) !=
        !           722:                                     VM_OBJECT_NULL)) {
        !           723:                                        continue;
        !           724:                                }
        !           725:                                return;
        !           726:                        }
        !           727: 
        !           728: #if    MIGHT_NOT_CACHE_SHADOWS
        !           729:                        /*
        !           730:                         *      Remove shadow now if we don't
        !           731:                         *      want to cache shadows.
        !           732:                         */
        !           733:                        if (! cache_shadows) {
        !           734:                                shadow = object->shadow;
        !           735:                                object->shadow = VM_OBJECT_NULL;
        !           736:                        }
        !           737: #endif /* MIGHT_NOT_CACHE_SHADOWS */
        !           738: 
        !           739:                        /*
        !           740:                         *      Enter the object onto the queue of
        !           741:                         *      cached objects, and deactivate
        !           742:                         *      all of its pages.
        !           743:                         */
        !           744:                        assert(object->shadow == VM_OBJECT_NULL);
        !           745:                        VM_OBJ_RES_DECR(object);
        !           746:                        XPR(XPR_VM_OBJECT,
        !           747:                      "vm_o_deallocate: adding %x to cache, queue = (%x, %x)\n",
        !           748:                                (integer_t)object,
        !           749:                                (integer_t)vm_object_cached_list.next,
        !           750:                                (integer_t)vm_object_cached_list.prev,0,0);
        !           751: 
        !           752:                        vm_object_cached_count++;
        !           753:                        if (vm_object_cached_count > vm_object_cached_high)
        !           754:                                vm_object_cached_high = vm_object_cached_count;
        !           755:                        queue_enter(&vm_object_cached_list, object,
        !           756:                                vm_object_t, cached_list);
        !           757:                        vm_object_cache_unlock();
        !           758:                        vm_object_deactivate_pages(object);
        !           759:                        vm_object_unlock(object);
        !           760: 
        !           761: #if    MIGHT_NOT_CACHE_SHADOWS
        !           762:                        /*
        !           763:                         *      If we have a shadow that we need
        !           764:                         *      to deallocate, do so now, remembering
        !           765:                         *      to trim the cache later.
        !           766:                         */
        !           767:                        if (! cache_shadows && shadow != VM_OBJECT_NULL) {
        !           768:                                object = shadow;
        !           769:                                retry_cache_trim = TRUE;
        !           770:                                continue;
        !           771:                        }
        !           772: #endif /* MIGHT_NOT_CACHE_SHADOWS */
        !           773: 
        !           774:                        /*
        !           775:                         *      Trim the cache. If the cache trim
        !           776:                         *      returns with a shadow for us to deallocate,
        !           777:                         *      then remember to retry the cache trim
        !           778:                         *      when we are done deallocating the shadow.
        !           779:                         *      Otherwise, we are done.
        !           780:                         */
        !           781: 
        !           782:                        object = vm_object_cache_trim(TRUE);
        !           783:                        if (object == VM_OBJECT_NULL) {
        !           784:                                return;
        !           785:                        }
        !           786:                        retry_cache_trim = TRUE;
        !           787: 
        !           788:                } else {
        !           789:                        /*
        !           790:                         *      This object is not cachable; terminate it.
        !           791:                         */
        !           792:                        XPR(XPR_VM_OBJECT,
        !           793:         "vm_o_deallocate: !cacheable 0x%X res %d paging_ops %d thread 0x%lX ref %d\n",
        !           794:                                (integer_t)object, object->resident_page_count,
        !           795:                                object->paging_in_progress,
        !           796:                                (natural_t)current_thread(),object->ref_count);
        !           797: 
        !           798:                        VM_OBJ_RES_DECR(object);        /* XXX ? */
        !           799:                        /*
        !           800:                         *      Terminate this object. If it had a shadow,
        !           801:                         *      then deallocate it; otherwise, if we need
        !           802:                         *      to retry a cache trim, do so now; otherwise,
        !           803:                         *      we are done. "pageout" objects have a shadow,
        !           804:                         *      but maintain a "paging reference" rather than
        !           805:                         *      a normal reference.
        !           806:                         */
        !           807:                        shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
        !           808:                        vm_object_terminate(object);
        !           809:                        if (shadow != VM_OBJECT_NULL) {
        !           810:                                object = shadow;
        !           811:                                continue;
        !           812:                        }
        !           813:                        if (retry_cache_trim &&
        !           814:                            ((object = vm_object_cache_trim(TRUE)) !=
        !           815:                             VM_OBJECT_NULL)) {
        !           816:                                continue;
        !           817:                        }
        !           818:                        return;
        !           819:                }
        !           820:        }
        !           821:        assert(! retry_cache_trim);
        !           822: }
        !           823: 
        !           824: /*
        !           825:  *     Check to see whether we really need to trim
        !           826:  *     down the cache. If so, remove an object from
        !           827:  *     the cache, terminate it, and repeat.
        !           828:  *
        !           829:  *     Called with, and returns with, cache lock unlocked.
        !           830:  */
        !           831: vm_object_t
        !           832: vm_object_cache_trim(
        !           833:        boolean_t called_from_vm_object_deallocate)
        !           834: {
        !           835:        register vm_object_t object = VM_OBJECT_NULL;
        !           836:        vm_object_t shadow;
        !           837: 
        !           838:        for (;;) {
        !           839: 
        !           840:                /*
        !           841:                 *      If we no longer need to trim the cache,
        !           842:                 *      then we are done.
        !           843:                 */
        !           844: 
        !           845:                vm_object_cache_lock();
        !           846:                if (vm_object_cached_count <= vm_object_cached_max) {
        !           847:                        vm_object_cache_unlock();
        !           848:                        return VM_OBJECT_NULL;
        !           849:                }
        !           850: 
        !           851:                /*
        !           852:                 *      We must trim down the cache, so remove
        !           853:                 *      the first object in the cache.
        !           854:                 */
        !           855:                XPR(XPR_VM_OBJECT,
        !           856:                "vm_object_cache_trim: removing from front of cache (%x, %x)\n",
        !           857:                        (integer_t)vm_object_cached_list.next,
        !           858:                        (integer_t)vm_object_cached_list.prev, 0, 0, 0);
        !           859: 
        !           860:                object = (vm_object_t) queue_first(&vm_object_cached_list);
        !           861:                vm_object_lock(object);
        !           862:                queue_remove(&vm_object_cached_list, object, vm_object_t,
        !           863:                             cached_list);
        !           864:                vm_object_cached_count--;
        !           865: 
        !           866:                /*
        !           867:                 *      Since this object is in the cache, we know
        !           868:                 *      that it is initialized and has no references.
        !           869:                 *      Take a reference to avoid recursive deallocations.
        !           870:                 */
        !           871: 
        !           872:                assert(object->pager_initialized);
        !           873:                assert(object->ref_count == 0);
        !           874:                object->ref_count++;
        !           875: 
        !           876:                /*
        !           877:                 *      Terminate the object.
        !           878:                 *      If the object had a shadow, we let vm_object_deallocate
        !           879:                 *      deallocate it. "pageout" objects have a shadow, but
        !           880:                 *      maintain a "paging reference" rather than a normal
        !           881:                 *      reference.
        !           882:                 *      (We are careful here to limit recursion.)
        !           883:                 */
        !           884:                shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
        !           885:                vm_object_terminate(object);
        !           886:                if (shadow != VM_OBJECT_NULL) {
        !           887:                        if (called_from_vm_object_deallocate) {
        !           888:                                return shadow;
        !           889:                        } else {
        !           890:                                vm_object_deallocate(shadow);
        !           891:                        }
        !           892:                }
        !           893:        }
        !           894: }
        !           895: 
        !           896: boolean_t      vm_object_terminate_remove_all = FALSE;
        !           897: 
        !           898: /*
        !           899:  *     Routine:        vm_object_terminate
        !           900:  *     Purpose:
        !           901:  *             Free all resources associated with a vm_object.
        !           902:  *     In/out conditions:
        !           903:  *             Upon entry, the object and the cache must be locked,
        !           904:  *             and the object must have exactly one reference.
        !           905:  *
        !           906:  *             The shadow object reference is left alone.
        !           907:  *
        !           908:  *             Upon exit, the cache will be unlocked, and the
        !           909:  *             object will cease to exist.
        !           910:  */
        !           911: void
        !           912: vm_object_terminate(
        !           913:        register vm_object_t    object)
        !           914: {
        !           915:        register vm_page_t      p;
        !           916:        vm_object_t             shadow_object;
        !           917: 
        !           918:        XPR(XPR_VM_OBJECT, "vm_object_terminate, object 0x%X ref %d\n",
        !           919:                (integer_t)object, object->ref_count, 0, 0, 0);
        !           920: 
        !           921:        /*
        !           922:         *      Make sure the object isn't already being terminated
        !           923:         */
        !           924: 
        !           925:        assert(object->alive);
        !           926:        object->alive = FALSE;
        !           927: 
        !           928:        /*
        !           929:         *      Make sure no one can look us up now.
        !           930:         */
        !           931: 
        !           932:        if(object->pager != IP_NULL) {
        !           933:                vm_object_hash_entry_t  entry;
        !           934: 
        !           935:                entry = vm_object_hash_lookup(object->pager, FALSE);
        !           936:                if (entry != VM_OBJECT_HASH_ENTRY_NULL)
        !           937:                        entry->object = VM_OBJECT_NULL;
        !           938:        }
        !           939: 
        !           940:        vm_object_cache_unlock();
        !           941: 
        !           942:        /*
        !           943:         *      Detach the object from its shadow if we are the shadow's
        !           944:         *      copy.
        !           945:         */
        !           946:        if (((shadow_object = object->shadow) != VM_OBJECT_NULL) &&
        !           947:            !(object->pageout)) {
        !           948:                vm_object_lock(shadow_object);
        !           949:                assert((shadow_object->copy == object) ||
        !           950:                       (shadow_object->copy == VM_OBJECT_NULL));
        !           951:                shadow_object->copy = VM_OBJECT_NULL;
        !           952:                vm_object_unlock(shadow_object);
        !           953:        }
        !           954: 
        !           955:        /*
        !           956:         *      The pageout daemon might be playing with our pages.
        !           957:         *      Now that the object is dead, it won't touch any more
        !           958:         *      pages, but some pages might already be on their way out.
        !           959:         *      Hence, we wait until the active paging activities have ceased.
        !           960:         */
        !           961:        vm_object_paging_wait(object, THREAD_UNINT);
        !           962:        object->ref_count--;
        !           963: #if    TASK_SWAPPER
        !           964:        assert(object->res_count == 0);
        !           965: #endif /* TASK_SWAPPER */
        !           966: 
        !           967: Restart:
        !           968:        assert (object->ref_count == 0);
        !           969: 
        !           970:        /*
        !           971:         *      Clean or free the pages, as appropriate.
        !           972:         *      It is possible for us to find busy/absent pages,
        !           973:         *      if some faults on this object were aborted.
        !           974:         */
        !           975:        if (object->pageout) {
        !           976:                assert(shadow_object != VM_OBJECT_NULL);
        !           977:                assert(shadow_object == object->shadow);
        !           978: 
        !           979:                vm_pageout_object_terminate(object);
        !           980: 
        !           981:        } else if (object->temporary && ! object->can_persist ||
        !           982:            object->pager == IP_NULL) {
        !           983:                while (!queue_empty(&object->memq)) {
        !           984:                        p = (vm_page_t) queue_first(&object->memq);
        !           985: 
        !           986:                        VM_PAGE_CHECK(p);
        !           987: 
        !           988:                        if (p->busy && !p->absent)
        !           989:                                panic("vm_object_terminate.2 0x%x 0x%x",
        !           990:                                      object, p);
        !           991: 
        !           992:                        VM_PAGE_FREE(p);
        !           993:                }
        !           994:        } else while (!queue_empty(&object->memq)) {
        !           995:                /*
        !           996:                 * Clear pager_trusted bit so that the pages get yanked
        !           997:                 * out of the object instead of cleaned in place.  This
        !           998:                 * prevents a deadlock in XMM and makes more sense anyway.
        !           999:                 */
        !          1000:                object->pager_trusted = FALSE;
        !          1001: 
        !          1002:                p = (vm_page_t) queue_first(&object->memq);
        !          1003: 
        !          1004:                VM_PAGE_CHECK(p);
        !          1005: 
        !          1006:                if (p->busy && !p->absent)
        !          1007:                        panic("vm_object_terminate.3 0x%x 0x%x", object, p);
        !          1008: 
        !          1009:                vm_page_lock_queues();
        !          1010:                VM_PAGE_QUEUES_REMOVE(p);
        !          1011:                vm_page_unlock_queues();
        !          1012: 
        !          1013:                if (p->absent || p->private) {
        !          1014: 
        !          1015:                        /*
        !          1016:                         *      For private pages, VM_PAGE_FREE just
        !          1017:                         *      leaves the page structure around for
        !          1018:                         *      its owner to clean up.  For absent
        !          1019:                         *      pages, the structure is returned to
        !          1020:                         *      the appropriate pool.
        !          1021:                         */
        !          1022: 
        !          1023:                        goto free_page;
        !          1024:                }
        !          1025: 
        !          1026:                if (p->fictitious)
        !          1027:                        panic("vm_object_terminate.4 0x%x 0x%x", object, p);
        !          1028: 
        !          1029:                if (!p->dirty)
        !          1030:                        p->dirty = pmap_is_modified(p->phys_addr);
        !          1031: 
        !          1032:                if (p->dirty || p->precious) {
        !          1033:                        p->busy = TRUE;
        !          1034:                        vm_object_paging_begin(object);
        !          1035:                        vm_object_unlock(object);
        !          1036:                        vm_pageout_cluster(p); /* flush page */
        !          1037:                        vm_object_lock(object);
        !          1038:                        vm_object_paging_wait(object, THREAD_UNINT);
        !          1039: 
        !          1040:                        XPR(XPR_VM_OBJECT,
        !          1041:                            "vm_object_terminate restart, object 0x%X ref %d\n",
        !          1042:                            (integer_t)object, object->ref_count, 0, 0, 0);
        !          1043: 
        !          1044:                        goto Restart;
        !          1045:                } else {
        !          1046:                    free_page:
        !          1047:                        VM_PAGE_FREE(p);
        !          1048:                }
        !          1049:        }
        !          1050: 
        !          1051:        assert(object->paging_in_progress == 0);
        !          1052:        assert(object->ref_count == 0);
        !          1053: 
        !          1054:        vm_object_remove(object);
        !          1055: 
        !          1056:        /*
        !          1057:         *      Throw away port rights... note that they may
        !          1058:         *      already have been thrown away (by vm_object_destroy
        !          1059:         *      or memory_object_destroy).
        !          1060:         *
        !          1061:         *      Instead of destroying the control port,
        !          1062:         *      we send all rights off to the memory manager,
        !          1063:         *      using memory_object_terminate.
        !          1064:         */
        !          1065: 
        !          1066:        vm_object_unlock(object);
        !          1067:        if (object->pager != IP_NULL) {
        !          1068:           /* consumes our rights for pager, pager_request */
        !          1069:          memory_object_release(object->pager, object->pager_request);
        !          1070:        }
        !          1071: 
        !          1072: #if    MACH_PAGEMAP
        !          1073:        vm_external_destroy(object->existence_map, object->size);
        !          1074: #endif /* MACH_PAGEMAP */
        !          1075: 
        !          1076:        /*
        !          1077:         *      Free the space for the object.
        !          1078:         */
        !          1079: 
        !          1080:        zfree(vm_object_zone, (vm_offset_t) object);
        !          1081: }
        !          1082: 
        !          1083: /*
        !          1084:  *     Routine:        vm_object_pager_wakeup
        !          1085:  *     Purpose:        Wake up anyone waiting for termination of a pager.
        !          1086:  */
        !          1087: 
        !          1088: void
        !          1089: vm_object_pager_wakeup(
        !          1090:        ipc_port_t      pager)
        !          1091: {
        !          1092:        vm_object_hash_entry_t  entry;
        !          1093:        boolean_t               waiting = FALSE;
        !          1094: 
        !          1095:        /*
        !          1096:         *      If anyone was waiting for the memory_object_terminate
        !          1097:         *      to be queued, wake them up now.
        !          1098:         */
        !          1099:        vm_object_cache_lock();
        !          1100:        entry = vm_object_hash_lookup(pager, TRUE);
        !          1101:        if (entry != VM_OBJECT_HASH_ENTRY_NULL)
        !          1102:                waiting = entry->waiting;
        !          1103:        vm_object_cache_unlock();
        !          1104:        if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
        !          1105:                if (waiting)
        !          1106:                        thread_wakeup((event_t) pager);
        !          1107:                vm_object_hash_entry_free(entry);
        !          1108:        }
        !          1109: }
        !          1110: 
        !          1111: /*
        !          1112:  *     Routine:        memory_object_release
        !          1113:  *     Purpose:        Terminate the pager and release port rights,
        !          1114:  *                     just like memory_object_terminate, except
        !          1115:  *                     that we wake up anyone blocked in vm_object_enter
        !          1116:  *                     waiting for termination message to be queued
        !          1117:  *                     before calling memory_object_init.
        !          1118:  */
        !          1119: void
        !          1120: memory_object_release(
        !          1121:        ipc_port_t      pager,
        !          1122:        pager_request_t pager_request)
        !          1123: {
        !          1124: #ifdef MACH_BSD
        !          1125:        kern_return_t vnode_pager_terminate(ipc_port_t, ipc_port_t);
        !          1126: #endif
        !          1127: 
        !          1128:        /*
        !          1129:         *      Keep a reference to pager port;
        !          1130:         *      the terminate might otherwise release all references.
        !          1131:         */
        !          1132:        ipc_port_copy_send(pager);
        !          1133: 
        !          1134:        /*
        !          1135:         *      Terminate the pager.
        !          1136:         */
        !          1137: 
        !          1138: #ifdef MACH_BSD
        !          1139:        if(((rpc_subsystem_t)pager_mux_hash_lookup(pager)) == 
        !          1140:                ((rpc_subsystem_t) &vnode_pager_workaround)) {
        !          1141:                (void) vnode_pager_terminate(pager, pager_request);
        !          1142:        } else {
        !          1143:                (void) memory_object_terminate(pager, pager_request);
        !          1144:        }
        !          1145: #else
        !          1146:        (void) memory_object_terminate(pager, pager_request);
        !          1147: #endif
        !          1148: 
        !          1149:        /*
        !          1150:         *      Wakeup anyone waiting for this terminate
        !          1151:         */
        !          1152:        vm_object_pager_wakeup(pager);
        !          1153: 
        !          1154:        /*
        !          1155:         *      Release reference to pager port.
        !          1156:         */
        !          1157:        ipc_port_release_send(pager);
        !          1158: }
        !          1159: 
        !          1160: /*
        !          1161:  *     Routine:        vm_object_abort_activity [internal use only]
        !          1162:  *     Purpose:
        !          1163:  *             Abort paging requests pending on this object.
        !          1164:  *     In/out conditions:
        !          1165:  *             The object is locked on entry and exit.
        !          1166:  */
        !          1167: void
        !          1168: vm_object_abort_activity(
        !          1169:        vm_object_t     object)
        !          1170: {
        !          1171:        register
        !          1172:        vm_page_t       p;
        !          1173:        vm_page_t       next;
        !          1174: 
        !          1175:        XPR(XPR_VM_OBJECT, "vm_object_abort_activity, object 0x%X\n",
        !          1176:                (integer_t)object, 0, 0, 0, 0);
        !          1177: 
        !          1178:        /*
        !          1179:         *      Abort all activity that would be waiting
        !          1180:         *      for a result on this memory object.
        !          1181:         *
        !          1182:         *      We could also choose to destroy all pages
        !          1183:         *      that we have in memory for this object, but
        !          1184:         *      we don't.
        !          1185:         */
        !          1186: 
        !          1187:        p = (vm_page_t) queue_first(&object->memq);
        !          1188:        while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !          1189:                next = (vm_page_t) queue_next(&p->listq);
        !          1190: 
        !          1191:                /*
        !          1192:                 *      If it's being paged in, destroy it.
        !          1193:                 *      If an unlock has been requested, start it again.
        !          1194:                 */
        !          1195: 
        !          1196:                if (p->busy && p->absent) {
        !          1197:                        VM_PAGE_FREE(p);
        !          1198:                }
        !          1199:                 else {
        !          1200:                        if (p->unlock_request != VM_PROT_NONE)
        !          1201:                                p->unlock_request = VM_PROT_NONE;
        !          1202:                        PAGE_WAKEUP(p);
        !          1203:                }
        !          1204:                
        !          1205:                p = next;
        !          1206:        }
        !          1207: 
        !          1208:        /*
        !          1209:         *      Wake up threads waiting for the memory object to
        !          1210:         *      become ready.
        !          1211:         */
        !          1212: 
        !          1213:        object->pager_ready = TRUE;
        !          1214:        vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
        !          1215: }
        !          1216: 
        !          1217: /*
        !          1218:  *     Routine:        memory_object_destroy [user interface]
        !          1219:  *     Purpose:
        !          1220:  *             Shut down a memory object, despite the
        !          1221:  *             presence of address map (or other) references
        !          1222:  *             to the vm_object.
        !          1223:  */
        !          1224: kern_return_t
        !          1225: memory_object_destroy(
        !          1226:        register vm_object_t    object,
        !          1227:        kern_return_t           reason)
        !          1228: {
        !          1229:        ipc_port_t      old_object;
        !          1230:        pager_request_t old_pager_request;
        !          1231: 
        !          1232: #ifdef lint
        !          1233:        reason++;
        !          1234: #endif /* lint */
        !          1235: 
        !          1236:        if (object == VM_OBJECT_NULL)
        !          1237:                return(KERN_SUCCESS);
        !          1238: 
        !          1239:        /*
        !          1240:         *      Remove the port associations immediately.
        !          1241:         *
        !          1242:         *      This will prevent the memory manager from further
        !          1243:         *      meddling.  [If it wanted to flush data or make
        !          1244:         *      other changes, it should have done so before performing
        !          1245:         *      the destroy call.]
        !          1246:         */
        !          1247: 
        !          1248:        vm_object_cache_lock();
        !          1249:        vm_object_lock(object);
        !          1250:        vm_object_remove(object);
        !          1251:        object->can_persist = FALSE;
        !          1252:        vm_object_cache_unlock();
        !          1253: 
        !          1254:        /*
        !          1255:         *      Rip out the ports from the vm_object now... this
        !          1256:         *      will prevent new memory_object calls from succeeding.
        !          1257:         */
        !          1258: 
        !          1259:        old_object = object->pager;
        !          1260:        old_pager_request = object->pager_request;
        !          1261: 
        !          1262:        object->pager = IP_NULL;
        !          1263:        object->pager_request = PAGER_REQUEST_NULL;
        !          1264: 
        !          1265:        /*
        !          1266:         *      Wait for existing paging activity (that might
        !          1267:         *      have the old ports) to subside.
        !          1268:         */
        !          1269: 
        !          1270:        vm_object_paging_wait(object, THREAD_UNINT);
        !          1271:        vm_object_unlock(object);
        !          1272: 
        !          1273:        /*
        !          1274:         *      Shut down the ports now.
        !          1275:         *
        !          1276:         *      [Paging operations may be proceeding concurrently --
        !          1277:         *      they'll get the null values established above.]
        !          1278:         */
        !          1279: 
        !          1280:        if (old_object != IP_NULL) {
        !          1281:                /* consumes our rights for object, control */
        !          1282:                memory_object_release(old_object, old_pager_request);
        !          1283:        }
        !          1284: 
        !          1285:        /*
        !          1286:         *      Lose the reference that was donated for this routine
        !          1287:         */
        !          1288: 
        !          1289:        vm_object_deallocate(object);
        !          1290: 
        !          1291:        return(KERN_SUCCESS);
        !          1292: }
        !          1293: 
        !          1294: /*
        !          1295:  *     vm_object_deactivate_pages
        !          1296:  *
        !          1297:  *     Deactivate all pages in the specified object.  (Keep its pages
        !          1298:  *     in memory even though it is no longer referenced.)
        !          1299:  *
        !          1300:  *     The object must be locked.
        !          1301:  */
        !          1302: void
        !          1303: vm_object_deactivate_pages(
        !          1304:        register vm_object_t    object)
        !          1305: {
        !          1306:        register vm_page_t      p;
        !          1307: 
        !          1308:        queue_iterate(&object->memq, p, vm_page_t, listq) {
        !          1309:                vm_page_lock_queues();
        !          1310:                if (!p->busy)
        !          1311:                        vm_page_deactivate(p);
        !          1312:                vm_page_unlock_queues();
        !          1313:        }
        !          1314: }
        !          1315: 
        !          1316: 
        !          1317: /*
        !          1318:  *     Routine:        vm_object_pmap_protect
        !          1319:  *
        !          1320:  *     Purpose:
        !          1321:  *             Reduces the permission for all physical
        !          1322:  *             pages in the specified object range.
        !          1323:  *
        !          1324:  *             If removing write permission only, it is
        !          1325:  *             sufficient to protect only the pages in
        !          1326:  *             the top-level object; only those pages may
        !          1327:  *             have write permission.
        !          1328:  *
        !          1329:  *             If removing all access, we must follow the
        !          1330:  *             shadow chain from the top-level object to
        !          1331:  *             remove access to all pages in shadowed objects.
        !          1332:  *
        !          1333:  *             The object must *not* be locked.  The object must
        !          1334:  *             be temporary/internal.  
        !          1335:  *
        !          1336:  *              If pmap is not NULL, this routine assumes that
        !          1337:  *              the only mappings for the pages are in that
        !          1338:  *              pmap.
        !          1339:  */
        !          1340: 
        !          1341: void
        !          1342: vm_object_pmap_protect(
        !          1343:        register vm_object_t    object,
        !          1344:        register vm_offset_t    offset,
        !          1345:        vm_offset_t             size,
        !          1346:        pmap_t                  pmap,
        !          1347:        vm_offset_t             pmap_start,
        !          1348:        vm_prot_t               prot)
        !          1349: {
        !          1350:        if (object == VM_OBJECT_NULL)
        !          1351:            return;
        !          1352: 
        !          1353:        vm_object_lock(object);
        !          1354: 
        !          1355:        assert(object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
        !          1356: 
        !          1357:        while (TRUE) {
        !          1358:            if (object->resident_page_count > atop(size) / 2 &&
        !          1359:                    pmap != PMAP_NULL) {
        !          1360:                vm_object_unlock(object);
        !          1361:                pmap_protect(pmap, pmap_start, pmap_start + size, prot);
        !          1362:                return;
        !          1363:            }
        !          1364: 
        !          1365:            {
        !          1366:                register vm_page_t      p;
        !          1367:                register vm_offset_t    end;
        !          1368: 
        !          1369:                end = offset + size;
        !          1370: 
        !          1371:                if (pmap != PMAP_NULL) {
        !          1372:                  queue_iterate(&object->memq, p, vm_page_t, listq) {
        !          1373:                    if (!p->fictitious &&
        !          1374:                        (offset <= p->offset) && (p->offset < end)) {
        !          1375: 
        !          1376:                        vm_offset_t start = pmap_start + (p->offset - offset);
        !          1377: 
        !          1378:                        pmap_protect(pmap, start, start + PAGE_SIZE, prot);
        !          1379:                    }
        !          1380:                  }
        !          1381:                } else {
        !          1382:                  queue_iterate(&object->memq, p, vm_page_t, listq) {
        !          1383:                    if (!p->fictitious &&
        !          1384:                        (offset <= p->offset) && (p->offset < end)) {
        !          1385: 
        !          1386:                            pmap_page_protect(p->phys_addr,
        !          1387:                                              prot & ~p->page_lock);
        !          1388:                    }
        !          1389:                  }
        !          1390:                }
        !          1391:            }
        !          1392: 
        !          1393:            if (prot == VM_PROT_NONE) {
        !          1394:                /*
        !          1395:                 * Must follow shadow chain to remove access
        !          1396:                 * to pages in shadowed objects.
        !          1397:                 */
        !          1398:                register vm_object_t    next_object;
        !          1399: 
        !          1400:                next_object = object->shadow;
        !          1401:                if (next_object != VM_OBJECT_NULL) {
        !          1402:                    offset += object->shadow_offset;
        !          1403:                    vm_object_lock(next_object);
        !          1404:                    vm_object_unlock(object);
        !          1405:                    object = next_object;
        !          1406:                }
        !          1407:                else {
        !          1408:                    /*
        !          1409:                     * End of chain - we are done.
        !          1410:                     */
        !          1411:                    break;
        !          1412:                }
        !          1413:            }
        !          1414:            else {
        !          1415:                /*
        !          1416:                 * Pages in shadowed objects may never have
        !          1417:                 * write permission - we may stop here.
        !          1418:                 */
        !          1419:                break;
        !          1420:            }
        !          1421:        }
        !          1422: 
        !          1423:        vm_object_unlock(object);
        !          1424: }
        !          1425: 
        !          1426: /*
        !          1427:  *     Routine:        vm_object_copy_slowly
        !          1428:  *
        !          1429:  *     Description:
        !          1430:  *             Copy the specified range of the source
        !          1431:  *             virtual memory object without using
        !          1432:  *             protection-based optimizations (such
        !          1433:  *             as copy-on-write).  The pages in the
        !          1434:  *             region are actually copied.
        !          1435:  *
        !          1436:  *     In/out conditions:
        !          1437:  *             The caller must hold a reference and a lock
        !          1438:  *             for the source virtual memory object.  The source
        !          1439:  *             object will be returned *unlocked*.
        !          1440:  *
        !          1441:  *     Results:
        !          1442:  *             If the copy is completed successfully, KERN_SUCCESS is
        !          1443:  *             returned.  If the caller asserted the interruptible
        !          1444:  *             argument, and an interruption occurred while waiting
        !          1445:  *             for a user-generated event, MACH_SEND_INTERRUPTED is
        !          1446:  *             returned.  Other values may be returned to indicate
        !          1447:  *             hard errors during the copy operation.
        !          1448:  *
        !          1449:  *             A new virtual memory object is returned in a
        !          1450:  *             parameter (_result_object).  The contents of this
        !          1451:  *             new object, starting at a zero offset, are a copy
        !          1452:  *             of the source memory region.  In the event of
        !          1453:  *             an error, this parameter will contain the value
        !          1454:  *             VM_OBJECT_NULL.
        !          1455:  */
        !          1456: kern_return_t
        !          1457: vm_object_copy_slowly(
        !          1458:        register vm_object_t    src_object,
        !          1459:        vm_offset_t             src_offset,
        !          1460:        vm_size_t               size,
        !          1461:        boolean_t               interruptible,
        !          1462:        vm_object_t             *_result_object)        /* OUT */
        !          1463: {
        !          1464:        vm_object_t     new_object;
        !          1465:        vm_offset_t     new_offset;
        !          1466: 
        !          1467:        vm_offset_t     src_lo_offset = src_offset;
        !          1468:        vm_offset_t     src_hi_offset = src_offset + size;
        !          1469: 
        !          1470:        XPR(XPR_VM_OBJECT, "v_o_c_slowly obj 0x%x off 0x%x size 0x%x\n",
        !          1471:            src_object, src_offset, size, 0, 0);
        !          1472: 
        !          1473:        if (size == 0) {
        !          1474:                vm_object_unlock(src_object);
        !          1475:                *_result_object = VM_OBJECT_NULL;
        !          1476:                return(KERN_INVALID_ARGUMENT);
        !          1477:        }
        !          1478: 
        !          1479:        /*
        !          1480:         *      Prevent destruction of the source object while we copy.
        !          1481:         */
        !          1482: 
        !          1483:        assert(src_object->ref_count > 0);
        !          1484:        src_object->ref_count++;
        !          1485:        VM_OBJ_RES_INCR(src_object);
        !          1486:        vm_object_unlock(src_object);
        !          1487: 
        !          1488:        /*
        !          1489:         *      Create a new object to hold the copied pages.
        !          1490:         *      A few notes:
        !          1491:         *              We fill the new object starting at offset 0,
        !          1492:         *               regardless of the input offset.
        !          1493:         *              We don't bother to lock the new object within
        !          1494:         *               this routine, since we have the only reference.
        !          1495:         */
        !          1496: 
        !          1497:        new_object = vm_object_allocate(size);
        !          1498:        new_offset = 0;
        !          1499: 
        !          1500:        assert(size == trunc_page(size));       /* Will the loop terminate? */
        !          1501: 
        !          1502:        for ( ;
        !          1503:            size != 0 ;
        !          1504:            src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
        !          1505:            ) {
        !          1506:                vm_page_t       new_page;
        !          1507:                vm_fault_return_t result;
        !          1508: 
        !          1509:                while ((new_page = vm_page_alloc(new_object, new_offset))
        !          1510:                                == VM_PAGE_NULL) {
        !          1511:                        VM_PAGE_WAIT();
        !          1512:                }
        !          1513: 
        !          1514:                do {
        !          1515:                        vm_prot_t       prot = VM_PROT_READ;
        !          1516:                        vm_page_t       _result_page;
        !          1517:                        vm_page_t       top_page;
        !          1518:                        register
        !          1519:                        vm_page_t       result_page;
        !          1520:                        kern_return_t   error_code;
        !          1521: 
        !          1522:                        vm_object_lock(src_object);
        !          1523:                        vm_object_paging_begin(src_object);
        !          1524: 
        !          1525:                        XPR(XPR_VM_FAULT,"vm_object_copy_slowly -> vm_fault_page",0,0,0,0,0);
        !          1526:                        result = vm_fault_page(src_object, src_offset,
        !          1527:                                VM_PROT_READ, FALSE, interruptible,
        !          1528:                                src_lo_offset, src_hi_offset,
        !          1529:                                VM_BEHAVIOR_SEQUENTIAL,
        !          1530:                                &prot, &_result_page, &top_page,
        !          1531:                                (int *)0,
        !          1532:                                &error_code, FALSE, FALSE);
        !          1533: 
        !          1534:                        switch(result) {
        !          1535:                                case VM_FAULT_SUCCESS:
        !          1536:                                        result_page = _result_page;
        !          1537: 
        !          1538:                                        /*
        !          1539:                                         *      We don't need to hold the object
        !          1540:                                         *      lock -- the busy page will be enough.
        !          1541:                                         *      [We don't care about picking up any
        !          1542:                                         *      new modifications.]
        !          1543:                                         *
        !          1544:                                         *      Copy the page to the new object.
        !          1545:                                         *
        !          1546:                                         *      POLICY DECISION:
        !          1547:                                         *              If result_page is clean,
        !          1548:                                         *              we could steal it instead
        !          1549:                                         *              of copying.
        !          1550:                                         */
        !          1551: 
        !          1552:                                        vm_object_unlock(result_page->object);
        !          1553:                                        vm_page_copy(result_page, new_page);
        !          1554: 
        !          1555:                                        /*
        !          1556:                                         *      Let go of both pages (make them
        !          1557:                                         *      not busy, perform wakeup, activate).
        !          1558:                                         */
        !          1559: 
        !          1560:                                        new_page->busy = FALSE;
        !          1561:                                        new_page->dirty = TRUE;
        !          1562:                                        vm_object_lock(result_page->object);
        !          1563:                                        PAGE_WAKEUP_DONE(result_page);
        !          1564: 
        !          1565:                                        vm_page_lock_queues();
        !          1566:                                        if (!result_page->active &&
        !          1567:                                            !result_page->inactive)
        !          1568:                                                vm_page_activate(result_page);
        !          1569:                                        vm_page_activate(new_page);
        !          1570:                                        vm_page_unlock_queues();
        !          1571: 
        !          1572:                                        /*
        !          1573:                                         *      Release paging references and
        !          1574:                                         *      top-level placeholder page, if any.
        !          1575:                                         */
        !          1576: 
        !          1577:                                        vm_fault_cleanup(result_page->object,
        !          1578:                                                        top_page);
        !          1579: 
        !          1580:                                        break;
        !          1581:                                
        !          1582:                                case VM_FAULT_RETRY:
        !          1583:                                        break;
        !          1584: 
        !          1585:                                case VM_FAULT_MEMORY_SHORTAGE:
        !          1586:                                        VM_PAGE_WAIT();
        !          1587:                                        break;
        !          1588: 
        !          1589:                                case VM_FAULT_FICTITIOUS_SHORTAGE:
        !          1590:                                        vm_page_more_fictitious();
        !          1591:                                        break;
        !          1592: 
        !          1593:                                case VM_FAULT_INTERRUPTED:
        !          1594:                                        vm_page_free(new_page);
        !          1595:                                        vm_object_deallocate(new_object);
        !          1596:                                        vm_object_deallocate(src_object);
        !          1597:                                        *_result_object = VM_OBJECT_NULL;
        !          1598:                                        return(MACH_SEND_INTERRUPTED);
        !          1599: 
        !          1600:                                case VM_FAULT_MEMORY_ERROR:
        !          1601:                                        /*
        !          1602:                                         * A policy choice:
        !          1603:                                         *      (a) ignore pages that we can't
        !          1604:                                         *          copy
        !          1605:                                         *      (b) return the null object if
        !          1606:                                         *          any page fails [chosen]
        !          1607:                                         */
        !          1608: 
        !          1609:                                        vm_page_free(new_page);
        !          1610:                                        vm_object_deallocate(new_object);
        !          1611:                                        vm_object_deallocate(src_object);
        !          1612:                                        *_result_object = VM_OBJECT_NULL;
        !          1613:                                        return(error_code ? error_code:
        !          1614:                                                KERN_MEMORY_ERROR);
        !          1615:                        }
        !          1616:                } while (result != VM_FAULT_SUCCESS);
        !          1617:        }
        !          1618: 
        !          1619:        /*
        !          1620:         *      Lose the extra reference, and return our object.
        !          1621:         */
        !          1622: 
        !          1623:        vm_object_deallocate(src_object);
        !          1624:        *_result_object = new_object;
        !          1625:        return(KERN_SUCCESS);
        !          1626: }
        !          1627: 
        !          1628: /*
        !          1629:  *     Routine:        vm_object_copy_quickly
        !          1630:  *
        !          1631:  *     Purpose:
        !          1632:  *             Copy the specified range of the source virtual
        !          1633:  *             memory object, if it can be done without waiting
        !          1634:  *             for user-generated events.
        !          1635:  *
        !          1636:  *     Results:
        !          1637:  *             If the copy is successful, the copy is returned in
        !          1638:  *             the arguments; otherwise, the arguments are not
        !          1639:  *             affected.
        !          1640:  *
        !          1641:  *     In/out conditions:
        !          1642:  *             The object should be unlocked on entry and exit.
        !          1643:  */
        !          1644: 
        !          1645: /*ARGSUSED*/
        !          1646: boolean_t
        !          1647: vm_object_copy_quickly(
        !          1648:        vm_object_t     *_object,               /* INOUT */
        !          1649:        vm_offset_t     offset,                 /* IN */
        !          1650:        vm_size_t       size,                   /* IN */
        !          1651:        boolean_t       *_src_needs_copy,       /* OUT */
        !          1652:        boolean_t       *_dst_needs_copy)       /* OUT */
        !          1653: {
        !          1654:        vm_object_t     object = *_object;
        !          1655:        memory_object_copy_strategy_t copy_strategy;
        !          1656: 
        !          1657:        XPR(XPR_VM_OBJECT, "v_o_c_quickly obj 0x%x off 0x%x size 0x%x\n",
        !          1658:            *_object, offset, size, 0, 0);
        !          1659:        if (object == VM_OBJECT_NULL) {
        !          1660:                *_src_needs_copy = FALSE;
        !          1661:                *_dst_needs_copy = FALSE;
        !          1662:                return(TRUE);
        !          1663:        }
        !          1664: 
        !          1665:        vm_object_lock(object);
        !          1666: 
        !          1667:        copy_strategy = object->copy_strategy;
        !          1668: 
        !          1669:        switch (copy_strategy) {
        !          1670:        case MEMORY_OBJECT_COPY_SYMMETRIC:
        !          1671: 
        !          1672:                /*
        !          1673:                 *      Symmetric copy strategy.
        !          1674:                 *      Make another reference to the object.
        !          1675:                 *      Leave object/offset unchanged.
        !          1676:                 */
        !          1677: 
        !          1678:                assert(object->ref_count > 0);
        !          1679:                object->ref_count++;
        !          1680:                vm_object_res_reference(object);
        !          1681:                object->shadowed = TRUE;
        !          1682:                vm_object_unlock(object);
        !          1683: 
        !          1684:                /*
        !          1685:                 *      Both source and destination must make
        !          1686:                 *      shadows, and the source must be made
        !          1687:                 *      read-only if not already.
        !          1688:                 */
        !          1689: 
        !          1690:                *_src_needs_copy = TRUE;
        !          1691:                *_dst_needs_copy = TRUE;
        !          1692: 
        !          1693:                break;
        !          1694: 
        !          1695:        case MEMORY_OBJECT_COPY_DELAY:
        !          1696:                vm_object_unlock(object);
        !          1697:                return(FALSE);
        !          1698: 
        !          1699:        default:
        !          1700:                vm_object_unlock(object);
        !          1701:                return(FALSE);
        !          1702:        }
        !          1703:        return(TRUE);
        !          1704: }
        !          1705: 
        !          1706: int copy_call_count = 0;
        !          1707: int copy_call_sleep_count = 0;
        !          1708: int copy_call_restart_count = 0;
        !          1709: 
        !          1710: /*
        !          1711:  *     Routine:        vm_object_copy_call [internal]
        !          1712:  *
        !          1713:  *     Description:
        !          1714:  *             Copy the source object (src_object), using the
        !          1715:  *             user-managed copy algorithm.
        !          1716:  *
        !          1717:  *     In/out conditions:
        !          1718:  *             The source object must be locked on entry.  It
        !          1719:  *             will be *unlocked* on exit.
        !          1720:  *
        !          1721:  *     Results:
        !          1722:  *             If the copy is successful, KERN_SUCCESS is returned.
        !          1723:  *             A new object that represents the copied virtual
        !          1724:  *             memory is returned in a parameter (*_result_object).
        !          1725:  *             If the return value indicates an error, this parameter
        !          1726:  *             is not valid.
        !          1727:  */
        !          1728: kern_return_t
        !          1729: vm_object_copy_call(
        !          1730:        vm_object_t     src_object,
        !          1731:        vm_offset_t     src_offset,
        !          1732:        vm_size_t       size,
        !          1733:        vm_object_t     *_result_object)        /* OUT */
        !          1734: {
        !          1735:        kern_return_t   kr;
        !          1736:        vm_object_t     copy;
        !          1737:        boolean_t       check_ready = FALSE;
        !          1738: 
        !          1739:        /*
        !          1740:         *      If a copy is already in progress, wait and retry.
        !          1741:         *
        !          1742:         *      XXX
        !          1743:         *      Consider making this call interruptable, as Mike
        !          1744:         *      intended it to be.
        !          1745:         *
        !          1746:         *      XXXO
        !          1747:         *      Need a counter or version or something to allow
        !          1748:         *      us to use the copy that the currently requesting
        !          1749:         *      thread is obtaining -- is it worth adding to the
        !          1750:         *      vm object structure? Depends how common this case it.
        !          1751:         */
        !          1752:        copy_call_count++;
        !          1753:        while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
        !          1754:                vm_object_wait(src_object, VM_OBJECT_EVENT_COPY_CALL,
        !          1755:                               THREAD_UNINT);
        !          1756:                vm_object_lock(src_object);
        !          1757:                copy_call_restart_count++;
        !          1758:        }
        !          1759: 
        !          1760:        /*
        !          1761:         *      Indicate (for the benefit of memory_object_create_copy)
        !          1762:         *      that we want a copy for src_object. (Note that we cannot
        !          1763:         *      do a real assert_wait before calling memory_object_copy,
        !          1764:         *      so we simply set the flag.)
        !          1765:         */
        !          1766: 
        !          1767:        vm_object_set_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL);
        !          1768:        vm_object_unlock(src_object);
        !          1769: 
        !          1770:        /*
        !          1771:         *      Ask the memory manager to give us a memory object
        !          1772:         *      which represents a copy of the src object.
        !          1773:         *      The memory manager may give us a memory object
        !          1774:         *      which we already have, or it may give us a
        !          1775:         *      new memory object. This memory object will arrive
        !          1776:         *      via memory_object_create_copy.
        !          1777:         */
        !          1778: 
        !          1779:        kr = KERN_FAILURE;      /* XXX need to change memory_object.defs */
        !          1780:        if (kr != KERN_SUCCESS) {
        !          1781:                return kr;
        !          1782:        }
        !          1783: 
        !          1784:        /*
        !          1785:         *      Wait for the copy to arrive.
        !          1786:         */
        !          1787:        vm_object_lock(src_object);
        !          1788:        while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
        !          1789:                vm_object_wait(src_object, VM_OBJECT_EVENT_COPY_CALL,
        !          1790:                               THREAD_UNINT);
        !          1791:                vm_object_lock(src_object);
        !          1792:                copy_call_sleep_count++;
        !          1793:        }
        !          1794: Retry:
        !          1795:        assert(src_object->copy != VM_OBJECT_NULL);
        !          1796:        copy = src_object->copy;
        !          1797:        if (!vm_object_lock_try(copy)) {
        !          1798:                vm_object_unlock(src_object);
        !          1799:                mutex_pause();  /* wait a bit */
        !          1800:                vm_object_lock(src_object);
        !          1801:                goto Retry;
        !          1802:        }
        !          1803:        if (copy->size < src_offset+size)
        !          1804:                copy->size = src_offset+size;
        !          1805: 
        !          1806:        if (!copy->pager_ready)
        !          1807:                check_ready = TRUE;
        !          1808: 
        !          1809:        /*
        !          1810:         *      Return the copy.
        !          1811:         */
        !          1812:        *_result_object = copy;
        !          1813:        vm_object_unlock(copy);
        !          1814:        vm_object_unlock(src_object);
        !          1815: 
        !          1816:        /* Wait for the copy to be ready. */
        !          1817:        if (check_ready == TRUE) {
        !          1818:                vm_object_lock(copy);
        !          1819:                while (!copy->pager_ready) {
        !          1820:                        vm_object_wait(copy, VM_OBJECT_EVENT_PAGER_READY,
        !          1821:                                                                        FALSE);
        !          1822:                        vm_object_lock(copy);
        !          1823:                }
        !          1824:                vm_object_unlock(copy);
        !          1825:        }
        !          1826: 
        !          1827:        return KERN_SUCCESS;
        !          1828: }
        !          1829: 
        !          1830: int copy_delayed_lock_collisions = 0;
        !          1831: int copy_delayed_max_collisions = 0;
        !          1832: int copy_delayed_lock_contention = 0;
        !          1833: int copy_delayed_protect_iterate = 0;
        !          1834: int copy_delayed_protect_lookup = 0;
        !          1835: int copy_delayed_protect_lookup_wait = 0;
        !          1836: 
        !          1837: /*
        !          1838:  *     Routine:        vm_object_copy_delayed [internal]
        !          1839:  *
        !          1840:  *     Description:
        !          1841:  *             Copy the specified virtual memory object, using
        !          1842:  *             the asymmetric copy-on-write algorithm.
        !          1843:  *
        !          1844:  *     In/out conditions:
        !          1845:  *             The object must be unlocked on entry.
        !          1846:  *
        !          1847:  *             This routine will not block waiting for user-generated
        !          1848:  *             events.  It is not interruptible.
        !          1849:  */
        !          1850: vm_object_t
        !          1851: vm_object_copy_delayed(
        !          1852:        vm_object_t     src_object,
        !          1853:        vm_offset_t     src_offset,
        !          1854:        vm_size_t       size)
        !          1855: {
        !          1856:        vm_object_t     new_copy = VM_OBJECT_NULL;
        !          1857:        vm_object_t     old_copy;
        !          1858:        vm_page_t       p;
        !          1859:        vm_size_t       copy_size;
        !          1860: 
        !          1861:        int collisions = 0;
        !          1862:        /*
        !          1863:         *      The user-level memory manager wants to see all of the changes
        !          1864:         *      to this object, but it has promised not to make any changes on
        !          1865:         *      its own.
        !          1866:         *
        !          1867:         *      Perform an asymmetric copy-on-write, as follows:
        !          1868:         *              Create a new object, called a "copy object" to hold
        !          1869:         *               pages modified by the new mapping  (i.e., the copy,
        !          1870:         *               not the original mapping).
        !          1871:         *              Record the original object as the backing object for
        !          1872:         *               the copy object.  If the original mapping does not
        !          1873:         *               change a page, it may be used read-only by the copy.
        !          1874:         *              Record the copy object in the original object.
        !          1875:         *               When the original mapping causes a page to be modified,
        !          1876:         *               it must be copied to a new page that is "pushed" to
        !          1877:         *               the copy object.
        !          1878:         *              Mark the new mapping (the copy object) copy-on-write.
        !          1879:         *               This makes the copy object itself read-only, allowing
        !          1880:         *               it to be reused if the original mapping makes no
        !          1881:         *               changes, and simplifying the synchronization required
        !          1882:         *               in the "push" operation described above.
        !          1883:         *
        !          1884:         *      The copy-on-write is said to be assymetric because the original
        !          1885:         *      object is *not* marked copy-on-write. A copied page is pushed
        !          1886:         *      to the copy object, regardless which party attempted to modify
        !          1887:         *      the page.
        !          1888:         *
        !          1889:         *      Repeated asymmetric copy operations may be done. If the
        !          1890:         *      original object has not been changed since the last copy, its
        !          1891:         *      copy object can be reused. Otherwise, a new copy object can be
        !          1892:         *      inserted between the original object and its previous copy
        !          1893:         *      object.  Since any copy object is read-only, this cannot affect
        !          1894:         *      affect the contents of the previous copy object.
        !          1895:         *
        !          1896:         *      Note that a copy object is higher in the object tree than the
        !          1897:         *      original object; therefore, use of the copy object recorded in
        !          1898:         *      the original object must be done carefully, to avoid deadlock.
        !          1899:         */
        !          1900: 
        !          1901:  Retry:
        !          1902:        vm_object_lock(src_object);
        !          1903:  
        !          1904:        /*
        !          1905:         *      See whether we can reuse the result of a previous
        !          1906:         *      copy operation.
        !          1907:         */
        !          1908: 
        !          1909:        old_copy = src_object->copy;
        !          1910:        if (old_copy != VM_OBJECT_NULL) {
        !          1911:                /*
        !          1912:                 *      Try to get the locks (out of order)
        !          1913:                 */
        !          1914:                if (!vm_object_lock_try(old_copy)) {
        !          1915:                        vm_object_unlock(src_object);
        !          1916:                        mutex_pause();
        !          1917: 
        !          1918:                        /* Heisenberg Rules */
        !          1919:                        copy_delayed_lock_collisions++;
        !          1920:                        if (collisions++ == 0)
        !          1921:                                copy_delayed_lock_contention++;
        !          1922: 
        !          1923:                        if (collisions > copy_delayed_max_collisions)
        !          1924:                                copy_delayed_max_collisions = collisions;
        !          1925: 
        !          1926:                        goto Retry;
        !          1927:                }
        !          1928: 
        !          1929:                /*
        !          1930:                 *      Determine whether the old copy object has
        !          1931:                 *      been modified.
        !          1932:                 */
        !          1933: 
        !          1934:                if (old_copy->resident_page_count == 0 &&
        !          1935:                    !old_copy->pager_created) {
        !          1936:                        /*
        !          1937:                         *      It has not been modified.
        !          1938:                         *
        !          1939:                         *      Return another reference to
        !          1940:                         *      the existing copy-object.
        !          1941:                         */
        !          1942:                        assert(old_copy->ref_count > 0);
        !          1943:                        old_copy->ref_count++;
        !          1944: 
        !          1945:                        if (old_copy->size < src_offset+size)
        !          1946:                                old_copy->size = src_offset+size;
        !          1947: 
        !          1948: #if    TASK_SWAPPER
        !          1949:                        /*
        !          1950:                         * We have to reproduce some of the code from
        !          1951:                         * vm_object_res_reference because we've taken
        !          1952:                         * the locks out of order here, and deadlock
        !          1953:                         * would result if we simply called that function.
        !          1954:                         */
        !          1955:                        if (++old_copy->res_count == 1) {
        !          1956:                                assert(old_copy->shadow == src_object);
        !          1957:                                vm_object_res_reference(src_object);
        !          1958:                        }
        !          1959: #endif /* TASK_SWAPPER */
        !          1960: 
        !          1961:                        vm_object_unlock(old_copy);
        !          1962:                        vm_object_unlock(src_object);
        !          1963: 
        !          1964:                        if (new_copy != VM_OBJECT_NULL) {
        !          1965:                                vm_object_unlock(new_copy);
        !          1966:                                vm_object_deallocate(new_copy);
        !          1967:                        }
        !          1968: 
        !          1969:                        return(old_copy);
        !          1970:                }
        !          1971:                if (new_copy == VM_OBJECT_NULL) {
        !          1972:                        vm_object_unlock(old_copy);
        !          1973:                        vm_object_unlock(src_object);
        !          1974:                        new_copy = vm_object_allocate(src_offset + size);
        !          1975:                        vm_object_lock(new_copy);
        !          1976:                        goto Retry;
        !          1977:                }
        !          1978: 
        !          1979:                /*
        !          1980:                 * Adjust the size argument so that the newly-created 
        !          1981:                 * copy object will be large enough to back either the
        !          1982:                 * new old copy object or the new mapping.
        !          1983:                 */
        !          1984:                if (old_copy->size > src_offset+size)
        !          1985:                        size =  old_copy->size - src_offset;
        !          1986: 
        !          1987:                /*
        !          1988:                 *      The copy-object is always made large enough to
        !          1989:                 *      completely shadow the original object, since
        !          1990:                 *      it may have several users who want to shadow
        !          1991:                 *      the original object at different points.
        !          1992:                 */
        !          1993: 
        !          1994:                assert((old_copy->shadow == src_object) &&
        !          1995:                    (old_copy->shadow_offset == (vm_offset_t) 0));
        !          1996: 
        !          1997:                /*
        !          1998:                 *      Make the old copy-object shadow the new one.
        !          1999:                 *      It will receive no more pages from the original
        !          2000:                 *      object.
        !          2001:                 */
        !          2002: 
        !          2003:                src_object->ref_count--;        /* remove ref. from old_copy */
        !          2004:                assert(src_object->ref_count > 0);
        !          2005:                old_copy->shadow = new_copy;
        !          2006:                assert(new_copy->ref_count > 0);
        !          2007:                new_copy->ref_count++;          /* for old_copy->shadow ref. */
        !          2008: 
        !          2009: #if TASK_SWAPPER
        !          2010:                if (old_copy->res_count) {
        !          2011:                        VM_OBJ_RES_INCR(new_copy);
        !          2012:                        VM_OBJ_RES_DECR(src_object);
        !          2013:                }
        !          2014: #endif
        !          2015: 
        !          2016:                vm_object_unlock(old_copy);     /* done with old_copy */
        !          2017:        } else if (new_copy == VM_OBJECT_NULL) {
        !          2018:                vm_object_unlock(src_object);
        !          2019:                new_copy = vm_object_allocate(src_offset + size);
        !          2020:                vm_object_lock(new_copy);
        !          2021:                goto Retry;
        !          2022:        }
        !          2023: 
        !          2024:        /*
        !          2025:         * Readjust the copy-object size if necessary.
        !          2026:         */
        !          2027:        copy_size = new_copy->size;
        !          2028:        if (copy_size < src_offset+size) {
        !          2029:                copy_size = src_offset+size;
        !          2030:                new_copy->size = copy_size;
        !          2031:        }
        !          2032: 
        !          2033:        /*
        !          2034:         *      Point the new copy at the existing object.
        !          2035:         */
        !          2036: 
        !          2037:        new_copy->shadow = src_object;
        !          2038:        new_copy->shadow_offset = 0;
        !          2039:        new_copy->shadowed = TRUE;      /* caller must set needs_copy */
        !          2040:        assert(src_object->ref_count > 0);
        !          2041:        src_object->ref_count++;
        !          2042:        VM_OBJ_RES_INCR(src_object);
        !          2043:        src_object->copy = new_copy;
        !          2044:        vm_object_unlock(new_copy);
        !          2045: 
        !          2046:        /*
        !          2047:         *      Mark all (current) pages of the existing object copy-on-write.
        !          2048:         *      This object may have a shadow chain below it, but
        !          2049:         *      those pages will already be marked copy-on-write.
        !          2050:         */
        !          2051: 
        !          2052:        vm_object_paging_wait(src_object, THREAD_UNINT);
        !          2053:        copy_delayed_protect_iterate++;
        !          2054:        queue_iterate(&src_object->memq, p, vm_page_t, listq) {
        !          2055:            if (!p->fictitious)
        !          2056:                pmap_page_protect(p->phys_addr, 
        !          2057:                                  (VM_PROT_ALL & ~VM_PROT_WRITE &
        !          2058:                                   ~p->page_lock));
        !          2059:        }
        !          2060:        vm_object_unlock(src_object);
        !          2061:        XPR(XPR_VM_OBJECT,
        !          2062:                "vm_object_copy_delayed: used copy object %X for source %X\n",
        !          2063:                (integer_t)new_copy, (integer_t)src_object, 0, 0, 0);
        !          2064: 
        !          2065:        return(new_copy);
        !          2066: }
        !          2067: 
        !          2068: /*
        !          2069:  *     Routine:        vm_object_copy_strategically
        !          2070:  *
        !          2071:  *     Purpose:
        !          2072:  *             Perform a copy according to the source object's
        !          2073:  *             declared strategy.  This operation may block,
        !          2074:  *             and may be interrupted.
        !          2075:  */
        !          2076: kern_return_t
        !          2077: vm_object_copy_strategically(
        !          2078:        register vm_object_t    src_object,
        !          2079:        vm_offset_t             src_offset,
        !          2080:        vm_size_t               size,
        !          2081:        vm_object_t             *dst_object,    /* OUT */
        !          2082:        vm_offset_t             *dst_offset,    /* OUT */
        !          2083:        boolean_t               *dst_needs_copy) /* OUT */
        !          2084: {
        !          2085:        boolean_t       result;
        !          2086:        boolean_t       interruptible = THREAD_ABORTSAFE; /* XXX */
        !          2087:        memory_object_copy_strategy_t copy_strategy;
        !          2088: 
        !          2089:        assert(src_object != VM_OBJECT_NULL);
        !          2090: 
        !          2091:        vm_object_lock(src_object);
        !          2092: 
        !          2093:        /*
        !          2094:         *      The copy strategy is only valid if the memory manager
        !          2095:         *      is "ready". Internal objects are always ready.
        !          2096:         */
        !          2097: 
        !          2098:        while (!src_object->internal && !src_object->pager_ready) {
        !          2099: 
        !          2100:                vm_object_wait( src_object,
        !          2101:                                VM_OBJECT_EVENT_PAGER_READY,
        !          2102:                                interruptible);
        !          2103:                if (interruptible &&
        !          2104:                    (current_thread()->wait_result != THREAD_AWAKENED)) {
        !          2105:                        *dst_object = VM_OBJECT_NULL;
        !          2106:                        *dst_offset = 0;
        !          2107:                        *dst_needs_copy = FALSE;
        !          2108:                        return(MACH_SEND_INTERRUPTED);
        !          2109:                }
        !          2110:                vm_object_lock(src_object);
        !          2111:        }
        !          2112: 
        !          2113:        copy_strategy = src_object->copy_strategy;
        !          2114: 
        !          2115:        /*
        !          2116:         *      Use the appropriate copy strategy.
        !          2117:         */
        !          2118: 
        !          2119:        switch (copy_strategy) {
        !          2120:            case MEMORY_OBJECT_COPY_NONE:
        !          2121:                result = vm_object_copy_slowly(src_object, src_offset, size,
        !          2122:                                               interruptible, dst_object);
        !          2123:                if (result == KERN_SUCCESS) {
        !          2124:                        *dst_offset = 0;
        !          2125:                        *dst_needs_copy = FALSE;
        !          2126:                }
        !          2127:                break;
        !          2128: 
        !          2129:            case MEMORY_OBJECT_COPY_CALL:
        !          2130:                result = vm_object_copy_call(src_object, src_offset, size,
        !          2131:                                dst_object);
        !          2132:                if (result == KERN_SUCCESS) {
        !          2133:                        *dst_offset = src_offset;
        !          2134:                        *dst_needs_copy = TRUE;
        !          2135:                }
        !          2136:                break;
        !          2137: 
        !          2138:            case MEMORY_OBJECT_COPY_DELAY:
        !          2139:                vm_object_unlock(src_object);
        !          2140:                *dst_object = vm_object_copy_delayed(src_object,
        !          2141:                                                     src_offset, size);
        !          2142:                *dst_offset = src_offset;
        !          2143:                *dst_needs_copy = TRUE;
        !          2144:                result = KERN_SUCCESS;
        !          2145:                break;
        !          2146: 
        !          2147:            case MEMORY_OBJECT_COPY_SYMMETRIC:
        !          2148:                XPR(XPR_VM_OBJECT, "v_o_c_strategically obj 0x%x off 0x%x size 0x%x\n",(natural_t)src_object, src_offset, size, 0, 0);
        !          2149:                vm_object_unlock(src_object);
        !          2150:                result = KERN_MEMORY_RESTART_COPY;
        !          2151:                break;
        !          2152: 
        !          2153:            default:
        !          2154:                panic("copy_strategically: bad strategy");
        !          2155:                result = KERN_INVALID_ARGUMENT;
        !          2156:        }
        !          2157:        return(result);
        !          2158: }
        !          2159: 
        !          2160: /*
        !          2161:  *     vm_object_shadow:
        !          2162:  *
        !          2163:  *     Create a new object which is backed by the
        !          2164:  *     specified existing object range.  The source
        !          2165:  *     object reference is deallocated.
        !          2166:  *
        !          2167:  *     The new object and offset into that object
        !          2168:  *     are returned in the source parameters.
        !          2169:  */
        !          2170: boolean_t vm_object_shadow_check = FALSE;
        !          2171: 
        !          2172: boolean_t
        !          2173: vm_object_shadow(
        !          2174:        vm_object_t     *object,        /* IN/OUT */
        !          2175:        vm_offset_t     *offset,        /* IN/OUT */
        !          2176:        vm_size_t       length)
        !          2177: {
        !          2178:        register vm_object_t    source;
        !          2179:        register vm_object_t    result;
        !          2180: 
        !          2181:        source = *object;
        !          2182:        assert(source->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
        !          2183: 
        !          2184:        /*
        !          2185:         *      Determine if we really need a shadow.
        !          2186:         */
        !          2187: 
        !          2188:        if (vm_object_shadow_check && source->ref_count == 1 &&
        !          2189:            (source->shadow == VM_OBJECT_NULL ||
        !          2190:             source->shadow->copy == VM_OBJECT_NULL))
        !          2191:        {
        !          2192:                source->shadowed = FALSE;
        !          2193:                return FALSE;
        !          2194:        }
        !          2195: 
        !          2196:        /*
        !          2197:         *      Allocate a new object with the given length
        !          2198:         */
        !          2199: 
        !          2200:        if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
        !          2201:                panic("vm_object_shadow: no object for shadowing");
        !          2202: 
        !          2203:        /*
        !          2204:         *      The new object shadows the source object, adding
        !          2205:         *      a reference to it.  Our caller changes his reference
        !          2206:         *      to point to the new object, removing a reference to
        !          2207:         *      the source object.  Net result: no change of reference
        !          2208:         *      count.
        !          2209:         */
        !          2210:        result->shadow = source;
        !          2211:        
        !          2212:        /*
        !          2213:         *      Store the offset into the source object,
        !          2214:         *      and fix up the offset into the new object.
        !          2215:         */
        !          2216: 
        !          2217:        result->shadow_offset = *offset;
        !          2218: 
        !          2219:        /*
        !          2220:         *      Return the new things
        !          2221:         */
        !          2222: 
        !          2223:        *offset = 0;
        !          2224:        *object = result;
        !          2225:        return TRUE;
        !          2226: }
        !          2227: 
        !          2228: /*
        !          2229:  *     The relationship between vm_object structures and
        !          2230:  *     the memory_object ports requires careful synchronization.
        !          2231:  *
        !          2232:  *     All associations are created by vm_object_enter.  All three
        !          2233:  *     port fields are filled in, as follows:
        !          2234:  *             pager:  the memory_object port itself, supplied by
        !          2235:  *                     the user requesting a mapping (or the kernel,
        !          2236:  *                     when initializing internal objects); the
        !          2237:  *                     kernel simulates holding send rights by keeping
        !          2238:  *                     a port reference;
        !          2239:  *             pager_request:
        !          2240:  *                     the memory object control port,
        !          2241:  *                     created by the kernel; the kernel holds
        !          2242:  *                     receive (and ownership) rights to this
        !          2243:  *                     port, but no other references.
        !          2244:  *     All of the ports are referenced by their global names.
        !          2245:  *
        !          2246:  *     When initialization is complete, the "initialized" field
        !          2247:  *     is asserted.  Other mappings using a particular memory object,
        !          2248:  *     and any references to the vm_object gained through the
        !          2249:  *     port association must wait for this initialization to occur.
        !          2250:  *
        !          2251:  *     In order to allow the memory manager to set attributes before
        !          2252:  *     requests (notably virtual copy operations, but also data or
        !          2253:  *     unlock requests) are made, a "ready" attribute is made available.
        !          2254:  *     Only the memory manager may affect the value of this attribute.
        !          2255:  *     Its value does not affect critical kernel functions, such as
        !          2256:  *     internal object initialization or destruction.  [Furthermore,
        !          2257:  *     memory objects created by the kernel are assumed to be ready
        !          2258:  *     immediately; the default memory manager need not explicitly
        !          2259:  *     set the "ready" attribute.]
        !          2260:  *
        !          2261:  *     [Both the "initialized" and "ready" attribute wait conditions
        !          2262:  *     use the "pager" field as the wait event.]
        !          2263:  *
        !          2264:  *     The port associations can be broken down by any of the
        !          2265:  *     following routines:
        !          2266:  *             vm_object_terminate:
        !          2267:  *                     No references to the vm_object remain, and
        !          2268:  *                     the object cannot (or will not) be cached.
        !          2269:  *                     This is the normal case, and is done even
        !          2270:  *                     though one of the other cases has already been
        !          2271:  *                     done.
        !          2272:  *             vm_object_destroy:
        !          2273:  *                     The memory_object port has been destroyed,
        !          2274:  *                     meaning that the kernel cannot flush dirty
        !          2275:  *                     pages or request new data or unlock existing
        !          2276:  *                     data.
        !          2277:  *             memory_object_destroy:
        !          2278:  *                     The memory manager has requested that the
        !          2279:  *                     kernel relinquish rights to the memory object
        !          2280:  *                     port.  [The memory manager may not want to
        !          2281:  *                     destroy the port, but may wish to refuse or
        !          2282:  *                     tear down existing memory mappings.]
        !          2283:  *     Each routine that breaks an association must break all of
        !          2284:  *     them at once.  At some later time, that routine must clear
        !          2285:  *     the vm_object port fields and release the port rights.
        !          2286:  *     [Furthermore, each routine must cope with the simultaneous
        !          2287:  *     or previous operations of the others.]
        !          2288:  *
        !          2289:  *     In addition to the lock on the object, the vm_object_cache_lock
        !          2290:  *     governs the port associations.  References gained through the
        !          2291:  *     port association require use of the cache lock.
        !          2292:  *
        !          2293:  *     Because the port fields may be cleared spontaneously, they
        !          2294:  *     cannot be used to determine whether a memory object has
        !          2295:  *     ever been associated with a particular vm_object.  [This
        !          2296:  *     knowledge is important to the shadow object mechanism.]
        !          2297:  *     For this reason, an additional "created" attribute is
        !          2298:  *     provided.
        !          2299:  *
        !          2300:  *     During various paging operations, the port values found in the
        !          2301:  *     vm_object must be valid.  To prevent these port rights from being
        !          2302:  *     released, and to prevent the port associations from changing
        !          2303:  *     (other than being removed, i.e., made null), routines may use
        !          2304:  *     the vm_object_paging_begin/end routines [actually, macros].
        !          2305:  *     The implementation uses the "paging_in_progress" and "wanted" fields.
        !          2306:  *     [Operations that alter the validity of the port values include the
        !          2307:  *     termination routines and vm_object_collapse.]
        !          2308:  */
        !          2309: 
        !          2310: #define        IKOT_PAGER_LOOKUP_TYPE  IKOT_PAGING_REQUEST
        !          2311: 
        !          2312: vm_object_t
        !          2313: vm_object_lookup(
        !          2314:        ipc_port_t      port)
        !          2315: {
        !          2316:        vm_object_t     object;
        !          2317: 
        !          2318: start_over:
        !          2319:        object = VM_OBJECT_NULL;
        !          2320: 
        !          2321:        if (IP_VALID(port)) {
        !          2322:                vm_object_cache_lock();
        !          2323:                ip_lock(port);
        !          2324:                if (ip_active(port) &&
        !          2325:                    (ip_kotype(port) == IKOT_PAGER_LOOKUP_TYPE)) {
        !          2326:                        object = (vm_object_t) port->ip_kobject;
        !          2327:                        if (!vm_object_lock_try(object)) {
        !          2328:                                /*
        !          2329:                                 * failed to acquire object lock.  Drop the
        !          2330:                                 * other two locks and wait for it, then go
        !          2331:                                 * back and start over in case the port
        !          2332:                                 * associations changed in the interim.
        !          2333:                                 */
        !          2334:                                ip_unlock(port);
        !          2335:                                vm_object_cache_unlock();
        !          2336:                                vm_object_lock(object);
        !          2337:                                vm_object_unlock(object);
        !          2338:                                goto start_over;
        !          2339:                        }
        !          2340: 
        !          2341:                        assert(object->alive);
        !          2342: 
        !          2343:                        if (object->ref_count == 0) {
        !          2344:                                queue_remove(&vm_object_cached_list, object,
        !          2345:                                             vm_object_t, cached_list);
        !          2346:                                vm_object_cached_count--;
        !          2347:                                XPR(XPR_VM_OBJECT_CACHE,
        !          2348:                               "vm_object_lookup: removing %X, head (%X, %X)\n",
        !          2349:                                    (integer_t)object, 
        !          2350:                                    (integer_t)vm_object_cached_list.next,
        !          2351:                                    (integer_t)vm_object_cached_list.prev, 0,0);
        !          2352:                        }
        !          2353: 
        !          2354:                        object->ref_count++;
        !          2355:                        vm_object_res_reference(object);
        !          2356:                        vm_object_unlock(object);
        !          2357:                }
        !          2358:                ip_unlock(port);
        !          2359:                vm_object_cache_unlock();
        !          2360:        }
        !          2361: 
        !          2362:        return object;
        !          2363: }
        !          2364: 
        !          2365: 
        !          2366: 
        !          2367: void
        !          2368: vm_object_destroy(
        !          2369:        ipc_port_t      pager)
        !          2370: {
        !          2371:        vm_object_t             object;
        !          2372:        vm_object_hash_entry_t  entry;
        !          2373:        pager_request_t         old_pager_request;
        !          2374: 
        !          2375:        /*
        !          2376:         *      Perform essentially the same operations as in vm_object_lookup,
        !          2377:         *      except that this time we look up based on the memory_object
        !          2378:         *      port, not the control port.
        !          2379:         */
        !          2380:        vm_object_cache_lock();
        !          2381:        entry = vm_object_hash_lookup(pager, FALSE);
        !          2382:        if (entry == VM_OBJECT_HASH_ENTRY_NULL ||
        !          2383:                        entry->object == VM_OBJECT_NULL) {
        !          2384:                vm_object_cache_unlock();
        !          2385:                return;
        !          2386:        }
        !          2387: 
        !          2388:        object = entry->object;
        !          2389:        entry->object = VM_OBJECT_NULL;
        !          2390: 
        !          2391:        vm_object_lock(object);
        !          2392:        if (object->ref_count == 0) {
        !          2393:                XPR(XPR_VM_OBJECT_CACHE,
        !          2394:                   "vm_object_destroy: removing %x from cache, head (%x, %x)\n",
        !          2395:                        (integer_t)object,
        !          2396:                        (integer_t)vm_object_cached_list.next,
        !          2397:                        (integer_t)vm_object_cached_list.prev, 0,0);
        !          2398: 
        !          2399:                queue_remove(&vm_object_cached_list, object,
        !          2400:                                vm_object_t, cached_list);
        !          2401:                vm_object_cached_count--;
        !          2402:        }
        !          2403:        object->ref_count++;
        !          2404:        vm_object_res_reference(object);
        !          2405: 
        !          2406:        object->can_persist = FALSE;
        !          2407: 
        !          2408:        assert(object->pager == pager);
        !          2409: 
        !          2410:        /*
        !          2411:         *      Remove the port associations.
        !          2412:         *
        !          2413:         *      Note that the memory_object itself is dead, so
        !          2414:         *      we don't bother with it.
        !          2415:         */
        !          2416: 
        !          2417:        object->pager = IP_NULL;
        !          2418:        vm_object_remove(object);
        !          2419: 
        !          2420:        old_pager_request = object->pager_request;
        !          2421: 
        !          2422:        object->pager_request = PAGER_REQUEST_NULL;
        !          2423: 
        !          2424:        vm_object_unlock(object);
        !          2425:        vm_object_cache_unlock();
        !          2426: 
        !          2427:        vm_object_pager_wakeup(pager);
        !          2428: 
        !          2429:        /*
        !          2430:         *      Clean up the port references.  Note that there's no
        !          2431:         *      point in trying the memory_object_terminate call
        !          2432:         *      because the memory_object itself is dead.
        !          2433:         */
        !          2434: 
        !          2435:        ipc_port_release_send(pager);
        !          2436: 
        !          2437:        if ((ipc_port_t)old_pager_request != IP_NULL)
        !          2438:                ipc_port_dealloc_kernel((ipc_port_t)old_pager_request);
        !          2439: 
        !          2440:        /*
        !          2441:         *      Restart pending page requests
        !          2442:         */
        !          2443:        vm_object_lock(object);
        !          2444: 
        !          2445:        vm_object_abort_activity(object);
        !          2446: 
        !          2447:        vm_object_unlock(object);
        !          2448: 
        !          2449:        /*
        !          2450:         *      Lose the object reference.
        !          2451:         */
        !          2452: 
        !          2453:        vm_object_deallocate(object);
        !          2454: }
        !          2455: 
        !          2456: /*
        !          2457:  *     Routine:        vm_object_enter
        !          2458:  *     Purpose:
        !          2459:  *             Find a VM object corresponding to the given
        !          2460:  *             pager; if no such object exists, create one,
        !          2461:  *             and initialize the pager.
        !          2462:  */
        !          2463: vm_object_t
        !          2464: vm_object_enter(
        !          2465:        ipc_port_t              pager,
        !          2466:        vm_size_t               size,
        !          2467:        boolean_t               internal,
        !          2468:        boolean_t               init)
        !          2469: {
        !          2470:        register vm_object_t    object;
        !          2471:        vm_object_t             new_object;
        !          2472:        boolean_t               must_init;
        !          2473:        ipc_port_t              pager_request;
        !          2474:        vm_object_hash_entry_t  entry, new_entry;
        !          2475: #ifdef MACH_BSD
        !          2476: kern_return_t   vnode_pager_init( ipc_port_t, ipc_port_t, vm_size_t);
        !          2477: #endif
        !          2478: 
        !          2479:        if (!IP_VALID(pager))
        !          2480:                return(vm_object_allocate(size));
        !          2481: 
        !          2482:        new_object = VM_OBJECT_NULL;
        !          2483:        new_entry = VM_OBJECT_HASH_ENTRY_NULL;
        !          2484:        must_init = init;
        !          2485: 
        !          2486:        /*
        !          2487:         *      Look for an object associated with this port.
        !          2488:         */
        !          2489: 
        !          2490: restart:
        !          2491:        vm_object_cache_lock();
        !          2492:        for (;;) {
        !          2493:                entry = vm_object_hash_lookup(pager, FALSE);
        !          2494: 
        !          2495:                /*
        !          2496:                 *      If a previous object is being terminated,
        !          2497:                 *      we must wait for the termination message
        !          2498:                 *      to be queued.
        !          2499:                 *
        !          2500:                 *      We set kobject to a non-null value to let the
        !          2501:                 *      terminator know that someone is waiting.
        !          2502:                 *      Among the possibilities is that the port
        !          2503:                 *      could die while we're waiting.  Must restart
        !          2504:                 *      instead of continuing the loop.
        !          2505:                 */
        !          2506: 
        !          2507:                if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
        !          2508:                        if (entry->object != VM_OBJECT_NULL)
        !          2509:                                break;
        !          2510: 
        !          2511:                        entry->waiting = TRUE;
        !          2512:                        assert_wait((event_t) pager, THREAD_UNINT);
        !          2513:                        vm_object_cache_unlock();
        !          2514:                        thread_block((void (*)(void))0);
        !          2515:                        goto restart;
        !          2516:                }
        !          2517: 
        !          2518:                /*
        !          2519:                 *      We must unlock to create a new object;
        !          2520:                 *      if we do so, we must try the lookup again.
        !          2521:                 */
        !          2522: 
        !          2523:                if (new_object == VM_OBJECT_NULL) {
        !          2524:                        vm_object_cache_unlock();
        !          2525:                        assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL);
        !          2526:                        new_entry = vm_object_hash_entry_alloc(pager);
        !          2527:                        new_object = vm_object_allocate(size);
        !          2528:                        vm_object_cache_lock();
        !          2529:                } else {
        !          2530:                        /*
        !          2531:                         *      Lookup failed twice, and we have something
        !          2532:                         *      to insert; set the object.
        !          2533:                         */
        !          2534: 
        !          2535:                        if (entry == VM_OBJECT_HASH_ENTRY_NULL) {
        !          2536:                                vm_object_hash_insert(new_entry);
        !          2537:                                entry = new_entry;
        !          2538:                                new_entry = VM_OBJECT_HASH_ENTRY_NULL;
        !          2539:                        }
        !          2540: 
        !          2541:                        entry->object = new_object;
        !          2542:                        new_object = VM_OBJECT_NULL;
        !          2543:                        must_init = TRUE;
        !          2544:                }
        !          2545:        }
        !          2546: 
        !          2547:        object = entry->object;
        !          2548:        assert(object != VM_OBJECT_NULL);
        !          2549: 
        !          2550:        if (!must_init) {
        !          2551:                vm_object_lock(object);
        !          2552:                assert(object->pager_created);
        !          2553:                assert(!internal || object->internal);
        !          2554:                if (object->ref_count == 0) {
        !          2555:                        XPR(XPR_VM_OBJECT_CACHE,
        !          2556:                    "vm_object_enter: removing %x from cache, head (%x, %x)\n",
        !          2557:                                (integer_t)object,
        !          2558:                                (integer_t)vm_object_cached_list.next,
        !          2559:                                (integer_t)vm_object_cached_list.prev, 0,0);
        !          2560:                        queue_remove(&vm_object_cached_list, object,
        !          2561:                                     vm_object_t, cached_list);
        !          2562:                        vm_object_cached_count--;
        !          2563:                }
        !          2564:                object->ref_count++;
        !          2565:                vm_object_res_reference(object);
        !          2566:                vm_object_unlock(object);
        !          2567: 
        !          2568:                VM_STAT(hits++);
        !          2569:        } 
        !          2570:        assert(object->ref_count > 0);
        !          2571: 
        !          2572:        VM_STAT(lookups++);
        !          2573: 
        !          2574:        vm_object_cache_unlock();
        !          2575: 
        !          2576:        XPR(XPR_VM_OBJECT,
        !          2577:                "vm_o_enter: pager 0x%x obj 0x%x must_init %d\n",
        !          2578:                (integer_t)pager, (integer_t)object, must_init, 0, 0);
        !          2579: 
        !          2580:        /*
        !          2581:         *      If we raced to create a vm_object but lost, let's
        !          2582:         *      throw away ours.
        !          2583:         */
        !          2584: 
        !          2585:        if (new_object != VM_OBJECT_NULL)
        !          2586:                vm_object_deallocate(new_object);
        !          2587: 
        !          2588:        if (new_entry != VM_OBJECT_HASH_ENTRY_NULL)
        !          2589:                vm_object_hash_entry_free(new_entry);
        !          2590: 
        !          2591:        if (must_init) {
        !          2592: 
        !          2593:                /*
        !          2594:                 *      Allocate request port.
        !          2595:                 */
        !          2596: 
        !          2597:                pager_request = ipc_port_alloc_kernel();
        !          2598:                assert (pager_request != IP_NULL);
        !          2599:                ipc_kobject_set(pager_request, (ipc_kobject_t) object,
        !          2600:                                IKOT_PAGING_REQUEST);
        !          2601: 
        !          2602:                vm_object_lock(object);
        !          2603: 
        !          2604:                /*
        !          2605:                 *      Copy the naked send right we were given.
        !          2606:                 */
        !          2607: 
        !          2608:                pager = ipc_port_copy_send(pager);
        !          2609:                if (!IP_VALID(pager))
        !          2610:                        panic("vm_object_enter: port died"); /* XXX */
        !          2611: 
        !          2612:                object->pager_created = TRUE;
        !          2613:                object->pager = pager;
        !          2614:                object->internal = internal;
        !          2615:                object->pager_trusted = internal;
        !          2616:                if (!internal) {
        !          2617:                        /* copy strategy invalid until set by memory manager */
        !          2618:                        object->copy_strategy = MEMORY_OBJECT_COPY_INVALID;
        !          2619:                }
        !          2620:                object->pager_request = pager_request;
        !          2621:                object->pager_ready = FALSE;
        !          2622: 
        !          2623:                vm_object_unlock(object);
        !          2624: 
        !          2625:                /*
        !          2626:                 *      Let the pager know we're using it.
        !          2627:                 */
        !          2628: 
        !          2629: #ifdef MACH_BSD
        !          2630:                if(((rpc_subsystem_t)pager_mux_hash_lookup(pager)) == 
        !          2631:                        ((rpc_subsystem_t) &vnode_pager_workaround)) {
        !          2632:                        (void) vnode_pager_init(pager,
        !          2633:                                object->pager_request,
        !          2634:                                PAGE_SIZE);
        !          2635:                } else {
        !          2636:                        (void) memory_object_init(pager,
        !          2637:                                object->pager_request,
        !          2638:                                PAGE_SIZE);
        !          2639:                }
        !          2640: #else
        !          2641:                        (void) memory_object_init(pager,
        !          2642:                                object->pager_request,
        !          2643:                                PAGE_SIZE);
        !          2644: #endif
        !          2645: 
        !          2646:                vm_object_lock(object);
        !          2647:                if (internal) {
        !          2648:                        object->pager_ready = TRUE;
        !          2649:                        vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
        !          2650:                }
        !          2651: 
        !          2652:                object->pager_initialized = TRUE;
        !          2653:                vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
        !          2654:        } else {
        !          2655:                vm_object_lock(object);
        !          2656:        }
        !          2657: 
        !          2658:        /*
        !          2659:         *      [At this point, the object must be locked]
        !          2660:         */
        !          2661: 
        !          2662:        /*
        !          2663:         *      Wait for the work above to be done by the first
        !          2664:         *      thread to map this object.
        !          2665:         */
        !          2666: 
        !          2667:        while (!object->pager_initialized) {
        !          2668:                vm_object_wait( object,
        !          2669:                                VM_OBJECT_EVENT_INITIALIZED,
        !          2670:                                THREAD_UNINT);
        !          2671:                vm_object_lock(object);
        !          2672:        }
        !          2673:        vm_object_unlock(object);
        !          2674: 
        !          2675:        XPR(XPR_VM_OBJECT,
        !          2676:            "vm_object_enter: vm_object %x, memory_object %x, internal %d\n",
        !          2677:            (integer_t)object, (integer_t)object->pager, internal, 0,0);
        !          2678:        return(object);
        !          2679: }
        !          2680: 
        !          2681: /*
        !          2682:  *     Routine:        vm_object_pager_create
        !          2683:  *     Purpose:
        !          2684:  *             Create a memory object for an internal object.
        !          2685:  *     In/out conditions:
        !          2686:  *             The object is locked on entry and exit;
        !          2687:  *             it may be unlocked within this call.
        !          2688:  *     Limitations:
        !          2689:  *             Only one thread may be performing a
        !          2690:  *             vm_object_pager_create on an object at
        !          2691:  *             a time.  Presumably, only the pageout
        !          2692:  *             daemon will be using this routine.
        !          2693:  */
        !          2694: 
        !          2695: void
        !          2696: vm_object_pager_create(
        !          2697:        register vm_object_t    object)
        !          2698: {
        !          2699:        ipc_port_t              pager;
        !          2700:        vm_object_hash_entry_t  entry;
        !          2701: #if    MACH_PAGEMAP
        !          2702:        vm_size_t               size;
        !          2703:        vm_external_map_t       map;
        !          2704: #endif /* MACH_PAGEMAP */
        !          2705: 
        !          2706:        XPR(XPR_VM_OBJECT, "vm_object_pager_create, object 0x%X\n",
        !          2707:                (integer_t)object, 0,0,0,0);
        !          2708: 
        !          2709:        if (memory_manager_default_check() != KERN_SUCCESS)
        !          2710:                return;
        !          2711: 
        !          2712:        /*
        !          2713:         *      Prevent collapse or termination by holding a paging reference
        !          2714:         */
        !          2715: 
        !          2716:        vm_object_paging_begin(object);
        !          2717:        if (object->pager_created) {
        !          2718:                /*
        !          2719:                 *      Someone else got to it first...
        !          2720:                 *      wait for them to finish initializing the ports
        !          2721:                 */
        !          2722:                while (!object->pager_initialized) {
        !          2723:                        vm_object_wait( object,
        !          2724:                                       VM_OBJECT_EVENT_INITIALIZED,
        !          2725:                                       THREAD_UNINT);
        !          2726:                        vm_object_lock(object);
        !          2727:                }
        !          2728:                vm_object_paging_end(object);
        !          2729:                return;
        !          2730:        }
        !          2731: 
        !          2732:        /*
        !          2733:         *      Indicate that a memory object has been assigned
        !          2734:         *      before dropping the lock, to prevent a race.
        !          2735:         */
        !          2736: 
        !          2737:        object->pager_created = TRUE;
        !          2738:        object->paging_offset = 0;
        !          2739:                
        !          2740: #if    MACH_PAGEMAP
        !          2741:        size = object->size;
        !          2742: #endif /* MACH_PAGEMAP */
        !          2743:        vm_object_unlock(object);
        !          2744: 
        !          2745: #if    MACH_PAGEMAP
        !          2746:        map = vm_external_create(size);
        !          2747:        vm_object_lock(object);
        !          2748:        assert(object->size == size);
        !          2749:        object->existence_map = map;
        !          2750:        vm_object_unlock(object);
        !          2751: #endif /* MACH_PAGEMAP */
        !          2752: 
        !          2753:        /*
        !          2754:         *      Create the pager ports, and associate them with this object.
        !          2755:         *
        !          2756:         *      We make the port association here so that vm_object_enter()
        !          2757:         *      can look up the object to complete initializing it.  No
        !          2758:         *      user will ever map this object.
        !          2759:         */
        !          2760:        {
        !          2761:                ipc_port_t      DMM;
        !          2762:                vm_size_t       cluster_size;
        !          2763: 
        !          2764:                /* acquire a naked send right for the DMM */
        !          2765:                DMM = memory_manager_default_reference(&cluster_size);
        !          2766:                assert(cluster_size >= PAGE_SIZE);
        !          2767: 
        !          2768:                object->cluster_size = cluster_size; /* XXX ??? */
        !          2769:                assert(object->temporary);
        !          2770: 
        !          2771:                /* consumes the naked send right for DMM */
        !          2772:                (void) memory_object_create(DMM, &pager, object->size);
        !          2773:                assert(IP_VALID(pager));
        !          2774:        }
        !          2775: 
        !          2776:        entry = vm_object_hash_entry_alloc(pager);
        !          2777: 
        !          2778:        vm_object_cache_lock();
        !          2779:        vm_object_hash_insert(entry);
        !          2780: 
        !          2781:        entry->object = object;
        !          2782:        vm_object_cache_unlock();
        !          2783: 
        !          2784:        /*
        !          2785:         *      A naked send right was returned by
        !          2786:         *      memory_object_create(), and it is
        !          2787:         *      copied by vm_object_enter().
        !          2788:         */
        !          2789: 
        !          2790:        if (vm_object_enter(pager, object->size, TRUE, TRUE) != object)
        !          2791:                panic("vm_object_pager_create: mismatch");
        !          2792: 
        !          2793:        /*
        !          2794:         *      Drop the naked send right.
        !          2795:         */
        !          2796:        ipc_port_release_send(pager);
        !          2797: 
        !          2798:        vm_object_lock(object);
        !          2799: 
        !          2800:        /*
        !          2801:         *      Release the paging reference
        !          2802:         */
        !          2803:        vm_object_paging_end(object);
        !          2804: }
        !          2805: 
        !          2806: /*
        !          2807:  *     Routine:        vm_object_remove
        !          2808:  *     Purpose:
        !          2809:  *             Eliminate the pager/object association
        !          2810:  *             for this pager.
        !          2811:  *     Conditions:
        !          2812:  *             The object cache must be locked.
        !          2813:  */
        !          2814: void
        !          2815: vm_object_remove(
        !          2816:        vm_object_t     object)
        !          2817: {
        !          2818:        ipc_port_t port;
        !          2819: 
        !          2820:        if ((port = object->pager) != IP_NULL) {
        !          2821:                vm_object_hash_entry_t  entry;
        !          2822: 
        !          2823:                entry = vm_object_hash_lookup(port, FALSE);
        !          2824:                if (entry != VM_OBJECT_HASH_ENTRY_NULL)
        !          2825:                        entry->object = VM_OBJECT_NULL;
        !          2826:        }
        !          2827: 
        !          2828:        if ((port = object->pager_request) != IP_NULL) {
        !          2829:                if (ip_kotype(port) == IKOT_PAGING_REQUEST)
        !          2830:                        ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
        !          2831:                 else if (ip_kotype(port) != IKOT_NONE)
        !          2832:                        panic("vm_object_remove: bad request port");
        !          2833:        }
        !          2834: }
        !          2835: 
        !          2836: /*
        !          2837:  *     Global variables for vm_object_collapse():
        !          2838:  *
        !          2839:  *             Counts for normal collapses and bypasses.
        !          2840:  *             Debugging variables, to watch or disable collapse.
        !          2841:  */
        !          2842: long   object_collapses = 0;
        !          2843: long   object_bypasses  = 0;
        !          2844: 
        !          2845: boolean_t      vm_object_collapse_allowed = TRUE;
        !          2846: boolean_t      vm_object_bypass_allowed = TRUE;
        !          2847: 
        !          2848: int    vm_external_discarded;
        !          2849: int    vm_external_collapsed;
        !          2850: /*
        !          2851:  *     vm_object_do_collapse:
        !          2852:  *
        !          2853:  *     Collapse an object with the object backing it.
        !          2854:  *     Pages in the backing object are moved into the
        !          2855:  *     parent, and the backing object is deallocated.
        !          2856:  *
        !          2857:  *     Both objects and the cache are locked; the page
        !          2858:  *     queues are unlocked.
        !          2859:  *
        !          2860:  */
        !          2861: void
        !          2862: vm_object_do_collapse(
        !          2863:        vm_object_t object,
        !          2864:        vm_object_t backing_object)
        !          2865: {
        !          2866:        vm_page_t p, pp;
        !          2867:        vm_offset_t new_offset, backing_offset;
        !          2868:        vm_size_t size;
        !          2869: 
        !          2870:        backing_offset = object->shadow_offset;
        !          2871:        size = object->size;
        !          2872: 
        !          2873: 
        !          2874:        /*
        !          2875:         *      Move all in-memory pages from backing_object
        !          2876:         *      to the parent.  Pages that have been paged out
        !          2877:         *      will be overwritten by any of the parent's
        !          2878:         *      pages that shadow them.
        !          2879:         */
        !          2880:        
        !          2881:        while (!queue_empty(&backing_object->memq)) {
        !          2882:                
        !          2883:                p = (vm_page_t) queue_first(&backing_object->memq);
        !          2884:                
        !          2885:                new_offset = (p->offset - backing_offset);
        !          2886:                
        !          2887:                assert(!p->busy || p->absent);
        !          2888:                
        !          2889:                /*
        !          2890:                 *      If the parent has a page here, or if
        !          2891:                 *      this page falls outside the parent,
        !          2892:                 *      dispose of it.
        !          2893:                 *
        !          2894:                 *      Otherwise, move it as planned.
        !          2895:                 */
        !          2896:                
        !          2897:                if (p->offset < backing_offset || new_offset >= size) {
        !          2898:                        VM_PAGE_FREE(p);
        !          2899:                } else {
        !          2900:                        pp = vm_page_lookup(object, new_offset);
        !          2901:                        if (pp == VM_PAGE_NULL) {
        !          2902: 
        !          2903:                                /*
        !          2904:                                 *      Parent now has no page.
        !          2905:                                 *      Move the backing object's page up.
        !          2906:                                 */
        !          2907: 
        !          2908:                                vm_page_rename(p, object, new_offset);
        !          2909: #if    MACH_PAGEMAP
        !          2910:                        } else if (pp->absent) {
        !          2911: 
        !          2912:                                /*
        !          2913:                                 *      Parent has an absent page...
        !          2914:                                 *      it's not being paged in, so
        !          2915:                                 *      it must really be missing from
        !          2916:                                 *      the parent.
        !          2917:                                 *
        !          2918:                                 *      Throw out the absent page...
        !          2919:                                 *      any faults looking for that
        !          2920:                                 *      page will restart with the new
        !          2921:                                 *      one.
        !          2922:                                 */
        !          2923: 
        !          2924:                                VM_PAGE_FREE(pp);
        !          2925:                                vm_page_rename(p, object, new_offset);
        !          2926: #endif /* MACH_PAGEMAP */
        !          2927:                        } else {
        !          2928:                                assert(! pp->absent);
        !          2929: 
        !          2930:                                /*
        !          2931:                                 *      Parent object has a real page.
        !          2932:                                 *      Throw away the backing object's
        !          2933:                                 *      page.
        !          2934:                                 */
        !          2935:                                VM_PAGE_FREE(p);
        !          2936:                        }
        !          2937:                }
        !          2938:        }
        !          2939:        
        !          2940:        assert(object->pager == IP_NULL || backing_object->pager == IP_NULL);
        !          2941: 
        !          2942:        if (backing_object->pager != IP_NULL) {
        !          2943:                vm_object_hash_entry_t  entry;
        !          2944: 
        !          2945:                /*
        !          2946:                 *      Move the pager from backing_object to object.
        !          2947:                 *
        !          2948:                 *      XXX We're only using part of the paging space
        !          2949:                 *      for keeps now... we ought to discard the
        !          2950:                 *      unused portion.
        !          2951:                 */
        !          2952: 
        !          2953:                object->pager = backing_object->pager;
        !          2954:                entry = vm_object_hash_lookup(object->pager, FALSE);
        !          2955:                assert(entry != VM_OBJECT_HASH_ENTRY_NULL);
        !          2956:                entry->object = object;
        !          2957:                object->pager_created = backing_object->pager_created;
        !          2958:                object->pager_request = backing_object->pager_request;
        !          2959:                object->pager_ready = backing_object->pager_ready;
        !          2960:                object->pager_initialized = backing_object->pager_initialized;
        !          2961:                object->cluster_size = backing_object->cluster_size;
        !          2962:                object->paging_offset =
        !          2963:                    backing_object->paging_offset + backing_offset;
        !          2964:                if (object->pager_request != IP_NULL) {
        !          2965:                        ipc_kobject_set(object->pager_request,
        !          2966:                                        (ipc_kobject_t) object,
        !          2967:                                        IKOT_PAGING_REQUEST);
        !          2968:                }
        !          2969:        }
        !          2970: 
        !          2971:        vm_object_cache_unlock();
        !          2972: 
        !          2973:        object->paging_offset = backing_object->paging_offset + backing_offset;
        !          2974: 
        !          2975: #if    MACH_PAGEMAP
        !          2976:        /*
        !          2977:         *      If the shadow offset is 0, the use the existence map from
        !          2978:         *      the backing object if there is one. If the shadow offset is
        !          2979:         *      not zero, toss it.
        !          2980:         *
        !          2981:         *      XXX - If the shadow offset is not 0 then a bit copy is needed
        !          2982:         *      if the map is to be salvaged.  For now, we just just toss the
        !          2983:         *      old map, giving the collapsed object no map. This means that
        !          2984:         *      the pager is invoked for zero fill pages.  If analysis shows
        !          2985:         *      that this happens frequently and is a performance hit, then
        !          2986:         *      this code should be fixed to salvage the map.
        !          2987:         */
        !          2988:        assert(object->existence_map == VM_EXTERNAL_NULL);
        !          2989:        if (backing_offset || (size != backing_object->size)) {
        !          2990:                vm_external_discarded++;
        !          2991:                vm_external_destroy(backing_object->existence_map,
        !          2992:                        backing_object->size);
        !          2993:        }
        !          2994:        else {
        !          2995:                vm_external_collapsed++;
        !          2996:                object->existence_map = backing_object->existence_map;
        !          2997:        }
        !          2998:        backing_object->existence_map = VM_EXTERNAL_NULL;
        !          2999: #endif /* MACH_PAGEMAP */
        !          3000: 
        !          3001:        /*
        !          3002:         *      Object now shadows whatever backing_object did.
        !          3003:         *      Note that the reference to backing_object->shadow
        !          3004:         *      moves from within backing_object to within object.
        !          3005:         */
        !          3006:        
        !          3007:        object->shadow = backing_object->shadow;
        !          3008:        object->shadow_offset += backing_object->shadow_offset;
        !          3009:        assert((object->shadow == VM_OBJECT_NULL) ||
        !          3010:               (object->shadow->copy == VM_OBJECT_NULL));
        !          3011: 
        !          3012:        /*
        !          3013:         *      Discard backing_object.
        !          3014:         *
        !          3015:         *      Since the backing object has no pages, no
        !          3016:         *      pager left, and no object references within it,
        !          3017:         *      all that is necessary is to dispose of it.
        !          3018:         */
        !          3019:        
        !          3020:        assert((backing_object->ref_count == 1) &&
        !          3021:               (backing_object->resident_page_count == 0) &&
        !          3022:               (backing_object->paging_in_progress == 0));
        !          3023: 
        !          3024:        assert(backing_object->alive);
        !          3025:        backing_object->alive = FALSE;
        !          3026:        vm_object_unlock(backing_object);
        !          3027: 
        !          3028:        XPR(XPR_VM_OBJECT, "vm_object_collapse, collapsed 0x%X\n",
        !          3029:                (integer_t)backing_object, 0,0,0,0);
        !          3030: 
        !          3031:        zfree(vm_object_zone, (vm_offset_t) backing_object);
        !          3032:        
        !          3033:        object_collapses++;
        !          3034: }
        !          3035: 
        !          3036: void
        !          3037: vm_object_do_bypass(
        !          3038:        vm_object_t object,
        !          3039:        vm_object_t backing_object)
        !          3040: {
        !          3041:        /*
        !          3042:         *      Make the parent shadow the next object
        !          3043:         *      in the chain.
        !          3044:         */
        !          3045:        
        !          3046: #if    TASK_SWAPPER
        !          3047:        /*
        !          3048:         *      Do object reference in-line to 
        !          3049:         *      conditionally increment shadow's
        !          3050:         *      residence count.  If object is not
        !          3051:         *      resident, leave residence count
        !          3052:         *      on shadow alone.
        !          3053:         */
        !          3054:        if (backing_object->shadow != VM_OBJECT_NULL) {
        !          3055:                vm_object_lock(backing_object->shadow);
        !          3056:                backing_object->shadow->ref_count++;
        !          3057:                if (object->res_count != 0)
        !          3058:                        vm_object_res_reference(backing_object->shadow);
        !          3059:                vm_object_unlock(backing_object->shadow);
        !          3060:        }
        !          3061: #else  /* TASK_SWAPPER */
        !          3062:        vm_object_reference(backing_object->shadow);
        !          3063: #endif /* TASK_SWAPPER */
        !          3064: 
        !          3065:        object->shadow = backing_object->shadow;
        !          3066:        object->shadow_offset += backing_object->shadow_offset;
        !          3067:        
        !          3068:        /*
        !          3069:         *      Backing object might have had a copy pointer
        !          3070:         *      to us.  If it did, clear it. 
        !          3071:         */
        !          3072:        if (backing_object->copy == object) {
        !          3073:                backing_object->copy = VM_OBJECT_NULL;
        !          3074:        }
        !          3075:        
        !          3076:        /*
        !          3077:         *      Drop the reference count on backing_object.
        !          3078: #if    TASK_SWAPPER
        !          3079:         *      Since its ref_count was at least 2, it
        !          3080:         *      will not vanish; so we don't need to call
        !          3081:         *      vm_object_deallocate.
        !          3082:         *      [FBDP: that doesn't seem to be true any more]
        !          3083:         * 
        !          3084:         *      The res_count on the backing object is
        !          3085:         *      conditionally decremented.  It's possible
        !          3086:         *      (via vm_pageout_scan) to get here with
        !          3087:         *      a "swapped" object, which has a 0 res_count,
        !          3088:         *      in which case, the backing object res_count
        !          3089:         *      is already down by one.
        !          3090: #else
        !          3091:         *      Don't call vm_object_deallocate unless
        !          3092:         *      ref_count drops to zero.
        !          3093:         *
        !          3094:         *      The ref_count can drop to zero here if the
        !          3095:         *      backing object could be bypassed but not
        !          3096:         *      collapsed, such as when the backing object
        !          3097:         *      is temporary and cachable.
        !          3098: #endif
        !          3099:         */
        !          3100:        if (backing_object->ref_count > 1) {
        !          3101:                backing_object->ref_count--;
        !          3102: #if    TASK_SWAPPER
        !          3103:                if (object->res_count != 0)
        !          3104:                        vm_object_res_deallocate(backing_object);
        !          3105:                assert(backing_object->ref_count > 0);
        !          3106: #endif /* TASK_SWAPPER */
        !          3107:                vm_object_unlock(backing_object);
        !          3108:        } else {
        !          3109: 
        !          3110:                /*
        !          3111:                 *      Drop locks so that we can deallocate
        !          3112:                 *      the backing object.
        !          3113:                 */
        !          3114: 
        !          3115: #if    TASK_SWAPPER
        !          3116:                if (object->res_count == 0) {
        !          3117:                        /* XXX get a reference for the deallocate below */
        !          3118:                        vm_object_res_reference(backing_object);
        !          3119:                }
        !          3120: #endif /* TASK_SWAPPER */
        !          3121:                vm_object_unlock(object);
        !          3122:                vm_object_unlock(backing_object);
        !          3123:                vm_object_deallocate(backing_object);
        !          3124: 
        !          3125:                /*
        !          3126:                 *      Relock object. We don't have to reverify
        !          3127:                 *      its state since vm_object_collapse will
        !          3128:                 *      do that for us as it starts at the
        !          3129:                 *      top of its loop.
        !          3130:                 */
        !          3131: 
        !          3132:                vm_object_lock(object);
        !          3133:        }
        !          3134:        
        !          3135:        object_bypasses++;
        !          3136: }
        !          3137:                
        !          3138: /*
        !          3139:  *     vm_object_collapse:
        !          3140:  *
        !          3141:  *     Perform an object collapse or an object bypass if appropriate.
        !          3142:  *     The real work of collapsing and bypassing is performed in
        !          3143:  *     the routines vm_object_do_collapse and vm_object_do_bypass.
        !          3144:  *
        !          3145:  *     Requires that the object be locked and the page queues be unlocked.
        !          3146:  *
        !          3147:  */
        !          3148: void
        !          3149: vm_object_collapse(
        !          3150:        register vm_object_t    object)
        !          3151: {
        !          3152:        register vm_object_t    backing_object;
        !          3153:        register vm_offset_t    backing_offset;
        !          3154:        register vm_size_t      size;
        !          3155:        register vm_offset_t    new_offset;
        !          3156:        register vm_page_t      p;
        !          3157: 
        !          3158:        if (! vm_object_collapse_allowed && ! vm_object_bypass_allowed) {
        !          3159:                return;
        !          3160:        }
        !          3161: 
        !          3162:        XPR(XPR_VM_OBJECT, "vm_object_collapse, obj 0x%X\n", 
        !          3163:                (integer_t)object, 0,0,0,0);
        !          3164: 
        !          3165:        while (TRUE) {
        !          3166:                /*
        !          3167:                 *      Verify that the conditions are right for either
        !          3168:                 *      collapse or bypass:
        !          3169:                 *
        !          3170:                 *      The object exists and no pages in it are currently
        !          3171:                 *      being paged out, and
        !          3172:                 */
        !          3173:                if (object == VM_OBJECT_NULL ||
        !          3174:                    object->paging_in_progress != 0 ||
        !          3175:                    object->absent_count != 0)
        !          3176:                        return;
        !          3177: 
        !          3178:                /*
        !          3179:                 *      There is a backing object, and
        !          3180:                 */
        !          3181:        
        !          3182:                if ((backing_object = object->shadow) == VM_OBJECT_NULL)
        !          3183:                        return;
        !          3184:        
        !          3185:                vm_object_lock(backing_object);
        !          3186: 
        !          3187:                /*
        !          3188:                 *      ...
        !          3189:                 *              The backing object is not read_only,
        !          3190:                 *              and no pages in the backing object are
        !          3191:                 *              currently being paged out.
        !          3192:                 *              The backing object is internal.
        !          3193:                 *
        !          3194:                 */
        !          3195:        
        !          3196:                if (!backing_object->internal ||
        !          3197:                    backing_object->paging_in_progress != 0) {
        !          3198:                        vm_object_unlock(backing_object);
        !          3199:                        return;
        !          3200:                }
        !          3201:        
        !          3202:                /*
        !          3203:                 *      The backing object can't be a copy-object:
        !          3204:                 *      the shadow_offset for the copy-object must stay
        !          3205:                 *      as 0.  Furthermore (for the 'we have all the
        !          3206:                 *      pages' case), if we bypass backing_object and
        !          3207:                 *      just shadow the next object in the chain, old
        !          3208:                 *      pages from that object would then have to be copied
        !          3209:                 *      BOTH into the (former) backing_object and into the
        !          3210:                 *      parent object.
        !          3211:                 */
        !          3212:                if (backing_object->shadow != VM_OBJECT_NULL &&
        !          3213:                    backing_object->shadow->copy != VM_OBJECT_NULL) {
        !          3214:                        vm_object_unlock(backing_object);
        !          3215:                        return;
        !          3216:                }
        !          3217: 
        !          3218:                /*
        !          3219:                 *      We can now try to either collapse the backing
        !          3220:                 *      object (if the parent is the only reference to
        !          3221:                 *      it) or (perhaps) remove the parent's reference
        !          3222:                 *      to it.
        !          3223:                 */
        !          3224: 
        !          3225:                /*
        !          3226:                 *      If there is exactly one reference to the backing
        !          3227:                 *      object, we may be able to collapse it into the parent.
        !          3228:                 *
        !          3229:                 *      XXXO (norma vm):
        !          3230:                 *
        !          3231:                 *      The backing object must not have a pager
        !          3232:                 *      created for it, since collapsing an object
        !          3233:                 *      into a backing_object dumps new pages into
        !          3234:                 *      the backing_object that its pager doesn't
        !          3235:                 *      know about, and we've already declared pages.
        !          3236:                 *      This page dumping is deadly if other kernels
        !          3237:                 *      are shadowing this object; this is the
        !          3238:                 *      distributed equivalent of the ref_count == 1
        !          3239:                 *      condition.
        !          3240:                 *
        !          3241:                 *      With some work, we could downgrade this
        !          3242:                 *      restriction to the backing object must not
        !          3243:                 *      be cachable, since when a temporary object
        !          3244:                 *      is uncachable we are allowed to do anything
        !          3245:                 *      to it. We would have to do something like
        !          3246:                 *      call declare_pages again, and we would have
        !          3247:                 *      to be prepared for the memory manager
        !          3248:                 *      disabling temporary termination, which right
        !          3249:                 *      now is a difficult race to deal with, since
        !          3250:                 *      the memory manager currently assumes that
        !          3251:                 *      termination is the only possible failure
        !          3252:                 *      for disabling temporary termination.
        !          3253:                 */
        !          3254: 
        !          3255:                if (backing_object->ref_count == 1 &&
        !          3256:                    ! object->pager_created &&
        !          3257:                    vm_object_collapse_allowed) {
        !          3258: 
        !          3259:                        XPR(XPR_VM_OBJECT, 
        !          3260:                   "vm_object_collapse: %x to %x, pager %x, pager_request %x\n",
        !          3261:                                (integer_t)backing_object, (integer_t)object,
        !          3262:                                (integer_t)backing_object->pager, 
        !          3263:                                (integer_t)backing_object->pager_request, 0);
        !          3264: 
        !          3265:                        /*
        !          3266:                         *      We need the cache lock for collapsing,
        !          3267:                         *      but we must not deadlock.
        !          3268:                         */
        !          3269:                        
        !          3270:                        if (! vm_object_cache_lock_try()) {
        !          3271:                                vm_object_unlock(backing_object);
        !          3272:                                return;
        !          3273:                        }
        !          3274: 
        !          3275:                        /*
        !          3276:                         *      Collapse the object with its backing
        !          3277:                         *      object, and try again with the object's
        !          3278:                         *      new backing object.
        !          3279:                         */
        !          3280: 
        !          3281:                        vm_object_do_collapse(object, backing_object);
        !          3282:                        continue;
        !          3283:                }
        !          3284: 
        !          3285: 
        !          3286:                /*
        !          3287:                 *      Collapsing the backing object was not possible
        !          3288:                 *      or permitted, so let's try bypassing it.
        !          3289:                 */
        !          3290: 
        !          3291:                if (! vm_object_bypass_allowed) {
        !          3292:                        vm_object_unlock(backing_object);
        !          3293:                        return;
        !          3294:                }
        !          3295: 
        !          3296:                /*
        !          3297:                 *      If the backing object has a pager but no pagemap,
        !          3298:                 *      then we cannot bypass it, because we don't know
        !          3299:                 *      what pages it has.
        !          3300:                 */
        !          3301:                if (backing_object->pager_created
        !          3302: #if    MACH_PAGEMAP
        !          3303:                    && (backing_object->existence_map == VM_EXTERNAL_NULL)
        !          3304: #endif /* MACH_PAGEMAP */
        !          3305:                    ) {
        !          3306:                        vm_object_unlock(backing_object);
        !          3307:                        return;
        !          3308:                }
        !          3309: 
        !          3310:                backing_offset = object->shadow_offset;
        !          3311:                size = object->size;
        !          3312: 
        !          3313:                /*
        !          3314:                 *      If all of the pages in the backing object are
        !          3315:                 *      shadowed by the parent object, the parent
        !          3316:                 *      object no longer has to shadow the backing
        !          3317:                 *      object; it can shadow the next one in the
        !          3318:                 *      chain.
        !          3319:                 *
        !          3320:                 *      If the backing object has existence info,
        !          3321:                 *      we must check examine its existence info
        !          3322:                 *      as well.
        !          3323:                 *
        !          3324:                 *      XXX
        !          3325:                 *      Should have a check for a 'small' number
        !          3326:                 *      of pages here.
        !          3327:                 */
        !          3328: 
        !          3329:                /*
        !          3330:                 *      First, check pages resident in the backing object.
        !          3331:                 */
        !          3332: 
        !          3333:                queue_iterate(&backing_object->memq, p, vm_page_t, listq) {
        !          3334:                        
        !          3335:                        /*
        !          3336:                         *      If the parent has a page here, or if
        !          3337:                         *      this page falls outside the parent,
        !          3338:                         *      keep going.
        !          3339:                         *
        !          3340:                         *      Otherwise, the backing_object must be
        !          3341:                         *      left in the chain.
        !          3342:                         */
        !          3343:                        
        !          3344:                        new_offset = (p->offset - backing_offset);
        !          3345:                        if (p->offset < backing_offset || new_offset >= size) {
        !          3346: 
        !          3347:                                /*
        !          3348:                                 *      Page falls outside of parent.
        !          3349:                                 *      Keep going.
        !          3350:                                 */
        !          3351: 
        !          3352:                                continue;
        !          3353:                        }
        !          3354: 
        !          3355:                        if ((vm_page_lookup(object, new_offset) == VM_PAGE_NULL)
        !          3356: #if    MACH_PAGEMAP
        !          3357:                            &&
        !          3358:                            (vm_external_state_get(object->existence_map,
        !          3359:                                        new_offset)
        !          3360:                                        != VM_EXTERNAL_STATE_EXISTS)
        !          3361: #endif /* MACH_PAGEMAP */
        !          3362:                            ) {
        !          3363: 
        !          3364:                                /*
        !          3365:                                 *      Page still needed.
        !          3366:                                 *      Can't go any further.
        !          3367:                                 */
        !          3368: 
        !          3369:                                vm_object_unlock(backing_object);
        !          3370:                                return;
        !          3371:                        }
        !          3372:                }
        !          3373: 
        !          3374: #if    MACH_PAGEMAP
        !          3375:                /*
        !          3376:                 *      Next, if backing object has been paged out,
        !          3377:                 *      we must check its existence info for pages
        !          3378:                 *      that the parent doesn't have.
        !          3379:                 */
        !          3380: 
        !          3381:                if (backing_object->pager_created) {
        !          3382:                        assert(backing_object->existence_map
        !          3383:                                        != VM_EXTERNAL_NULL);
        !          3384:                        for (new_offset = 0; new_offset < object->size;
        !          3385:                             new_offset += PAGE_SIZE) {
        !          3386:                                vm_offset_t
        !          3387:                                offset = new_offset + backing_offset;
        !          3388: 
        !          3389:                                /*
        !          3390:                                 *      If this page doesn't exist in
        !          3391:                                 *      the backing object's existence
        !          3392:                                 *      info, then continue.
        !          3393:                                 */
        !          3394: 
        !          3395:                                if (vm_external_state_get(
        !          3396:                                        backing_object->existence_map,
        !          3397:                                        offset) == VM_EXTERNAL_STATE_ABSENT) {
        !          3398:                                        continue;
        !          3399:                                }
        !          3400: 
        !          3401:                                /*
        !          3402:                                 *      If this page is neither resident
        !          3403:                                 *      in the parent nor paged out to
        !          3404:                                 *      the parent's pager, then we cannot
        !          3405:                                 *      bypass the backing object.
        !          3406:                                 */
        !          3407: 
        !          3408:                                if ((vm_page_lookup(object, new_offset) ==
        !          3409:                                     VM_PAGE_NULL) &&
        !          3410:                                    ((object->existence_map == VM_EXTERNAL_NULL)
        !          3411:                                        || (vm_external_state_get(
        !          3412:                                        object->existence_map, new_offset)
        !          3413:                                        == VM_EXTERNAL_STATE_ABSENT))) {
        !          3414:                                        vm_object_unlock(backing_object);
        !          3415:                                        return;
        !          3416:                                }
        !          3417:                        }
        !          3418:                }
        !          3419: #else  /* MACH_PAGEMAP */
        !          3420:                assert(! backing_object->pager_created);
        !          3421: #endif /* MACH_PAGEMAP */
        !          3422: 
        !          3423:                /*
        !          3424:                 *      All interesting pages in the backing object
        !          3425:                 *      already live in the parent or its pager.
        !          3426:                 *      Thus we can bypass the backing object.
        !          3427:                 */
        !          3428: 
        !          3429:                vm_object_do_bypass(object, backing_object);
        !          3430: 
        !          3431:                /*
        !          3432:                 *      Try again with this object's new backing object.
        !          3433:                 */
        !          3434: 
        !          3435:                continue;
        !          3436:        }
        !          3437: }
        !          3438: 
        !          3439: /*
        !          3440:  *     Routine:        vm_object_page_remove: [internal]
        !          3441:  *     Purpose:
        !          3442:  *             Removes all physical pages in the specified
        !          3443:  *             object range from the object's list of pages.
        !          3444:  *
        !          3445:  *     In/out conditions:
        !          3446:  *             The object must be locked.
        !          3447:  *             The object must not have paging_in_progress, usually
        !          3448:  *             guaranteed by not having a pager.
        !          3449:  */
        !          3450: unsigned int vm_object_page_remove_lookup = 0;
        !          3451: unsigned int vm_object_page_remove_iterate = 0;
        !          3452: 
        !          3453: void
        !          3454: vm_object_page_remove(
        !          3455:        register vm_object_t    object,
        !          3456:        register vm_offset_t    start,
        !          3457:        register vm_offset_t    end)
        !          3458: {
        !          3459:        register vm_page_t      p, next;
        !          3460: 
        !          3461:        /*
        !          3462:         *      One and two page removals are most popular.
        !          3463:         *      The factor of 16 here is somewhat arbitrary.
        !          3464:         *      It balances vm_object_lookup vs iteration.
        !          3465:         */
        !          3466: 
        !          3467:        if (atop(end - start) < (unsigned)object->resident_page_count/16) {
        !          3468:                vm_object_page_remove_lookup++;
        !          3469: 
        !          3470:                for (; start < end; start += PAGE_SIZE) {
        !          3471:                        p = vm_page_lookup(object, start);
        !          3472:                        if (p != VM_PAGE_NULL) {
        !          3473:                                assert(!p->cleaning && !p->pageout);
        !          3474:                                if (!p->fictitious)
        !          3475:                                        pmap_page_protect(p->phys_addr,
        !          3476:                                                          VM_PROT_NONE);
        !          3477:                                VM_PAGE_FREE(p);
        !          3478:                        }
        !          3479:                }
        !          3480:        } else {
        !          3481:                vm_object_page_remove_iterate++;
        !          3482: 
        !          3483:                p = (vm_page_t) queue_first(&object->memq);
        !          3484:                while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !          3485:                        next = (vm_page_t) queue_next(&p->listq);
        !          3486:                        if ((start <= p->offset) && (p->offset < end)) {
        !          3487:                                assert(!p->cleaning && !p->pageout);
        !          3488:                                if (!p->fictitious)
        !          3489:                                    pmap_page_protect(p->phys_addr,
        !          3490:                                                      VM_PROT_NONE);
        !          3491:                                VM_PAGE_FREE(p);
        !          3492:                        }
        !          3493:                        p = next;
        !          3494:                }
        !          3495:        }
        !          3496: }
        !          3497: 
        !          3498: /*
        !          3499:  *     Routine:        vm_object_coalesce
        !          3500:  *     Function:       Coalesces two objects backing up adjoining
        !          3501:  *                     regions of memory into a single object.
        !          3502:  *
        !          3503:  *     returns TRUE if objects were combined.
        !          3504:  *
        !          3505:  *     NOTE:   Only works at the moment if the second object is NULL -
        !          3506:  *             if it's not, which object do we lock first?
        !          3507:  *
        !          3508:  *     Parameters:
        !          3509:  *             prev_object     First object to coalesce
        !          3510:  *             prev_offset     Offset into prev_object
        !          3511:  *             next_object     Second object into coalesce
        !          3512:  *             next_offset     Offset into next_object
        !          3513:  *
        !          3514:  *             prev_size       Size of reference to prev_object
        !          3515:  *             next_size       Size of reference to next_object
        !          3516:  *
        !          3517:  *     Conditions:
        !          3518:  *     The object(s) must *not* be locked. The map must be locked
        !          3519:  *     to preserve the reference to the object(s).
        !          3520:  */
        !          3521: int vm_object_coalesce_count = 0;
        !          3522: 
        !          3523: boolean_t
        !          3524: vm_object_coalesce(
        !          3525:        register vm_object_t    prev_object,
        !          3526:        vm_object_t             next_object,
        !          3527:        vm_offset_t             prev_offset,
        !          3528:        vm_offset_t             next_offset,
        !          3529:        vm_size_t               prev_size,
        !          3530:        vm_size_t               next_size)
        !          3531: {
        !          3532:        vm_size_t       newsize;
        !          3533: 
        !          3534: #ifdef lint
        !          3535:        next_offset++;
        !          3536: #endif /* lint */
        !          3537: 
        !          3538:        if (next_object != VM_OBJECT_NULL) {
        !          3539:                return(FALSE);
        !          3540:        }
        !          3541: 
        !          3542:        if (prev_object == VM_OBJECT_NULL) {
        !          3543:                return(TRUE);
        !          3544:        }
        !          3545: 
        !          3546:        XPR(XPR_VM_OBJECT,
        !          3547:        "vm_object_coalesce: 0x%X prev_off 0x%X prev_size 0x%X next_size 0x%X\n",
        !          3548:                (integer_t)prev_object, prev_offset, prev_size, next_size, 0);
        !          3549: 
        !          3550:        vm_object_lock(prev_object);
        !          3551: 
        !          3552:        /*
        !          3553:         *      Try to collapse the object first
        !          3554:         */
        !          3555:        vm_object_collapse(prev_object);
        !          3556: 
        !          3557:        /*
        !          3558:         *      Can't coalesce if pages not mapped to
        !          3559:         *      prev_entry may be in use any way:
        !          3560:         *      . more than one reference
        !          3561:         *      . paged out
        !          3562:         *      . shadows another object
        !          3563:         *      . has a copy elsewhere
        !          3564:         *      . paging references (pages might be in page-list)
        !          3565:         */
        !          3566: 
        !          3567:        if ((prev_object->ref_count > 1) ||
        !          3568:            prev_object->pager_created ||
        !          3569:            (prev_object->shadow != VM_OBJECT_NULL) ||
        !          3570:            (prev_object->copy != VM_OBJECT_NULL) ||
        !          3571:            (prev_object->true_share != FALSE) ||
        !          3572:            (prev_object->paging_in_progress != 0)) {
        !          3573:                vm_object_unlock(prev_object);
        !          3574:                return(FALSE);
        !          3575:        }
        !          3576: 
        !          3577:        vm_object_coalesce_count++;
        !          3578: 
        !          3579:        /*
        !          3580:         *      Remove any pages that may still be in the object from
        !          3581:         *      a previous deallocation.
        !          3582:         */
        !          3583:        vm_object_page_remove(prev_object,
        !          3584:                prev_offset + prev_size,
        !          3585:                prev_offset + prev_size + next_size);
        !          3586: 
        !          3587:        /*
        !          3588:         *      Extend the object if necessary.
        !          3589:         */
        !          3590:        newsize = prev_offset + prev_size + next_size;
        !          3591:        if (newsize > prev_object->size) {
        !          3592: #if    MACH_PAGEMAP
        !          3593:                /*
        !          3594:                 *      We cannot extend an object that has existence info,
        !          3595:                 *      since the existence info might then fail to cover
        !          3596:                 *      the entire object.
        !          3597:                 *
        !          3598:                 *      This assertion must be true because the object
        !          3599:                 *      has no pager, and we only create existence info
        !          3600:                 *      for objects with pagers.
        !          3601:                 */
        !          3602:                assert(prev_object->existence_map == VM_EXTERNAL_NULL);
        !          3603: #endif /* MACH_PAGEMAP */
        !          3604:                prev_object->size = newsize;
        !          3605:        }
        !          3606: 
        !          3607:        vm_object_unlock(prev_object);
        !          3608:        return(TRUE);
        !          3609: }
        !          3610: 
        !          3611: /*
        !          3612:  *     Attach a set of physical pages to an object, so that they can
        !          3613:  *     be mapped by mapping the object.  Typically used to map IO memory.
        !          3614:  *
        !          3615:  *     The mapping function and its private data are used to obtain the
        !          3616:  *     physical addresses for each page to be mapped.
        !          3617:  */
        !          3618: void
        !          3619: vm_object_page_map(
        !          3620:        vm_object_t     object,
        !          3621:        vm_offset_t     offset,
        !          3622:        vm_size_t       size,
        !          3623:        vm_offset_t     (*map_fn)(void *map_fn_data, vm_offset_t offset),
        !          3624:        void            *map_fn_data)   /* private to map_fn */
        !          3625: {
        !          3626:        int     num_pages;
        !          3627:        int     i;
        !          3628:        vm_page_t       m;
        !          3629:        vm_page_t       old_page;
        !          3630:        vm_offset_t     addr;
        !          3631: 
        !          3632:        num_pages = atop(size);
        !          3633: 
        !          3634:        for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
        !          3635: 
        !          3636:            addr = (*map_fn)(map_fn_data, offset);
        !          3637: 
        !          3638:            while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
        !          3639:                vm_page_more_fictitious();
        !          3640: 
        !          3641:            vm_object_lock(object);
        !          3642:            if ((old_page = vm_page_lookup(object, offset))
        !          3643:                        != VM_PAGE_NULL)
        !          3644:            {
        !          3645:                vm_page_lock_queues();
        !          3646:                vm_page_free(old_page);
        !          3647:                vm_page_unlock_queues();
        !          3648:            }
        !          3649: 
        !          3650:            vm_page_init(m, addr);
        !          3651:            m->private = TRUE;          /* don`t free page */
        !          3652:            m->wire_count = 1;
        !          3653:            vm_page_insert(m, object, offset);
        !          3654: 
        !          3655:            PAGE_WAKEUP_DONE(m);
        !          3656:            vm_object_unlock(object);
        !          3657:        }
        !          3658: }
        !          3659: 
        !          3660: #include <mach_kdb.h>
        !          3661: 
        !          3662: #if    MACH_KDB
        !          3663: #include <ddb/db_output.h>
        !          3664: #include <vm/vm_print.h>
        !          3665: 
        !          3666: #define printf kdbprintf
        !          3667: 
        !          3668: extern boolean_t       vm_object_cached(
        !          3669:                                vm_object_t object);
        !          3670: 
        !          3671: extern void            print_bitstring(
        !          3672:                                char byte);
        !          3673: 
        !          3674: boolean_t      vm_object_print_pages = FALSE;
        !          3675: 
        !          3676: void
        !          3677: print_bitstring(
        !          3678:        char byte)
        !          3679: {
        !          3680:        printf("%c%c%c%c%c%c%c%c",
        !          3681:               ((byte & (1 << 0)) ? '1' : '0'),
        !          3682:               ((byte & (1 << 1)) ? '1' : '0'),
        !          3683:               ((byte & (1 << 2)) ? '1' : '0'),
        !          3684:               ((byte & (1 << 3)) ? '1' : '0'),
        !          3685:               ((byte & (1 << 4)) ? '1' : '0'),
        !          3686:               ((byte & (1 << 5)) ? '1' : '0'),
        !          3687:               ((byte & (1 << 6)) ? '1' : '0'),
        !          3688:               ((byte & (1 << 7)) ? '1' : '0'));
        !          3689: }
        !          3690: 
        !          3691: boolean_t
        !          3692: vm_object_cached(
        !          3693:        register vm_object_t object)
        !          3694: {
        !          3695:        register vm_object_t o;
        !          3696: 
        !          3697:        queue_iterate(&vm_object_cached_list, o, vm_object_t, cached_list) {
        !          3698:                if (object == o) {
        !          3699:                        return TRUE;
        !          3700:                }
        !          3701:        }
        !          3702:        return FALSE;
        !          3703: }
        !          3704: 
        !          3705: #if    MACH_PAGEMAP
        !          3706: /*
        !          3707:  *     vm_external_print:      [ debug ]
        !          3708:  */
        !          3709: void
        !          3710: vm_external_print(
        !          3711:        vm_external_map_t map,
        !          3712:        vm_size_t       size)
        !          3713: {
        !          3714:        if (map == VM_EXTERNAL_NULL) {
        !          3715:                printf("0  ");
        !          3716:        } else {
        !          3717:                vm_size_t existence_size = stob(size);
        !          3718:                printf("{ size=%d, map=[", existence_size);
        !          3719:                if (existence_size > 0) {
        !          3720:                        print_bitstring(map[0]);
        !          3721:                }
        !          3722:                if (existence_size > 1) {
        !          3723:                        print_bitstring(map[1]);
        !          3724:                }
        !          3725:                if (existence_size > 2) {
        !          3726:                        printf("...");
        !          3727:                        print_bitstring(map[existence_size-1]);
        !          3728:                }
        !          3729:                printf("] }\n");
        !          3730:        }
        !          3731:        return;
        !          3732: }
        !          3733: #endif /* MACH_PAGEMAP */
        !          3734: 
        !          3735: int
        !          3736: vm_follow_object(
        !          3737:        vm_object_t object)
        !          3738: {
        !          3739:        extern db_indent;
        !          3740: 
        !          3741:        int count = 1;
        !          3742: 
        !          3743:        if (object == VM_OBJECT_NULL)
        !          3744:                return 0;
        !          3745: 
        !          3746:        iprintf("object 0x%x", object);
        !          3747:        printf(", shadow=0x%x", object->shadow);
        !          3748:        printf(", copy=0x%x", object->copy);
        !          3749:        printf(", pager=0x%x", object->pager);
        !          3750:        printf(", ref=%d\n", object->ref_count);
        !          3751: 
        !          3752:        db_indent += 2;
        !          3753:        if (object->shadow)
        !          3754:            count += vm_follow_object(object->shadow);
        !          3755: 
        !          3756:        db_indent -= 2;
        !          3757:        return count;
        !          3758: }
        !          3759: 
        !          3760: /*
        !          3761:  *     vm_object_print:        [ debug ]
        !          3762:  */
        !          3763: void
        !          3764: vm_object_print(
        !          3765:        vm_object_t     object,
        !          3766:        boolean_t       have_addr,
        !          3767:        int             arg_count,
        !          3768:        char            *modif)
        !          3769: {
        !          3770:        register vm_page_t p;
        !          3771:        extern db_indent;
        !          3772:        char *s;
        !          3773: 
        !          3774:        register int count;
        !          3775: 
        !          3776:        if (object == VM_OBJECT_NULL)
        !          3777:                return;
        !          3778: 
        !          3779:        iprintf("object 0x%x\n", object);
        !          3780: 
        !          3781:        db_indent += 2;
        !          3782: 
        !          3783:        iprintf("size=0x%x", object->size);
        !          3784:        printf(", cluster=0x%x", object->cluster_size);
        !          3785:        printf(", frozen=0x%x", object->frozen_size);
        !          3786:        printf(", ref_count=%d\n", object->ref_count);
        !          3787:        iprintf("");
        !          3788: #if    TASK_SWAPPER
        !          3789:        printf("res_count=%d, ", object->res_count);
        !          3790: #endif /* TASK_SWAPPER */
        !          3791:        printf("resident_page_count=%d\n", object->resident_page_count);
        !          3792: 
        !          3793:        iprintf("shadow=0x%x", object->shadow);
        !          3794:        if (object->shadow) {
        !          3795:                register int i = 0;
        !          3796:                vm_object_t shadow = object;
        !          3797:                while(shadow = shadow->shadow)
        !          3798:                        i++;
        !          3799:                printf(" (depth %d)", i);
        !          3800:        }
        !          3801:        printf(", copy=0x%x", object->copy);
        !          3802:        printf(", shadow_offset=0x%x", object->shadow_offset);
        !          3803:        printf(", last_alloc=0x%x\n", object->last_alloc);
        !          3804: 
        !          3805:        iprintf("pager=0x%x", object->pager);
        !          3806:        printf(", paging_offset=0x%x", object->paging_offset);
        !          3807:        printf(", pager_request=0x%x\n", object->pager_request);
        !          3808: 
        !          3809:        iprintf("copy_strategy=%d[", object->copy_strategy);
        !          3810:        switch (object->copy_strategy) {
        !          3811:                case MEMORY_OBJECT_COPY_NONE:
        !          3812:                printf("copy_none");
        !          3813:                break;
        !          3814: 
        !          3815:                case MEMORY_OBJECT_COPY_CALL:
        !          3816:                printf("copy_call");
        !          3817:                break;
        !          3818: 
        !          3819:                case MEMORY_OBJECT_COPY_DELAY:
        !          3820:                printf("copy_delay");
        !          3821:                break;
        !          3822: 
        !          3823:                case MEMORY_OBJECT_COPY_SYMMETRIC:
        !          3824:                printf("copy_symmetric");
        !          3825:                break;
        !          3826: 
        !          3827:                case MEMORY_OBJECT_COPY_INVALID:
        !          3828:                printf("copy_invalid");
        !          3829:                break;
        !          3830: 
        !          3831:                default:
        !          3832:                printf("?");
        !          3833:        }
        !          3834:        printf("]");
        !          3835:        printf(", absent_count=%d\n", object->absent_count);
        !          3836: 
        !          3837:        iprintf("all_wanted=0x%x<", object->all_wanted);
        !          3838:        s = "";
        !          3839:        if (vm_object_wanted(object, VM_OBJECT_EVENT_INITIALIZED)) {
        !          3840:                printf("%sinit", s);
        !          3841:                s = ",";
        !          3842:        }
        !          3843:        if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGER_READY)) {
        !          3844:                printf("%sready", s);
        !          3845:                s = ",";
        !          3846:        }
        !          3847:        if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGING_IN_PROGRESS)) {
        !          3848:                printf("%spaging", s);
        !          3849:                s = ",";
        !          3850:        }
        !          3851:        if (vm_object_wanted(object, VM_OBJECT_EVENT_ABSENT_COUNT)) {
        !          3852:                printf("%sabsent", s);
        !          3853:                s = ",";
        !          3854:        }
        !          3855:        if (vm_object_wanted(object, VM_OBJECT_EVENT_LOCK_IN_PROGRESS)) {
        !          3856:                printf("%slock", s);
        !          3857:                s = ",";
        !          3858:        }
        !          3859:        if (vm_object_wanted(object, VM_OBJECT_EVENT_UNCACHING)) {
        !          3860:                printf("%suncaching", s);
        !          3861:                s = ",";
        !          3862:        }
        !          3863:        if (vm_object_wanted(object, VM_OBJECT_EVENT_COPY_CALL)) {
        !          3864:                printf("%scopy_call", s);
        !          3865:                s = ",";
        !          3866:        }
        !          3867:        if (vm_object_wanted(object, VM_OBJECT_EVENT_CACHING)) {
        !          3868:                printf("%scaching", s);
        !          3869:                s = ",";
        !          3870:        }
        !          3871:        printf(">");
        !          3872:        printf(", paging_in_progress=%d\n", object->paging_in_progress);
        !          3873: 
        !          3874:        iprintf("%screated, %sinit, %sready, %spersist, %strusted, %spageout, %s, %s\n",
        !          3875:                (object->pager_created ? "" : "!"),
        !          3876:                (object->pager_initialized ? "" : "!"),
        !          3877:                (object->pager_ready ? "" : "!"),
        !          3878:                (object->can_persist ? "" : "!"),
        !          3879:                (object->pager_trusted ? "" : "!"),
        !          3880:                (object->pageout ? "" : "!"),
        !          3881:                (object->internal ? "internal" : "external"),
        !          3882:                (object->temporary ? "temporary" : "permanent"));
        !          3883:        iprintf("%salive, %slock_in_progress, %slock_restart, %sshadowed, %scached, %sprivate\n",
        !          3884:                (object->alive ? "" : "!"),
        !          3885:                (object->lock_in_progress ? "" : "!"),
        !          3886:                (object->lock_restart ? "" : "!"),
        !          3887:                (object->shadowed ? "" : "!"),
        !          3888:                (vm_object_cached(object) ? "" : "!"),
        !          3889:                (object->private ? "" : "!"));
        !          3890:        iprintf("%sadvisory_pageout, %ssilent_overwrite\n",
        !          3891:                (object->advisory_pageout ? "" : "!"),
        !          3892:                (object->silent_overwrite ? "" : "!"));
        !          3893: 
        !          3894: #if    MACH_PAGEMAP
        !          3895:        iprintf("existence_map=");
        !          3896:        vm_external_print(object->existence_map, object->size);
        !          3897: #endif /* MACH_PAGEMAP */
        !          3898: #if    MACH_ASSERT
        !          3899:        iprintf("paging_object=0x%x\n", object->paging_object);
        !          3900: #endif /* MACH_ASSERT */
        !          3901: 
        !          3902:        if (vm_object_print_pages) {
        !          3903:                count = 0;
        !          3904:                p = (vm_page_t) queue_first(&object->memq);
        !          3905:                while (!queue_end(&object->memq, (queue_entry_t) p)) {
        !          3906:                        if (count == 0) {
        !          3907:                                iprintf("memory:=");
        !          3908:                        } else if (count == 2) {
        !          3909:                                printf("\n");
        !          3910:                                iprintf(" ...");
        !          3911:                                count = 0;
        !          3912:                        } else {
        !          3913:                                printf(",");
        !          3914:                        }
        !          3915:                        count++;
        !          3916: 
        !          3917:                        printf("(off=0x%X,page=0x%X)", p->offset, (integer_t) p);
        !          3918:                        p = (vm_page_t) queue_next(&p->listq);
        !          3919:                }
        !          3920:                if (count != 0) {
        !          3921:                        printf("\n");
        !          3922:                }
        !          3923:        }
        !          3924:        db_indent -= 2;
        !          3925: }
        !          3926: 
        !          3927: 
        !          3928: /*
        !          3929:  *     vm_object_find          [ debug ]
        !          3930:  *
        !          3931:  *     Find all tasks which reference the given vm_object.
        !          3932:  */
        !          3933: 
        !          3934: boolean_t vm_object_find(vm_object_t object);
        !          3935: boolean_t vm_object_print_verbose = FALSE;
        !          3936: 
        !          3937: boolean_t
        !          3938: vm_object_find(
        !          3939:        vm_object_t     object)
        !          3940: {
        !          3941:         task_t task;
        !          3942:        vm_map_t map;
        !          3943:        vm_map_entry_t entry;
        !          3944:         processor_set_t pset;
        !          3945:        boolean_t found = FALSE;
        !          3946: 
        !          3947:         queue_iterate(&all_psets, pset, processor_set_t, all_psets) {
        !          3948:             queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
        !          3949:                map = task->map;
        !          3950:                for (entry = vm_map_first_entry(map);
        !          3951:                      entry && entry != vm_map_to_entry(map);
        !          3952:                      entry = entry->vme_next) {
        !          3953: 
        !          3954:                        vm_object_t obj;
        !          3955: 
        !          3956:                        /* 
        !          3957:                         * For the time being skip submaps,
        !          3958:                         * only the kernel can have submaps,
        !          3959:                         * and unless we are interested in 
        !          3960:                         * kernel objects, we can simply skip 
        !          3961:                         * submaps. See sb/dejan/nmk18b7/src/mach_kernel/vm
        !          3962:                         * for a full solution.
        !          3963:                         */
        !          3964:                        if (entry->is_sub_map)
        !          3965:                                continue;
        !          3966:                        if (entry) 
        !          3967:                                obj = entry->object.vm_object;
        !          3968:                        else 
        !          3969:                                continue;
        !          3970: 
        !          3971:                        while (obj != VM_OBJECT_NULL) {
        !          3972:                                if (obj == object) {
        !          3973:                                        if (!found) {
        !          3974:                                                printf("TASK\t\tMAP\t\tENTRY\n");
        !          3975:                                                found = TRUE;
        !          3976:                                        }
        !          3977:                                        printf("0x%x\t0x%x\t0x%x\n", 
        !          3978:                                                        task, map, entry);
        !          3979:                                }
        !          3980:                                obj = obj->shadow;
        !          3981:                        }
        !          3982:                }
        !          3983:             }
        !          3984:         }
        !          3985: 
        !          3986:        return(found);
        !          3987: }
        !          3988: 
        !          3989: #endif /* MACH_KDB */
        !          3990: 
        !          3991: /*
        !          3992:  *     memory_object_free_from_cache:
        !          3993:  *
        !          3994:  *     Walk the vm_object cache list, removing and freeing vm_objects 
        !          3995:  *     which are backed by the pager identified by the caller, (pager_id).  
        !          3996:  *     Remove up to "count" objects, if there are that may available
        !          3997:  *     in the cache.
        !          3998:  *     Walk the list at most once, return the number of vm_objects
        !          3999:  *     actually freed.
        !          4000:  *
        !          4001:  */
        !          4002: 
        !          4003: kern_return_t
        !          4004: memory_object_free_from_cache(
        !          4005:        host_t          host,
        !          4006:        int             pager_id,
        !          4007:        int             *count)
        !          4008: {
        !          4009: 
        !          4010:        int     object_released = 0;
        !          4011:        int     i;
        !          4012: 
        !          4013:        register vm_object_t object = VM_OBJECT_NULL;
        !          4014:        vm_object_t shadow;
        !          4015: 
        !          4016:        if(host == HOST_NULL)
        !          4017:                return(KERN_INVALID_ARGUMENT);
        !          4018: 
        !          4019:  try_again:
        !          4020:        vm_object_cache_lock();
        !          4021: 
        !          4022:        queue_iterate(&vm_object_cached_list, object, 
        !          4023:                                        vm_object_t, cached_list) {
        !          4024:                if (pager_id == (int) pager_mux_hash_lookup(
        !          4025:                                        (ipc_port_t)object->pager)) {
        !          4026:                        vm_object_lock(object);
        !          4027:                        queue_remove(&vm_object_cached_list, object, 
        !          4028:                                        vm_object_t, cached_list);
        !          4029:                        vm_object_cached_count--;
        !          4030: 
        !          4031:                        /*
        !          4032:                        *       Since this object is in the cache, we know
        !          4033:                        *       that it is initialized and has no references.
        !          4034:                        *       Take a reference to avoid recursive 
        !          4035:                        *       deallocations.
        !          4036:                        */
        !          4037: 
        !          4038:                        assert(object->pager_initialized);
        !          4039:                        assert(object->ref_count == 0);
        !          4040:                        object->ref_count++;
        !          4041: 
        !          4042:                        /*
        !          4043:                        *       Terminate the object.
        !          4044:                        *       If the object had a shadow, we let 
        !          4045:                        *       vm_object_deallocate deallocate it. 
        !          4046:                        *       "pageout" objects have a shadow, but
        !          4047:                        *       maintain a "paging reference" rather 
        !          4048:                        *       than a normal reference.
        !          4049:                        *       (We are careful here to limit recursion.)
        !          4050:                        */
        !          4051:                        shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
        !          4052:                        vm_object_terminate(object);
        !          4053:                        if (shadow != VM_OBJECT_NULL) {
        !          4054:                                vm_object_deallocate(shadow);
        !          4055:                        }
        !          4056:                
        !          4057:                        if(object_released++ == *count)
        !          4058:                                return KERN_SUCCESS;
        !          4059:                        goto try_again;
        !          4060:                }
        !          4061:        }
        !          4062:        vm_object_cache_unlock();
        !          4063:        *count  = object_released;
        !          4064:        return KERN_SUCCESS;
        !          4065: }
        !          4066: 
        !          4067: /*
        !          4068:  *     memory_object_remove_cached_object:
        !          4069:  *
        !          4070:  *     Check for the existance of a memory object represented by the
        !          4071:  *     supplied port.  If one exists and it is not in use, remove the 
        !          4072:  *     memory object from the vm_object cache.
        !          4073:  *     If the memory object is in use, turn off the the "can_persist"
        !          4074:  *     property so that it will not go in the cache when the last user 
        !          4075:  *     gives it up.
        !          4076:  *
        !          4077:  */
        !          4078: 
        !          4079: kern_return_t
        !          4080: memory_object_remove_cached_object(
        !          4081:        ipc_port_t      port)
        !          4082: {
        !          4083:        vm_object_t     object;
        !          4084:        vm_object_t     shadow;
        !          4085: 
        !          4086: repeat_lock_acquire:
        !          4087:        object = VM_OBJECT_NULL;
        !          4088: 
        !          4089:        if (IP_VALID(port)) {
        !          4090:                vm_object_cache_lock();
        !          4091:                ip_lock(port);
        !          4092:                if (ip_active(port) &&
        !          4093:                    (ip_kotype(port) == IKOT_PAGER_LOOKUP_TYPE)) {
        !          4094:                        object = (vm_object_t) port->ip_kobject;
        !          4095:                        if (!vm_object_lock_try(object)) {
        !          4096:                                /*
        !          4097:                                 * failed to acquire object lock.  Drop the
        !          4098:                                 * other two locks and wait for it, then go
        !          4099:                                 * back and start over in case the port
        !          4100:                                 * associations changed in the interim.
        !          4101:                                 */
        !          4102:                                ip_unlock(port);
        !          4103:                                vm_object_cache_unlock();
        !          4104:                                vm_object_lock(object);
        !          4105:                                vm_object_unlock(object);
        !          4106:                                goto repeat_lock_acquire;
        !          4107:                        }
        !          4108: 
        !          4109:                        assert(object->alive);
        !          4110:                        ip_unlock(port);
        !          4111: 
        !          4112:                        if (object->ref_count == 0) {
        !          4113:                                queue_remove(&vm_object_cached_list, object,
        !          4114:                                             vm_object_t, cached_list);
        !          4115:                                vm_object_cached_count--;
        !          4116:                                object->ref_count++;
        !          4117:                                /*
        !          4118:                                *       Terminate the object.
        !          4119:                                *       If the object had a shadow, we let
        !          4120:                                *       vm_object_deallocate deallocate it.
        !          4121:                                *       "pageout" objects have a shadow, but
        !          4122:                                *       maintain a "paging reference" rather
        !          4123:                                *       than a normal reference.
        !          4124:                                *       (We are careful here to limit 
        !          4125:                                *       recursion.)
        !          4126:                                */
        !          4127:                                shadow = object->pageout?
        !          4128:                                        VM_OBJECT_NULL:object->shadow;
        !          4129:                                /* will do the vm_object_cache_unlock */
        !          4130:                                vm_object_terminate(object);
        !          4131:                                if (shadow != VM_OBJECT_NULL) {
        !          4132:                                        /* will lock and unlock cache_lock */
        !          4133:                                        vm_object_deallocate(shadow);
        !          4134:                                }
        !          4135:                        }
        !          4136:                        else {
        !          4137:                                /* 
        !          4138:                                *       We cannot free object but we can
        !          4139:                                *       make sure it doesn't go into the 
        !          4140:                                *       cache when it is no longer in
        !          4141:                                *       use.
        !          4142:                                */ 
        !          4143:                                object->can_persist = FALSE;
        !          4144: 
        !          4145:                                vm_object_unlock(object);
        !          4146:                                vm_object_cache_unlock();
        !          4147:                                return KERN_RIGHT_EXISTS;
        !          4148:                        }
        !          4149: 
        !          4150: 
        !          4151:                }
        !          4152:                else {
        !          4153:                        ip_unlock(port);
        !          4154:                        vm_object_cache_unlock();
        !          4155:                }
        !          4156:        } else {
        !          4157:                return KERN_INVALID_ARGUMENT;
        !          4158:        }
        !          4159: 
        !          4160: 
        !          4161:        return KERN_SUCCESS;
        !          4162: }

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.