![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_object.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52: /*
53: * File: vm/vm_object.c
54: * Author: Avadis Tevanian, Jr., Michael Wayne Young
55: *
56: * Virtual memory object module.
57: */
58:
59: #ifdef MACH_BSD
60: /* remove as part of compoenent support merge */
61: extern int vnode_pager_workaround;
62: #endif
63:
64: #include <mach_pagemap.h>
65: #include <task_swapper.h>
66:
67: #include <mach/memory_object.h>
68: #include <mach/memory_object_default.h>
69: #include <mach/memory_object_control_server.h>
70: #include <mach/vm_param.h>
71: #include <ipc/ipc_port.h>
72: #include <ipc/ipc_space.h>
73: #include <kern/assert.h>
74: #include <kern/lock.h>
75: #include <kern/queue.h>
76: #include <kern/xpr.h>
77: #include <kern/zalloc.h>
78: #include <kern/host.h>
79: #include <kern/host_statistics.h>
80: #include <vm/memory_object.h>
81: #include <vm/vm_fault.h>
82: #include <vm/vm_map.h>
83: #include <vm/vm_object.h>
84: #include <vm/vm_page.h>
85: #include <vm/vm_pageout.h>
86: #include <kern/misc_protos.h>
87:
88: /*
89: * Virtual memory objects maintain the actual data
90: * associated with allocated virtual memory. A given
91: * page of memory exists within exactly one object.
92: *
93: * An object is only deallocated when all "references"
94: * are given up. Only one "reference" to a given
95: * region of an object should be writeable.
96: *
97: * Associated with each object is a list of all resident
98: * memory pages belonging to that object; this list is
99: * maintained by the "vm_page" module, but locked by the object's
100: * lock.
101: *
102: * Each object also records the memory object port
103: * that is used by the kernel to request and write
104: * back data (the memory object port, field "pager"),
105: * and the ports provided to the memory manager, the server that
106: * manages that data, to return data and control its
107: * use (the memory object control port, field "pager_request")
108: * and for naming (the memory object name port, field "pager_name").
109: *
110: * Virtual memory objects are allocated to provide
111: * zero-filled memory (vm_allocate) or map a user-defined
112: * memory object into a virtual address space (vm_map).
113: *
114: * Virtual memory objects that refer to a user-defined
115: * memory object are called "permanent", because all changes
116: * made in virtual memory are reflected back to the
117: * memory manager, which may then store it permanently.
118: * Other virtual memory objects are called "temporary",
119: * meaning that changes need be written back only when
120: * necessary to reclaim pages, and that storage associated
121: * with the object can be discarded once it is no longer
122: * mapped.
123: *
124: * A permanent memory object may be mapped into more
125: * than one virtual address space. Moreover, two threads
126: * may attempt to make the first mapping of a memory
127: * object concurrently. Only one thread is allowed to
128: * complete this mapping; all others wait for the
129: * "pager_initialized" field is asserted, indicating
130: * that the first thread has initialized all of the
131: * necessary fields in the virtual memory object structure.
132: *
133: * The kernel relies on a *default memory manager* to
134: * provide backing storage for the zero-filled virtual
135: * memory objects. The memory object ports associated
136: * with these temporary virtual memory objects are only
137: * generated and passed to the default memory manager
138: * when it becomes necessary. Virtual memory objects
139: * that depend on the default memory manager are called
140: * "internal". The "pager_created" field is provided to
141: * indicate whether these ports have ever been allocated.
142: *
143: * The kernel may also create virtual memory objects to
144: * hold changed pages after a copy-on-write operation.
145: * In this case, the virtual memory object (and its
146: * backing storage -- its memory object) only contain
147: * those pages that have been changed. The "shadow"
148: * field refers to the virtual memory object that contains
149: * the remainder of the contents. The "shadow_offset"
150: * field indicates where in the "shadow" these contents begin.
151: * The "copy" field refers to a virtual memory object
152: * to which changed pages must be copied before changing
153: * this object, in order to implement another form
154: * of copy-on-write optimization.
155: *
156: * The virtual memory object structure also records
157: * the attributes associated with its memory object.
158: * The "pager_ready", "can_persist" and "copy_strategy"
159: * fields represent those attributes. The "cached_list"
160: * field is used in the implementation of the persistence
161: * attribute.
162: *
163: * ZZZ Continue this comment.
164: */
165:
166: /* Forward declarations for internal functions. */
167: extern void _vm_object_allocate(
168: vm_size_t size,
169: vm_object_t object);
170:
171: extern void vm_object_terminate(
172: vm_object_t object);
173:
174: extern void vm_object_remove(
175: vm_object_t object);
176:
177: extern vm_object_t vm_object_cache_trim(
178: boolean_t called_from_vm_object_deallocate);
179:
180: extern void vm_object_deactivate_pages(
181: vm_object_t object);
182:
183: extern void vm_object_abort_activity(
184: vm_object_t object);
185:
186: extern kern_return_t vm_object_copy_call(
187: vm_object_t src_object,
188: vm_offset_t src_offset,
189: vm_size_t size,
190: vm_object_t *_result_object);
191:
192: extern void vm_object_do_collapse(
193: vm_object_t object,
194: vm_object_t backing_object);
195:
196: extern void vm_object_do_bypass(
197: vm_object_t object,
198: vm_object_t backing_object);
199:
200: extern void memory_object_release(
201: ipc_port_t pager,
202: pager_request_t pager_request);
203:
204: zone_t vm_object_zone; /* vm backing store zone */
205:
206: /*
207: * All wired-down kernel memory belongs to a single virtual
208: * memory object (kernel_object) to avoid wasting data structures.
209: */
210: struct vm_object kernel_object_store;
211: vm_object_t kernel_object = &kernel_object_store;
212:
213: /*
214: * The submap object is used as a placeholder for vm_map_submap
215: * operations. The object is declared in vm_map.c because it
216: * is exported by the vm_map module. The storage is declared
217: * here because it must be initialized here.
218: */
219: struct vm_object vm_submap_object_store;
220:
221: /*
222: * Virtual memory objects are initialized from
223: * a template (see vm_object_allocate).
224: *
225: * When adding a new field to the virtual memory
226: * object structure, be sure to add initialization
227: * (see vm_object_init).
228: */
229: struct vm_object vm_object_template;
230:
231: /*
232: * Virtual memory objects that are not referenced by
233: * any address maps, but that are allowed to persist
234: * (an attribute specified by the associated memory manager),
235: * are kept in a queue (vm_object_cached_list).
236: *
237: * When an object from this queue is referenced again,
238: * for example to make another address space mapping,
239: * it must be removed from the queue. That is, the
240: * queue contains *only* objects with zero references.
241: *
242: * The kernel may choose to terminate objects from this
243: * queue in order to reclaim storage. The current policy
244: * is to permit a fixed maximum number of unreferenced
245: * objects (vm_object_cached_max).
246: *
247: * A spin lock (accessed by routines
248: * vm_object_cache_{lock,lock_try,unlock}) governs the
249: * object cache. It must be held when objects are
250: * added to or removed from the cache (in vm_object_terminate).
251: * The routines that acquire a reference to a virtual
252: * memory object based on one of the memory object ports
253: * must also lock the cache.
254: *
255: * Ideally, the object cache should be more isolated
256: * from the reference mechanism, so that the lock need
257: * not be held to make simple references.
258: */
259: queue_head_t vm_object_cached_list;
260: int vm_object_cached_count;
261: int vm_object_cached_high; /* highest # of cached objects */
262: int vm_object_cached_max = 500; /* may be patched*/
263:
264: decl_mutex_data(,vm_object_cached_lock_data)
265:
266: #define vm_object_cache_lock() \
267: mutex_lock(&vm_object_cached_lock_data)
268: #define vm_object_cache_lock_try() \
269: mutex_try(&vm_object_cached_lock_data)
270: #define vm_object_cache_unlock() \
271: mutex_unlock(&vm_object_cached_lock_data)
272:
273: #define VM_OBJECT_HASH_COUNT 1024
274: queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT];
275: struct zone *vm_object_hash_zone;
276:
277: struct vm_object_hash_entry {
278: queue_chain_t hash_link; /* hash chain link */
279: ipc_port_t pager; /* pager we represent */
280: vm_object_t object; /* corresponding object */
281: boolean_t waiting; /* someone waiting for
282: * termination */
283: };
284:
285: typedef struct vm_object_hash_entry *vm_object_hash_entry_t;
286: #define VM_OBJECT_HASH_ENTRY_NULL ((vm_object_hash_entry_t) 0)
287:
288: #define VM_OBJECT_HASH_SHIFT 8
289: #define vm_object_hash(pager) \
290: ((((unsigned)pager) >> VM_OBJECT_HASH_SHIFT) % VM_OBJECT_HASH_COUNT)
291:
292: /*
293: * vm_object_hash_lookup looks up a pager in the hashtable
294: * and returns the corresponding entry, with optional removal.
295: */
296:
297: vm_object_hash_entry_t
298: vm_object_hash_lookup(
299: ipc_port_t pager,
300: boolean_t remove_entry)
301: {
302: register queue_t bucket;
303: register vm_object_hash_entry_t entry;
304:
305: bucket = &vm_object_hashtable[vm_object_hash(pager)];
306:
307: entry = (vm_object_hash_entry_t)queue_first(bucket);
308: while (!queue_end(bucket, (queue_entry_t)entry)) {
309: if (entry->pager == pager && !remove_entry)
310: return(entry);
311: else if (entry->pager == pager) {
312: queue_remove(bucket, entry,
313: vm_object_hash_entry_t, hash_link);
314: return(entry);
315: }
316:
317: entry = (vm_object_hash_entry_t)queue_next(&entry->hash_link);
318: }
319:
320: return(VM_OBJECT_HASH_ENTRY_NULL);
321: }
322:
323: /*
324: * vm_object_hash_enter enters the specified
325: * pager / cache object association in the hashtable.
326: */
327:
328: void
329: vm_object_hash_insert(
330: vm_object_hash_entry_t entry)
331: {
332: register queue_t bucket;
333:
334: bucket = &vm_object_hashtable[vm_object_hash(entry->pager)];
335:
336: queue_enter(bucket, entry, vm_object_hash_entry_t, hash_link);
337: }
338:
339: vm_object_hash_entry_t
340: vm_object_hash_entry_alloc(
341: ipc_port_t pager)
342: {
343: vm_object_hash_entry_t entry;
344:
345: entry = (vm_object_hash_entry_t)zalloc(vm_object_hash_zone);
346: entry->pager = pager;
347: entry->object = VM_OBJECT_NULL;
348: entry->waiting = FALSE;
349:
350: return(entry);
351: }
352:
353: void
354: vm_object_hash_entry_free(
355: vm_object_hash_entry_t entry)
356: {
357: zfree(vm_object_hash_zone, (vm_offset_t)entry);
358: }
359:
360: /*
361: * vm_object_allocate:
362: *
363: * Returns a new object with the given size.
364: */
365:
366: void
367: _vm_object_allocate(
368: vm_size_t size,
369: vm_object_t object)
370: {
371: XPR(XPR_VM_OBJECT,
372: "vm_object_allocate, object 0x%X size 0x%X\n",
373: (integer_t)object, size, 0,0,0);
374:
375: *object = vm_object_template;
376: queue_init(&object->memq);
377: queue_init(&object->msr_q);
378: vm_object_lock_init(object);
379: object->size = size;
380: }
381:
382: vm_object_t
383: vm_object_allocate(
384: vm_size_t size)
385: {
386: register vm_object_t object;
387: register ipc_port_t port;
388:
389: object = (vm_object_t) zalloc(vm_object_zone);
390:
391: // dbgLog(object, size, 0, 2); /* (TEST/DEBUG) */
392:
393: _vm_object_allocate(size, object);
394:
395: return object;
396: }
397:
398: /*
399: * vm_object_bootstrap:
400: *
401: * Initialize the VM objects module.
402: */
403: void
404: vm_object_bootstrap(void)
405: {
406: register i;
407:
408: vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
409: round_page(512*1024),
410: round_page(12*1024),
411: "vm objects");
412:
413: queue_init(&vm_object_cached_list);
414: mutex_init(&vm_object_cached_lock_data, ETAP_VM_OBJ_CACHE);
415:
416: vm_object_hash_zone =
417: zinit((vm_size_t) sizeof (struct vm_object_hash_entry),
418: round_page(512*1024),
419: round_page(12*1024),
420: "vm object hash entries");
421:
422: for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
423: queue_init(&vm_object_hashtable[i]);
424:
425: /*
426: * Fill in a template object, for quick initialization
427: */
428:
429: /* memq; Lock; init after allocation */
430: vm_object_template.size = 0;
431: vm_object_template.frozen_size = 0;
432: vm_object_template.ref_count = 1;
433: #if TASK_SWAPPER
434: vm_object_template.res_count = 1;
435: #endif /* TASK_SWAPPER */
436: vm_object_template.resident_page_count = 0;
437: vm_object_template.copy = VM_OBJECT_NULL;
438: vm_object_template.shadow = VM_OBJECT_NULL;
439: vm_object_template.shadow_offset = (vm_offset_t) 0;
440: vm_object_template.true_share = FALSE;
441:
442: vm_object_template.pager = IP_NULL;
443: vm_object_template.paging_offset = 0;
444: vm_object_template.pager_request = PAGER_REQUEST_NULL;
445: /* msr_q; init after allocation */
446:
447: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_SYMMETRIC;
448: vm_object_template.absent_count = 0;
449: vm_object_template.paging_in_progress = 0;
450:
451: /* Begin bitfields */
452: vm_object_template.all_wanted = 0; /* all bits FALSE */
453: vm_object_template.pager_created = FALSE;
454: vm_object_template.pager_initialized = FALSE;
455: vm_object_template.pager_ready = FALSE;
456: vm_object_template.pager_trusted = FALSE;
457: vm_object_template.can_persist = FALSE;
458: vm_object_template.internal = TRUE;
459: vm_object_template.temporary = TRUE;
460: vm_object_template.private = FALSE;
461: vm_object_template.pageout = FALSE;
462: vm_object_template.alive = TRUE;
463: vm_object_template.lock_in_progress = FALSE;
464: vm_object_template.lock_restart = FALSE;
465: vm_object_template.silent_overwrite = FALSE;
466: vm_object_template.advisory_pageout = FALSE;
467: vm_object_template.shadowed = FALSE;
468: /* End bitfields */
469:
470: /* cached_list; init after allocation */
471: vm_object_template.last_alloc = (vm_offset_t) 0;
472: vm_object_template.cluster_size = 0;
473: #if MACH_PAGEMAP
474: vm_object_template.existence_map = VM_EXTERNAL_NULL;
475: #endif /* MACH_PAGEMAP */
476: #if MACH_ASSERT
477: vm_object_template.paging_object = VM_OBJECT_NULL;
478: #endif /* MACH_ASSERT */
479:
480: /*
481: * Initialize the "kernel object"
482: */
483:
484: kernel_object = &kernel_object_store;
485:
486: /*
487: * Note that in the following size specifications, we need to add 1 because
488: * VM_MAX_KERNEL_ADDRESS is a maximum address, not a size.
489: */
490: _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
491: kernel_object);
492:
493: /*
494: * Initialize the "submap object". Make it as large as the
495: * kernel object so that no limit is imposed on submap sizes.
496: */
497:
498: vm_submap_object = &vm_submap_object_store;
499: _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
500: vm_submap_object);
501: /*
502: * Create an "extra" reference to this object so that we never
503: * try to deallocate it; zfree doesn't like to be called with
504: * non-zone memory.
505: */
506: vm_object_reference(vm_submap_object);
507:
508: #if MACH_PAGEMAP
509: vm_external_module_initialize();
510: #endif /* MACH_PAGEMAP */
511: }
512:
513: void
514: vm_object_init(void)
515: {
516: /*
517: * Finish initializing the kernel object.
518: */
519: }
520:
521: #if TASK_SWAPPER
522: /*
523: * vm_object_res_deallocate
524: *
525: * (recursively) decrement residence counts on vm objects and their shadows.
526: * Called from vm_object_deallocate and when swapping out an object.
527: *
528: * The object is locked, and remains locked throughout the function,
529: * even as we iterate down the shadow chain. Locks on intermediate objects
530: * will be dropped, but not the original object.
531: *
532: * NOTE: this function used to use recursion, rather than iteration.
533: */
534:
535: void
536: vm_object_res_deallocate(
537: vm_object_t object)
538: {
539: vm_object_t orig_object = object;
540: /*
541: * Object is locked so it can be called directly
542: * from vm_object_deallocate. Original object is never
543: * unlocked.
544: */
545: assert(object->res_count > 0);
546: while (--object->res_count == 0) {
547: assert(object->ref_count >= object->res_count);
548: vm_object_deactivate_pages(object);
549: /* iterate on shadow, if present */
550: if (object->shadow != VM_OBJECT_NULL) {
551: vm_object_t tmp_object = object->shadow;
552: vm_object_lock(tmp_object);
553: if (object != orig_object)
554: vm_object_unlock(object);
555: object = tmp_object;
556: assert(object->res_count > 0);
557: } else
558: break;
559: }
560: if (object != orig_object)
561: vm_object_unlock(object);
562: }
563:
564: /*
565: * vm_object_res_reference
566: *
567: * Internal function to increment residence count on a vm object
568: * and its shadows. It is called only from vm_object_reference, and
569: * when swapping in a vm object, via vm_map_swap.
570: *
571: * The object is locked, and remains locked throughout the function,
572: * even as we iterate down the shadow chain. Locks on intermediate objects
573: * will be dropped, but not the original object.
574: *
575: * NOTE: this function used to use recursion, rather than iteration.
576: */
577:
578: void
579: vm_object_res_reference(
580: vm_object_t object)
581: {
582: vm_object_t orig_object = object;
583: /*
584: * Object is locked, so this can be called directly
585: * from vm_object_reference. This lock is never released.
586: */
587: while ((++object->res_count == 1) &&
588: (object->shadow != VM_OBJECT_NULL)) {
589: vm_object_t tmp_object = object->shadow;
590:
591: assert(object->ref_count >= object->res_count);
592: vm_object_lock(tmp_object);
593: if (object != orig_object)
594: vm_object_unlock(object);
595: object = tmp_object;
596: }
597: if (object != orig_object)
598: vm_object_unlock(object);
599: assert(orig_object->ref_count >= orig_object->res_count);
600: }
601: #endif /* TASK_SWAPPER */
602:
603: #if MACH_ASSERT
604: /*
605: * vm_object_reference:
606: *
607: * Gets another reference to the given object.
608: */
609: void
610: vm_object_reference(
611: register vm_object_t object)
612: {
613: if (object == VM_OBJECT_NULL)
614: return;
615:
616: vm_object_lock(object);
617: assert(object->ref_count > 0);
618: object->ref_count++;
619: vm_object_res_reference(object);
620: vm_object_unlock(object);
621: }
622: #endif /* MACH_ASSERT */
623:
624: #define MIGHT_NOT_CACHE_SHADOWS 1
625: #if MIGHT_NOT_CACHE_SHADOWS
626: int cache_shadows = TRUE;
627: #endif /* MIGHT_NOT_CACHE_SHADOWS */
628:
629: /*
630: * vm_object_deallocate:
631: *
632: * Release a reference to the specified object,
633: * gained either through a vm_object_allocate
634: * or a vm_object_reference call. When all references
635: * are gone, storage associated with this object
636: * may be relinquished.
637: *
638: * No object may be locked.
639: */
640: void
641: vm_object_deallocate(
642: register vm_object_t object)
643: {
644: boolean_t retry_cache_trim = FALSE;
645: vm_object_t shadow;
646:
647: // if(object)dbgLog(object, object->ref_count, object->can_persist, 3); /* (TEST/DEBUG) */
648: // else dbgLog(object, 0, 0, 3); /* (TEST/DEBUG) */
649:
650:
651: while (object != VM_OBJECT_NULL) {
652:
653: /*
654: * The cache holds a reference (uncounted) to
655: * the object; we must lock it before removing
656: * the object.
657: */
658:
659: vm_object_cache_lock();
660: vm_object_lock(object);
661: assert(object->alive);
662:
663: /*
664: * Lose the reference. If other references
665: * remain, then we are done, unless we need
666: * to retry a cache trim.
667: * If it is the last reference, then keep it
668: * until any pending initialization is completed.
669: */
670:
671: assert(object->ref_count > 0);
672: if (object->ref_count > 1) {
673: object->ref_count--;
674: vm_object_res_deallocate(object);
675: vm_object_unlock(object);
676: vm_object_cache_unlock();
677: if (retry_cache_trim &&
678: ((object = vm_object_cache_trim(TRUE)) !=
679: VM_OBJECT_NULL)) {
680: continue;
681: }
682: return;
683: }
684:
685: /*
686: * We have to wait for initialization
687: * before destroying or caching the object.
688: */
689:
690: if (object->pager_created && ! object->pager_initialized) {
691: assert(! object->can_persist);
692: vm_object_assert_wait(object,
693: VM_OBJECT_EVENT_INITIALIZED,
694: THREAD_UNINT);
695: vm_object_unlock(object);
696: vm_object_cache_unlock();
697: thread_block((void (*)(void))0);
698: continue;
699: }
700:
701: /*
702: * If this object can persist, then enter it in
703: * the cache. Otherwise, terminate it.
704: *
705: * NOTE: Only permanent objects are cached, and
706: * permanent objects cannot have shadows. This
707: * affects the residence counting logic in a minor
708: * way (can do it in-line, mostly).
709: */
710:
711: if (object->can_persist) {
712: /*
713: * Now it is safe to decrement reference count,
714: * and to return if reference count is > 0.
715: */
716: if (--object->ref_count > 0) {
717: vm_object_res_deallocate(object);
718: vm_object_unlock(object);
719: vm_object_cache_unlock();
720: if (retry_cache_trim &&
721: ((object = vm_object_cache_trim(TRUE)) !=
722: VM_OBJECT_NULL)) {
723: continue;
724: }
725: return;
726: }
727:
728: #if MIGHT_NOT_CACHE_SHADOWS
729: /*
730: * Remove shadow now if we don't
731: * want to cache shadows.
732: */
733: if (! cache_shadows) {
734: shadow = object->shadow;
735: object->shadow = VM_OBJECT_NULL;
736: }
737: #endif /* MIGHT_NOT_CACHE_SHADOWS */
738:
739: /*
740: * Enter the object onto the queue of
741: * cached objects, and deactivate
742: * all of its pages.
743: */
744: assert(object->shadow == VM_OBJECT_NULL);
745: VM_OBJ_RES_DECR(object);
746: XPR(XPR_VM_OBJECT,
747: "vm_o_deallocate: adding %x to cache, queue = (%x, %x)\n",
748: (integer_t)object,
749: (integer_t)vm_object_cached_list.next,
750: (integer_t)vm_object_cached_list.prev,0,0);
751:
752: vm_object_cached_count++;
753: if (vm_object_cached_count > vm_object_cached_high)
754: vm_object_cached_high = vm_object_cached_count;
755: queue_enter(&vm_object_cached_list, object,
756: vm_object_t, cached_list);
757: vm_object_cache_unlock();
758: vm_object_deactivate_pages(object);
759: vm_object_unlock(object);
760:
761: #if MIGHT_NOT_CACHE_SHADOWS
762: /*
763: * If we have a shadow that we need
764: * to deallocate, do so now, remembering
765: * to trim the cache later.
766: */
767: if (! cache_shadows && shadow != VM_OBJECT_NULL) {
768: object = shadow;
769: retry_cache_trim = TRUE;
770: continue;
771: }
772: #endif /* MIGHT_NOT_CACHE_SHADOWS */
773:
774: /*
775: * Trim the cache. If the cache trim
776: * returns with a shadow for us to deallocate,
777: * then remember to retry the cache trim
778: * when we are done deallocating the shadow.
779: * Otherwise, we are done.
780: */
781:
782: object = vm_object_cache_trim(TRUE);
783: if (object == VM_OBJECT_NULL) {
784: return;
785: }
786: retry_cache_trim = TRUE;
787:
788: } else {
789: /*
790: * This object is not cachable; terminate it.
791: */
792: XPR(XPR_VM_OBJECT,
793: "vm_o_deallocate: !cacheable 0x%X res %d paging_ops %d thread 0x%lX ref %d\n",
794: (integer_t)object, object->resident_page_count,
795: object->paging_in_progress,
796: (natural_t)current_thread(),object->ref_count);
797:
798: VM_OBJ_RES_DECR(object); /* XXX ? */
799: /*
800: * Terminate this object. If it had a shadow,
801: * then deallocate it; otherwise, if we need
802: * to retry a cache trim, do so now; otherwise,
803: * we are done. "pageout" objects have a shadow,
804: * but maintain a "paging reference" rather than
805: * a normal reference.
806: */
807: shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
808: vm_object_terminate(object);
809: if (shadow != VM_OBJECT_NULL) {
810: object = shadow;
811: continue;
812: }
813: if (retry_cache_trim &&
814: ((object = vm_object_cache_trim(TRUE)) !=
815: VM_OBJECT_NULL)) {
816: continue;
817: }
818: return;
819: }
820: }
821: assert(! retry_cache_trim);
822: }
823:
824: /*
825: * Check to see whether we really need to trim
826: * down the cache. If so, remove an object from
827: * the cache, terminate it, and repeat.
828: *
829: * Called with, and returns with, cache lock unlocked.
830: */
831: vm_object_t
832: vm_object_cache_trim(
833: boolean_t called_from_vm_object_deallocate)
834: {
835: register vm_object_t object = VM_OBJECT_NULL;
836: vm_object_t shadow;
837:
838: for (;;) {
839:
840: /*
841: * If we no longer need to trim the cache,
842: * then we are done.
843: */
844:
845: vm_object_cache_lock();
846: if (vm_object_cached_count <= vm_object_cached_max) {
847: vm_object_cache_unlock();
848: return VM_OBJECT_NULL;
849: }
850:
851: /*
852: * We must trim down the cache, so remove
853: * the first object in the cache.
854: */
855: XPR(XPR_VM_OBJECT,
856: "vm_object_cache_trim: removing from front of cache (%x, %x)\n",
857: (integer_t)vm_object_cached_list.next,
858: (integer_t)vm_object_cached_list.prev, 0, 0, 0);
859:
860: object = (vm_object_t) queue_first(&vm_object_cached_list);
861: vm_object_lock(object);
862: queue_remove(&vm_object_cached_list, object, vm_object_t,
863: cached_list);
864: vm_object_cached_count--;
865:
866: /*
867: * Since this object is in the cache, we know
868: * that it is initialized and has no references.
869: * Take a reference to avoid recursive deallocations.
870: */
871:
872: assert(object->pager_initialized);
873: assert(object->ref_count == 0);
874: object->ref_count++;
875:
876: /*
877: * Terminate the object.
878: * If the object had a shadow, we let vm_object_deallocate
879: * deallocate it. "pageout" objects have a shadow, but
880: * maintain a "paging reference" rather than a normal
881: * reference.
882: * (We are careful here to limit recursion.)
883: */
884: shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
885: vm_object_terminate(object);
886: if (shadow != VM_OBJECT_NULL) {
887: if (called_from_vm_object_deallocate) {
888: return shadow;
889: } else {
890: vm_object_deallocate(shadow);
891: }
892: }
893: }
894: }
895:
896: boolean_t vm_object_terminate_remove_all = FALSE;
897:
898: /*
899: * Routine: vm_object_terminate
900: * Purpose:
901: * Free all resources associated with a vm_object.
902: * In/out conditions:
903: * Upon entry, the object and the cache must be locked,
904: * and the object must have exactly one reference.
905: *
906: * The shadow object reference is left alone.
907: *
908: * Upon exit, the cache will be unlocked, and the
909: * object will cease to exist.
910: */
911: void
912: vm_object_terminate(
913: register vm_object_t object)
914: {
915: register vm_page_t p;
916: vm_object_t shadow_object;
917:
918: XPR(XPR_VM_OBJECT, "vm_object_terminate, object 0x%X ref %d\n",
919: (integer_t)object, object->ref_count, 0, 0, 0);
920:
921: /*
922: * Make sure the object isn't already being terminated
923: */
924:
925: assert(object->alive);
926: object->alive = FALSE;
927:
928: /*
929: * Make sure no one can look us up now.
930: */
931:
932: if(object->pager != IP_NULL) {
933: vm_object_hash_entry_t entry;
934:
935: entry = vm_object_hash_lookup(object->pager, FALSE);
936: if (entry != VM_OBJECT_HASH_ENTRY_NULL)
937: entry->object = VM_OBJECT_NULL;
938: }
939:
940: vm_object_cache_unlock();
941:
942: /*
943: * Detach the object from its shadow if we are the shadow's
944: * copy.
945: */
946: if (((shadow_object = object->shadow) != VM_OBJECT_NULL) &&
947: !(object->pageout)) {
948: vm_object_lock(shadow_object);
949: assert((shadow_object->copy == object) ||
950: (shadow_object->copy == VM_OBJECT_NULL));
951: shadow_object->copy = VM_OBJECT_NULL;
952: vm_object_unlock(shadow_object);
953: }
954:
955: /*
956: * The pageout daemon might be playing with our pages.
957: * Now that the object is dead, it won't touch any more
958: * pages, but some pages might already be on their way out.
959: * Hence, we wait until the active paging activities have ceased.
960: */
961: vm_object_paging_wait(object, THREAD_UNINT);
962: object->ref_count--;
963: #if TASK_SWAPPER
964: assert(object->res_count == 0);
965: #endif /* TASK_SWAPPER */
966:
967: Restart:
968: assert (object->ref_count == 0);
969:
970: /*
971: * Clean or free the pages, as appropriate.
972: * It is possible for us to find busy/absent pages,
973: * if some faults on this object were aborted.
974: */
975: if (object->pageout) {
976: assert(shadow_object != VM_OBJECT_NULL);
977: assert(shadow_object == object->shadow);
978:
979: vm_pageout_object_terminate(object);
980:
981: } else if (object->temporary && ! object->can_persist ||
982: object->pager == IP_NULL) {
983: while (!queue_empty(&object->memq)) {
984: p = (vm_page_t) queue_first(&object->memq);
985:
986: VM_PAGE_CHECK(p);
987:
988: if (p->busy && !p->absent)
989: panic("vm_object_terminate.2 0x%x 0x%x",
990: object, p);
991:
992: VM_PAGE_FREE(p);
993: }
994: } else while (!queue_empty(&object->memq)) {
995: /*
996: * Clear pager_trusted bit so that the pages get yanked
997: * out of the object instead of cleaned in place. This
998: * prevents a deadlock in XMM and makes more sense anyway.
999: */
1000: object->pager_trusted = FALSE;
1001:
1002: p = (vm_page_t) queue_first(&object->memq);
1003:
1004: VM_PAGE_CHECK(p);
1005:
1006: if (p->busy && !p->absent)
1007: panic("vm_object_terminate.3 0x%x 0x%x", object, p);
1008:
1009: vm_page_lock_queues();
1010: VM_PAGE_QUEUES_REMOVE(p);
1011: vm_page_unlock_queues();
1012:
1013: if (p->absent || p->private) {
1014:
1015: /*
1016: * For private pages, VM_PAGE_FREE just
1017: * leaves the page structure around for
1018: * its owner to clean up. For absent
1019: * pages, the structure is returned to
1020: * the appropriate pool.
1021: */
1022:
1023: goto free_page;
1024: }
1025:
1026: if (p->fictitious)
1027: panic("vm_object_terminate.4 0x%x 0x%x", object, p);
1028:
1029: if (!p->dirty)
1030: p->dirty = pmap_is_modified(p->phys_addr);
1031:
1032: if (p->dirty || p->precious) {
1033: p->busy = TRUE;
1034: vm_object_paging_begin(object);
1035: vm_object_unlock(object);
1036: vm_pageout_cluster(p); /* flush page */
1037: vm_object_lock(object);
1038: vm_object_paging_wait(object, THREAD_UNINT);
1039:
1040: XPR(XPR_VM_OBJECT,
1041: "vm_object_terminate restart, object 0x%X ref %d\n",
1042: (integer_t)object, object->ref_count, 0, 0, 0);
1043:
1044: goto Restart;
1045: } else {
1046: free_page:
1047: VM_PAGE_FREE(p);
1048: }
1049: }
1050:
1051: assert(object->paging_in_progress == 0);
1052: assert(object->ref_count == 0);
1053:
1054: vm_object_remove(object);
1055:
1056: /*
1057: * Throw away port rights... note that they may
1058: * already have been thrown away (by vm_object_destroy
1059: * or memory_object_destroy).
1060: *
1061: * Instead of destroying the control port,
1062: * we send all rights off to the memory manager,
1063: * using memory_object_terminate.
1064: */
1065:
1066: vm_object_unlock(object);
1067: if (object->pager != IP_NULL) {
1068: /* consumes our rights for pager, pager_request */
1069: memory_object_release(object->pager, object->pager_request);
1070: }
1071:
1072: #if MACH_PAGEMAP
1073: vm_external_destroy(object->existence_map, object->size);
1074: #endif /* MACH_PAGEMAP */
1075:
1076: /*
1077: * Free the space for the object.
1078: */
1079:
1080: zfree(vm_object_zone, (vm_offset_t) object);
1081: }
1082:
1083: /*
1084: * Routine: vm_object_pager_wakeup
1085: * Purpose: Wake up anyone waiting for termination of a pager.
1086: */
1087:
1088: void
1089: vm_object_pager_wakeup(
1090: ipc_port_t pager)
1091: {
1092: vm_object_hash_entry_t entry;
1093: boolean_t waiting = FALSE;
1094:
1095: /*
1096: * If anyone was waiting for the memory_object_terminate
1097: * to be queued, wake them up now.
1098: */
1099: vm_object_cache_lock();
1100: entry = vm_object_hash_lookup(pager, TRUE);
1101: if (entry != VM_OBJECT_HASH_ENTRY_NULL)
1102: waiting = entry->waiting;
1103: vm_object_cache_unlock();
1104: if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
1105: if (waiting)
1106: thread_wakeup((event_t) pager);
1107: vm_object_hash_entry_free(entry);
1108: }
1109: }
1110:
1111: /*
1112: * Routine: memory_object_release
1113: * Purpose: Terminate the pager and release port rights,
1114: * just like memory_object_terminate, except
1115: * that we wake up anyone blocked in vm_object_enter
1116: * waiting for termination message to be queued
1117: * before calling memory_object_init.
1118: */
1119: void
1120: memory_object_release(
1121: ipc_port_t pager,
1122: pager_request_t pager_request)
1123: {
1124: #ifdef MACH_BSD
1125: kern_return_t vnode_pager_terminate(ipc_port_t, ipc_port_t);
1126: #endif
1127:
1128: /*
1129: * Keep a reference to pager port;
1130: * the terminate might otherwise release all references.
1131: */
1132: ipc_port_copy_send(pager);
1133:
1134: /*
1135: * Terminate the pager.
1136: */
1137:
1138: #ifdef MACH_BSD
1139: if(((rpc_subsystem_t)pager_mux_hash_lookup(pager)) ==
1140: ((rpc_subsystem_t) &vnode_pager_workaround)) {
1141: (void) vnode_pager_terminate(pager, pager_request);
1142: } else {
1143: (void) memory_object_terminate(pager, pager_request);
1144: }
1145: #else
1146: (void) memory_object_terminate(pager, pager_request);
1147: #endif
1148:
1149: /*
1150: * Wakeup anyone waiting for this terminate
1151: */
1152: vm_object_pager_wakeup(pager);
1153:
1154: /*
1155: * Release reference to pager port.
1156: */
1157: ipc_port_release_send(pager);
1158: }
1159:
1160: /*
1161: * Routine: vm_object_abort_activity [internal use only]
1162: * Purpose:
1163: * Abort paging requests pending on this object.
1164: * In/out conditions:
1165: * The object is locked on entry and exit.
1166: */
1167: void
1168: vm_object_abort_activity(
1169: vm_object_t object)
1170: {
1171: register
1172: vm_page_t p;
1173: vm_page_t next;
1174:
1175: XPR(XPR_VM_OBJECT, "vm_object_abort_activity, object 0x%X\n",
1176: (integer_t)object, 0, 0, 0, 0);
1177:
1178: /*
1179: * Abort all activity that would be waiting
1180: * for a result on this memory object.
1181: *
1182: * We could also choose to destroy all pages
1183: * that we have in memory for this object, but
1184: * we don't.
1185: */
1186:
1187: p = (vm_page_t) queue_first(&object->memq);
1188: while (!queue_end(&object->memq, (queue_entry_t) p)) {
1189: next = (vm_page_t) queue_next(&p->listq);
1190:
1191: /*
1192: * If it's being paged in, destroy it.
1193: * If an unlock has been requested, start it again.
1194: */
1195:
1196: if (p->busy && p->absent) {
1197: VM_PAGE_FREE(p);
1198: }
1199: else {
1200: if (p->unlock_request != VM_PROT_NONE)
1201: p->unlock_request = VM_PROT_NONE;
1202: PAGE_WAKEUP(p);
1203: }
1204:
1205: p = next;
1206: }
1207:
1208: /*
1209: * Wake up threads waiting for the memory object to
1210: * become ready.
1211: */
1212:
1213: object->pager_ready = TRUE;
1214: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
1215: }
1216:
1217: /*
1218: * Routine: memory_object_destroy [user interface]
1219: * Purpose:
1220: * Shut down a memory object, despite the
1221: * presence of address map (or other) references
1222: * to the vm_object.
1223: */
1224: kern_return_t
1225: memory_object_destroy(
1226: register vm_object_t object,
1227: kern_return_t reason)
1228: {
1229: ipc_port_t old_object;
1230: pager_request_t old_pager_request;
1231:
1232: #ifdef lint
1233: reason++;
1234: #endif /* lint */
1235:
1236: if (object == VM_OBJECT_NULL)
1237: return(KERN_SUCCESS);
1238:
1239: /*
1240: * Remove the port associations immediately.
1241: *
1242: * This will prevent the memory manager from further
1243: * meddling. [If it wanted to flush data or make
1244: * other changes, it should have done so before performing
1245: * the destroy call.]
1246: */
1247:
1248: vm_object_cache_lock();
1249: vm_object_lock(object);
1250: vm_object_remove(object);
1251: object->can_persist = FALSE;
1252: vm_object_cache_unlock();
1253:
1254: /*
1255: * Rip out the ports from the vm_object now... this
1256: * will prevent new memory_object calls from succeeding.
1257: */
1258:
1259: old_object = object->pager;
1260: old_pager_request = object->pager_request;
1261:
1262: object->pager = IP_NULL;
1263: object->pager_request = PAGER_REQUEST_NULL;
1264:
1265: /*
1266: * Wait for existing paging activity (that might
1267: * have the old ports) to subside.
1268: */
1269:
1270: vm_object_paging_wait(object, THREAD_UNINT);
1271: vm_object_unlock(object);
1272:
1273: /*
1274: * Shut down the ports now.
1275: *
1276: * [Paging operations may be proceeding concurrently --
1277: * they'll get the null values established above.]
1278: */
1279:
1280: if (old_object != IP_NULL) {
1281: /* consumes our rights for object, control */
1282: memory_object_release(old_object, old_pager_request);
1283: }
1284:
1285: /*
1286: * Lose the reference that was donated for this routine
1287: */
1288:
1289: vm_object_deallocate(object);
1290:
1291: return(KERN_SUCCESS);
1292: }
1293:
1294: /*
1295: * vm_object_deactivate_pages
1296: *
1297: * Deactivate all pages in the specified object. (Keep its pages
1298: * in memory even though it is no longer referenced.)
1299: *
1300: * The object must be locked.
1301: */
1302: void
1303: vm_object_deactivate_pages(
1304: register vm_object_t object)
1305: {
1306: register vm_page_t p;
1307:
1308: queue_iterate(&object->memq, p, vm_page_t, listq) {
1309: vm_page_lock_queues();
1310: if (!p->busy)
1311: vm_page_deactivate(p);
1312: vm_page_unlock_queues();
1313: }
1314: }
1315:
1316:
1317: /*
1318: * Routine: vm_object_pmap_protect
1319: *
1320: * Purpose:
1321: * Reduces the permission for all physical
1322: * pages in the specified object range.
1323: *
1324: * If removing write permission only, it is
1325: * sufficient to protect only the pages in
1326: * the top-level object; only those pages may
1327: * have write permission.
1328: *
1329: * If removing all access, we must follow the
1330: * shadow chain from the top-level object to
1331: * remove access to all pages in shadowed objects.
1332: *
1333: * The object must *not* be locked. The object must
1334: * be temporary/internal.
1335: *
1336: * If pmap is not NULL, this routine assumes that
1337: * the only mappings for the pages are in that
1338: * pmap.
1339: */
1340:
1341: void
1342: vm_object_pmap_protect(
1343: register vm_object_t object,
1344: register vm_offset_t offset,
1345: vm_offset_t size,
1346: pmap_t pmap,
1347: vm_offset_t pmap_start,
1348: vm_prot_t prot)
1349: {
1350: if (object == VM_OBJECT_NULL)
1351: return;
1352:
1353: vm_object_lock(object);
1354:
1355: assert(object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
1356:
1357: while (TRUE) {
1358: if (object->resident_page_count > atop(size) / 2 &&
1359: pmap != PMAP_NULL) {
1360: vm_object_unlock(object);
1361: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
1362: return;
1363: }
1364:
1365: {
1366: register vm_page_t p;
1367: register vm_offset_t end;
1368:
1369: end = offset + size;
1370:
1371: if (pmap != PMAP_NULL) {
1372: queue_iterate(&object->memq, p, vm_page_t, listq) {
1373: if (!p->fictitious &&
1374: (offset <= p->offset) && (p->offset < end)) {
1375:
1376: vm_offset_t start = pmap_start + (p->offset - offset);
1377:
1378: pmap_protect(pmap, start, start + PAGE_SIZE, prot);
1379: }
1380: }
1381: } else {
1382: queue_iterate(&object->memq, p, vm_page_t, listq) {
1383: if (!p->fictitious &&
1384: (offset <= p->offset) && (p->offset < end)) {
1385:
1386: pmap_page_protect(p->phys_addr,
1387: prot & ~p->page_lock);
1388: }
1389: }
1390: }
1391: }
1392:
1393: if (prot == VM_PROT_NONE) {
1394: /*
1395: * Must follow shadow chain to remove access
1396: * to pages in shadowed objects.
1397: */
1398: register vm_object_t next_object;
1399:
1400: next_object = object->shadow;
1401: if (next_object != VM_OBJECT_NULL) {
1402: offset += object->shadow_offset;
1403: vm_object_lock(next_object);
1404: vm_object_unlock(object);
1405: object = next_object;
1406: }
1407: else {
1408: /*
1409: * End of chain - we are done.
1410: */
1411: break;
1412: }
1413: }
1414: else {
1415: /*
1416: * Pages in shadowed objects may never have
1417: * write permission - we may stop here.
1418: */
1419: break;
1420: }
1421: }
1422:
1423: vm_object_unlock(object);
1424: }
1425:
1426: /*
1427: * Routine: vm_object_copy_slowly
1428: *
1429: * Description:
1430: * Copy the specified range of the source
1431: * virtual memory object without using
1432: * protection-based optimizations (such
1433: * as copy-on-write). The pages in the
1434: * region are actually copied.
1435: *
1436: * In/out conditions:
1437: * The caller must hold a reference and a lock
1438: * for the source virtual memory object. The source
1439: * object will be returned *unlocked*.
1440: *
1441: * Results:
1442: * If the copy is completed successfully, KERN_SUCCESS is
1443: * returned. If the caller asserted the interruptible
1444: * argument, and an interruption occurred while waiting
1445: * for a user-generated event, MACH_SEND_INTERRUPTED is
1446: * returned. Other values may be returned to indicate
1447: * hard errors during the copy operation.
1448: *
1449: * A new virtual memory object is returned in a
1450: * parameter (_result_object). The contents of this
1451: * new object, starting at a zero offset, are a copy
1452: * of the source memory region. In the event of
1453: * an error, this parameter will contain the value
1454: * VM_OBJECT_NULL.
1455: */
1456: kern_return_t
1457: vm_object_copy_slowly(
1458: register vm_object_t src_object,
1459: vm_offset_t src_offset,
1460: vm_size_t size,
1461: boolean_t interruptible,
1462: vm_object_t *_result_object) /* OUT */
1463: {
1464: vm_object_t new_object;
1465: vm_offset_t new_offset;
1466:
1467: vm_offset_t src_lo_offset = src_offset;
1468: vm_offset_t src_hi_offset = src_offset + size;
1469:
1470: XPR(XPR_VM_OBJECT, "v_o_c_slowly obj 0x%x off 0x%x size 0x%x\n",
1471: src_object, src_offset, size, 0, 0);
1472:
1473: if (size == 0) {
1474: vm_object_unlock(src_object);
1475: *_result_object = VM_OBJECT_NULL;
1476: return(KERN_INVALID_ARGUMENT);
1477: }
1478:
1479: /*
1480: * Prevent destruction of the source object while we copy.
1481: */
1482:
1483: assert(src_object->ref_count > 0);
1484: src_object->ref_count++;
1485: VM_OBJ_RES_INCR(src_object);
1486: vm_object_unlock(src_object);
1487:
1488: /*
1489: * Create a new object to hold the copied pages.
1490: * A few notes:
1491: * We fill the new object starting at offset 0,
1492: * regardless of the input offset.
1493: * We don't bother to lock the new object within
1494: * this routine, since we have the only reference.
1495: */
1496:
1497: new_object = vm_object_allocate(size);
1498: new_offset = 0;
1499:
1500: assert(size == trunc_page(size)); /* Will the loop terminate? */
1501:
1502: for ( ;
1503: size != 0 ;
1504: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1505: ) {
1506: vm_page_t new_page;
1507: vm_fault_return_t result;
1508:
1509: while ((new_page = vm_page_alloc(new_object, new_offset))
1510: == VM_PAGE_NULL) {
1511: VM_PAGE_WAIT();
1512: }
1513:
1514: do {
1515: vm_prot_t prot = VM_PROT_READ;
1516: vm_page_t _result_page;
1517: vm_page_t top_page;
1518: register
1519: vm_page_t result_page;
1520: kern_return_t error_code;
1521:
1522: vm_object_lock(src_object);
1523: vm_object_paging_begin(src_object);
1524:
1525: XPR(XPR_VM_FAULT,"vm_object_copy_slowly -> vm_fault_page",0,0,0,0,0);
1526: result = vm_fault_page(src_object, src_offset,
1527: VM_PROT_READ, FALSE, interruptible,
1528: src_lo_offset, src_hi_offset,
1529: VM_BEHAVIOR_SEQUENTIAL,
1530: &prot, &_result_page, &top_page,
1531: (int *)0,
1532: &error_code, FALSE, FALSE);
1533:
1534: switch(result) {
1535: case VM_FAULT_SUCCESS:
1536: result_page = _result_page;
1537:
1538: /*
1539: * We don't need to hold the object
1540: * lock -- the busy page will be enough.
1541: * [We don't care about picking up any
1542: * new modifications.]
1543: *
1544: * Copy the page to the new object.
1545: *
1546: * POLICY DECISION:
1547: * If result_page is clean,
1548: * we could steal it instead
1549: * of copying.
1550: */
1551:
1552: vm_object_unlock(result_page->object);
1553: vm_page_copy(result_page, new_page);
1554:
1555: /*
1556: * Let go of both pages (make them
1557: * not busy, perform wakeup, activate).
1558: */
1559:
1560: new_page->busy = FALSE;
1561: new_page->dirty = TRUE;
1562: vm_object_lock(result_page->object);
1563: PAGE_WAKEUP_DONE(result_page);
1564:
1565: vm_page_lock_queues();
1566: if (!result_page->active &&
1567: !result_page->inactive)
1568: vm_page_activate(result_page);
1569: vm_page_activate(new_page);
1570: vm_page_unlock_queues();
1571:
1572: /*
1573: * Release paging references and
1574: * top-level placeholder page, if any.
1575: */
1576:
1577: vm_fault_cleanup(result_page->object,
1578: top_page);
1579:
1580: break;
1581:
1582: case VM_FAULT_RETRY:
1583: break;
1584:
1585: case VM_FAULT_MEMORY_SHORTAGE:
1586: VM_PAGE_WAIT();
1587: break;
1588:
1589: case VM_FAULT_FICTITIOUS_SHORTAGE:
1590: vm_page_more_fictitious();
1591: break;
1592:
1593: case VM_FAULT_INTERRUPTED:
1594: vm_page_free(new_page);
1595: vm_object_deallocate(new_object);
1596: vm_object_deallocate(src_object);
1597: *_result_object = VM_OBJECT_NULL;
1598: return(MACH_SEND_INTERRUPTED);
1599:
1600: case VM_FAULT_MEMORY_ERROR:
1601: /*
1602: * A policy choice:
1603: * (a) ignore pages that we can't
1604: * copy
1605: * (b) return the null object if
1606: * any page fails [chosen]
1607: */
1608:
1609: vm_page_free(new_page);
1610: vm_object_deallocate(new_object);
1611: vm_object_deallocate(src_object);
1612: *_result_object = VM_OBJECT_NULL;
1613: return(error_code ? error_code:
1614: KERN_MEMORY_ERROR);
1615: }
1616: } while (result != VM_FAULT_SUCCESS);
1617: }
1618:
1619: /*
1620: * Lose the extra reference, and return our object.
1621: */
1622:
1623: vm_object_deallocate(src_object);
1624: *_result_object = new_object;
1625: return(KERN_SUCCESS);
1626: }
1627:
1628: /*
1629: * Routine: vm_object_copy_quickly
1630: *
1631: * Purpose:
1632: * Copy the specified range of the source virtual
1633: * memory object, if it can be done without waiting
1634: * for user-generated events.
1635: *
1636: * Results:
1637: * If the copy is successful, the copy is returned in
1638: * the arguments; otherwise, the arguments are not
1639: * affected.
1640: *
1641: * In/out conditions:
1642: * The object should be unlocked on entry and exit.
1643: */
1644:
1645: /*ARGSUSED*/
1646: boolean_t
1647: vm_object_copy_quickly(
1648: vm_object_t *_object, /* INOUT */
1649: vm_offset_t offset, /* IN */
1650: vm_size_t size, /* IN */
1651: boolean_t *_src_needs_copy, /* OUT */
1652: boolean_t *_dst_needs_copy) /* OUT */
1653: {
1654: vm_object_t object = *_object;
1655: memory_object_copy_strategy_t copy_strategy;
1656:
1657: XPR(XPR_VM_OBJECT, "v_o_c_quickly obj 0x%x off 0x%x size 0x%x\n",
1658: *_object, offset, size, 0, 0);
1659: if (object == VM_OBJECT_NULL) {
1660: *_src_needs_copy = FALSE;
1661: *_dst_needs_copy = FALSE;
1662: return(TRUE);
1663: }
1664:
1665: vm_object_lock(object);
1666:
1667: copy_strategy = object->copy_strategy;
1668:
1669: switch (copy_strategy) {
1670: case MEMORY_OBJECT_COPY_SYMMETRIC:
1671:
1672: /*
1673: * Symmetric copy strategy.
1674: * Make another reference to the object.
1675: * Leave object/offset unchanged.
1676: */
1677:
1678: assert(object->ref_count > 0);
1679: object->ref_count++;
1680: vm_object_res_reference(object);
1681: object->shadowed = TRUE;
1682: vm_object_unlock(object);
1683:
1684: /*
1685: * Both source and destination must make
1686: * shadows, and the source must be made
1687: * read-only if not already.
1688: */
1689:
1690: *_src_needs_copy = TRUE;
1691: *_dst_needs_copy = TRUE;
1692:
1693: break;
1694:
1695: case MEMORY_OBJECT_COPY_DELAY:
1696: vm_object_unlock(object);
1697: return(FALSE);
1698:
1699: default:
1700: vm_object_unlock(object);
1701: return(FALSE);
1702: }
1703: return(TRUE);
1704: }
1705:
1706: int copy_call_count = 0;
1707: int copy_call_sleep_count = 0;
1708: int copy_call_restart_count = 0;
1709:
1710: /*
1711: * Routine: vm_object_copy_call [internal]
1712: *
1713: * Description:
1714: * Copy the source object (src_object), using the
1715: * user-managed copy algorithm.
1716: *
1717: * In/out conditions:
1718: * The source object must be locked on entry. It
1719: * will be *unlocked* on exit.
1720: *
1721: * Results:
1722: * If the copy is successful, KERN_SUCCESS is returned.
1723: * A new object that represents the copied virtual
1724: * memory is returned in a parameter (*_result_object).
1725: * If the return value indicates an error, this parameter
1726: * is not valid.
1727: */
1728: kern_return_t
1729: vm_object_copy_call(
1730: vm_object_t src_object,
1731: vm_offset_t src_offset,
1732: vm_size_t size,
1733: vm_object_t *_result_object) /* OUT */
1734: {
1735: kern_return_t kr;
1736: vm_object_t copy;
1737: boolean_t check_ready = FALSE;
1738:
1739: /*
1740: * If a copy is already in progress, wait and retry.
1741: *
1742: * XXX
1743: * Consider making this call interruptable, as Mike
1744: * intended it to be.
1745: *
1746: * XXXO
1747: * Need a counter or version or something to allow
1748: * us to use the copy that the currently requesting
1749: * thread is obtaining -- is it worth adding to the
1750: * vm object structure? Depends how common this case it.
1751: */
1752: copy_call_count++;
1753: while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
1754: vm_object_wait(src_object, VM_OBJECT_EVENT_COPY_CALL,
1755: THREAD_UNINT);
1756: vm_object_lock(src_object);
1757: copy_call_restart_count++;
1758: }
1759:
1760: /*
1761: * Indicate (for the benefit of memory_object_create_copy)
1762: * that we want a copy for src_object. (Note that we cannot
1763: * do a real assert_wait before calling memory_object_copy,
1764: * so we simply set the flag.)
1765: */
1766:
1767: vm_object_set_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL);
1768: vm_object_unlock(src_object);
1769:
1770: /*
1771: * Ask the memory manager to give us a memory object
1772: * which represents a copy of the src object.
1773: * The memory manager may give us a memory object
1774: * which we already have, or it may give us a
1775: * new memory object. This memory object will arrive
1776: * via memory_object_create_copy.
1777: */
1778:
1779: kr = KERN_FAILURE; /* XXX need to change memory_object.defs */
1780: if (kr != KERN_SUCCESS) {
1781: return kr;
1782: }
1783:
1784: /*
1785: * Wait for the copy to arrive.
1786: */
1787: vm_object_lock(src_object);
1788: while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
1789: vm_object_wait(src_object, VM_OBJECT_EVENT_COPY_CALL,
1790: THREAD_UNINT);
1791: vm_object_lock(src_object);
1792: copy_call_sleep_count++;
1793: }
1794: Retry:
1795: assert(src_object->copy != VM_OBJECT_NULL);
1796: copy = src_object->copy;
1797: if (!vm_object_lock_try(copy)) {
1798: vm_object_unlock(src_object);
1799: mutex_pause(); /* wait a bit */
1800: vm_object_lock(src_object);
1801: goto Retry;
1802: }
1803: if (copy->size < src_offset+size)
1804: copy->size = src_offset+size;
1805:
1806: if (!copy->pager_ready)
1807: check_ready = TRUE;
1808:
1809: /*
1810: * Return the copy.
1811: */
1812: *_result_object = copy;
1813: vm_object_unlock(copy);
1814: vm_object_unlock(src_object);
1815:
1816: /* Wait for the copy to be ready. */
1817: if (check_ready == TRUE) {
1818: vm_object_lock(copy);
1819: while (!copy->pager_ready) {
1820: vm_object_wait(copy, VM_OBJECT_EVENT_PAGER_READY,
1821: FALSE);
1822: vm_object_lock(copy);
1823: }
1824: vm_object_unlock(copy);
1825: }
1826:
1827: return KERN_SUCCESS;
1828: }
1829:
1830: int copy_delayed_lock_collisions = 0;
1831: int copy_delayed_max_collisions = 0;
1832: int copy_delayed_lock_contention = 0;
1833: int copy_delayed_protect_iterate = 0;
1834: int copy_delayed_protect_lookup = 0;
1835: int copy_delayed_protect_lookup_wait = 0;
1836:
1837: /*
1838: * Routine: vm_object_copy_delayed [internal]
1839: *
1840: * Description:
1841: * Copy the specified virtual memory object, using
1842: * the asymmetric copy-on-write algorithm.
1843: *
1844: * In/out conditions:
1845: * The object must be unlocked on entry.
1846: *
1847: * This routine will not block waiting for user-generated
1848: * events. It is not interruptible.
1849: */
1850: vm_object_t
1851: vm_object_copy_delayed(
1852: vm_object_t src_object,
1853: vm_offset_t src_offset,
1854: vm_size_t size)
1855: {
1856: vm_object_t new_copy = VM_OBJECT_NULL;
1857: vm_object_t old_copy;
1858: vm_page_t p;
1859: vm_size_t copy_size;
1860:
1861: int collisions = 0;
1862: /*
1863: * The user-level memory manager wants to see all of the changes
1864: * to this object, but it has promised not to make any changes on
1865: * its own.
1866: *
1867: * Perform an asymmetric copy-on-write, as follows:
1868: * Create a new object, called a "copy object" to hold
1869: * pages modified by the new mapping (i.e., the copy,
1870: * not the original mapping).
1871: * Record the original object as the backing object for
1872: * the copy object. If the original mapping does not
1873: * change a page, it may be used read-only by the copy.
1874: * Record the copy object in the original object.
1875: * When the original mapping causes a page to be modified,
1876: * it must be copied to a new page that is "pushed" to
1877: * the copy object.
1878: * Mark the new mapping (the copy object) copy-on-write.
1879: * This makes the copy object itself read-only, allowing
1880: * it to be reused if the original mapping makes no
1881: * changes, and simplifying the synchronization required
1882: * in the "push" operation described above.
1883: *
1884: * The copy-on-write is said to be assymetric because the original
1885: * object is *not* marked copy-on-write. A copied page is pushed
1886: * to the copy object, regardless which party attempted to modify
1887: * the page.
1888: *
1889: * Repeated asymmetric copy operations may be done. If the
1890: * original object has not been changed since the last copy, its
1891: * copy object can be reused. Otherwise, a new copy object can be
1892: * inserted between the original object and its previous copy
1893: * object. Since any copy object is read-only, this cannot affect
1894: * affect the contents of the previous copy object.
1895: *
1896: * Note that a copy object is higher in the object tree than the
1897: * original object; therefore, use of the copy object recorded in
1898: * the original object must be done carefully, to avoid deadlock.
1899: */
1900:
1901: Retry:
1902: vm_object_lock(src_object);
1903:
1904: /*
1905: * See whether we can reuse the result of a previous
1906: * copy operation.
1907: */
1908:
1909: old_copy = src_object->copy;
1910: if (old_copy != VM_OBJECT_NULL) {
1911: /*
1912: * Try to get the locks (out of order)
1913: */
1914: if (!vm_object_lock_try(old_copy)) {
1915: vm_object_unlock(src_object);
1916: mutex_pause();
1917:
1918: /* Heisenberg Rules */
1919: copy_delayed_lock_collisions++;
1920: if (collisions++ == 0)
1921: copy_delayed_lock_contention++;
1922:
1923: if (collisions > copy_delayed_max_collisions)
1924: copy_delayed_max_collisions = collisions;
1925:
1926: goto Retry;
1927: }
1928:
1929: /*
1930: * Determine whether the old copy object has
1931: * been modified.
1932: */
1933:
1934: if (old_copy->resident_page_count == 0 &&
1935: !old_copy->pager_created) {
1936: /*
1937: * It has not been modified.
1938: *
1939: * Return another reference to
1940: * the existing copy-object.
1941: */
1942: assert(old_copy->ref_count > 0);
1943: old_copy->ref_count++;
1944:
1945: if (old_copy->size < src_offset+size)
1946: old_copy->size = src_offset+size;
1947:
1948: #if TASK_SWAPPER
1949: /*
1950: * We have to reproduce some of the code from
1951: * vm_object_res_reference because we've taken
1952: * the locks out of order here, and deadlock
1953: * would result if we simply called that function.
1954: */
1955: if (++old_copy->res_count == 1) {
1956: assert(old_copy->shadow == src_object);
1957: vm_object_res_reference(src_object);
1958: }
1959: #endif /* TASK_SWAPPER */
1960:
1961: vm_object_unlock(old_copy);
1962: vm_object_unlock(src_object);
1963:
1964: if (new_copy != VM_OBJECT_NULL) {
1965: vm_object_unlock(new_copy);
1966: vm_object_deallocate(new_copy);
1967: }
1968:
1969: return(old_copy);
1970: }
1971: if (new_copy == VM_OBJECT_NULL) {
1972: vm_object_unlock(old_copy);
1973: vm_object_unlock(src_object);
1974: new_copy = vm_object_allocate(src_offset + size);
1975: vm_object_lock(new_copy);
1976: goto Retry;
1977: }
1978:
1979: /*
1980: * Adjust the size argument so that the newly-created
1981: * copy object will be large enough to back either the
1982: * new old copy object or the new mapping.
1983: */
1984: if (old_copy->size > src_offset+size)
1985: size = old_copy->size - src_offset;
1986:
1987: /*
1988: * The copy-object is always made large enough to
1989: * completely shadow the original object, since
1990: * it may have several users who want to shadow
1991: * the original object at different points.
1992: */
1993:
1994: assert((old_copy->shadow == src_object) &&
1995: (old_copy->shadow_offset == (vm_offset_t) 0));
1996:
1997: /*
1998: * Make the old copy-object shadow the new one.
1999: * It will receive no more pages from the original
2000: * object.
2001: */
2002:
2003: src_object->ref_count--; /* remove ref. from old_copy */
2004: assert(src_object->ref_count > 0);
2005: old_copy->shadow = new_copy;
2006: assert(new_copy->ref_count > 0);
2007: new_copy->ref_count++; /* for old_copy->shadow ref. */
2008:
2009: #if TASK_SWAPPER
2010: if (old_copy->res_count) {
2011: VM_OBJ_RES_INCR(new_copy);
2012: VM_OBJ_RES_DECR(src_object);
2013: }
2014: #endif
2015:
2016: vm_object_unlock(old_copy); /* done with old_copy */
2017: } else if (new_copy == VM_OBJECT_NULL) {
2018: vm_object_unlock(src_object);
2019: new_copy = vm_object_allocate(src_offset + size);
2020: vm_object_lock(new_copy);
2021: goto Retry;
2022: }
2023:
2024: /*
2025: * Readjust the copy-object size if necessary.
2026: */
2027: copy_size = new_copy->size;
2028: if (copy_size < src_offset+size) {
2029: copy_size = src_offset+size;
2030: new_copy->size = copy_size;
2031: }
2032:
2033: /*
2034: * Point the new copy at the existing object.
2035: */
2036:
2037: new_copy->shadow = src_object;
2038: new_copy->shadow_offset = 0;
2039: new_copy->shadowed = TRUE; /* caller must set needs_copy */
2040: assert(src_object->ref_count > 0);
2041: src_object->ref_count++;
2042: VM_OBJ_RES_INCR(src_object);
2043: src_object->copy = new_copy;
2044: vm_object_unlock(new_copy);
2045:
2046: /*
2047: * Mark all (current) pages of the existing object copy-on-write.
2048: * This object may have a shadow chain below it, but
2049: * those pages will already be marked copy-on-write.
2050: */
2051:
2052: vm_object_paging_wait(src_object, THREAD_UNINT);
2053: copy_delayed_protect_iterate++;
2054: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
2055: if (!p->fictitious)
2056: pmap_page_protect(p->phys_addr,
2057: (VM_PROT_ALL & ~VM_PROT_WRITE &
2058: ~p->page_lock));
2059: }
2060: vm_object_unlock(src_object);
2061: XPR(XPR_VM_OBJECT,
2062: "vm_object_copy_delayed: used copy object %X for source %X\n",
2063: (integer_t)new_copy, (integer_t)src_object, 0, 0, 0);
2064:
2065: return(new_copy);
2066: }
2067:
2068: /*
2069: * Routine: vm_object_copy_strategically
2070: *
2071: * Purpose:
2072: * Perform a copy according to the source object's
2073: * declared strategy. This operation may block,
2074: * and may be interrupted.
2075: */
2076: kern_return_t
2077: vm_object_copy_strategically(
2078: register vm_object_t src_object,
2079: vm_offset_t src_offset,
2080: vm_size_t size,
2081: vm_object_t *dst_object, /* OUT */
2082: vm_offset_t *dst_offset, /* OUT */
2083: boolean_t *dst_needs_copy) /* OUT */
2084: {
2085: boolean_t result;
2086: boolean_t interruptible = THREAD_ABORTSAFE; /* XXX */
2087: memory_object_copy_strategy_t copy_strategy;
2088:
2089: assert(src_object != VM_OBJECT_NULL);
2090:
2091: vm_object_lock(src_object);
2092:
2093: /*
2094: * The copy strategy is only valid if the memory manager
2095: * is "ready". Internal objects are always ready.
2096: */
2097:
2098: while (!src_object->internal && !src_object->pager_ready) {
2099:
2100: vm_object_wait( src_object,
2101: VM_OBJECT_EVENT_PAGER_READY,
2102: interruptible);
2103: if (interruptible &&
2104: (current_thread()->wait_result != THREAD_AWAKENED)) {
2105: *dst_object = VM_OBJECT_NULL;
2106: *dst_offset = 0;
2107: *dst_needs_copy = FALSE;
2108: return(MACH_SEND_INTERRUPTED);
2109: }
2110: vm_object_lock(src_object);
2111: }
2112:
2113: copy_strategy = src_object->copy_strategy;
2114:
2115: /*
2116: * Use the appropriate copy strategy.
2117: */
2118:
2119: switch (copy_strategy) {
2120: case MEMORY_OBJECT_COPY_NONE:
2121: result = vm_object_copy_slowly(src_object, src_offset, size,
2122: interruptible, dst_object);
2123: if (result == KERN_SUCCESS) {
2124: *dst_offset = 0;
2125: *dst_needs_copy = FALSE;
2126: }
2127: break;
2128:
2129: case MEMORY_OBJECT_COPY_CALL:
2130: result = vm_object_copy_call(src_object, src_offset, size,
2131: dst_object);
2132: if (result == KERN_SUCCESS) {
2133: *dst_offset = src_offset;
2134: *dst_needs_copy = TRUE;
2135: }
2136: break;
2137:
2138: case MEMORY_OBJECT_COPY_DELAY:
2139: vm_object_unlock(src_object);
2140: *dst_object = vm_object_copy_delayed(src_object,
2141: src_offset, size);
2142: *dst_offset = src_offset;
2143: *dst_needs_copy = TRUE;
2144: result = KERN_SUCCESS;
2145: break;
2146:
2147: case MEMORY_OBJECT_COPY_SYMMETRIC:
2148: XPR(XPR_VM_OBJECT, "v_o_c_strategically obj 0x%x off 0x%x size 0x%x\n",(natural_t)src_object, src_offset, size, 0, 0);
2149: vm_object_unlock(src_object);
2150: result = KERN_MEMORY_RESTART_COPY;
2151: break;
2152:
2153: default:
2154: panic("copy_strategically: bad strategy");
2155: result = KERN_INVALID_ARGUMENT;
2156: }
2157: return(result);
2158: }
2159:
2160: /*
2161: * vm_object_shadow:
2162: *
2163: * Create a new object which is backed by the
2164: * specified existing object range. The source
2165: * object reference is deallocated.
2166: *
2167: * The new object and offset into that object
2168: * are returned in the source parameters.
2169: */
2170: boolean_t vm_object_shadow_check = FALSE;
2171:
2172: boolean_t
2173: vm_object_shadow(
2174: vm_object_t *object, /* IN/OUT */
2175: vm_offset_t *offset, /* IN/OUT */
2176: vm_size_t length)
2177: {
2178: register vm_object_t source;
2179: register vm_object_t result;
2180:
2181: source = *object;
2182: assert(source->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
2183:
2184: /*
2185: * Determine if we really need a shadow.
2186: */
2187:
2188: if (vm_object_shadow_check && source->ref_count == 1 &&
2189: (source->shadow == VM_OBJECT_NULL ||
2190: source->shadow->copy == VM_OBJECT_NULL))
2191: {
2192: source->shadowed = FALSE;
2193: return FALSE;
2194: }
2195:
2196: /*
2197: * Allocate a new object with the given length
2198: */
2199:
2200: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
2201: panic("vm_object_shadow: no object for shadowing");
2202:
2203: /*
2204: * The new object shadows the source object, adding
2205: * a reference to it. Our caller changes his reference
2206: * to point to the new object, removing a reference to
2207: * the source object. Net result: no change of reference
2208: * count.
2209: */
2210: result->shadow = source;
2211:
2212: /*
2213: * Store the offset into the source object,
2214: * and fix up the offset into the new object.
2215: */
2216:
2217: result->shadow_offset = *offset;
2218:
2219: /*
2220: * Return the new things
2221: */
2222:
2223: *offset = 0;
2224: *object = result;
2225: return TRUE;
2226: }
2227:
2228: /*
2229: * The relationship between vm_object structures and
2230: * the memory_object ports requires careful synchronization.
2231: *
2232: * All associations are created by vm_object_enter. All three
2233: * port fields are filled in, as follows:
2234: * pager: the memory_object port itself, supplied by
2235: * the user requesting a mapping (or the kernel,
2236: * when initializing internal objects); the
2237: * kernel simulates holding send rights by keeping
2238: * a port reference;
2239: * pager_request:
2240: * the memory object control port,
2241: * created by the kernel; the kernel holds
2242: * receive (and ownership) rights to this
2243: * port, but no other references.
2244: * All of the ports are referenced by their global names.
2245: *
2246: * When initialization is complete, the "initialized" field
2247: * is asserted. Other mappings using a particular memory object,
2248: * and any references to the vm_object gained through the
2249: * port association must wait for this initialization to occur.
2250: *
2251: * In order to allow the memory manager to set attributes before
2252: * requests (notably virtual copy operations, but also data or
2253: * unlock requests) are made, a "ready" attribute is made available.
2254: * Only the memory manager may affect the value of this attribute.
2255: * Its value does not affect critical kernel functions, such as
2256: * internal object initialization or destruction. [Furthermore,
2257: * memory objects created by the kernel are assumed to be ready
2258: * immediately; the default memory manager need not explicitly
2259: * set the "ready" attribute.]
2260: *
2261: * [Both the "initialized" and "ready" attribute wait conditions
2262: * use the "pager" field as the wait event.]
2263: *
2264: * The port associations can be broken down by any of the
2265: * following routines:
2266: * vm_object_terminate:
2267: * No references to the vm_object remain, and
2268: * the object cannot (or will not) be cached.
2269: * This is the normal case, and is done even
2270: * though one of the other cases has already been
2271: * done.
2272: * vm_object_destroy:
2273: * The memory_object port has been destroyed,
2274: * meaning that the kernel cannot flush dirty
2275: * pages or request new data or unlock existing
2276: * data.
2277: * memory_object_destroy:
2278: * The memory manager has requested that the
2279: * kernel relinquish rights to the memory object
2280: * port. [The memory manager may not want to
2281: * destroy the port, but may wish to refuse or
2282: * tear down existing memory mappings.]
2283: * Each routine that breaks an association must break all of
2284: * them at once. At some later time, that routine must clear
2285: * the vm_object port fields and release the port rights.
2286: * [Furthermore, each routine must cope with the simultaneous
2287: * or previous operations of the others.]
2288: *
2289: * In addition to the lock on the object, the vm_object_cache_lock
2290: * governs the port associations. References gained through the
2291: * port association require use of the cache lock.
2292: *
2293: * Because the port fields may be cleared spontaneously, they
2294: * cannot be used to determine whether a memory object has
2295: * ever been associated with a particular vm_object. [This
2296: * knowledge is important to the shadow object mechanism.]
2297: * For this reason, an additional "created" attribute is
2298: * provided.
2299: *
2300: * During various paging operations, the port values found in the
2301: * vm_object must be valid. To prevent these port rights from being
2302: * released, and to prevent the port associations from changing
2303: * (other than being removed, i.e., made null), routines may use
2304: * the vm_object_paging_begin/end routines [actually, macros].
2305: * The implementation uses the "paging_in_progress" and "wanted" fields.
2306: * [Operations that alter the validity of the port values include the
2307: * termination routines and vm_object_collapse.]
2308: */
2309:
2310: #define IKOT_PAGER_LOOKUP_TYPE IKOT_PAGING_REQUEST
2311:
2312: vm_object_t
2313: vm_object_lookup(
2314: ipc_port_t port)
2315: {
2316: vm_object_t object;
2317:
2318: start_over:
2319: object = VM_OBJECT_NULL;
2320:
2321: if (IP_VALID(port)) {
2322: vm_object_cache_lock();
2323: ip_lock(port);
2324: if (ip_active(port) &&
2325: (ip_kotype(port) == IKOT_PAGER_LOOKUP_TYPE)) {
2326: object = (vm_object_t) port->ip_kobject;
2327: if (!vm_object_lock_try(object)) {
2328: /*
2329: * failed to acquire object lock. Drop the
2330: * other two locks and wait for it, then go
2331: * back and start over in case the port
2332: * associations changed in the interim.
2333: */
2334: ip_unlock(port);
2335: vm_object_cache_unlock();
2336: vm_object_lock(object);
2337: vm_object_unlock(object);
2338: goto start_over;
2339: }
2340:
2341: assert(object->alive);
2342:
2343: if (object->ref_count == 0) {
2344: queue_remove(&vm_object_cached_list, object,
2345: vm_object_t, cached_list);
2346: vm_object_cached_count--;
2347: XPR(XPR_VM_OBJECT_CACHE,
2348: "vm_object_lookup: removing %X, head (%X, %X)\n",
2349: (integer_t)object,
2350: (integer_t)vm_object_cached_list.next,
2351: (integer_t)vm_object_cached_list.prev, 0,0);
2352: }
2353:
2354: object->ref_count++;
2355: vm_object_res_reference(object);
2356: vm_object_unlock(object);
2357: }
2358: ip_unlock(port);
2359: vm_object_cache_unlock();
2360: }
2361:
2362: return object;
2363: }
2364:
2365:
2366:
2367: void
2368: vm_object_destroy(
2369: ipc_port_t pager)
2370: {
2371: vm_object_t object;
2372: vm_object_hash_entry_t entry;
2373: pager_request_t old_pager_request;
2374:
2375: /*
2376: * Perform essentially the same operations as in vm_object_lookup,
2377: * except that this time we look up based on the memory_object
2378: * port, not the control port.
2379: */
2380: vm_object_cache_lock();
2381: entry = vm_object_hash_lookup(pager, FALSE);
2382: if (entry == VM_OBJECT_HASH_ENTRY_NULL ||
2383: entry->object == VM_OBJECT_NULL) {
2384: vm_object_cache_unlock();
2385: return;
2386: }
2387:
2388: object = entry->object;
2389: entry->object = VM_OBJECT_NULL;
2390:
2391: vm_object_lock(object);
2392: if (object->ref_count == 0) {
2393: XPR(XPR_VM_OBJECT_CACHE,
2394: "vm_object_destroy: removing %x from cache, head (%x, %x)\n",
2395: (integer_t)object,
2396: (integer_t)vm_object_cached_list.next,
2397: (integer_t)vm_object_cached_list.prev, 0,0);
2398:
2399: queue_remove(&vm_object_cached_list, object,
2400: vm_object_t, cached_list);
2401: vm_object_cached_count--;
2402: }
2403: object->ref_count++;
2404: vm_object_res_reference(object);
2405:
2406: object->can_persist = FALSE;
2407:
2408: assert(object->pager == pager);
2409:
2410: /*
2411: * Remove the port associations.
2412: *
2413: * Note that the memory_object itself is dead, so
2414: * we don't bother with it.
2415: */
2416:
2417: object->pager = IP_NULL;
2418: vm_object_remove(object);
2419:
2420: old_pager_request = object->pager_request;
2421:
2422: object->pager_request = PAGER_REQUEST_NULL;
2423:
2424: vm_object_unlock(object);
2425: vm_object_cache_unlock();
2426:
2427: vm_object_pager_wakeup(pager);
2428:
2429: /*
2430: * Clean up the port references. Note that there's no
2431: * point in trying the memory_object_terminate call
2432: * because the memory_object itself is dead.
2433: */
2434:
2435: ipc_port_release_send(pager);
2436:
2437: if ((ipc_port_t)old_pager_request != IP_NULL)
2438: ipc_port_dealloc_kernel((ipc_port_t)old_pager_request);
2439:
2440: /*
2441: * Restart pending page requests
2442: */
2443: vm_object_lock(object);
2444:
2445: vm_object_abort_activity(object);
2446:
2447: vm_object_unlock(object);
2448:
2449: /*
2450: * Lose the object reference.
2451: */
2452:
2453: vm_object_deallocate(object);
2454: }
2455:
2456: /*
2457: * Routine: vm_object_enter
2458: * Purpose:
2459: * Find a VM object corresponding to the given
2460: * pager; if no such object exists, create one,
2461: * and initialize the pager.
2462: */
2463: vm_object_t
2464: vm_object_enter(
2465: ipc_port_t pager,
2466: vm_size_t size,
2467: boolean_t internal,
2468: boolean_t init)
2469: {
2470: register vm_object_t object;
2471: vm_object_t new_object;
2472: boolean_t must_init;
2473: ipc_port_t pager_request;
2474: vm_object_hash_entry_t entry, new_entry;
2475: #ifdef MACH_BSD
2476: kern_return_t vnode_pager_init( ipc_port_t, ipc_port_t, vm_size_t);
2477: #endif
2478:
2479: if (!IP_VALID(pager))
2480: return(vm_object_allocate(size));
2481:
2482: new_object = VM_OBJECT_NULL;
2483: new_entry = VM_OBJECT_HASH_ENTRY_NULL;
2484: must_init = init;
2485:
2486: /*
2487: * Look for an object associated with this port.
2488: */
2489:
2490: restart:
2491: vm_object_cache_lock();
2492: for (;;) {
2493: entry = vm_object_hash_lookup(pager, FALSE);
2494:
2495: /*
2496: * If a previous object is being terminated,
2497: * we must wait for the termination message
2498: * to be queued.
2499: *
2500: * We set kobject to a non-null value to let the
2501: * terminator know that someone is waiting.
2502: * Among the possibilities is that the port
2503: * could die while we're waiting. Must restart
2504: * instead of continuing the loop.
2505: */
2506:
2507: if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
2508: if (entry->object != VM_OBJECT_NULL)
2509: break;
2510:
2511: entry->waiting = TRUE;
2512: assert_wait((event_t) pager, THREAD_UNINT);
2513: vm_object_cache_unlock();
2514: thread_block((void (*)(void))0);
2515: goto restart;
2516: }
2517:
2518: /*
2519: * We must unlock to create a new object;
2520: * if we do so, we must try the lookup again.
2521: */
2522:
2523: if (new_object == VM_OBJECT_NULL) {
2524: vm_object_cache_unlock();
2525: assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL);
2526: new_entry = vm_object_hash_entry_alloc(pager);
2527: new_object = vm_object_allocate(size);
2528: vm_object_cache_lock();
2529: } else {
2530: /*
2531: * Lookup failed twice, and we have something
2532: * to insert; set the object.
2533: */
2534:
2535: if (entry == VM_OBJECT_HASH_ENTRY_NULL) {
2536: vm_object_hash_insert(new_entry);
2537: entry = new_entry;
2538: new_entry = VM_OBJECT_HASH_ENTRY_NULL;
2539: }
2540:
2541: entry->object = new_object;
2542: new_object = VM_OBJECT_NULL;
2543: must_init = TRUE;
2544: }
2545: }
2546:
2547: object = entry->object;
2548: assert(object != VM_OBJECT_NULL);
2549:
2550: if (!must_init) {
2551: vm_object_lock(object);
2552: assert(object->pager_created);
2553: assert(!internal || object->internal);
2554: if (object->ref_count == 0) {
2555: XPR(XPR_VM_OBJECT_CACHE,
2556: "vm_object_enter: removing %x from cache, head (%x, %x)\n",
2557: (integer_t)object,
2558: (integer_t)vm_object_cached_list.next,
2559: (integer_t)vm_object_cached_list.prev, 0,0);
2560: queue_remove(&vm_object_cached_list, object,
2561: vm_object_t, cached_list);
2562: vm_object_cached_count--;
2563: }
2564: object->ref_count++;
2565: vm_object_res_reference(object);
2566: vm_object_unlock(object);
2567:
2568: VM_STAT(hits++);
2569: }
2570: assert(object->ref_count > 0);
2571:
2572: VM_STAT(lookups++);
2573:
2574: vm_object_cache_unlock();
2575:
2576: XPR(XPR_VM_OBJECT,
2577: "vm_o_enter: pager 0x%x obj 0x%x must_init %d\n",
2578: (integer_t)pager, (integer_t)object, must_init, 0, 0);
2579:
2580: /*
2581: * If we raced to create a vm_object but lost, let's
2582: * throw away ours.
2583: */
2584:
2585: if (new_object != VM_OBJECT_NULL)
2586: vm_object_deallocate(new_object);
2587:
2588: if (new_entry != VM_OBJECT_HASH_ENTRY_NULL)
2589: vm_object_hash_entry_free(new_entry);
2590:
2591: if (must_init) {
2592:
2593: /*
2594: * Allocate request port.
2595: */
2596:
2597: pager_request = ipc_port_alloc_kernel();
2598: assert (pager_request != IP_NULL);
2599: ipc_kobject_set(pager_request, (ipc_kobject_t) object,
2600: IKOT_PAGING_REQUEST);
2601:
2602: vm_object_lock(object);
2603:
2604: /*
2605: * Copy the naked send right we were given.
2606: */
2607:
2608: pager = ipc_port_copy_send(pager);
2609: if (!IP_VALID(pager))
2610: panic("vm_object_enter: port died"); /* XXX */
2611:
2612: object->pager_created = TRUE;
2613: object->pager = pager;
2614: object->internal = internal;
2615: object->pager_trusted = internal;
2616: if (!internal) {
2617: /* copy strategy invalid until set by memory manager */
2618: object->copy_strategy = MEMORY_OBJECT_COPY_INVALID;
2619: }
2620: object->pager_request = pager_request;
2621: object->pager_ready = FALSE;
2622:
2623: vm_object_unlock(object);
2624:
2625: /*
2626: * Let the pager know we're using it.
2627: */
2628:
2629: #ifdef MACH_BSD
2630: if(((rpc_subsystem_t)pager_mux_hash_lookup(pager)) ==
2631: ((rpc_subsystem_t) &vnode_pager_workaround)) {
2632: (void) vnode_pager_init(pager,
2633: object->pager_request,
2634: PAGE_SIZE);
2635: } else {
2636: (void) memory_object_init(pager,
2637: object->pager_request,
2638: PAGE_SIZE);
2639: }
2640: #else
2641: (void) memory_object_init(pager,
2642: object->pager_request,
2643: PAGE_SIZE);
2644: #endif
2645:
2646: vm_object_lock(object);
2647: if (internal) {
2648: object->pager_ready = TRUE;
2649: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
2650: }
2651:
2652: object->pager_initialized = TRUE;
2653: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2654: } else {
2655: vm_object_lock(object);
2656: }
2657:
2658: /*
2659: * [At this point, the object must be locked]
2660: */
2661:
2662: /*
2663: * Wait for the work above to be done by the first
2664: * thread to map this object.
2665: */
2666:
2667: while (!object->pager_initialized) {
2668: vm_object_wait( object,
2669: VM_OBJECT_EVENT_INITIALIZED,
2670: THREAD_UNINT);
2671: vm_object_lock(object);
2672: }
2673: vm_object_unlock(object);
2674:
2675: XPR(XPR_VM_OBJECT,
2676: "vm_object_enter: vm_object %x, memory_object %x, internal %d\n",
2677: (integer_t)object, (integer_t)object->pager, internal, 0,0);
2678: return(object);
2679: }
2680:
2681: /*
2682: * Routine: vm_object_pager_create
2683: * Purpose:
2684: * Create a memory object for an internal object.
2685: * In/out conditions:
2686: * The object is locked on entry and exit;
2687: * it may be unlocked within this call.
2688: * Limitations:
2689: * Only one thread may be performing a
2690: * vm_object_pager_create on an object at
2691: * a time. Presumably, only the pageout
2692: * daemon will be using this routine.
2693: */
2694:
2695: void
2696: vm_object_pager_create(
2697: register vm_object_t object)
2698: {
2699: ipc_port_t pager;
2700: vm_object_hash_entry_t entry;
2701: #if MACH_PAGEMAP
2702: vm_size_t size;
2703: vm_external_map_t map;
2704: #endif /* MACH_PAGEMAP */
2705:
2706: XPR(XPR_VM_OBJECT, "vm_object_pager_create, object 0x%X\n",
2707: (integer_t)object, 0,0,0,0);
2708:
2709: if (memory_manager_default_check() != KERN_SUCCESS)
2710: return;
2711:
2712: /*
2713: * Prevent collapse or termination by holding a paging reference
2714: */
2715:
2716: vm_object_paging_begin(object);
2717: if (object->pager_created) {
2718: /*
2719: * Someone else got to it first...
2720: * wait for them to finish initializing the ports
2721: */
2722: while (!object->pager_initialized) {
2723: vm_object_wait( object,
2724: VM_OBJECT_EVENT_INITIALIZED,
2725: THREAD_UNINT);
2726: vm_object_lock(object);
2727: }
2728: vm_object_paging_end(object);
2729: return;
2730: }
2731:
2732: /*
2733: * Indicate that a memory object has been assigned
2734: * before dropping the lock, to prevent a race.
2735: */
2736:
2737: object->pager_created = TRUE;
2738: object->paging_offset = 0;
2739:
2740: #if MACH_PAGEMAP
2741: size = object->size;
2742: #endif /* MACH_PAGEMAP */
2743: vm_object_unlock(object);
2744:
2745: #if MACH_PAGEMAP
2746: map = vm_external_create(size);
2747: vm_object_lock(object);
2748: assert(object->size == size);
2749: object->existence_map = map;
2750: vm_object_unlock(object);
2751: #endif /* MACH_PAGEMAP */
2752:
2753: /*
2754: * Create the pager ports, and associate them with this object.
2755: *
2756: * We make the port association here so that vm_object_enter()
2757: * can look up the object to complete initializing it. No
2758: * user will ever map this object.
2759: */
2760: {
2761: ipc_port_t DMM;
2762: vm_size_t cluster_size;
2763:
2764: /* acquire a naked send right for the DMM */
2765: DMM = memory_manager_default_reference(&cluster_size);
2766: assert(cluster_size >= PAGE_SIZE);
2767:
2768: object->cluster_size = cluster_size; /* XXX ??? */
2769: assert(object->temporary);
2770:
2771: /* consumes the naked send right for DMM */
2772: (void) memory_object_create(DMM, &pager, object->size);
2773: assert(IP_VALID(pager));
2774: }
2775:
2776: entry = vm_object_hash_entry_alloc(pager);
2777:
2778: vm_object_cache_lock();
2779: vm_object_hash_insert(entry);
2780:
2781: entry->object = object;
2782: vm_object_cache_unlock();
2783:
2784: /*
2785: * A naked send right was returned by
2786: * memory_object_create(), and it is
2787: * copied by vm_object_enter().
2788: */
2789:
2790: if (vm_object_enter(pager, object->size, TRUE, TRUE) != object)
2791: panic("vm_object_pager_create: mismatch");
2792:
2793: /*
2794: * Drop the naked send right.
2795: */
2796: ipc_port_release_send(pager);
2797:
2798: vm_object_lock(object);
2799:
2800: /*
2801: * Release the paging reference
2802: */
2803: vm_object_paging_end(object);
2804: }
2805:
2806: /*
2807: * Routine: vm_object_remove
2808: * Purpose:
2809: * Eliminate the pager/object association
2810: * for this pager.
2811: * Conditions:
2812: * The object cache must be locked.
2813: */
2814: void
2815: vm_object_remove(
2816: vm_object_t object)
2817: {
2818: ipc_port_t port;
2819:
2820: if ((port = object->pager) != IP_NULL) {
2821: vm_object_hash_entry_t entry;
2822:
2823: entry = vm_object_hash_lookup(port, FALSE);
2824: if (entry != VM_OBJECT_HASH_ENTRY_NULL)
2825: entry->object = VM_OBJECT_NULL;
2826: }
2827:
2828: if ((port = object->pager_request) != IP_NULL) {
2829: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2830: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2831: else if (ip_kotype(port) != IKOT_NONE)
2832: panic("vm_object_remove: bad request port");
2833: }
2834: }
2835:
2836: /*
2837: * Global variables for vm_object_collapse():
2838: *
2839: * Counts for normal collapses and bypasses.
2840: * Debugging variables, to watch or disable collapse.
2841: */
2842: long object_collapses = 0;
2843: long object_bypasses = 0;
2844:
2845: boolean_t vm_object_collapse_allowed = TRUE;
2846: boolean_t vm_object_bypass_allowed = TRUE;
2847:
2848: int vm_external_discarded;
2849: int vm_external_collapsed;
2850: /*
2851: * vm_object_do_collapse:
2852: *
2853: * Collapse an object with the object backing it.
2854: * Pages in the backing object are moved into the
2855: * parent, and the backing object is deallocated.
2856: *
2857: * Both objects and the cache are locked; the page
2858: * queues are unlocked.
2859: *
2860: */
2861: void
2862: vm_object_do_collapse(
2863: vm_object_t object,
2864: vm_object_t backing_object)
2865: {
2866: vm_page_t p, pp;
2867: vm_offset_t new_offset, backing_offset;
2868: vm_size_t size;
2869:
2870: backing_offset = object->shadow_offset;
2871: size = object->size;
2872:
2873:
2874: /*
2875: * Move all in-memory pages from backing_object
2876: * to the parent. Pages that have been paged out
2877: * will be overwritten by any of the parent's
2878: * pages that shadow them.
2879: */
2880:
2881: while (!queue_empty(&backing_object->memq)) {
2882:
2883: p = (vm_page_t) queue_first(&backing_object->memq);
2884:
2885: new_offset = (p->offset - backing_offset);
2886:
2887: assert(!p->busy || p->absent);
2888:
2889: /*
2890: * If the parent has a page here, or if
2891: * this page falls outside the parent,
2892: * dispose of it.
2893: *
2894: * Otherwise, move it as planned.
2895: */
2896:
2897: if (p->offset < backing_offset || new_offset >= size) {
2898: VM_PAGE_FREE(p);
2899: } else {
2900: pp = vm_page_lookup(object, new_offset);
2901: if (pp == VM_PAGE_NULL) {
2902:
2903: /*
2904: * Parent now has no page.
2905: * Move the backing object's page up.
2906: */
2907:
2908: vm_page_rename(p, object, new_offset);
2909: #if MACH_PAGEMAP
2910: } else if (pp->absent) {
2911:
2912: /*
2913: * Parent has an absent page...
2914: * it's not being paged in, so
2915: * it must really be missing from
2916: * the parent.
2917: *
2918: * Throw out the absent page...
2919: * any faults looking for that
2920: * page will restart with the new
2921: * one.
2922: */
2923:
2924: VM_PAGE_FREE(pp);
2925: vm_page_rename(p, object, new_offset);
2926: #endif /* MACH_PAGEMAP */
2927: } else {
2928: assert(! pp->absent);
2929:
2930: /*
2931: * Parent object has a real page.
2932: * Throw away the backing object's
2933: * page.
2934: */
2935: VM_PAGE_FREE(p);
2936: }
2937: }
2938: }
2939:
2940: assert(object->pager == IP_NULL || backing_object->pager == IP_NULL);
2941:
2942: if (backing_object->pager != IP_NULL) {
2943: vm_object_hash_entry_t entry;
2944:
2945: /*
2946: * Move the pager from backing_object to object.
2947: *
2948: * XXX We're only using part of the paging space
2949: * for keeps now... we ought to discard the
2950: * unused portion.
2951: */
2952:
2953: object->pager = backing_object->pager;
2954: entry = vm_object_hash_lookup(object->pager, FALSE);
2955: assert(entry != VM_OBJECT_HASH_ENTRY_NULL);
2956: entry->object = object;
2957: object->pager_created = backing_object->pager_created;
2958: object->pager_request = backing_object->pager_request;
2959: object->pager_ready = backing_object->pager_ready;
2960: object->pager_initialized = backing_object->pager_initialized;
2961: object->cluster_size = backing_object->cluster_size;
2962: object->paging_offset =
2963: backing_object->paging_offset + backing_offset;
2964: if (object->pager_request != IP_NULL) {
2965: ipc_kobject_set(object->pager_request,
2966: (ipc_kobject_t) object,
2967: IKOT_PAGING_REQUEST);
2968: }
2969: }
2970:
2971: vm_object_cache_unlock();
2972:
2973: object->paging_offset = backing_object->paging_offset + backing_offset;
2974:
2975: #if MACH_PAGEMAP
2976: /*
2977: * If the shadow offset is 0, the use the existence map from
2978: * the backing object if there is one. If the shadow offset is
2979: * not zero, toss it.
2980: *
2981: * XXX - If the shadow offset is not 0 then a bit copy is needed
2982: * if the map is to be salvaged. For now, we just just toss the
2983: * old map, giving the collapsed object no map. This means that
2984: * the pager is invoked for zero fill pages. If analysis shows
2985: * that this happens frequently and is a performance hit, then
2986: * this code should be fixed to salvage the map.
2987: */
2988: assert(object->existence_map == VM_EXTERNAL_NULL);
2989: if (backing_offset || (size != backing_object->size)) {
2990: vm_external_discarded++;
2991: vm_external_destroy(backing_object->existence_map,
2992: backing_object->size);
2993: }
2994: else {
2995: vm_external_collapsed++;
2996: object->existence_map = backing_object->existence_map;
2997: }
2998: backing_object->existence_map = VM_EXTERNAL_NULL;
2999: #endif /* MACH_PAGEMAP */
3000:
3001: /*
3002: * Object now shadows whatever backing_object did.
3003: * Note that the reference to backing_object->shadow
3004: * moves from within backing_object to within object.
3005: */
3006:
3007: object->shadow = backing_object->shadow;
3008: object->shadow_offset += backing_object->shadow_offset;
3009: assert((object->shadow == VM_OBJECT_NULL) ||
3010: (object->shadow->copy == VM_OBJECT_NULL));
3011:
3012: /*
3013: * Discard backing_object.
3014: *
3015: * Since the backing object has no pages, no
3016: * pager left, and no object references within it,
3017: * all that is necessary is to dispose of it.
3018: */
3019:
3020: assert((backing_object->ref_count == 1) &&
3021: (backing_object->resident_page_count == 0) &&
3022: (backing_object->paging_in_progress == 0));
3023:
3024: assert(backing_object->alive);
3025: backing_object->alive = FALSE;
3026: vm_object_unlock(backing_object);
3027:
3028: XPR(XPR_VM_OBJECT, "vm_object_collapse, collapsed 0x%X\n",
3029: (integer_t)backing_object, 0,0,0,0);
3030:
3031: zfree(vm_object_zone, (vm_offset_t) backing_object);
3032:
3033: object_collapses++;
3034: }
3035:
3036: void
3037: vm_object_do_bypass(
3038: vm_object_t object,
3039: vm_object_t backing_object)
3040: {
3041: /*
3042: * Make the parent shadow the next object
3043: * in the chain.
3044: */
3045:
3046: #if TASK_SWAPPER
3047: /*
3048: * Do object reference in-line to
3049: * conditionally increment shadow's
3050: * residence count. If object is not
3051: * resident, leave residence count
3052: * on shadow alone.
3053: */
3054: if (backing_object->shadow != VM_OBJECT_NULL) {
3055: vm_object_lock(backing_object->shadow);
3056: backing_object->shadow->ref_count++;
3057: if (object->res_count != 0)
3058: vm_object_res_reference(backing_object->shadow);
3059: vm_object_unlock(backing_object->shadow);
3060: }
3061: #else /* TASK_SWAPPER */
3062: vm_object_reference(backing_object->shadow);
3063: #endif /* TASK_SWAPPER */
3064:
3065: object->shadow = backing_object->shadow;
3066: object->shadow_offset += backing_object->shadow_offset;
3067:
3068: /*
3069: * Backing object might have had a copy pointer
3070: * to us. If it did, clear it.
3071: */
3072: if (backing_object->copy == object) {
3073: backing_object->copy = VM_OBJECT_NULL;
3074: }
3075:
3076: /*
3077: * Drop the reference count on backing_object.
3078: #if TASK_SWAPPER
3079: * Since its ref_count was at least 2, it
3080: * will not vanish; so we don't need to call
3081: * vm_object_deallocate.
3082: * [FBDP: that doesn't seem to be true any more]
3083: *
3084: * The res_count on the backing object is
3085: * conditionally decremented. It's possible
3086: * (via vm_pageout_scan) to get here with
3087: * a "swapped" object, which has a 0 res_count,
3088: * in which case, the backing object res_count
3089: * is already down by one.
3090: #else
3091: * Don't call vm_object_deallocate unless
3092: * ref_count drops to zero.
3093: *
3094: * The ref_count can drop to zero here if the
3095: * backing object could be bypassed but not
3096: * collapsed, such as when the backing object
3097: * is temporary and cachable.
3098: #endif
3099: */
3100: if (backing_object->ref_count > 1) {
3101: backing_object->ref_count--;
3102: #if TASK_SWAPPER
3103: if (object->res_count != 0)
3104: vm_object_res_deallocate(backing_object);
3105: assert(backing_object->ref_count > 0);
3106: #endif /* TASK_SWAPPER */
3107: vm_object_unlock(backing_object);
3108: } else {
3109:
3110: /*
3111: * Drop locks so that we can deallocate
3112: * the backing object.
3113: */
3114:
3115: #if TASK_SWAPPER
3116: if (object->res_count == 0) {
3117: /* XXX get a reference for the deallocate below */
3118: vm_object_res_reference(backing_object);
3119: }
3120: #endif /* TASK_SWAPPER */
3121: vm_object_unlock(object);
3122: vm_object_unlock(backing_object);
3123: vm_object_deallocate(backing_object);
3124:
3125: /*
3126: * Relock object. We don't have to reverify
3127: * its state since vm_object_collapse will
3128: * do that for us as it starts at the
3129: * top of its loop.
3130: */
3131:
3132: vm_object_lock(object);
3133: }
3134:
3135: object_bypasses++;
3136: }
3137:
3138: /*
3139: * vm_object_collapse:
3140: *
3141: * Perform an object collapse or an object bypass if appropriate.
3142: * The real work of collapsing and bypassing is performed in
3143: * the routines vm_object_do_collapse and vm_object_do_bypass.
3144: *
3145: * Requires that the object be locked and the page queues be unlocked.
3146: *
3147: */
3148: void
3149: vm_object_collapse(
3150: register vm_object_t object)
3151: {
3152: register vm_object_t backing_object;
3153: register vm_offset_t backing_offset;
3154: register vm_size_t size;
3155: register vm_offset_t new_offset;
3156: register vm_page_t p;
3157:
3158: if (! vm_object_collapse_allowed && ! vm_object_bypass_allowed) {
3159: return;
3160: }
3161:
3162: XPR(XPR_VM_OBJECT, "vm_object_collapse, obj 0x%X\n",
3163: (integer_t)object, 0,0,0,0);
3164:
3165: while (TRUE) {
3166: /*
3167: * Verify that the conditions are right for either
3168: * collapse or bypass:
3169: *
3170: * The object exists and no pages in it are currently
3171: * being paged out, and
3172: */
3173: if (object == VM_OBJECT_NULL ||
3174: object->paging_in_progress != 0 ||
3175: object->absent_count != 0)
3176: return;
3177:
3178: /*
3179: * There is a backing object, and
3180: */
3181:
3182: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
3183: return;
3184:
3185: vm_object_lock(backing_object);
3186:
3187: /*
3188: * ...
3189: * The backing object is not read_only,
3190: * and no pages in the backing object are
3191: * currently being paged out.
3192: * The backing object is internal.
3193: *
3194: */
3195:
3196: if (!backing_object->internal ||
3197: backing_object->paging_in_progress != 0) {
3198: vm_object_unlock(backing_object);
3199: return;
3200: }
3201:
3202: /*
3203: * The backing object can't be a copy-object:
3204: * the shadow_offset for the copy-object must stay
3205: * as 0. Furthermore (for the 'we have all the
3206: * pages' case), if we bypass backing_object and
3207: * just shadow the next object in the chain, old
3208: * pages from that object would then have to be copied
3209: * BOTH into the (former) backing_object and into the
3210: * parent object.
3211: */
3212: if (backing_object->shadow != VM_OBJECT_NULL &&
3213: backing_object->shadow->copy != VM_OBJECT_NULL) {
3214: vm_object_unlock(backing_object);
3215: return;
3216: }
3217:
3218: /*
3219: * We can now try to either collapse the backing
3220: * object (if the parent is the only reference to
3221: * it) or (perhaps) remove the parent's reference
3222: * to it.
3223: */
3224:
3225: /*
3226: * If there is exactly one reference to the backing
3227: * object, we may be able to collapse it into the parent.
3228: *
3229: * XXXO (norma vm):
3230: *
3231: * The backing object must not have a pager
3232: * created for it, since collapsing an object
3233: * into a backing_object dumps new pages into
3234: * the backing_object that its pager doesn't
3235: * know about, and we've already declared pages.
3236: * This page dumping is deadly if other kernels
3237: * are shadowing this object; this is the
3238: * distributed equivalent of the ref_count == 1
3239: * condition.
3240: *
3241: * With some work, we could downgrade this
3242: * restriction to the backing object must not
3243: * be cachable, since when a temporary object
3244: * is uncachable we are allowed to do anything
3245: * to it. We would have to do something like
3246: * call declare_pages again, and we would have
3247: * to be prepared for the memory manager
3248: * disabling temporary termination, which right
3249: * now is a difficult race to deal with, since
3250: * the memory manager currently assumes that
3251: * termination is the only possible failure
3252: * for disabling temporary termination.
3253: */
3254:
3255: if (backing_object->ref_count == 1 &&
3256: ! object->pager_created &&
3257: vm_object_collapse_allowed) {
3258:
3259: XPR(XPR_VM_OBJECT,
3260: "vm_object_collapse: %x to %x, pager %x, pager_request %x\n",
3261: (integer_t)backing_object, (integer_t)object,
3262: (integer_t)backing_object->pager,
3263: (integer_t)backing_object->pager_request, 0);
3264:
3265: /*
3266: * We need the cache lock for collapsing,
3267: * but we must not deadlock.
3268: */
3269:
3270: if (! vm_object_cache_lock_try()) {
3271: vm_object_unlock(backing_object);
3272: return;
3273: }
3274:
3275: /*
3276: * Collapse the object with its backing
3277: * object, and try again with the object's
3278: * new backing object.
3279: */
3280:
3281: vm_object_do_collapse(object, backing_object);
3282: continue;
3283: }
3284:
3285:
3286: /*
3287: * Collapsing the backing object was not possible
3288: * or permitted, so let's try bypassing it.
3289: */
3290:
3291: if (! vm_object_bypass_allowed) {
3292: vm_object_unlock(backing_object);
3293: return;
3294: }
3295:
3296: /*
3297: * If the backing object has a pager but no pagemap,
3298: * then we cannot bypass it, because we don't know
3299: * what pages it has.
3300: */
3301: if (backing_object->pager_created
3302: #if MACH_PAGEMAP
3303: && (backing_object->existence_map == VM_EXTERNAL_NULL)
3304: #endif /* MACH_PAGEMAP */
3305: ) {
3306: vm_object_unlock(backing_object);
3307: return;
3308: }
3309:
3310: backing_offset = object->shadow_offset;
3311: size = object->size;
3312:
3313: /*
3314: * If all of the pages in the backing object are
3315: * shadowed by the parent object, the parent
3316: * object no longer has to shadow the backing
3317: * object; it can shadow the next one in the
3318: * chain.
3319: *
3320: * If the backing object has existence info,
3321: * we must check examine its existence info
3322: * as well.
3323: *
3324: * XXX
3325: * Should have a check for a 'small' number
3326: * of pages here.
3327: */
3328:
3329: /*
3330: * First, check pages resident in the backing object.
3331: */
3332:
3333: queue_iterate(&backing_object->memq, p, vm_page_t, listq) {
3334:
3335: /*
3336: * If the parent has a page here, or if
3337: * this page falls outside the parent,
3338: * keep going.
3339: *
3340: * Otherwise, the backing_object must be
3341: * left in the chain.
3342: */
3343:
3344: new_offset = (p->offset - backing_offset);
3345: if (p->offset < backing_offset || new_offset >= size) {
3346:
3347: /*
3348: * Page falls outside of parent.
3349: * Keep going.
3350: */
3351:
3352: continue;
3353: }
3354:
3355: if ((vm_page_lookup(object, new_offset) == VM_PAGE_NULL)
3356: #if MACH_PAGEMAP
3357: &&
3358: (vm_external_state_get(object->existence_map,
3359: new_offset)
3360: != VM_EXTERNAL_STATE_EXISTS)
3361: #endif /* MACH_PAGEMAP */
3362: ) {
3363:
3364: /*
3365: * Page still needed.
3366: * Can't go any further.
3367: */
3368:
3369: vm_object_unlock(backing_object);
3370: return;
3371: }
3372: }
3373:
3374: #if MACH_PAGEMAP
3375: /*
3376: * Next, if backing object has been paged out,
3377: * we must check its existence info for pages
3378: * that the parent doesn't have.
3379: */
3380:
3381: if (backing_object->pager_created) {
3382: assert(backing_object->existence_map
3383: != VM_EXTERNAL_NULL);
3384: for (new_offset = 0; new_offset < object->size;
3385: new_offset += PAGE_SIZE) {
3386: vm_offset_t
3387: offset = new_offset + backing_offset;
3388:
3389: /*
3390: * If this page doesn't exist in
3391: * the backing object's existence
3392: * info, then continue.
3393: */
3394:
3395: if (vm_external_state_get(
3396: backing_object->existence_map,
3397: offset) == VM_EXTERNAL_STATE_ABSENT) {
3398: continue;
3399: }
3400:
3401: /*
3402: * If this page is neither resident
3403: * in the parent nor paged out to
3404: * the parent's pager, then we cannot
3405: * bypass the backing object.
3406: */
3407:
3408: if ((vm_page_lookup(object, new_offset) ==
3409: VM_PAGE_NULL) &&
3410: ((object->existence_map == VM_EXTERNAL_NULL)
3411: || (vm_external_state_get(
3412: object->existence_map, new_offset)
3413: == VM_EXTERNAL_STATE_ABSENT))) {
3414: vm_object_unlock(backing_object);
3415: return;
3416: }
3417: }
3418: }
3419: #else /* MACH_PAGEMAP */
3420: assert(! backing_object->pager_created);
3421: #endif /* MACH_PAGEMAP */
3422:
3423: /*
3424: * All interesting pages in the backing object
3425: * already live in the parent or its pager.
3426: * Thus we can bypass the backing object.
3427: */
3428:
3429: vm_object_do_bypass(object, backing_object);
3430:
3431: /*
3432: * Try again with this object's new backing object.
3433: */
3434:
3435: continue;
3436: }
3437: }
3438:
3439: /*
3440: * Routine: vm_object_page_remove: [internal]
3441: * Purpose:
3442: * Removes all physical pages in the specified
3443: * object range from the object's list of pages.
3444: *
3445: * In/out conditions:
3446: * The object must be locked.
3447: * The object must not have paging_in_progress, usually
3448: * guaranteed by not having a pager.
3449: */
3450: unsigned int vm_object_page_remove_lookup = 0;
3451: unsigned int vm_object_page_remove_iterate = 0;
3452:
3453: void
3454: vm_object_page_remove(
3455: register vm_object_t object,
3456: register vm_offset_t start,
3457: register vm_offset_t end)
3458: {
3459: register vm_page_t p, next;
3460:
3461: /*
3462: * One and two page removals are most popular.
3463: * The factor of 16 here is somewhat arbitrary.
3464: * It balances vm_object_lookup vs iteration.
3465: */
3466:
3467: if (atop(end - start) < (unsigned)object->resident_page_count/16) {
3468: vm_object_page_remove_lookup++;
3469:
3470: for (; start < end; start += PAGE_SIZE) {
3471: p = vm_page_lookup(object, start);
3472: if (p != VM_PAGE_NULL) {
3473: assert(!p->cleaning && !p->pageout);
3474: if (!p->fictitious)
3475: pmap_page_protect(p->phys_addr,
3476: VM_PROT_NONE);
3477: VM_PAGE_FREE(p);
3478: }
3479: }
3480: } else {
3481: vm_object_page_remove_iterate++;
3482:
3483: p = (vm_page_t) queue_first(&object->memq);
3484: while (!queue_end(&object->memq, (queue_entry_t) p)) {
3485: next = (vm_page_t) queue_next(&p->listq);
3486: if ((start <= p->offset) && (p->offset < end)) {
3487: assert(!p->cleaning && !p->pageout);
3488: if (!p->fictitious)
3489: pmap_page_protect(p->phys_addr,
3490: VM_PROT_NONE);
3491: VM_PAGE_FREE(p);
3492: }
3493: p = next;
3494: }
3495: }
3496: }
3497:
3498: /*
3499: * Routine: vm_object_coalesce
3500: * Function: Coalesces two objects backing up adjoining
3501: * regions of memory into a single object.
3502: *
3503: * returns TRUE if objects were combined.
3504: *
3505: * NOTE: Only works at the moment if the second object is NULL -
3506: * if it's not, which object do we lock first?
3507: *
3508: * Parameters:
3509: * prev_object First object to coalesce
3510: * prev_offset Offset into prev_object
3511: * next_object Second object into coalesce
3512: * next_offset Offset into next_object
3513: *
3514: * prev_size Size of reference to prev_object
3515: * next_size Size of reference to next_object
3516: *
3517: * Conditions:
3518: * The object(s) must *not* be locked. The map must be locked
3519: * to preserve the reference to the object(s).
3520: */
3521: int vm_object_coalesce_count = 0;
3522:
3523: boolean_t
3524: vm_object_coalesce(
3525: register vm_object_t prev_object,
3526: vm_object_t next_object,
3527: vm_offset_t prev_offset,
3528: vm_offset_t next_offset,
3529: vm_size_t prev_size,
3530: vm_size_t next_size)
3531: {
3532: vm_size_t newsize;
3533:
3534: #ifdef lint
3535: next_offset++;
3536: #endif /* lint */
3537:
3538: if (next_object != VM_OBJECT_NULL) {
3539: return(FALSE);
3540: }
3541:
3542: if (prev_object == VM_OBJECT_NULL) {
3543: return(TRUE);
3544: }
3545:
3546: XPR(XPR_VM_OBJECT,
3547: "vm_object_coalesce: 0x%X prev_off 0x%X prev_size 0x%X next_size 0x%X\n",
3548: (integer_t)prev_object, prev_offset, prev_size, next_size, 0);
3549:
3550: vm_object_lock(prev_object);
3551:
3552: /*
3553: * Try to collapse the object first
3554: */
3555: vm_object_collapse(prev_object);
3556:
3557: /*
3558: * Can't coalesce if pages not mapped to
3559: * prev_entry may be in use any way:
3560: * . more than one reference
3561: * . paged out
3562: * . shadows another object
3563: * . has a copy elsewhere
3564: * . paging references (pages might be in page-list)
3565: */
3566:
3567: if ((prev_object->ref_count > 1) ||
3568: prev_object->pager_created ||
3569: (prev_object->shadow != VM_OBJECT_NULL) ||
3570: (prev_object->copy != VM_OBJECT_NULL) ||
3571: (prev_object->true_share != FALSE) ||
3572: (prev_object->paging_in_progress != 0)) {
3573: vm_object_unlock(prev_object);
3574: return(FALSE);
3575: }
3576:
3577: vm_object_coalesce_count++;
3578:
3579: /*
3580: * Remove any pages that may still be in the object from
3581: * a previous deallocation.
3582: */
3583: vm_object_page_remove(prev_object,
3584: prev_offset + prev_size,
3585: prev_offset + prev_size + next_size);
3586:
3587: /*
3588: * Extend the object if necessary.
3589: */
3590: newsize = prev_offset + prev_size + next_size;
3591: if (newsize > prev_object->size) {
3592: #if MACH_PAGEMAP
3593: /*
3594: * We cannot extend an object that has existence info,
3595: * since the existence info might then fail to cover
3596: * the entire object.
3597: *
3598: * This assertion must be true because the object
3599: * has no pager, and we only create existence info
3600: * for objects with pagers.
3601: */
3602: assert(prev_object->existence_map == VM_EXTERNAL_NULL);
3603: #endif /* MACH_PAGEMAP */
3604: prev_object->size = newsize;
3605: }
3606:
3607: vm_object_unlock(prev_object);
3608: return(TRUE);
3609: }
3610:
3611: /*
3612: * Attach a set of physical pages to an object, so that they can
3613: * be mapped by mapping the object. Typically used to map IO memory.
3614: *
3615: * The mapping function and its private data are used to obtain the
3616: * physical addresses for each page to be mapped.
3617: */
3618: void
3619: vm_object_page_map(
3620: vm_object_t object,
3621: vm_offset_t offset,
3622: vm_size_t size,
3623: vm_offset_t (*map_fn)(void *map_fn_data, vm_offset_t offset),
3624: void *map_fn_data) /* private to map_fn */
3625: {
3626: int num_pages;
3627: int i;
3628: vm_page_t m;
3629: vm_page_t old_page;
3630: vm_offset_t addr;
3631:
3632: num_pages = atop(size);
3633:
3634: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
3635:
3636: addr = (*map_fn)(map_fn_data, offset);
3637:
3638: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
3639: vm_page_more_fictitious();
3640:
3641: vm_object_lock(object);
3642: if ((old_page = vm_page_lookup(object, offset))
3643: != VM_PAGE_NULL)
3644: {
3645: vm_page_lock_queues();
3646: vm_page_free(old_page);
3647: vm_page_unlock_queues();
3648: }
3649:
3650: vm_page_init(m, addr);
3651: m->private = TRUE; /* don`t free page */
3652: m->wire_count = 1;
3653: vm_page_insert(m, object, offset);
3654:
3655: PAGE_WAKEUP_DONE(m);
3656: vm_object_unlock(object);
3657: }
3658: }
3659:
3660: #include <mach_kdb.h>
3661:
3662: #if MACH_KDB
3663: #include <ddb/db_output.h>
3664: #include <vm/vm_print.h>
3665:
3666: #define printf kdbprintf
3667:
3668: extern boolean_t vm_object_cached(
3669: vm_object_t object);
3670:
3671: extern void print_bitstring(
3672: char byte);
3673:
3674: boolean_t vm_object_print_pages = FALSE;
3675:
3676: void
3677: print_bitstring(
3678: char byte)
3679: {
3680: printf("%c%c%c%c%c%c%c%c",
3681: ((byte & (1 << 0)) ? '1' : '0'),
3682: ((byte & (1 << 1)) ? '1' : '0'),
3683: ((byte & (1 << 2)) ? '1' : '0'),
3684: ((byte & (1 << 3)) ? '1' : '0'),
3685: ((byte & (1 << 4)) ? '1' : '0'),
3686: ((byte & (1 << 5)) ? '1' : '0'),
3687: ((byte & (1 << 6)) ? '1' : '0'),
3688: ((byte & (1 << 7)) ? '1' : '0'));
3689: }
3690:
3691: boolean_t
3692: vm_object_cached(
3693: register vm_object_t object)
3694: {
3695: register vm_object_t o;
3696:
3697: queue_iterate(&vm_object_cached_list, o, vm_object_t, cached_list) {
3698: if (object == o) {
3699: return TRUE;
3700: }
3701: }
3702: return FALSE;
3703: }
3704:
3705: #if MACH_PAGEMAP
3706: /*
3707: * vm_external_print: [ debug ]
3708: */
3709: void
3710: vm_external_print(
3711: vm_external_map_t map,
3712: vm_size_t size)
3713: {
3714: if (map == VM_EXTERNAL_NULL) {
3715: printf("0 ");
3716: } else {
3717: vm_size_t existence_size = stob(size);
3718: printf("{ size=%d, map=[", existence_size);
3719: if (existence_size > 0) {
3720: print_bitstring(map[0]);
3721: }
3722: if (existence_size > 1) {
3723: print_bitstring(map[1]);
3724: }
3725: if (existence_size > 2) {
3726: printf("...");
3727: print_bitstring(map[existence_size-1]);
3728: }
3729: printf("] }\n");
3730: }
3731: return;
3732: }
3733: #endif /* MACH_PAGEMAP */
3734:
3735: int
3736: vm_follow_object(
3737: vm_object_t object)
3738: {
3739: extern db_indent;
3740:
3741: int count = 1;
3742:
3743: if (object == VM_OBJECT_NULL)
3744: return 0;
3745:
3746: iprintf("object 0x%x", object);
3747: printf(", shadow=0x%x", object->shadow);
3748: printf(", copy=0x%x", object->copy);
3749: printf(", pager=0x%x", object->pager);
3750: printf(", ref=%d\n", object->ref_count);
3751:
3752: db_indent += 2;
3753: if (object->shadow)
3754: count += vm_follow_object(object->shadow);
3755:
3756: db_indent -= 2;
3757: return count;
3758: }
3759:
3760: /*
3761: * vm_object_print: [ debug ]
3762: */
3763: void
3764: vm_object_print(
3765: vm_object_t object,
3766: boolean_t have_addr,
3767: int arg_count,
3768: char *modif)
3769: {
3770: register vm_page_t p;
3771: extern db_indent;
3772: char *s;
3773:
3774: register int count;
3775:
3776: if (object == VM_OBJECT_NULL)
3777: return;
3778:
3779: iprintf("object 0x%x\n", object);
3780:
3781: db_indent += 2;
3782:
3783: iprintf("size=0x%x", object->size);
3784: printf(", cluster=0x%x", object->cluster_size);
3785: printf(", frozen=0x%x", object->frozen_size);
3786: printf(", ref_count=%d\n", object->ref_count);
3787: iprintf("");
3788: #if TASK_SWAPPER
3789: printf("res_count=%d, ", object->res_count);
3790: #endif /* TASK_SWAPPER */
3791: printf("resident_page_count=%d\n", object->resident_page_count);
3792:
3793: iprintf("shadow=0x%x", object->shadow);
3794: if (object->shadow) {
3795: register int i = 0;
3796: vm_object_t shadow = object;
3797: while(shadow = shadow->shadow)
3798: i++;
3799: printf(" (depth %d)", i);
3800: }
3801: printf(", copy=0x%x", object->copy);
3802: printf(", shadow_offset=0x%x", object->shadow_offset);
3803: printf(", last_alloc=0x%x\n", object->last_alloc);
3804:
3805: iprintf("pager=0x%x", object->pager);
3806: printf(", paging_offset=0x%x", object->paging_offset);
3807: printf(", pager_request=0x%x\n", object->pager_request);
3808:
3809: iprintf("copy_strategy=%d[", object->copy_strategy);
3810: switch (object->copy_strategy) {
3811: case MEMORY_OBJECT_COPY_NONE:
3812: printf("copy_none");
3813: break;
3814:
3815: case MEMORY_OBJECT_COPY_CALL:
3816: printf("copy_call");
3817: break;
3818:
3819: case MEMORY_OBJECT_COPY_DELAY:
3820: printf("copy_delay");
3821: break;
3822:
3823: case MEMORY_OBJECT_COPY_SYMMETRIC:
3824: printf("copy_symmetric");
3825: break;
3826:
3827: case MEMORY_OBJECT_COPY_INVALID:
3828: printf("copy_invalid");
3829: break;
3830:
3831: default:
3832: printf("?");
3833: }
3834: printf("]");
3835: printf(", absent_count=%d\n", object->absent_count);
3836:
3837: iprintf("all_wanted=0x%x<", object->all_wanted);
3838: s = "";
3839: if (vm_object_wanted(object, VM_OBJECT_EVENT_INITIALIZED)) {
3840: printf("%sinit", s);
3841: s = ",";
3842: }
3843: if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGER_READY)) {
3844: printf("%sready", s);
3845: s = ",";
3846: }
3847: if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGING_IN_PROGRESS)) {
3848: printf("%spaging", s);
3849: s = ",";
3850: }
3851: if (vm_object_wanted(object, VM_OBJECT_EVENT_ABSENT_COUNT)) {
3852: printf("%sabsent", s);
3853: s = ",";
3854: }
3855: if (vm_object_wanted(object, VM_OBJECT_EVENT_LOCK_IN_PROGRESS)) {
3856: printf("%slock", s);
3857: s = ",";
3858: }
3859: if (vm_object_wanted(object, VM_OBJECT_EVENT_UNCACHING)) {
3860: printf("%suncaching", s);
3861: s = ",";
3862: }
3863: if (vm_object_wanted(object, VM_OBJECT_EVENT_COPY_CALL)) {
3864: printf("%scopy_call", s);
3865: s = ",";
3866: }
3867: if (vm_object_wanted(object, VM_OBJECT_EVENT_CACHING)) {
3868: printf("%scaching", s);
3869: s = ",";
3870: }
3871: printf(">");
3872: printf(", paging_in_progress=%d\n", object->paging_in_progress);
3873:
3874: iprintf("%screated, %sinit, %sready, %spersist, %strusted, %spageout, %s, %s\n",
3875: (object->pager_created ? "" : "!"),
3876: (object->pager_initialized ? "" : "!"),
3877: (object->pager_ready ? "" : "!"),
3878: (object->can_persist ? "" : "!"),
3879: (object->pager_trusted ? "" : "!"),
3880: (object->pageout ? "" : "!"),
3881: (object->internal ? "internal" : "external"),
3882: (object->temporary ? "temporary" : "permanent"));
3883: iprintf("%salive, %slock_in_progress, %slock_restart, %sshadowed, %scached, %sprivate\n",
3884: (object->alive ? "" : "!"),
3885: (object->lock_in_progress ? "" : "!"),
3886: (object->lock_restart ? "" : "!"),
3887: (object->shadowed ? "" : "!"),
3888: (vm_object_cached(object) ? "" : "!"),
3889: (object->private ? "" : "!"));
3890: iprintf("%sadvisory_pageout, %ssilent_overwrite\n",
3891: (object->advisory_pageout ? "" : "!"),
3892: (object->silent_overwrite ? "" : "!"));
3893:
3894: #if MACH_PAGEMAP
3895: iprintf("existence_map=");
3896: vm_external_print(object->existence_map, object->size);
3897: #endif /* MACH_PAGEMAP */
3898: #if MACH_ASSERT
3899: iprintf("paging_object=0x%x\n", object->paging_object);
3900: #endif /* MACH_ASSERT */
3901:
3902: if (vm_object_print_pages) {
3903: count = 0;
3904: p = (vm_page_t) queue_first(&object->memq);
3905: while (!queue_end(&object->memq, (queue_entry_t) p)) {
3906: if (count == 0) {
3907: iprintf("memory:=");
3908: } else if (count == 2) {
3909: printf("\n");
3910: iprintf(" ...");
3911: count = 0;
3912: } else {
3913: printf(",");
3914: }
3915: count++;
3916:
3917: printf("(off=0x%X,page=0x%X)", p->offset, (integer_t) p);
3918: p = (vm_page_t) queue_next(&p->listq);
3919: }
3920: if (count != 0) {
3921: printf("\n");
3922: }
3923: }
3924: db_indent -= 2;
3925: }
3926:
3927:
3928: /*
3929: * vm_object_find [ debug ]
3930: *
3931: * Find all tasks which reference the given vm_object.
3932: */
3933:
3934: boolean_t vm_object_find(vm_object_t object);
3935: boolean_t vm_object_print_verbose = FALSE;
3936:
3937: boolean_t
3938: vm_object_find(
3939: vm_object_t object)
3940: {
3941: task_t task;
3942: vm_map_t map;
3943: vm_map_entry_t entry;
3944: processor_set_t pset;
3945: boolean_t found = FALSE;
3946:
3947: queue_iterate(&all_psets, pset, processor_set_t, all_psets) {
3948: queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
3949: map = task->map;
3950: for (entry = vm_map_first_entry(map);
3951: entry && entry != vm_map_to_entry(map);
3952: entry = entry->vme_next) {
3953:
3954: vm_object_t obj;
3955:
3956: /*
3957: * For the time being skip submaps,
3958: * only the kernel can have submaps,
3959: * and unless we are interested in
3960: * kernel objects, we can simply skip
3961: * submaps. See sb/dejan/nmk18b7/src/mach_kernel/vm
3962: * for a full solution.
3963: */
3964: if (entry->is_sub_map)
3965: continue;
3966: if (entry)
3967: obj = entry->object.vm_object;
3968: else
3969: continue;
3970:
3971: while (obj != VM_OBJECT_NULL) {
3972: if (obj == object) {
3973: if (!found) {
3974: printf("TASK\t\tMAP\t\tENTRY\n");
3975: found = TRUE;
3976: }
3977: printf("0x%x\t0x%x\t0x%x\n",
3978: task, map, entry);
3979: }
3980: obj = obj->shadow;
3981: }
3982: }
3983: }
3984: }
3985:
3986: return(found);
3987: }
3988:
3989: #endif /* MACH_KDB */
3990:
3991: /*
3992: * memory_object_free_from_cache:
3993: *
3994: * Walk the vm_object cache list, removing and freeing vm_objects
3995: * which are backed by the pager identified by the caller, (pager_id).
3996: * Remove up to "count" objects, if there are that may available
3997: * in the cache.
3998: * Walk the list at most once, return the number of vm_objects
3999: * actually freed.
4000: *
4001: */
4002:
4003: kern_return_t
4004: memory_object_free_from_cache(
4005: host_t host,
4006: int pager_id,
4007: int *count)
4008: {
4009:
4010: int object_released = 0;
4011: int i;
4012:
4013: register vm_object_t object = VM_OBJECT_NULL;
4014: vm_object_t shadow;
4015:
4016: if(host == HOST_NULL)
4017: return(KERN_INVALID_ARGUMENT);
4018:
4019: try_again:
4020: vm_object_cache_lock();
4021:
4022: queue_iterate(&vm_object_cached_list, object,
4023: vm_object_t, cached_list) {
4024: if (pager_id == (int) pager_mux_hash_lookup(
4025: (ipc_port_t)object->pager)) {
4026: vm_object_lock(object);
4027: queue_remove(&vm_object_cached_list, object,
4028: vm_object_t, cached_list);
4029: vm_object_cached_count--;
4030:
4031: /*
4032: * Since this object is in the cache, we know
4033: * that it is initialized and has no references.
4034: * Take a reference to avoid recursive
4035: * deallocations.
4036: */
4037:
4038: assert(object->pager_initialized);
4039: assert(object->ref_count == 0);
4040: object->ref_count++;
4041:
4042: /*
4043: * Terminate the object.
4044: * If the object had a shadow, we let
4045: * vm_object_deallocate deallocate it.
4046: * "pageout" objects have a shadow, but
4047: * maintain a "paging reference" rather
4048: * than a normal reference.
4049: * (We are careful here to limit recursion.)
4050: */
4051: shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
4052: vm_object_terminate(object);
4053: if (shadow != VM_OBJECT_NULL) {
4054: vm_object_deallocate(shadow);
4055: }
4056:
4057: if(object_released++ == *count)
4058: return KERN_SUCCESS;
4059: goto try_again;
4060: }
4061: }
4062: vm_object_cache_unlock();
4063: *count = object_released;
4064: return KERN_SUCCESS;
4065: }
4066:
4067: /*
4068: * memory_object_remove_cached_object:
4069: *
4070: * Check for the existance of a memory object represented by the
4071: * supplied port. If one exists and it is not in use, remove the
4072: * memory object from the vm_object cache.
4073: * If the memory object is in use, turn off the the "can_persist"
4074: * property so that it will not go in the cache when the last user
4075: * gives it up.
4076: *
4077: */
4078:
4079: kern_return_t
4080: memory_object_remove_cached_object(
4081: ipc_port_t port)
4082: {
4083: vm_object_t object;
4084: vm_object_t shadow;
4085:
4086: repeat_lock_acquire:
4087: object = VM_OBJECT_NULL;
4088:
4089: if (IP_VALID(port)) {
4090: vm_object_cache_lock();
4091: ip_lock(port);
4092: if (ip_active(port) &&
4093: (ip_kotype(port) == IKOT_PAGER_LOOKUP_TYPE)) {
4094: object = (vm_object_t) port->ip_kobject;
4095: if (!vm_object_lock_try(object)) {
4096: /*
4097: * failed to acquire object lock. Drop the
4098: * other two locks and wait for it, then go
4099: * back and start over in case the port
4100: * associations changed in the interim.
4101: */
4102: ip_unlock(port);
4103: vm_object_cache_unlock();
4104: vm_object_lock(object);
4105: vm_object_unlock(object);
4106: goto repeat_lock_acquire;
4107: }
4108:
4109: assert(object->alive);
4110: ip_unlock(port);
4111:
4112: if (object->ref_count == 0) {
4113: queue_remove(&vm_object_cached_list, object,
4114: vm_object_t, cached_list);
4115: vm_object_cached_count--;
4116: object->ref_count++;
4117: /*
4118: * Terminate the object.
4119: * If the object had a shadow, we let
4120: * vm_object_deallocate deallocate it.
4121: * "pageout" objects have a shadow, but
4122: * maintain a "paging reference" rather
4123: * than a normal reference.
4124: * (We are careful here to limit
4125: * recursion.)
4126: */
4127: shadow = object->pageout?
4128: VM_OBJECT_NULL:object->shadow;
4129: /* will do the vm_object_cache_unlock */
4130: vm_object_terminate(object);
4131: if (shadow != VM_OBJECT_NULL) {
4132: /* will lock and unlock cache_lock */
4133: vm_object_deallocate(shadow);
4134: }
4135: }
4136: else {
4137: /*
4138: * We cannot free object but we can
4139: * make sure it doesn't go into the
4140: * cache when it is no longer in
4141: * use.
4142: */
4143: object->can_persist = FALSE;
4144:
4145: vm_object_unlock(object);
4146: vm_object_cache_unlock();
4147: return KERN_RIGHT_EXISTS;
4148: }
4149:
4150:
4151: }
4152: else {
4153: ip_unlock(port);
4154: vm_object_cache_unlock();
4155: }
4156: } else {
4157: return KERN_INVALID_ARGUMENT;
4158: }
4159:
4160:
4161: return KERN_SUCCESS;
4162: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.