![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
1.1.1.2 root 36: #include <kern/printf.h>
37: #include <string.h>
1.1 root 38:
39: #include <mach/memory_object.h>
1.1.1.2 root 40: #include <vm/memory_object_default.user.h>
41: #include <vm/memory_object_user.user.h>
42: #include <machine/vm_param.h>
1.1 root 43: #include <ipc/ipc_port.h>
44: #include <ipc/ipc_space.h>
45: #include <kern/assert.h>
1.1.1.2 root 46: #include <kern/debug.h>
1.1 root 47: #include <kern/lock.h>
48: #include <kern/queue.h>
49: #include <kern/xpr.h>
1.1.1.2 root 50: #include <kern/slab.h>
1.1 root 51: #include <vm/memory_object.h>
52: #include <vm/vm_fault.h>
53: #include <vm/vm_map.h>
54: #include <vm/vm_object.h>
55: #include <vm/vm_page.h>
56: #include <vm/vm_pageout.h>
57:
1.1.1.2 root 58: #if MACH_KDB
59: #include <ddb/db_output.h>
60: #endif /* MACH_KDB */
61:
1.1.1.5 ! root 62: void memory_object_release(
! 63: ipc_port_t pager,
! 64: pager_request_t pager_request,
! 65: ipc_port_t pager_name); /* forward */
! 66:
1.1 root 67: /*
68: * Virtual memory objects maintain the actual data
69: * associated with allocated virtual memory. A given
70: * page of memory exists within exactly one object.
71: *
72: * An object is only deallocated when all "references"
73: * are given up. Only one "reference" to a given
74: * region of an object should be writeable.
75: *
76: * Associated with each object is a list of all resident
77: * memory pages belonging to that object; this list is
78: * maintained by the "vm_page" module, but locked by the object's
79: * lock.
80: *
81: * Each object also records the memory object port
82: * that is used by the kernel to request and write
83: * back data (the memory object port, field "pager"),
84: * and the ports provided to the memory manager, the server that
85: * manages that data, to return data and control its
86: * use (the memory object control port, field "pager_request")
87: * and for naming (the memory object name port, field "pager_name").
88: *
89: * Virtual memory objects are allocated to provide
90: * zero-filled memory (vm_allocate) or map a user-defined
91: * memory object into a virtual address space (vm_map).
92: *
93: * Virtual memory objects that refer to a user-defined
94: * memory object are called "permanent", because all changes
95: * made in virtual memory are reflected back to the
96: * memory manager, which may then store it permanently.
97: * Other virtual memory objects are called "temporary",
98: * meaning that changes need be written back only when
99: * necessary to reclaim pages, and that storage associated
100: * with the object can be discarded once it is no longer
101: * mapped.
102: *
103: * A permanent memory object may be mapped into more
104: * than one virtual address space. Moreover, two threads
105: * may attempt to make the first mapping of a memory
106: * object concurrently. Only one thread is allowed to
107: * complete this mapping; all others wait for the
108: * "pager_initialized" field is asserted, indicating
109: * that the first thread has initialized all of the
110: * necessary fields in the virtual memory object structure.
111: *
112: * The kernel relies on a *default memory manager* to
113: * provide backing storage for the zero-filled virtual
114: * memory objects. The memory object ports associated
115: * with these temporary virtual memory objects are only
116: * generated and passed to the default memory manager
117: * when it becomes necessary. Virtual memory objects
118: * that depend on the default memory manager are called
119: * "internal". The "pager_created" field is provided to
120: * indicate whether these ports have ever been allocated.
121: *
122: * The kernel may also create virtual memory objects to
123: * hold changed pages after a copy-on-write operation.
124: * In this case, the virtual memory object (and its
125: * backing storage -- its memory object) only contain
126: * those pages that have been changed. The "shadow"
127: * field refers to the virtual memory object that contains
128: * the remainder of the contents. The "shadow_offset"
129: * field indicates where in the "shadow" these contents begin.
130: * The "copy" field refers to a virtual memory object
131: * to which changed pages must be copied before changing
132: * this object, in order to implement another form
133: * of copy-on-write optimization.
134: *
135: * The virtual memory object structure also records
136: * the attributes associated with its memory object.
137: * The "pager_ready", "can_persist" and "copy_strategy"
138: * fields represent those attributes. The "cached_list"
139: * field is used in the implementation of the persistence
140: * attribute.
141: *
142: * ZZZ Continue this comment.
143: */
144:
1.1.1.2 root 145: struct kmem_cache vm_object_cache; /* vm backing store cache */
1.1 root 146:
147: /*
148: * All wired-down kernel memory belongs to a single virtual
149: * memory object (kernel_object) to avoid wasting data structures.
150: */
1.1.1.2 root 151: static struct vm_object kernel_object_store;
152: vm_object_t kernel_object = &kernel_object_store;
1.1 root 153:
154: /*
155: * Virtual memory objects that are not referenced by
156: * any address maps, but that are allowed to persist
157: * (an attribute specified by the associated memory manager),
158: * are kept in a queue (vm_object_cached_list).
159: *
160: * When an object from this queue is referenced again,
161: * for example to make another address space mapping,
162: * it must be removed from the queue. That is, the
163: * queue contains *only* objects with zero references.
164: *
165: * The kernel may choose to terminate objects from this
166: * queue in order to reclaim storage. The current policy
1.1.1.5 ! root 167: * is to let memory pressure dynamically adjust the number
! 168: * of unreferenced objects. The pageout daemon attempts to
! 169: * collect objects after removing pages from them.
1.1 root 170: *
171: * A simple lock (accessed by routines
172: * vm_object_cache_{lock,lock_try,unlock}) governs the
173: * object cache. It must be held when objects are
174: * added to or removed from the cache (in vm_object_terminate).
175: * The routines that acquire a reference to a virtual
176: * memory object based on one of the memory object ports
177: * must also lock the cache.
178: *
179: * Ideally, the object cache should be more isolated
180: * from the reference mechanism, so that the lock need
181: * not be held to make simple references.
182: */
183: queue_head_t vm_object_cached_list;
184: int vm_object_cached_count;
185:
186: decl_simple_lock_data(,vm_object_cached_lock_data)
187:
188: #define vm_object_cache_lock() \
189: simple_lock(&vm_object_cached_lock_data)
190: #define vm_object_cache_lock_try() \
191: simple_lock_try(&vm_object_cached_lock_data)
192: #define vm_object_cache_unlock() \
193: simple_unlock(&vm_object_cached_lock_data)
194:
195: /*
1.1.1.2 root 196: * Number of physical pages referenced by cached objects.
197: * This counter is protected by its own lock to work around
198: * lock ordering issues.
199: */
200: int vm_object_cached_pages;
201:
202: decl_simple_lock_data(,vm_object_cached_pages_lock_data)
203:
204: /*
1.1 root 205: * Virtual memory objects are initialized from
206: * a template (see vm_object_allocate).
207: *
208: * When adding a new field to the virtual memory
209: * object structure, be sure to add initialization
210: * (see vm_object_init).
211: */
1.1.1.2 root 212: struct vm_object vm_object_template;
1.1 root 213:
214: /*
215: * vm_object_allocate:
216: *
217: * Returns a new object with the given size.
218: */
219:
1.1.1.2 root 220: static void _vm_object_setup(
221: vm_object_t object,
222: vm_size_t size)
223: {
224: *object = vm_object_template;
225: queue_init(&object->memq);
226: vm_object_lock_init(object);
227: object->size = size;
228: }
229:
1.1 root 230: vm_object_t _vm_object_allocate(
231: vm_size_t size)
232: {
1.1.1.3 root 233: vm_object_t object;
1.1 root 234:
1.1.1.2 root 235: object = (vm_object_t) kmem_cache_alloc(&vm_object_cache);
1.1.1.5 ! root 236: if (!object)
! 237: return 0;
1.1 root 238:
1.1.1.2 root 239: _vm_object_setup(object, size);
1.1 root 240:
241: return object;
242: }
243:
244: vm_object_t vm_object_allocate(
245: vm_size_t size)
246: {
1.1.1.3 root 247: vm_object_t object;
248: ipc_port_t port;
1.1 root 249:
250: object = _vm_object_allocate(size);
1.1.1.5 ! root 251: if (object == 0)
! 252: panic("vm_object_allocate");
1.1 root 253: port = ipc_port_alloc_kernel();
254: if (port == IP_NULL)
255: panic("vm_object_allocate");
256: object->pager_name = port;
257: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
258:
259: return object;
260: }
261:
262: /*
263: * vm_object_bootstrap:
264: *
265: * Initialize the VM objects module.
266: */
267: void vm_object_bootstrap(void)
268: {
1.1.1.2 root 269: kmem_cache_init(&vm_object_cache, "vm_object",
1.1.1.5 ! root 270: sizeof(struct vm_object), 0, NULL, 0);
1.1 root 271:
272: queue_init(&vm_object_cached_list);
273: simple_lock_init(&vm_object_cached_lock_data);
274:
275: /*
276: * Fill in a template object, for quick initialization
277: */
278:
1.1.1.2 root 279: vm_object_template.ref_count = 1;
280: vm_object_template.size = 0;
281: vm_object_template.resident_page_count = 0;
282: vm_object_template.copy = VM_OBJECT_NULL;
283: vm_object_template.shadow = VM_OBJECT_NULL;
284: vm_object_template.shadow_offset = (vm_offset_t) 0;
285:
286: vm_object_template.pager = IP_NULL;
287: vm_object_template.paging_offset = 0;
288: vm_object_template.pager_request = PAGER_REQUEST_NULL;
289: vm_object_template.pager_name = IP_NULL;
290:
291: vm_object_template.pager_created = FALSE;
292: vm_object_template.pager_initialized = FALSE;
293: vm_object_template.pager_ready = FALSE;
1.1 root 294:
1.1.1.2 root 295: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE;
1.1 root 296: /* ignored if temporary, will be reset before
297: * permanent object becomes ready */
1.1.1.2 root 298: vm_object_template.use_shared_copy = FALSE;
299: vm_object_template.shadowed = FALSE;
1.1 root 300:
1.1.1.2 root 301: vm_object_template.absent_count = 0;
302: vm_object_template.all_wanted = 0; /* all bits FALSE */
1.1 root 303:
1.1.1.2 root 304: vm_object_template.paging_in_progress = 0;
1.1.1.5 ! root 305: vm_object_template.used_for_pageout = FALSE;
1.1.1.2 root 306: vm_object_template.can_persist = FALSE;
1.1.1.5 ! root 307: vm_object_template.cached = FALSE;
1.1.1.2 root 308: vm_object_template.internal = TRUE;
309: vm_object_template.temporary = TRUE;
310: vm_object_template.alive = TRUE;
311: vm_object_template.lock_in_progress = FALSE;
312: vm_object_template.lock_restart = FALSE;
313: vm_object_template.use_old_pageout = TRUE; /* XXX change later */
314: vm_object_template.last_alloc = (vm_offset_t) 0;
1.1 root 315:
316: #if MACH_PAGEMAP
1.1.1.2 root 317: vm_object_template.existence_info = VM_EXTERNAL_NULL;
1.1 root 318: #endif /* MACH_PAGEMAP */
319:
320: /*
321: * Initialize the "kernel object"
322: */
323:
1.1.1.2 root 324: _vm_object_setup(kernel_object,
1.1 root 325: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
326:
327: /*
328: * Initialize the "submap object". Make it as large as the
329: * kernel object so that no limit is imposed on submap sizes.
330: */
331:
1.1.1.2 root 332: _vm_object_setup(vm_submap_object,
1.1 root 333: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
334:
335: #if MACH_PAGEMAP
336: vm_external_module_initialize();
337: #endif /* MACH_PAGEMAP */
338: }
339:
340: void vm_object_init(void)
341: {
342: /*
343: * Finish initializing the kernel object.
344: * The submap object doesn't need a name port.
345: */
346:
347: kernel_object->pager_name = ipc_port_alloc_kernel();
348: ipc_kobject_set(kernel_object->pager_name,
349: (ipc_kobject_t) kernel_object,
350: IKOT_PAGING_NAME);
351: }
352:
353: /*
1.1.1.5 ! root 354: * Object cache management functions.
! 355: *
! 356: * Both the cache and the object must be locked
! 357: * before calling these functions.
! 358: */
! 359:
! 360: static void vm_object_cache_add(
! 361: vm_object_t object)
! 362: {
! 363: assert(!object->cached);
! 364: queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list);
! 365: vm_object_cached_count++;
! 366: vm_object_cached_pages_update(object->resident_page_count);
! 367: object->cached = TRUE;
! 368: }
! 369:
! 370: static void vm_object_cache_remove(
! 371: vm_object_t object)
! 372: {
! 373: assert(object->cached);
! 374: queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list);
! 375: vm_object_cached_count--;
! 376: vm_object_cached_pages_update(-object->resident_page_count);
! 377: object->cached = FALSE;
! 378: }
! 379:
! 380: void vm_object_collect(
! 381: register vm_object_t object)
! 382: {
! 383: vm_object_unlock(object);
! 384:
! 385: /*
! 386: * The cache lock must be acquired in the proper order.
! 387: */
! 388:
! 389: vm_object_cache_lock();
! 390: vm_object_lock(object);
! 391:
! 392: /*
! 393: * If the object was referenced while the lock was
! 394: * dropped, cancel the termination.
! 395: */
! 396:
! 397: if (!vm_object_collectable(object)) {
! 398: vm_object_unlock(object);
! 399: vm_object_cache_unlock();
! 400: return;
! 401: }
! 402:
! 403: vm_object_cache_remove(object);
! 404: vm_object_terminate(object);
! 405: }
! 406:
! 407: /*
1.1 root 408: * vm_object_reference:
409: *
410: * Gets another reference to the given object.
411: */
412: void vm_object_reference(
1.1.1.3 root 413: vm_object_t object)
1.1 root 414: {
415: if (object == VM_OBJECT_NULL)
416: return;
417:
418: vm_object_lock(object);
419: assert(object->ref_count > 0);
420: object->ref_count++;
421: vm_object_unlock(object);
422: }
423:
424: /*
425: * vm_object_deallocate:
426: *
427: * Release a reference to the specified object,
428: * gained either through a vm_object_allocate
429: * or a vm_object_reference call. When all references
430: * are gone, storage associated with this object
431: * may be relinquished.
432: *
433: * No object may be locked.
434: */
435: void vm_object_deallocate(
1.1.1.3 root 436: vm_object_t object)
1.1 root 437: {
438: vm_object_t temp;
439:
440: while (object != VM_OBJECT_NULL) {
441:
442: /*
443: * The cache holds a reference (uncounted) to
444: * the object; we must lock it before removing
445: * the object.
446: */
447:
448: vm_object_cache_lock();
449:
450: /*
451: * Lose the reference
452: */
453: vm_object_lock(object);
454: if (--(object->ref_count) > 0) {
455:
456: /*
457: * If there are still references, then
458: * we are done.
459: */
460: vm_object_unlock(object);
461: vm_object_cache_unlock();
462: return;
463: }
464:
465: /*
466: * See whether this object can persist. If so, enter
1.1.1.5 ! root 467: * it in the cache.
1.1 root 468: */
1.1.1.5 ! root 469: if (object->can_persist && (object->resident_page_count > 0)) {
! 470: vm_object_cache_add(object);
1.1 root 471: vm_object_cache_unlock();
472: vm_object_unlock(object);
1.1.1.5 ! root 473: return;
! 474: }
1.1 root 475:
1.1.1.5 ! root 476: if (object->pager_created &&
! 477: !object->pager_initialized) {
1.1 root 478:
479: /*
1.1.1.5 ! root 480: * Have to wait for initialization.
! 481: * Put reference back and retry
! 482: * when it's initialized.
1.1 root 483: */
484:
1.1.1.5 ! root 485: object->ref_count++;
! 486: vm_object_assert_wait(object,
! 487: VM_OBJECT_EVENT_INITIALIZED, FALSE);
! 488: vm_object_unlock(object);
! 489: vm_object_cache_unlock();
! 490: thread_block((void (*)()) 0);
! 491: continue;
1.1 root 492: }
493:
494: /*
495: * Take the reference to the shadow object
496: * out of the object to be destroyed.
497: */
498:
499: temp = object->shadow;
500:
501: /*
502: * Destroy the object; the cache lock will
503: * be released in the process.
504: */
505:
506: vm_object_terminate(object);
507:
508: /*
509: * Deallocate the reference to the shadow
510: * by continuing the loop with that object
511: * in place of the original.
512: */
513:
514: object = temp;
515: }
516: }
517:
518: /*
519: * Routine: vm_object_terminate
520: * Purpose:
521: * Free all resources associated with a vm_object.
522: * In/out conditions:
523: * Upon entry, the object and the cache must be locked,
524: * and the object must have no references.
525: *
526: * The shadow object reference is left alone.
527: *
528: * Upon exit, the cache will be unlocked, and the
529: * object will cease to exist.
530: */
531: void vm_object_terminate(
1.1.1.3 root 532: vm_object_t object)
1.1 root 533: {
1.1.1.3 root 534: vm_page_t p;
535: vm_object_t shadow_object;
1.1 root 536:
537: /*
538: * Make sure the object isn't already being terminated
539: */
540:
541: assert(object->alive);
542: object->alive = FALSE;
543:
544: /*
545: * Make sure no one can look us up now.
546: */
547:
548: vm_object_remove(object);
549: vm_object_cache_unlock();
550:
551: /*
552: * Detach the object from its shadow if we are the shadow's
553: * copy.
554: */
555: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
556: vm_object_lock(shadow_object);
557: assert((shadow_object->copy == object) ||
558: (shadow_object->copy == VM_OBJECT_NULL));
559: shadow_object->copy = VM_OBJECT_NULL;
560: vm_object_unlock(shadow_object);
561: }
562:
563: /*
564: * The pageout daemon might be playing with our pages.
565: * Now that the object is dead, it won't touch any more
566: * pages, but some pages might already be on their way out.
567: * Hence, we wait until the active paging activities have ceased.
568: */
569:
570: vm_object_paging_wait(object, FALSE);
571:
572: /*
573: * Clean or free the pages, as appropriate.
574: * It is possible for us to find busy/absent pages,
575: * if some faults on this object were aborted.
576: */
577:
578: if ((object->temporary) || (object->pager == IP_NULL)) {
579: while (!queue_empty(&object->memq)) {
580: p = (vm_page_t) queue_first(&object->memq);
581:
582: VM_PAGE_CHECK(p);
583:
584: VM_PAGE_FREE(p);
585: }
586: } else while (!queue_empty(&object->memq)) {
587: p = (vm_page_t) queue_first(&object->memq);
588:
589: VM_PAGE_CHECK(p);
590:
591: vm_page_lock_queues();
592: VM_PAGE_QUEUES_REMOVE(p);
593: vm_page_unlock_queues();
594:
595: if (p->absent || p->private) {
596:
597: /*
598: * For private pages, VM_PAGE_FREE just
599: * leaves the page structure around for
600: * its owner to clean up. For absent
601: * pages, the structure is returned to
602: * the appropriate pool.
603: */
604:
605: goto free_page;
606: }
607:
608: if (!p->dirty)
609: p->dirty = pmap_is_modified(p->phys_addr);
610:
611: if (p->dirty || p->precious) {
612: p->busy = TRUE;
613: vm_pageout_page(p, FALSE, TRUE); /* flush page */
614: } else {
615: free_page:
616: VM_PAGE_FREE(p);
617: }
618: }
619:
620: assert(object->ref_count == 0);
621: assert(object->paging_in_progress == 0);
1.1.1.5 ! root 622: assert(!object->cached);
1.1 root 623:
624: /*
625: * Throw away port rights... note that they may
626: * already have been thrown away (by vm_object_destroy
627: * or memory_object_destroy).
628: *
629: * Instead of destroying the control and name ports,
630: * we send all rights off to the memory manager instead,
631: * using memory_object_terminate.
632: */
633:
634: vm_object_unlock(object);
635:
636: if (object->pager != IP_NULL) {
637: /* consumes our rights for pager, pager_request, pager_name */
638: memory_object_release(object->pager,
639: object->pager_request,
640: object->pager_name);
641: } else if (object->pager_name != IP_NULL) {
642: /* consumes our right for pager_name */
643: ipc_port_dealloc_kernel(object->pager_name);
644: }
645:
646: #if MACH_PAGEMAP
647: vm_external_destroy(object->existence_info);
648: #endif /* MACH_PAGEMAP */
649:
650: /*
651: * Free the space for the object.
652: */
653:
1.1.1.2 root 654: kmem_cache_free(&vm_object_cache, (vm_offset_t) object);
1.1 root 655: }
656:
657: /*
658: * Routine: vm_object_pager_wakeup
659: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
660: */
661:
662: void
663: vm_object_pager_wakeup(
664: ipc_port_t pager)
665: {
666: boolean_t someone_waiting;
667:
668: /*
669: * If anyone was waiting for the memory_object_terminate
670: * to be queued, wake them up now.
671: */
672: vm_object_cache_lock();
673: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
674: someone_waiting = (pager->ip_kobject != IKO_NULL);
675: if (ip_active(pager))
676: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
677: vm_object_cache_unlock();
678: if (someone_waiting) {
679: thread_wakeup((event_t) pager);
680: }
681: }
682:
683: /*
684: * Routine: memory_object_release
685: * Purpose: Terminate the pager and release port rights,
686: * just like memory_object_terminate, except
687: * that we wake up anyone blocked in vm_object_enter
688: * waiting for termination message to be queued
689: * before calling memory_object_init.
690: */
691: void memory_object_release(
692: ipc_port_t pager,
693: pager_request_t pager_request,
694: ipc_port_t pager_name)
695: {
696:
697: /*
698: * Keep a reference to pager port;
699: * the terminate might otherwise release all references.
700: */
701: ip_reference(pager);
702:
703: /*
704: * Terminate the pager.
705: */
706: (void) memory_object_terminate(pager, pager_request, pager_name);
707:
708: /*
709: * Wakeup anyone waiting for this terminate
710: */
711: vm_object_pager_wakeup(pager);
712:
713: /*
714: * Release reference to pager port.
715: */
716: ip_release(pager);
717: }
718:
719: /*
720: * Routine: vm_object_abort_activity [internal use only]
721: * Purpose:
722: * Abort paging requests pending on this object.
723: * In/out conditions:
724: * The object is locked on entry and exit.
725: */
726: void vm_object_abort_activity(
727: vm_object_t object)
728: {
729: vm_page_t p;
730: vm_page_t next;
731:
732: /*
733: * Abort all activity that would be waiting
734: * for a result on this memory object.
735: *
736: * We could also choose to destroy all pages
737: * that we have in memory for this object, but
738: * we don't.
739: */
740:
741: p = (vm_page_t) queue_first(&object->memq);
742: while (!queue_end(&object->memq, (queue_entry_t) p)) {
743: next = (vm_page_t) queue_next(&p->listq);
744:
745: /*
746: * If it's being paged in, destroy it.
747: * If an unlock has been requested, start it again.
748: */
749:
750: if (p->busy && p->absent) {
751: VM_PAGE_FREE(p);
752: }
753: else {
754: if (p->unlock_request != VM_PROT_NONE)
755: p->unlock_request = VM_PROT_NONE;
756: PAGE_WAKEUP(p);
757: }
758:
759: p = next;
760: }
761:
762: /*
763: * Wake up threads waiting for the memory object to
764: * become ready.
765: */
766:
767: object->pager_ready = TRUE;
768: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
769: }
770:
771: /*
772: * Routine: memory_object_destroy [user interface]
773: * Purpose:
774: * Shut down a memory object, despite the
775: * presence of address map (or other) references
776: * to the vm_object.
777: * Note:
778: * This routine may be called either from the user interface,
779: * or from port destruction handling (via vm_object_destroy).
780: */
781: kern_return_t memory_object_destroy(
782: vm_object_t object,
783: kern_return_t reason)
784: {
785: ipc_port_t old_object, old_name;
786: pager_request_t old_control;
787:
788: if (object == VM_OBJECT_NULL)
789: return KERN_SUCCESS;
790:
791: /*
792: * Remove the port associations immediately.
793: *
794: * This will prevent the memory manager from further
795: * meddling. [If it wanted to flush data or make
796: * other changes, it should have done so before performing
797: * the destroy call.]
798: */
799:
800: vm_object_cache_lock();
801: vm_object_lock(object);
802: vm_object_remove(object);
803: object->can_persist = FALSE;
804: vm_object_cache_unlock();
805:
806: /*
807: * Rip out the ports from the vm_object now... this
808: * will prevent new memory_object calls from succeeding.
809: */
810:
811: old_object = object->pager;
812: object->pager = IP_NULL;
813:
814: old_control = object->pager_request;
815: object->pager_request = PAGER_REQUEST_NULL;
816:
817: old_name = object->pager_name;
818: object->pager_name = IP_NULL;
819:
820:
821: /*
822: * Wait for existing paging activity (that might
823: * have the old ports) to subside.
824: */
825:
826: vm_object_paging_wait(object, FALSE);
827: vm_object_unlock(object);
828:
829: /*
830: * Shut down the ports now.
831: *
832: * [Paging operations may be proceeding concurrently --
833: * they'll get the null values established above.]
834: */
835:
836: if (old_object != IP_NULL) {
837: /* consumes our rights for object, control, name */
838: memory_object_release(old_object, old_control,
839: old_name);
840: } else if (old_name != IP_NULL) {
841: /* consumes our right for name */
842: ipc_port_dealloc_kernel(object->pager_name);
843: }
844:
845: /*
846: * Lose the reference that was donated for this routine
847: */
848:
849: vm_object_deallocate(object);
850:
851: return KERN_SUCCESS;
852: }
853:
854: /*
855: * Routine: vm_object_pmap_protect
856: *
857: * Purpose:
858: * Reduces the permission for all physical
859: * pages in the specified object range.
860: *
861: * If removing write permission only, it is
862: * sufficient to protect only the pages in
863: * the top-level object; only those pages may
864: * have write permission.
865: *
866: * If removing all access, we must follow the
867: * shadow chain from the top-level object to
868: * remove access to all pages in shadowed objects.
869: *
870: * The object must *not* be locked. The object must
871: * be temporary/internal.
872: *
873: * If pmap is not NULL, this routine assumes that
874: * the only mappings for the pages are in that
875: * pmap.
876: */
877: boolean_t vm_object_pmap_protect_by_page = FALSE;
878:
879: void vm_object_pmap_protect(
1.1.1.3 root 880: vm_object_t object,
881: vm_offset_t offset,
1.1.1.2 root 882: vm_size_t size,
1.1 root 883: pmap_t pmap,
884: vm_offset_t pmap_start,
885: vm_prot_t prot)
886: {
887: if (object == VM_OBJECT_NULL)
888: return;
889:
890: vm_object_lock(object);
891:
892: assert(object->temporary && object->internal);
893:
894: while (TRUE) {
895: if (object->resident_page_count > atop(size) / 2 &&
896: pmap != PMAP_NULL) {
897: vm_object_unlock(object);
898: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
899: return;
900: }
901:
902: {
1.1.1.3 root 903: vm_page_t p;
904: vm_offset_t end;
1.1 root 905:
906: end = offset + size;
907:
908: queue_iterate(&object->memq, p, vm_page_t, listq) {
909: if (!p->fictitious &&
910: (offset <= p->offset) &&
911: (p->offset < end)) {
912: if ((pmap == PMAP_NULL) ||
913: vm_object_pmap_protect_by_page) {
914: pmap_page_protect(p->phys_addr,
915: prot & ~p->page_lock);
916: } else {
917: vm_offset_t start =
918: pmap_start +
919: (p->offset - offset);
920:
921: pmap_protect(pmap,
922: start,
923: start + PAGE_SIZE,
924: prot);
925: }
926: }
927: }
928: }
929:
930: if (prot == VM_PROT_NONE) {
931: /*
932: * Must follow shadow chain to remove access
933: * to pages in shadowed objects.
934: */
1.1.1.3 root 935: vm_object_t next_object;
1.1 root 936:
937: next_object = object->shadow;
938: if (next_object != VM_OBJECT_NULL) {
939: offset += object->shadow_offset;
940: vm_object_lock(next_object);
941: vm_object_unlock(object);
942: object = next_object;
943: }
944: else {
945: /*
946: * End of chain - we are done.
947: */
948: break;
949: }
950: }
951: else {
952: /*
953: * Pages in shadowed objects may never have
954: * write permission - we may stop here.
955: */
956: break;
957: }
958: }
959:
960: vm_object_unlock(object);
961: }
962:
963: /*
964: * vm_object_pmap_remove:
965: *
966: * Removes all physical pages in the specified
967: * object range from all physical maps.
968: *
969: * The object must *not* be locked.
970: */
971: void vm_object_pmap_remove(
1.1.1.3 root 972: vm_object_t object,
973: vm_offset_t start,
974: vm_offset_t end)
1.1 root 975: {
1.1.1.3 root 976: vm_page_t p;
1.1 root 977:
978: if (object == VM_OBJECT_NULL)
979: return;
980:
981: vm_object_lock(object);
982: queue_iterate(&object->memq, p, vm_page_t, listq) {
983: if (!p->fictitious &&
984: (start <= p->offset) &&
985: (p->offset < end))
986: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
987: }
988: vm_object_unlock(object);
989: }
990:
991: /*
992: * Routine: vm_object_copy_slowly
993: *
994: * Description:
995: * Copy the specified range of the source
996: * virtual memory object without using
997: * protection-based optimizations (such
998: * as copy-on-write). The pages in the
999: * region are actually copied.
1000: *
1001: * In/out conditions:
1002: * The caller must hold a reference and a lock
1003: * for the source virtual memory object. The source
1004: * object will be returned *unlocked*.
1005: *
1006: * Results:
1007: * If the copy is completed successfully, KERN_SUCCESS is
1008: * returned. If the caller asserted the interruptible
1009: * argument, and an interruption occurred while waiting
1010: * for a user-generated event, MACH_SEND_INTERRUPTED is
1011: * returned. Other values may be returned to indicate
1012: * hard errors during the copy operation.
1013: *
1014: * A new virtual memory object is returned in a
1015: * parameter (_result_object). The contents of this
1016: * new object, starting at a zero offset, are a copy
1017: * of the source memory region. In the event of
1018: * an error, this parameter will contain the value
1019: * VM_OBJECT_NULL.
1020: */
1021: kern_return_t vm_object_copy_slowly(
1022: vm_object_t src_object,
1023: vm_offset_t src_offset,
1024: vm_size_t size,
1025: boolean_t interruptible,
1026: vm_object_t *_result_object) /* OUT */
1027: {
1028: vm_object_t new_object;
1029: vm_offset_t new_offset;
1030:
1031: if (size == 0) {
1032: vm_object_unlock(src_object);
1033: *_result_object = VM_OBJECT_NULL;
1034: return KERN_INVALID_ARGUMENT;
1035: }
1036:
1037: /*
1038: * Prevent destruction of the source object while we copy.
1039: */
1040:
1041: assert(src_object->ref_count > 0);
1042: src_object->ref_count++;
1043: vm_object_unlock(src_object);
1044:
1045: /*
1046: * Create a new object to hold the copied pages.
1047: * A few notes:
1048: * We fill the new object starting at offset 0,
1049: * regardless of the input offset.
1050: * We don't bother to lock the new object within
1051: * this routine, since we have the only reference.
1052: */
1053:
1054: new_object = vm_object_allocate(size);
1055: new_offset = 0;
1056:
1057: assert(size == trunc_page(size)); /* Will the loop terminate? */
1058:
1059: for ( ;
1060: size != 0 ;
1061: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1062: ) {
1063: vm_page_t new_page;
1064: vm_fault_return_t result;
1065:
1066: while ((new_page = vm_page_alloc(new_object, new_offset))
1067: == VM_PAGE_NULL) {
1068: VM_PAGE_WAIT((void (*)()) 0);
1069: }
1070:
1071: do {
1072: vm_prot_t prot = VM_PROT_READ;
1073: vm_page_t _result_page;
1074: vm_page_t top_page;
1075: vm_page_t result_page;
1076:
1077: vm_object_lock(src_object);
1078: src_object->paging_in_progress++;
1079:
1080: result = vm_fault_page(src_object, src_offset,
1081: VM_PROT_READ, FALSE, interruptible,
1082: &prot, &_result_page, &top_page,
1083: FALSE, (void (*)()) 0);
1084:
1085: switch(result) {
1086: case VM_FAULT_SUCCESS:
1087: result_page = _result_page;
1088:
1089: /*
1090: * We don't need to hold the object
1091: * lock -- the busy page will be enough.
1092: * [We don't care about picking up any
1093: * new modifications.]
1094: *
1095: * Copy the page to the new object.
1096: *
1097: * POLICY DECISION:
1098: * If result_page is clean,
1099: * we could steal it instead
1100: * of copying.
1101: */
1102:
1103: vm_object_unlock(result_page->object);
1104: vm_page_copy(result_page, new_page);
1105:
1106: /*
1107: * Let go of both pages (make them
1108: * not busy, perform wakeup, activate).
1109: */
1110:
1111: new_page->busy = FALSE;
1112: new_page->dirty = TRUE;
1113: vm_object_lock(result_page->object);
1114: PAGE_WAKEUP_DONE(result_page);
1115:
1116: vm_page_lock_queues();
1117: if (!result_page->active &&
1118: !result_page->inactive)
1119: vm_page_activate(result_page);
1120: vm_page_activate(new_page);
1121: vm_page_unlock_queues();
1122:
1123: /*
1124: * Release paging references and
1125: * top-level placeholder page, if any.
1126: */
1127:
1128: vm_fault_cleanup(result_page->object,
1129: top_page);
1130:
1131: break;
1132:
1133: case VM_FAULT_RETRY:
1134: break;
1135:
1136: case VM_FAULT_MEMORY_SHORTAGE:
1137: VM_PAGE_WAIT((void (*)()) 0);
1138: break;
1139:
1140: case VM_FAULT_FICTITIOUS_SHORTAGE:
1141: vm_page_more_fictitious();
1142: break;
1143:
1144: case VM_FAULT_INTERRUPTED:
1145: vm_page_free(new_page);
1146: vm_object_deallocate(new_object);
1147: vm_object_deallocate(src_object);
1148: *_result_object = VM_OBJECT_NULL;
1149: return MACH_SEND_INTERRUPTED;
1150:
1151: case VM_FAULT_MEMORY_ERROR:
1152: /*
1153: * A policy choice:
1154: * (a) ignore pages that we can't
1155: * copy
1156: * (b) return the null object if
1157: * any page fails [chosen]
1158: */
1159:
1160: vm_page_free(new_page);
1161: vm_object_deallocate(new_object);
1162: vm_object_deallocate(src_object);
1163: *_result_object = VM_OBJECT_NULL;
1164: return KERN_MEMORY_ERROR;
1165: }
1166: } while (result != VM_FAULT_SUCCESS);
1167: }
1168:
1169: /*
1170: * Lose the extra reference, and return our object.
1171: */
1172:
1173: vm_object_deallocate(src_object);
1174: *_result_object = new_object;
1175: return KERN_SUCCESS;
1176: }
1177:
1178: /*
1179: * Routine: vm_object_copy_temporary
1180: *
1181: * Purpose:
1182: * Copy the specified range of the source virtual
1183: * memory object, if it can be done without blocking.
1184: *
1185: * Results:
1186: * If the copy is successful, the copy is returned in
1187: * the arguments; otherwise, the arguments are not
1188: * affected.
1189: *
1190: * In/out conditions:
1191: * The object should be unlocked on entry and exit.
1192: */
1193:
1194: boolean_t vm_object_copy_temporary(
1195: vm_object_t *_object, /* INOUT */
1196: vm_offset_t *_offset, /* INOUT */
1197: boolean_t *_src_needs_copy, /* OUT */
1198: boolean_t *_dst_needs_copy) /* OUT */
1199: {
1200: vm_object_t object = *_object;
1201:
1202: if (object == VM_OBJECT_NULL) {
1203: *_src_needs_copy = FALSE;
1204: *_dst_needs_copy = FALSE;
1205: return TRUE;
1206: }
1207:
1208: /*
1209: * If the object is temporary, we can perform
1210: * a symmetric copy-on-write without asking.
1211: */
1212:
1213: vm_object_lock(object);
1214: if (object->temporary) {
1215:
1216: /*
1217: * Shared objects use delayed copy
1218: */
1219: if (object->use_shared_copy) {
1220:
1221: /*
1222: * Asymmetric copy strategy. Destination
1223: * must be copied (to allow copy object reuse).
1224: * Source is unaffected.
1225: */
1226: vm_object_unlock(object);
1227: object = vm_object_copy_delayed(object);
1228: *_object = object;
1229: *_src_needs_copy = FALSE;
1230: *_dst_needs_copy = TRUE;
1231: return TRUE;
1232: }
1233:
1234: /*
1235: * Make another reference to the object.
1236: *
1237: * Leave object/offset unchanged.
1238: */
1239:
1240: assert(object->ref_count > 0);
1241: object->ref_count++;
1242: object->shadowed = TRUE;
1243: vm_object_unlock(object);
1244:
1245: /*
1246: * Both source and destination must make
1247: * shadows, and the source must be made
1248: * read-only if not already.
1249: */
1250:
1251: *_src_needs_copy = TRUE;
1252: *_dst_needs_copy = TRUE;
1253: return TRUE;
1254: }
1255:
1256: if (object->pager_ready &&
1257: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1258: /* XXX Do something intelligent (see temporary code above) */
1259: }
1260: vm_object_unlock(object);
1261:
1262: return FALSE;
1263: }
1264:
1265: /*
1266: * Routine: vm_object_copy_call [internal]
1267: *
1268: * Description:
1269: * Copy the specified (src_offset, size) portion
1270: * of the source object (src_object), using the
1271: * user-managed copy algorithm.
1272: *
1273: * In/out conditions:
1274: * The source object must be locked on entry. It
1275: * will be *unlocked* on exit.
1276: *
1277: * Results:
1278: * If the copy is successful, KERN_SUCCESS is returned.
1279: * This routine is interruptible; if a wait for
1280: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1281: * is returned. Other return values indicate hard errors
1282: * in creating the user-managed memory object for the copy.
1283: *
1284: * A new object that represents the copied virtual
1285: * memory is returned in a parameter (*_result_object).
1286: * If the return value indicates an error, this parameter
1287: * is not valid.
1288: */
1289: kern_return_t vm_object_copy_call(
1290: vm_object_t src_object,
1291: vm_offset_t src_offset,
1292: vm_size_t size,
1293: vm_object_t *_result_object) /* OUT */
1294: {
1295: vm_offset_t src_end = src_offset + size;
1296: ipc_port_t new_memory_object;
1297: vm_object_t new_object;
1298: vm_page_t p;
1299:
1300: /*
1301: * Create a memory object port to be associated
1302: * with this new vm_object.
1303: *
1304: * Since the kernel has the only rights to this
1305: * port, we need not hold the cache lock.
1306: *
1307: * Since we have the only object reference, we
1308: * need not be worried about collapse operations.
1309: *
1310: */
1311:
1312: new_memory_object = ipc_port_alloc_kernel();
1.1.1.4 root 1313: if (new_memory_object == IP_NULL)
1314: return KERN_RESOURCE_SHORTAGE;
1315:
1316: /*
1317: * Set the backing object for the new
1318: * temporary object.
1319: */
1320:
1321: assert(src_object->ref_count > 0);
1322: src_object->ref_count++;
1323: vm_object_paging_begin(src_object);
1324: vm_object_unlock(src_object);
1.1 root 1325:
1326: /* we hold a naked receive right for new_memory_object */
1327: (void) ipc_port_make_send(new_memory_object);
1328: /* now we also hold a naked send right for new_memory_object */
1329:
1330: /*
1331: * Let the memory manager know that a copy operation
1332: * is in progress. Note that we're using the old
1333: * memory object's ports (for which we're holding
1334: * a paging reference)... the memory manager cannot
1335: * yet affect the new memory object.
1336: */
1337:
1338: (void) memory_object_copy(src_object->pager,
1339: src_object->pager_request,
1340: src_offset, size,
1341: new_memory_object);
1342: /* no longer hold the naked receive right for new_memory_object */
1343:
1344: vm_object_lock(src_object);
1345: vm_object_paging_end(src_object);
1346:
1347: /*
1348: * Remove write access from all of the pages of
1349: * the old memory object that we can.
1350: */
1351:
1352: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1353: if (!p->fictitious &&
1354: (src_offset <= p->offset) &&
1355: (p->offset < src_end) &&
1356: !(p->page_lock & VM_PROT_WRITE)) {
1357: p->page_lock |= VM_PROT_WRITE;
1358: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1359: }
1360: }
1361:
1362: vm_object_unlock(src_object);
1363:
1364: /*
1365: * Initialize the rest of the paging stuff
1366: */
1367:
1368: new_object = vm_object_enter(new_memory_object, size, FALSE);
1369: new_object->shadow = src_object;
1370: new_object->shadow_offset = src_offset;
1371:
1372: /*
1373: * Drop the reference for new_memory_object taken above.
1374: */
1375:
1376: ipc_port_release_send(new_memory_object);
1377: /* no longer hold the naked send right for new_memory_object */
1378:
1379: *_result_object = new_object;
1380: return KERN_SUCCESS;
1381: }
1382:
1383: /*
1384: * Routine: vm_object_copy_delayed [internal]
1385: *
1386: * Description:
1387: * Copy the specified virtual memory object, using
1388: * the asymmetric copy-on-write algorithm.
1389: *
1390: * In/out conditions:
1391: * The object must be unlocked on entry.
1392: *
1393: * This routine will not block waiting for user-generated
1394: * events. It is not interruptible.
1395: */
1396: vm_object_t vm_object_copy_delayed(
1397: vm_object_t src_object)
1398: {
1399: vm_object_t new_copy;
1400: vm_object_t old_copy;
1401: vm_page_t p;
1402:
1403: /*
1404: * The user-level memory manager wants to see
1405: * all of the changes to this object, but it
1406: * has promised not to make any changes on its own.
1407: *
1408: * Perform an asymmetric copy-on-write, as follows:
1409: * Create a new object, called a "copy object"
1410: * to hold pages modified by the new mapping
1411: * (i.e., the copy, not the original mapping).
1412: * Record the original object as the backing
1413: * object for the copy object. If the
1414: * original mapping does not change a page,
1415: * it may be used read-only by the copy.
1416: * Record the copy object in the original
1417: * object. When the original mapping causes
1418: * a page to be modified, it must be copied
1419: * to a new page that is "pushed" to the
1420: * copy object.
1421: * Mark the new mapping (the copy object)
1422: * copy-on-write. This makes the copy
1423: * object itself read-only, allowing it
1424: * to be reused if the original mapping
1425: * makes no changes, and simplifying the
1426: * synchronization required in the "push"
1427: * operation described above.
1428: *
1.1.1.3 root 1429: * The copy-on-write is said to be asymmetric because
1.1 root 1430: * the original object is *not* marked copy-on-write.
1431: * A copied page is pushed to the copy object, regardless
1432: * which party attempted to modify the page.
1433: *
1434: * Repeated asymmetric copy operations may be done.
1435: * If the original object has not been changed since
1436: * the last copy, its copy object can be reused.
1437: * Otherwise, a new copy object can be inserted
1438: * between the original object and its previous
1439: * copy object. Since any copy object is read-only,
1440: * this cannot affect the contents of the previous copy
1441: * object.
1442: *
1443: * Note that a copy object is higher in the object
1444: * tree than the original object; therefore, use of
1445: * the copy object recorded in the original object
1446: * must be done carefully, to avoid deadlock.
1447: */
1448:
1449: /*
1450: * Allocate a new copy object before locking, even
1451: * though we may not need it later.
1452: */
1453:
1454: new_copy = vm_object_allocate(src_object->size);
1455:
1456: vm_object_lock(src_object);
1457:
1458: /*
1459: * See whether we can reuse the result of a previous
1460: * copy operation.
1461: */
1462: Retry:
1463: old_copy = src_object->copy;
1464: if (old_copy != VM_OBJECT_NULL) {
1465: /*
1466: * Try to get the locks (out of order)
1467: */
1468: if (!vm_object_lock_try(old_copy)) {
1469: vm_object_unlock(src_object);
1470:
1471: simple_lock_pause(); /* wait a bit */
1472:
1473: vm_object_lock(src_object);
1474: goto Retry;
1475: }
1476:
1477: /*
1478: * Determine whether the old copy object has
1479: * been modified.
1480: */
1481:
1482: if (old_copy->resident_page_count == 0 &&
1483: !old_copy->pager_created) {
1484: /*
1485: * It has not been modified.
1486: *
1487: * Return another reference to
1488: * the existing copy-object.
1489: */
1490: assert(old_copy->ref_count > 0);
1491: old_copy->ref_count++;
1492: vm_object_unlock(old_copy);
1493: vm_object_unlock(src_object);
1494:
1495: vm_object_deallocate(new_copy);
1496:
1497: return old_copy;
1498: }
1499:
1500: /*
1501: * The copy-object is always made large enough to
1502: * completely shadow the original object, since
1503: * it may have several users who want to shadow
1504: * the original object at different points.
1505: */
1506:
1507: assert((old_copy->shadow == src_object) &&
1508: (old_copy->shadow_offset == (vm_offset_t) 0));
1509:
1510: /*
1511: * Make the old copy-object shadow the new one.
1512: * It will receive no more pages from the original
1513: * object.
1514: */
1515:
1516: src_object->ref_count--; /* remove ref. from old_copy */
1517: assert(src_object->ref_count > 0);
1518: old_copy->shadow = new_copy;
1519: assert(new_copy->ref_count > 0);
1520: new_copy->ref_count++;
1521: vm_object_unlock(old_copy); /* done with old_copy */
1522: }
1523:
1524: /*
1525: * Point the new copy at the existing object.
1526: */
1527:
1528: new_copy->shadow = src_object;
1529: new_copy->shadow_offset = 0;
1530: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1531: assert(src_object->ref_count > 0);
1532: src_object->ref_count++;
1533: src_object->copy = new_copy;
1534:
1535: /*
1536: * Mark all pages of the existing object copy-on-write.
1537: * This object may have a shadow chain below it, but
1538: * those pages will already be marked copy-on-write.
1539: */
1540:
1541: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1542: if (!p->fictitious)
1543: pmap_page_protect(p->phys_addr,
1544: (VM_PROT_ALL & ~VM_PROT_WRITE &
1545: ~p->page_lock));
1546: }
1547:
1548: vm_object_unlock(src_object);
1549:
1550: return new_copy;
1551: }
1552:
1553: /*
1554: * Routine: vm_object_copy_strategically
1555: *
1556: * Purpose:
1557: * Perform a copy according to the source object's
1558: * declared strategy. This operation may block,
1559: * and may be interrupted.
1560: */
1561: kern_return_t vm_object_copy_strategically(
1562: vm_object_t src_object,
1563: vm_offset_t src_offset,
1564: vm_size_t size,
1565: vm_object_t *dst_object, /* OUT */
1566: vm_offset_t *dst_offset, /* OUT */
1567: boolean_t *dst_needs_copy) /* OUT */
1568: {
1569: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1570: boolean_t interruptible = TRUE; /* XXX */
1571:
1572: assert(src_object != VM_OBJECT_NULL);
1573:
1574: vm_object_lock(src_object);
1575:
1576: /* XXX assert(!src_object->temporary); JSB FIXME */
1577:
1578: /*
1579: * The copy strategy is only valid if the memory manager
1580: * is "ready".
1581: */
1582:
1583: while (!src_object->pager_ready) {
1584: vm_object_wait( src_object,
1585: VM_OBJECT_EVENT_PAGER_READY,
1586: interruptible);
1587: if (interruptible &&
1588: (current_thread()->wait_result != THREAD_AWAKENED)) {
1589: *dst_object = VM_OBJECT_NULL;
1590: *dst_offset = 0;
1591: *dst_needs_copy = FALSE;
1592: return MACH_SEND_INTERRUPTED;
1593: }
1594: vm_object_lock(src_object);
1595: }
1596:
1597: /*
1598: * The object may be temporary (even though it is external).
1599: * If so, do a symmetric copy.
1600: */
1601:
1602: if (src_object->temporary) {
1603: /*
1604: * XXX
1605: * This does not count as intelligent!
1606: * This buys us the object->temporary optimizations,
1607: * but we aren't using a symmetric copy,
1608: * which may confuse the vm code. The correct thing
1609: * to do here is to figure out what to call to get
1610: * a temporary shadowing set up.
1611: */
1612: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1613: }
1614:
1615: /*
1616: * The object is permanent. Use the appropriate copy strategy.
1617: */
1618:
1619: switch (src_object->copy_strategy) {
1620: case MEMORY_OBJECT_COPY_NONE:
1621: if ((result = vm_object_copy_slowly(
1622: src_object,
1623: src_offset,
1624: size,
1625: interruptible,
1626: dst_object))
1627: == KERN_SUCCESS) {
1628: *dst_offset = 0;
1629: *dst_needs_copy = FALSE;
1630: }
1631: break;
1632:
1633: case MEMORY_OBJECT_COPY_CALL:
1634: if ((result = vm_object_copy_call(
1635: src_object,
1636: src_offset,
1637: size,
1638: dst_object))
1639: == KERN_SUCCESS) {
1640: *dst_offset = 0;
1641: *dst_needs_copy = FALSE;
1642: }
1643: break;
1644:
1645: case MEMORY_OBJECT_COPY_DELAY:
1646: vm_object_unlock(src_object);
1647: *dst_object = vm_object_copy_delayed(src_object);
1648: *dst_offset = src_offset;
1649: *dst_needs_copy = TRUE;
1650:
1651: result = KERN_SUCCESS;
1652: break;
1653: }
1654:
1655: return result;
1656: }
1657:
1658: /*
1659: * vm_object_shadow:
1660: *
1661: * Create a new object which is backed by the
1662: * specified existing object range. The source
1663: * object reference is deallocated.
1664: *
1665: * The new object and offset into that object
1666: * are returned in the source parameters.
1667: */
1668:
1669: void vm_object_shadow(
1670: vm_object_t *object, /* IN/OUT */
1671: vm_offset_t *offset, /* IN/OUT */
1672: vm_size_t length)
1673: {
1.1.1.3 root 1674: vm_object_t source;
1675: vm_object_t result;
1.1 root 1676:
1677: source = *object;
1678:
1679: /*
1680: * Allocate a new object with the given length
1681: */
1682:
1683: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1684: panic("vm_object_shadow: no object for shadowing");
1685:
1686: /*
1687: * The new object shadows the source object, adding
1688: * a reference to it. Our caller changes his reference
1689: * to point to the new object, removing a reference to
1690: * the source object. Net result: no change of reference
1691: * count.
1692: */
1693: result->shadow = source;
1694:
1695: /*
1696: * Store the offset into the source object,
1697: * and fix up the offset into the new object.
1698: */
1699:
1700: result->shadow_offset = *offset;
1701:
1702: /*
1703: * Return the new things
1704: */
1705:
1706: *offset = 0;
1707: *object = result;
1708: }
1709:
1710: /*
1711: * The relationship between vm_object structures and
1712: * the memory_object ports requires careful synchronization.
1713: *
1714: * All associations are created by vm_object_enter. All three
1715: * port fields are filled in, as follows:
1716: * pager: the memory_object port itself, supplied by
1717: * the user requesting a mapping (or the kernel,
1718: * when initializing internal objects); the
1719: * kernel simulates holding send rights by keeping
1720: * a port reference;
1721: * pager_request:
1722: * pager_name:
1723: * the memory object control and name ports,
1724: * created by the kernel; the kernel holds
1725: * receive (and ownership) rights to these
1726: * ports, but no other references.
1727: * All of the ports are referenced by their global names.
1728: *
1729: * When initialization is complete, the "initialized" field
1730: * is asserted. Other mappings using a particular memory object,
1731: * and any references to the vm_object gained through the
1732: * port association must wait for this initialization to occur.
1733: *
1734: * In order to allow the memory manager to set attributes before
1735: * requests (notably virtual copy operations, but also data or
1736: * unlock requests) are made, a "ready" attribute is made available.
1737: * Only the memory manager may affect the value of this attribute.
1738: * Its value does not affect critical kernel functions, such as
1739: * internal object initialization or destruction. [Furthermore,
1740: * memory objects created by the kernel are assumed to be ready
1741: * immediately; the default memory manager need not explicitly
1742: * set the "ready" attribute.]
1743: *
1744: * [Both the "initialized" and "ready" attribute wait conditions
1745: * use the "pager" field as the wait event.]
1746: *
1747: * The port associations can be broken down by any of the
1748: * following routines:
1749: * vm_object_terminate:
1750: * No references to the vm_object remain, and
1751: * the object cannot (or will not) be cached.
1752: * This is the normal case, and is done even
1753: * though one of the other cases has already been
1754: * done.
1755: * vm_object_destroy:
1756: * The memory_object port has been destroyed,
1757: * meaning that the kernel cannot flush dirty
1758: * pages or request new data or unlock existing
1759: * data.
1760: * memory_object_destroy:
1761: * The memory manager has requested that the
1762: * kernel relinquish rights to the memory object
1763: * port. [The memory manager may not want to
1764: * destroy the port, but may wish to refuse or
1765: * tear down existing memory mappings.]
1766: * Each routine that breaks an association must break all of
1767: * them at once. At some later time, that routine must clear
1768: * the vm_object port fields and release the port rights.
1769: * [Furthermore, each routine must cope with the simultaneous
1770: * or previous operations of the others.]
1771: *
1772: * In addition to the lock on the object, the vm_object_cache_lock
1773: * governs the port associations. References gained through the
1774: * port association require use of the cache lock.
1775: *
1776: * Because the port fields may be cleared spontaneously, they
1777: * cannot be used to determine whether a memory object has
1778: * ever been associated with a particular vm_object. [This
1779: * knowledge is important to the shadow object mechanism.]
1780: * For this reason, an additional "created" attribute is
1781: * provided.
1782: *
1783: * During various paging operations, the port values found in the
1784: * vm_object must be valid. To prevent these port rights from being
1785: * released, and to prevent the port associations from changing
1786: * (other than being removed, i.e., made null), routines may use
1787: * the vm_object_paging_begin/end routines [actually, macros].
1788: * The implementation uses the "paging_in_progress" and "wanted" fields.
1789: * [Operations that alter the validity of the port values include the
1790: * termination routines and vm_object_collapse.]
1791: */
1792:
1793: vm_object_t vm_object_lookup(
1794: ipc_port_t port)
1795: {
1796: vm_object_t object = VM_OBJECT_NULL;
1797:
1798: if (IP_VALID(port)) {
1799: ip_lock(port);
1800: if (ip_active(port) &&
1801: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1802: vm_object_cache_lock();
1803: object = (vm_object_t) port->ip_kobject;
1804: vm_object_lock(object);
1805:
1806: assert(object->alive);
1807:
1.1.1.5 ! root 1808: if (object->ref_count == 0)
! 1809: vm_object_cache_remove(object);
1.1 root 1810:
1811: object->ref_count++;
1812: vm_object_unlock(object);
1813: vm_object_cache_unlock();
1814: }
1815: ip_unlock(port);
1816: }
1817:
1818: return object;
1819: }
1820:
1821: vm_object_t vm_object_lookup_name(
1822: ipc_port_t port)
1823: {
1824: vm_object_t object = VM_OBJECT_NULL;
1825:
1826: if (IP_VALID(port)) {
1827: ip_lock(port);
1828: if (ip_active(port) &&
1829: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1830: vm_object_cache_lock();
1831: object = (vm_object_t) port->ip_kobject;
1832: vm_object_lock(object);
1833:
1834: assert(object->alive);
1835:
1.1.1.5 ! root 1836: if (object->ref_count == 0)
! 1837: vm_object_cache_remove(object);
1.1 root 1838:
1839: object->ref_count++;
1840: vm_object_unlock(object);
1841: vm_object_cache_unlock();
1842: }
1843: ip_unlock(port);
1844: }
1845:
1846: return object;
1847: }
1848:
1849: void vm_object_destroy(
1850: ipc_port_t pager)
1851: {
1852: vm_object_t object;
1853: pager_request_t old_request;
1854: ipc_port_t old_name;
1855:
1856: /*
1857: * Perform essentially the same operations as in vm_object_lookup,
1858: * except that this time we look up based on the memory_object
1859: * port, not the control port.
1860: */
1861: vm_object_cache_lock();
1862: if (ip_kotype(pager) != IKOT_PAGER) {
1863: vm_object_cache_unlock();
1864: return;
1865: }
1866:
1867: object = (vm_object_t) pager->ip_kobject;
1868: vm_object_lock(object);
1.1.1.5 ! root 1869: if (object->ref_count == 0)
! 1870: vm_object_cache_remove(object);
1.1 root 1871: object->ref_count++;
1872:
1873: object->can_persist = FALSE;
1874:
1875: assert(object->pager == pager);
1876:
1877: /*
1878: * Remove the port associations.
1879: *
1880: * Note that the memory_object itself is dead, so
1881: * we don't bother with it.
1882: */
1883:
1884: object->pager = IP_NULL;
1885: vm_object_remove(object);
1886:
1887: old_request = object->pager_request;
1888: object->pager_request = PAGER_REQUEST_NULL;
1889:
1890: old_name = object->pager_name;
1891: object->pager_name = IP_NULL;
1892:
1893: vm_object_unlock(object);
1894: vm_object_cache_unlock();
1895:
1896: /*
1897: * Clean up the port references. Note that there's no
1898: * point in trying the memory_object_terminate call
1899: * because the memory_object itself is dead.
1900: */
1901:
1902: ipc_port_release_send(pager);
1903: if (old_request != IP_NULL)
1904: ipc_port_dealloc_kernel(old_request);
1905: if (old_name != IP_NULL)
1906: ipc_port_dealloc_kernel(old_name);
1907:
1908: /*
1909: * Restart pending page requests
1910: */
1911:
1912: vm_object_abort_activity(object);
1913:
1914: /*
1915: * Lose the object reference.
1916: */
1917:
1918: vm_object_deallocate(object);
1919: }
1920:
1921: boolean_t vm_object_accept_old_init_protocol = FALSE;
1922:
1923: /*
1924: * Routine: vm_object_enter
1925: * Purpose:
1926: * Find a VM object corresponding to the given
1927: * pager; if no such object exists, create one,
1928: * and initialize the pager.
1929: */
1930: vm_object_t vm_object_enter(
1931: ipc_port_t pager,
1932: vm_size_t size,
1933: boolean_t internal)
1934: {
1935: vm_object_t object;
1936: vm_object_t new_object;
1937: boolean_t must_init;
1938: ipc_kobject_type_t po;
1939:
1940: restart:
1941: if (!IP_VALID(pager))
1942: return vm_object_allocate(size);
1943:
1944: new_object = VM_OBJECT_NULL;
1945: must_init = FALSE;
1946:
1947: /*
1948: * Look for an object associated with this port.
1949: */
1950:
1951: vm_object_cache_lock();
1952: for (;;) {
1953: po = ip_kotype(pager);
1954:
1955: /*
1956: * If a previous object is being terminated,
1957: * we must wait for the termination message
1958: * to be queued.
1959: *
1960: * We set kobject to a non-null value to let the
1961: * terminator know that someone is waiting.
1962: * Among the possibilities is that the port
1963: * could die while we're waiting. Must restart
1964: * instead of continuing the loop.
1965: */
1966:
1967: if (po == IKOT_PAGER_TERMINATING) {
1968: pager->ip_kobject = (ipc_kobject_t) pager;
1969: assert_wait((event_t) pager, FALSE);
1970: vm_object_cache_unlock();
1971: thread_block((void (*)()) 0);
1972: goto restart;
1973: }
1974:
1975: /*
1976: * Bail if there is already a kobject associated
1977: * with the pager port.
1978: */
1979: if (po != IKOT_NONE) {
1980: break;
1981: }
1982:
1983: /*
1984: * We must unlock to create a new object;
1985: * if we do so, we must try the lookup again.
1986: */
1987:
1988: if (new_object == VM_OBJECT_NULL) {
1989: vm_object_cache_unlock();
1990: new_object = vm_object_allocate(size);
1991: vm_object_cache_lock();
1992: } else {
1993: /*
1994: * Lookup failed twice, and we have something
1995: * to insert; set the object.
1996: */
1997:
1998: ipc_kobject_set(pager,
1999: (ipc_kobject_t) new_object,
2000: IKOT_PAGER);
2001: new_object = VM_OBJECT_NULL;
2002: must_init = TRUE;
2003: }
2004: }
2005:
2006: if (internal)
2007: must_init = TRUE;
2008:
2009: /*
2010: * It's only good if it's a VM object!
2011: */
2012:
2013: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2014: : VM_OBJECT_NULL;
2015:
2016: if ((object != VM_OBJECT_NULL) && !must_init) {
2017: vm_object_lock(object);
1.1.1.5 ! root 2018: if (object->ref_count == 0)
! 2019: vm_object_cache_remove(object);
1.1 root 2020: object->ref_count++;
2021: vm_object_unlock(object);
2022:
2023: vm_stat.hits++;
2024: }
2025: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2026: ((object->paging_in_progress != 0) && internal));
2027:
2028: vm_stat.lookups++;
2029:
2030: vm_object_cache_unlock();
2031:
2032: /*
2033: * If we raced to create a vm_object but lost, let's
2034: * throw away ours.
2035: */
2036:
2037: if (new_object != VM_OBJECT_NULL)
2038: vm_object_deallocate(new_object);
2039:
2040: if (object == VM_OBJECT_NULL)
2041: return(object);
2042:
2043: if (must_init) {
2044: /*
2045: * Copy the naked send right we were given.
2046: */
2047:
2048: pager = ipc_port_copy_send(pager);
2049: if (!IP_VALID(pager))
2050: panic("vm_object_enter: port died"); /* XXX */
2051:
2052: object->pager_created = TRUE;
2053: object->pager = pager;
2054:
2055: /*
2056: * Allocate request port.
2057: */
2058:
2059: object->pager_request = ipc_port_alloc_kernel();
2060: if (object->pager_request == IP_NULL)
2061: panic("vm_object_enter: pager request alloc");
2062:
2063: ipc_kobject_set(object->pager_request,
2064: (ipc_kobject_t) object,
2065: IKOT_PAGING_REQUEST);
2066:
2067: /*
2068: * Let the pager know we're using it.
2069: */
2070:
2071: if (internal) {
2072: /* acquire a naked send right for the DMM */
2073: ipc_port_t DMM = memory_manager_default_reference();
2074:
2075: /* mark the object internal */
2076: object->internal = TRUE;
2077: assert(object->temporary);
2078:
2079: /* default-pager objects are ready immediately */
2080: object->pager_ready = TRUE;
2081:
2082: /* consumes the naked send right for DMM */
2083: (void) memory_object_create(DMM,
2084: pager,
2085: object->size,
2086: object->pager_request,
2087: object->pager_name,
2088: PAGE_SIZE);
2089: } else {
2090: /* the object is external and not temporary */
2091: object->internal = FALSE;
2092: object->temporary = FALSE;
2093:
2094: /* user pager objects are not ready until marked so */
2095: object->pager_ready = FALSE;
2096:
2097: (void) memory_object_init(pager,
2098: object->pager_request,
2099: object->pager_name,
2100: PAGE_SIZE);
2101:
2102: }
2103:
2104: vm_object_lock(object);
2105: object->pager_initialized = TRUE;
2106:
2107: if (vm_object_accept_old_init_protocol)
2108: object->pager_ready = TRUE;
2109:
2110: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2111: } else {
2112: vm_object_lock(object);
2113: }
2114: /*
2115: * [At this point, the object must be locked]
2116: */
2117:
2118: /*
2119: * Wait for the work above to be done by the first
2120: * thread to map this object.
2121: */
2122:
2123: while (!object->pager_initialized) {
2124: vm_object_wait( object,
2125: VM_OBJECT_EVENT_INITIALIZED,
2126: FALSE);
2127: vm_object_lock(object);
2128: }
2129: vm_object_unlock(object);
2130:
2131: return object;
2132: }
2133:
2134: /*
2135: * Routine: vm_object_pager_create
2136: * Purpose:
2137: * Create a memory object for an internal object.
2138: * In/out conditions:
2139: * The object is locked on entry and exit;
2140: * it may be unlocked within this call.
2141: * Limitations:
2142: * Only one thread may be performing a
2143: * vm_object_pager_create on an object at
2144: * a time. Presumably, only the pageout
2145: * daemon will be using this routine.
2146: */
2147: void vm_object_pager_create(
2148: vm_object_t object)
2149: {
2150: ipc_port_t pager;
2151:
2152: if (object->pager_created) {
2153: /*
2154: * Someone else got to it first...
2155: * wait for them to finish initializing
2156: */
2157:
2158: while (!object->pager_initialized) {
2159: vm_object_wait( object,
2160: VM_OBJECT_EVENT_PAGER_READY,
2161: FALSE);
2162: vm_object_lock(object);
2163: }
2164: return;
2165: }
2166:
2167: /*
2168: * Indicate that a memory object has been assigned
2169: * before dropping the lock, to prevent a race.
2170: */
2171:
2172: object->pager_created = TRUE;
2173:
2174: /*
2175: * Prevent collapse or termination by
2176: * holding a paging reference
2177: */
2178:
2179: vm_object_paging_begin(object);
2180: vm_object_unlock(object);
2181:
2182: #if MACH_PAGEMAP
2183: object->existence_info = vm_external_create(
2184: object->size +
2185: object->paging_offset);
2186: assert((object->size + object->paging_offset) >=
2187: object->size);
2188: #endif /* MACH_PAGEMAP */
2189:
2190: /*
2191: * Create the pager, and associate with it
2192: * this object.
2193: *
2194: * Note that we only make the port association
2195: * so that vm_object_enter can properly look up
2196: * the object to complete the initialization...
2197: * we do not expect any user to ever map this
2198: * object.
2199: *
2200: * Since the kernel has the only rights to the
2201: * port, it's safe to install the association
2202: * without holding the cache lock.
2203: */
2204:
2205: pager = ipc_port_alloc_kernel();
2206: if (pager == IP_NULL)
2207: panic("vm_object_pager_create: allocate pager port");
2208:
2209: (void) ipc_port_make_send(pager);
2210: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2211:
2212: /*
2213: * Initialize the rest of the paging stuff
2214: */
2215:
2216: if (vm_object_enter(pager, object->size, TRUE) != object)
2217: panic("vm_object_pager_create: mismatch");
2218:
2219: /*
2220: * Drop the naked send right taken above.
2221: */
2222:
2223: ipc_port_release_send(pager);
2224:
2225: /*
2226: * Release the paging reference
2227: */
2228:
2229: vm_object_lock(object);
2230: vm_object_paging_end(object);
2231: }
2232:
2233: /*
2234: * Routine: vm_object_remove
2235: * Purpose:
2236: * Eliminate the pager/object association
2237: * for this pager.
2238: * Conditions:
2239: * The object cache must be locked.
2240: */
2241: void vm_object_remove(
2242: vm_object_t object)
2243: {
2244: ipc_port_t port;
2245:
2246: if ((port = object->pager) != IP_NULL) {
2247: if (ip_kotype(port) == IKOT_PAGER)
2248: ipc_kobject_set(port, IKO_NULL,
2249: IKOT_PAGER_TERMINATING);
2250: else if (ip_kotype(port) != IKOT_NONE)
2251: panic("vm_object_remove: bad object port");
2252: }
2253: if ((port = object->pager_request) != IP_NULL) {
2254: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2255: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2256: else if (ip_kotype(port) != IKOT_NONE)
2257: panic("vm_object_remove: bad request port");
2258: }
2259: if ((port = object->pager_name) != IP_NULL) {
2260: if (ip_kotype(port) == IKOT_PAGING_NAME)
2261: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2262: else if (ip_kotype(port) != IKOT_NONE)
2263: panic("vm_object_remove: bad name port");
2264: }
2265: }
2266:
2267: /*
2268: * Global variables for vm_object_collapse():
2269: *
2270: * Counts for normal collapses and bypasses.
2271: * Debugging variables, to watch or disable collapse.
2272: */
2273: long object_collapses = 0;
2274: long object_bypasses = 0;
2275:
2276: int vm_object_collapse_debug = 0;
2277: boolean_t vm_object_collapse_allowed = TRUE;
2278: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2279:
2280: /*
2281: * vm_object_collapse:
2282: *
2283: * Collapse an object with the object backing it.
2284: * Pages in the backing object are moved into the
2285: * parent, and the backing object is deallocated.
2286: *
2287: * Requires that the object be locked and the page
2288: * queues be unlocked. May unlock/relock the object,
2289: * so the caller should hold a reference for the object.
2290: */
2291: void vm_object_collapse(
1.1.1.3 root 2292: vm_object_t object)
1.1 root 2293: {
1.1.1.3 root 2294: vm_object_t backing_object;
2295: vm_offset_t backing_offset;
2296: vm_size_t size;
2297: vm_offset_t new_offset;
2298: vm_page_t p, pp;
2299: ipc_port_t old_name_port;
1.1 root 2300:
2301: if (!vm_object_collapse_allowed)
2302: return;
2303:
2304: while (TRUE) {
2305: /*
2306: * Verify that the conditions are right for collapse:
2307: *
2308: * The object exists and no pages in it are currently
2309: * being paged out (or have ever been paged out).
2310: *
2311: * This check is probably overkill -- if a memory
2312: * object has not been created, the fault handler
2313: * shouldn't release the object lock while paging
2314: * is in progress or absent pages exist.
2315: */
2316: if (object == VM_OBJECT_NULL ||
2317: object->pager_created ||
2318: object->paging_in_progress != 0 ||
2319: object->absent_count != 0)
2320: return;
2321:
2322: /*
2323: * There is a backing object, and
2324: */
2325:
2326: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2327: return;
2328:
2329: vm_object_lock(backing_object);
2330: /*
2331: * ...
2332: * The backing object is not read_only,
2333: * and no pages in the backing object are
2334: * currently being paged out.
2335: * The backing object is internal.
2336: *
2337: * XXX It may be sufficient for the backing
2338: * XXX object to be temporary.
2339: */
2340:
2341: if (!backing_object->internal ||
2342: backing_object->paging_in_progress != 0) {
2343: vm_object_unlock(backing_object);
2344: return;
2345: }
2346:
2347: /*
2348: * The backing object can't be a copy-object:
2349: * the shadow_offset for the copy-object must stay
2350: * as 0. Furthermore (for the 'we have all the
2351: * pages' case), if we bypass backing_object and
2352: * just shadow the next object in the chain, old
2353: * pages from that object would then have to be copied
2354: * BOTH into the (former) backing_object and into the
2355: * parent object.
2356: */
2357: if (backing_object->shadow != VM_OBJECT_NULL &&
2358: backing_object->shadow->copy != VM_OBJECT_NULL) {
2359: vm_object_unlock(backing_object);
2360: return;
2361: }
2362:
2363: /*
2364: * We know that we can either collapse the backing
2365: * object (if the parent is the only reference to
2366: * it) or (perhaps) remove the parent's reference
2367: * to it.
2368: */
2369:
2370: backing_offset = object->shadow_offset;
2371: size = object->size;
2372:
2373: /*
2374: * If there is exactly one reference to the backing
2375: * object, we can collapse it into the parent.
2376: */
2377:
2378: if (backing_object->ref_count == 1) {
2379: if (!vm_object_cache_lock_try()) {
2380: vm_object_unlock(backing_object);
2381: return;
2382: }
2383:
2384: /*
2385: * We can collapse the backing object.
2386: *
2387: * Move all in-memory pages from backing_object
2388: * to the parent. Pages that have been paged out
2389: * will be overwritten by any of the parent's
2390: * pages that shadow them.
2391: */
2392:
2393: while (!queue_empty(&backing_object->memq)) {
2394:
2395: p = (vm_page_t)
2396: queue_first(&backing_object->memq);
2397:
2398: new_offset = (p->offset - backing_offset);
2399:
2400: assert(!p->busy || p->absent);
2401:
2402: /*
2403: * If the parent has a page here, or if
2404: * this page falls outside the parent,
2405: * dispose of it.
2406: *
2407: * Otherwise, move it as planned.
2408: */
2409:
2410: if (p->offset < backing_offset ||
2411: new_offset >= size) {
1.1.1.2 root 2412: VM_PAGE_FREE(p);
1.1 root 2413: } else {
2414: pp = vm_page_lookup(object, new_offset);
2415: if (pp != VM_PAGE_NULL && !pp->absent) {
2416: /*
2417: * Parent object has a real page.
2418: * Throw away the backing object's
2419: * page.
2420: */
1.1.1.2 root 2421: VM_PAGE_FREE(p);
1.1 root 2422: }
2423: else {
1.1.1.4 root 2424: assert(pp == VM_PAGE_NULL || !
2425: "vm_object_collapse: bad case");
2426:
1.1 root 2427: /*
2428: * Parent now has no page.
2429: * Move the backing object's page up.
2430: */
2431: vm_page_rename(p, object, new_offset);
2432: }
2433: }
2434: }
2435:
2436: /*
2437: * Move the pager from backing_object to object.
2438: *
2439: * XXX We're only using part of the paging space
2440: * for keeps now... we ought to discard the
2441: * unused portion.
2442: */
2443:
2444: switch (vm_object_collapse_debug) {
2445: case 0:
2446: break;
2447: case 1:
2448: if ((backing_object->pager == IP_NULL) &&
2449: (backing_object->pager_request ==
2450: PAGER_REQUEST_NULL))
2451: break;
2452: /* Fall through to... */
2453:
2454: default:
1.1.1.2 root 2455: printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n",
1.1 root 2456: backing_object, backing_object->pager, backing_object->pager_request,
2457: object);
2458: if (vm_object_collapse_debug > 2)
1.1.1.2 root 2459: SoftDebugger("vm_object_collapse");
1.1 root 2460: }
2461:
2462: object->pager = backing_object->pager;
2463: if (object->pager != IP_NULL)
2464: ipc_kobject_set(object->pager,
2465: (ipc_kobject_t) object,
2466: IKOT_PAGER);
2467: object->pager_initialized = backing_object->pager_initialized;
2468: object->pager_ready = backing_object->pager_ready;
2469: object->pager_created = backing_object->pager_created;
2470:
2471: object->pager_request = backing_object->pager_request;
2472: if (object->pager_request != IP_NULL)
2473: ipc_kobject_set(object->pager_request,
2474: (ipc_kobject_t) object,
2475: IKOT_PAGING_REQUEST);
2476: old_name_port = object->pager_name;
2477: if (old_name_port != IP_NULL)
2478: ipc_kobject_set(old_name_port,
2479: IKO_NULL, IKOT_NONE);
2480: object->pager_name = backing_object->pager_name;
2481: if (object->pager_name != IP_NULL)
2482: ipc_kobject_set(object->pager_name,
2483: (ipc_kobject_t) object,
2484: IKOT_PAGING_NAME);
2485:
2486: vm_object_cache_unlock();
2487:
2488: /*
2489: * If there is no pager, leave paging-offset alone.
2490: */
2491: if (object->pager != IP_NULL)
2492: object->paging_offset =
2493: backing_object->paging_offset +
2494: backing_offset;
2495:
2496: #if MACH_PAGEMAP
2497: assert(object->existence_info == VM_EXTERNAL_NULL);
2498: object->existence_info = backing_object->existence_info;
2499: #endif /* MACH_PAGEMAP */
2500:
2501: /*
2502: * Object now shadows whatever backing_object did.
2503: * Note that the reference to backing_object->shadow
2504: * moves from within backing_object to within object.
2505: */
2506:
2507: object->shadow = backing_object->shadow;
2508: object->shadow_offset += backing_object->shadow_offset;
2509: if (object->shadow != VM_OBJECT_NULL &&
2510: object->shadow->copy != VM_OBJECT_NULL) {
2511: panic("vm_object_collapse: we collapsed a copy-object!");
2512: }
2513: /*
2514: * Discard backing_object.
2515: *
2516: * Since the backing object has no pages, no
2517: * pager left, and no object references within it,
2518: * all that is necessary is to dispose of it.
2519: */
2520:
2521: assert(
2522: (backing_object->ref_count == 1) &&
2523: (backing_object->resident_page_count == 0) &&
2524: (backing_object->paging_in_progress == 0)
2525: );
2526:
2527: assert(backing_object->alive);
1.1.1.5 ! root 2528: assert(!backing_object->cached);
1.1 root 2529: backing_object->alive = FALSE;
2530: vm_object_unlock(backing_object);
2531:
2532: vm_object_unlock(object);
2533: if (old_name_port != IP_NULL)
2534: ipc_port_dealloc_kernel(old_name_port);
1.1.1.2 root 2535: kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object);
1.1 root 2536: vm_object_lock(object);
2537:
2538: object_collapses++;
2539: }
2540: else {
2541: if (!vm_object_collapse_bypass_allowed) {
2542: vm_object_unlock(backing_object);
2543: return;
2544: }
2545:
2546: /*
2547: * If all of the pages in the backing object are
2548: * shadowed by the parent object, the parent
2549: * object no longer has to shadow the backing
2550: * object; it can shadow the next one in the
2551: * chain.
2552: *
2553: * The backing object must not be paged out - we'd
2554: * have to check all of the paged-out pages, as
2555: * well.
2556: */
2557:
2558: if (backing_object->pager_created) {
2559: vm_object_unlock(backing_object);
2560: return;
2561: }
2562:
2563: /*
2564: * Should have a check for a 'small' number
2565: * of pages here.
2566: */
2567:
2568: queue_iterate(&backing_object->memq, p,
2569: vm_page_t, listq)
2570: {
2571: new_offset = (p->offset - backing_offset);
2572:
2573: /*
2574: * If the parent has a page here, or if
2575: * this page falls outside the parent,
2576: * keep going.
2577: *
2578: * Otherwise, the backing_object must be
2579: * left in the chain.
2580: */
2581:
2582: if (p->offset >= backing_offset &&
2583: new_offset <= size &&
2584: (pp = vm_page_lookup(object, new_offset))
2585: == VM_PAGE_NULL) {
2586: /*
2587: * Page still needed.
2588: * Can't go any further.
2589: */
2590: vm_object_unlock(backing_object);
2591: return;
2592: }
2593: }
2594:
2595: /*
2596: * Make the parent shadow the next object
2597: * in the chain. Deallocating backing_object
2598: * will not remove it, since its reference
2599: * count is at least 2.
2600: */
2601:
2602: vm_object_reference(object->shadow = backing_object->shadow);
2603: object->shadow_offset += backing_object->shadow_offset;
2604:
2605: /*
2606: * Backing object might have had a copy pointer
2607: * to us. If it did, clear it.
2608: */
2609: if (backing_object->copy == object)
2610: backing_object->copy = VM_OBJECT_NULL;
2611:
2612: /*
2613: * Drop the reference count on backing_object.
2614: * Since its ref_count was at least 2, it
2615: * will not vanish; so we don't need to call
2616: * vm_object_deallocate.
2617: */
2618: backing_object->ref_count--;
2619: assert(backing_object->ref_count > 0);
2620: vm_object_unlock(backing_object);
2621:
2622: object_bypasses ++;
2623:
2624: }
2625:
2626: /*
2627: * Try again with this object's new backing object.
2628: */
2629: }
2630: }
2631:
2632: /*
2633: * Routine: vm_object_page_remove: [internal]
2634: * Purpose:
2635: * Removes all physical pages in the specified
2636: * object range from the object's list of pages.
2637: *
2638: * In/out conditions:
2639: * The object must be locked.
2640: */
2641: unsigned int vm_object_page_remove_lookup = 0;
2642: unsigned int vm_object_page_remove_iterate = 0;
2643:
2644: void vm_object_page_remove(
1.1.1.3 root 2645: vm_object_t object,
2646: vm_offset_t start,
2647: vm_offset_t end)
1.1 root 2648: {
1.1.1.3 root 2649: vm_page_t p, next;
1.1 root 2650:
2651: /*
2652: * One and two page removals are most popular.
2653: * The factor of 16 here is somewhat arbitrary.
2654: * It balances vm_object_lookup vs iteration.
2655: */
2656:
1.1.1.5 ! root 2657: if (atop(end - start) < object->resident_page_count/16) {
1.1 root 2658: vm_object_page_remove_lookup++;
2659:
2660: for (; start < end; start += PAGE_SIZE) {
2661: p = vm_page_lookup(object, start);
2662: if (p != VM_PAGE_NULL) {
2663: if (!p->fictitious)
2664: pmap_page_protect(p->phys_addr,
2665: VM_PROT_NONE);
1.1.1.2 root 2666: VM_PAGE_FREE(p);
1.1 root 2667: }
2668: }
2669: } else {
2670: vm_object_page_remove_iterate++;
2671:
2672: p = (vm_page_t) queue_first(&object->memq);
2673: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2674: next = (vm_page_t) queue_next(&p->listq);
2675: if ((start <= p->offset) && (p->offset < end)) {
2676: if (!p->fictitious)
2677: pmap_page_protect(p->phys_addr,
2678: VM_PROT_NONE);
1.1.1.2 root 2679: VM_PAGE_FREE(p);
1.1 root 2680: }
2681: p = next;
2682: }
2683: }
2684: }
2685:
2686: /*
2687: * Routine: vm_object_coalesce
2688: * Function: Coalesces two objects backing up adjoining
2689: * regions of memory into a single object.
2690: *
2691: * returns TRUE if objects were combined.
2692: *
2693: * NOTE: Only works at the moment if the second object is NULL -
2694: * if it's not, which object do we lock first?
2695: *
2696: * Parameters:
2697: * prev_object First object to coalesce
2698: * prev_offset Offset into prev_object
2699: * next_object Second object into coalesce
2700: * next_offset Offset into next_object
2701: *
2702: * prev_size Size of reference to prev_object
2703: * next_size Size of reference to next_object
2704: *
2705: * Conditions:
2706: * The object must *not* be locked.
2707: */
2708:
2709: boolean_t vm_object_coalesce(
1.1.1.3 root 2710: vm_object_t prev_object,
1.1 root 2711: vm_object_t next_object,
2712: vm_offset_t prev_offset,
2713: vm_offset_t next_offset,
2714: vm_size_t prev_size,
2715: vm_size_t next_size)
2716: {
2717: vm_size_t newsize;
2718:
2719: if (next_object != VM_OBJECT_NULL) {
2720: return FALSE;
2721: }
2722:
2723: if (prev_object == VM_OBJECT_NULL) {
2724: return TRUE;
2725: }
2726:
2727: vm_object_lock(prev_object);
2728:
2729: /*
2730: * Try to collapse the object first
2731: */
2732: vm_object_collapse(prev_object);
2733:
2734: /*
2735: * Can't coalesce if pages not mapped to
2736: * prev_entry may be in use anyway:
2737: * . more than one reference
2738: * . paged out
2739: * . shadows another object
2740: * . has a copy elsewhere
2741: * . paging references (pages might be in page-list)
2742: */
2743:
2744: if ((prev_object->ref_count > 1) ||
2745: prev_object->pager_created ||
1.1.1.5 ! root 2746: prev_object->used_for_pageout ||
1.1 root 2747: (prev_object->shadow != VM_OBJECT_NULL) ||
2748: (prev_object->copy != VM_OBJECT_NULL) ||
2749: (prev_object->paging_in_progress != 0)) {
2750: vm_object_unlock(prev_object);
2751: return FALSE;
2752: }
2753:
2754: /*
2755: * Remove any pages that may still be in the object from
2756: * a previous deallocation.
2757: */
2758:
2759: vm_object_page_remove(prev_object,
2760: prev_offset + prev_size,
2761: prev_offset + prev_size + next_size);
2762:
2763: /*
2764: * Extend the object if necessary.
2765: */
2766: newsize = prev_offset + prev_size + next_size;
2767: if (newsize > prev_object->size)
2768: prev_object->size = newsize;
2769:
2770: vm_object_unlock(prev_object);
2771: return TRUE;
2772: }
2773:
2774: vm_object_t vm_object_request_object(
2775: ipc_port_t p)
2776: {
2777: return vm_object_lookup(p);
2778: }
2779:
2780: /*
2781: * Routine: vm_object_name
2782: * Purpose:
2783: * Returns a naked send right to the "name" port associated
2784: * with this object.
2785: */
2786: ipc_port_t vm_object_name(
2787: vm_object_t object)
2788: {
2789: ipc_port_t p;
2790:
2791: if (object == VM_OBJECT_NULL)
2792: return IP_NULL;
2793:
2794: vm_object_lock(object);
2795:
2796: while (object->shadow != VM_OBJECT_NULL) {
2797: vm_object_t new_object = object->shadow;
2798: vm_object_lock(new_object);
2799: vm_object_unlock(object);
2800: object = new_object;
2801: }
2802:
2803: p = object->pager_name;
2804: if (p != IP_NULL)
2805: p = ipc_port_make_send(p);
2806: vm_object_unlock(object);
2807:
2808: return p;
2809: }
2810:
2811: /*
2812: * Attach a set of physical pages to an object, so that they can
2813: * be mapped by mapping the object. Typically used to map IO memory.
2814: *
2815: * The mapping function and its private data are used to obtain the
2816: * physical addresses for each page to be mapped.
2817: */
2818: void
2819: vm_object_page_map(
2820: vm_object_t object,
2821: vm_offset_t offset,
2822: vm_size_t size,
2823: vm_offset_t (*map_fn)(void *, vm_offset_t),
2824: void * map_fn_data) /* private to map_fn */
2825: {
2826: int num_pages;
2827: int i;
2828: vm_page_t m;
2829: vm_page_t old_page;
2830: vm_offset_t addr;
2831:
2832: num_pages = atop(size);
2833:
2834: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2835:
2836: addr = (*map_fn)(map_fn_data, offset);
2837:
2838: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
2839: vm_page_more_fictitious();
2840:
2841: vm_object_lock(object);
2842: if ((old_page = vm_page_lookup(object, offset))
2843: != VM_PAGE_NULL)
2844: {
1.1.1.2 root 2845: VM_PAGE_FREE(old_page);
1.1 root 2846: }
2847:
1.1.1.5 ! root 2848: vm_page_init(m);
! 2849: m->phys_addr = addr;
1.1 root 2850: m->private = TRUE; /* don`t free page */
2851: m->wire_count = 1;
2852: vm_page_lock_queues();
2853: vm_page_insert(m, object, offset);
2854: vm_page_unlock_queues();
2855:
2856: PAGE_WAKEUP_DONE(m);
2857: vm_object_unlock(object);
2858: }
2859: }
2860:
2861:
2862: #if MACH_KDB
1.1.1.2 root 2863: #include <vm/vm_print.h>
1.1 root 2864: #define printf kdbprintf
2865:
2866: boolean_t vm_object_print_pages = FALSE;
2867:
2868: /*
2869: * vm_object_print: [ debug ]
2870: */
2871: void vm_object_print(
2872: vm_object_t object)
2873: {
1.1.1.3 root 2874: vm_page_t p;
1.1 root 2875:
1.1.1.3 root 2876: int count;
1.1 root 2877:
2878: if (object == VM_OBJECT_NULL)
2879: return;
2880:
1.1.1.5 ! root 2881: iprintf("Object 0x%X: size=0x%X, %d references",
! 2882: (vm_offset_t) object, (vm_offset_t) object->size,
! 2883: object->ref_count);
! 2884: printf("\n");
! 2885: iprintf("%lu resident pages,", object->resident_page_count);
1.1 root 2886: printf(" %d absent pages,", object->absent_count);
2887: printf(" %d paging ops\n", object->paging_in_progress);
1.1.1.5 ! root 2888: indent += 1;
1.1 root 2889: iprintf("memory object=0x%X (offset=0x%X),",
2890: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
2891: printf("control=0x%X, name=0x%X\n",
2892: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
2893: iprintf("%s%s",
2894: object->pager_ready ? " ready" : "",
2895: object->pager_created ? " created" : "");
2896: printf("%s,%s ",
2897: object->pager_initialized ? "" : "uninitialized",
2898: object->temporary ? "temporary" : "permanent");
2899: printf("%s%s,",
2900: object->internal ? "internal" : "external",
2901: object->can_persist ? " cacheable" : "");
2902: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
2903: iprintf("shadow=0x%X (offset=0x%X),",
2904: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
2905: printf("copy=0x%X\n", (vm_offset_t) object->copy);
2906:
1.1.1.5 ! root 2907: indent += 1;
1.1 root 2908:
2909: if (vm_object_print_pages) {
2910: count = 0;
2911: p = (vm_page_t) queue_first(&object->memq);
2912: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2913: if (count == 0) iprintf("memory:=");
2914: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
2915: else printf(",");
2916: count++;
2917:
2918: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
2919: p = (vm_page_t) queue_next(&p->listq);
2920: }
2921: if (count != 0)
2922: printf("\n");
2923: }
1.1.1.5 ! root 2924: indent -= 2;
1.1 root 2925: }
2926:
2927: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.