![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
1.1.1.2 root 36: #include <kern/printf.h>
37: #include <string.h>
1.1 root 38:
39: #include <mach/memory_object.h>
1.1.1.2 root 40: #include <vm/memory_object_default.user.h>
41: #include <vm/memory_object_user.user.h>
42: #include <machine/vm_param.h>
1.1 root 43: #include <ipc/ipc_port.h>
44: #include <ipc/ipc_space.h>
45: #include <kern/assert.h>
1.1.1.2 root 46: #include <kern/debug.h>
1.1 root 47: #include <kern/lock.h>
48: #include <kern/queue.h>
49: #include <kern/xpr.h>
1.1.1.2 root 50: #include <kern/slab.h>
1.1 root 51: #include <vm/memory_object.h>
52: #include <vm/vm_fault.h>
53: #include <vm/vm_map.h>
54: #include <vm/vm_object.h>
55: #include <vm/vm_page.h>
56: #include <vm/vm_pageout.h>
57:
1.1.1.2 root 58: #if MACH_KDB
59: #include <ddb/db_output.h>
60: #endif /* MACH_KDB */
61:
1.1.1.5 root 62: void memory_object_release(
63: ipc_port_t pager,
64: pager_request_t pager_request,
65: ipc_port_t pager_name); /* forward */
66:
1.1 root 67: /*
68: * Virtual memory objects maintain the actual data
69: * associated with allocated virtual memory. A given
70: * page of memory exists within exactly one object.
71: *
72: * An object is only deallocated when all "references"
73: * are given up. Only one "reference" to a given
74: * region of an object should be writeable.
75: *
76: * Associated with each object is a list of all resident
77: * memory pages belonging to that object; this list is
78: * maintained by the "vm_page" module, but locked by the object's
79: * lock.
80: *
81: * Each object also records the memory object port
82: * that is used by the kernel to request and write
83: * back data (the memory object port, field "pager"),
84: * and the ports provided to the memory manager, the server that
85: * manages that data, to return data and control its
86: * use (the memory object control port, field "pager_request")
87: * and for naming (the memory object name port, field "pager_name").
88: *
89: * Virtual memory objects are allocated to provide
90: * zero-filled memory (vm_allocate) or map a user-defined
91: * memory object into a virtual address space (vm_map).
92: *
93: * Virtual memory objects that refer to a user-defined
94: * memory object are called "permanent", because all changes
95: * made in virtual memory are reflected back to the
96: * memory manager, which may then store it permanently.
97: * Other virtual memory objects are called "temporary",
98: * meaning that changes need be written back only when
99: * necessary to reclaim pages, and that storage associated
100: * with the object can be discarded once it is no longer
101: * mapped.
102: *
103: * A permanent memory object may be mapped into more
104: * than one virtual address space. Moreover, two threads
105: * may attempt to make the first mapping of a memory
106: * object concurrently. Only one thread is allowed to
107: * complete this mapping; all others wait for the
108: * "pager_initialized" field is asserted, indicating
109: * that the first thread has initialized all of the
110: * necessary fields in the virtual memory object structure.
111: *
112: * The kernel relies on a *default memory manager* to
113: * provide backing storage for the zero-filled virtual
114: * memory objects. The memory object ports associated
115: * with these temporary virtual memory objects are only
116: * generated and passed to the default memory manager
117: * when it becomes necessary. Virtual memory objects
118: * that depend on the default memory manager are called
119: * "internal". The "pager_created" field is provided to
120: * indicate whether these ports have ever been allocated.
121: *
122: * The kernel may also create virtual memory objects to
123: * hold changed pages after a copy-on-write operation.
124: * In this case, the virtual memory object (and its
125: * backing storage -- its memory object) only contain
126: * those pages that have been changed. The "shadow"
127: * field refers to the virtual memory object that contains
128: * the remainder of the contents. The "shadow_offset"
129: * field indicates where in the "shadow" these contents begin.
130: * The "copy" field refers to a virtual memory object
131: * to which changed pages must be copied before changing
132: * this object, in order to implement another form
133: * of copy-on-write optimization.
134: *
135: * The virtual memory object structure also records
136: * the attributes associated with its memory object.
137: * The "pager_ready", "can_persist" and "copy_strategy"
138: * fields represent those attributes. The "cached_list"
139: * field is used in the implementation of the persistence
140: * attribute.
141: *
142: * ZZZ Continue this comment.
143: */
144:
1.1.1.2 root 145: struct kmem_cache vm_object_cache; /* vm backing store cache */
1.1 root 146:
147: /*
148: * All wired-down kernel memory belongs to a single virtual
149: * memory object (kernel_object) to avoid wasting data structures.
150: */
1.1.1.2 root 151: static struct vm_object kernel_object_store;
152: vm_object_t kernel_object = &kernel_object_store;
1.1 root 153:
154: /*
155: * Virtual memory objects that are not referenced by
156: * any address maps, but that are allowed to persist
157: * (an attribute specified by the associated memory manager),
158: * are kept in a queue (vm_object_cached_list).
159: *
160: * When an object from this queue is referenced again,
161: * for example to make another address space mapping,
162: * it must be removed from the queue. That is, the
163: * queue contains *only* objects with zero references.
164: *
165: * The kernel may choose to terminate objects from this
166: * queue in order to reclaim storage. The current policy
1.1.1.5 root 167: * is to let memory pressure dynamically adjust the number
168: * of unreferenced objects. The pageout daemon attempts to
169: * collect objects after removing pages from them.
1.1 root 170: *
171: * A simple lock (accessed by routines
172: * vm_object_cache_{lock,lock_try,unlock}) governs the
173: * object cache. It must be held when objects are
174: * added to or removed from the cache (in vm_object_terminate).
175: * The routines that acquire a reference to a virtual
176: * memory object based on one of the memory object ports
177: * must also lock the cache.
178: *
179: * Ideally, the object cache should be more isolated
180: * from the reference mechanism, so that the lock need
181: * not be held to make simple references.
182: */
183: queue_head_t vm_object_cached_list;
184:
185: decl_simple_lock_data(,vm_object_cached_lock_data)
186:
187: #define vm_object_cache_lock() \
188: simple_lock(&vm_object_cached_lock_data)
189: #define vm_object_cache_lock_try() \
190: simple_lock_try(&vm_object_cached_lock_data)
191: #define vm_object_cache_unlock() \
192: simple_unlock(&vm_object_cached_lock_data)
193:
194: /*
1.1.1.2 root 195: * Number of physical pages referenced by cached objects.
196: * This counter is protected by its own lock to work around
197: * lock ordering issues.
198: */
199: int vm_object_cached_pages;
200:
201: decl_simple_lock_data(,vm_object_cached_pages_lock_data)
202:
203: /*
1.1 root 204: * Virtual memory objects are initialized from
205: * a template (see vm_object_allocate).
206: *
207: * When adding a new field to the virtual memory
208: * object structure, be sure to add initialization
209: * (see vm_object_init).
210: */
1.1.1.2 root 211: struct vm_object vm_object_template;
1.1 root 212:
213: /*
214: * vm_object_allocate:
215: *
216: * Returns a new object with the given size.
217: */
218:
1.1.1.2 root 219: static void _vm_object_setup(
220: vm_object_t object,
221: vm_size_t size)
222: {
223: *object = vm_object_template;
224: queue_init(&object->memq);
225: vm_object_lock_init(object);
226: object->size = size;
227: }
228:
1.1 root 229: vm_object_t _vm_object_allocate(
230: vm_size_t size)
231: {
1.1.1.3 root 232: vm_object_t object;
1.1 root 233:
1.1.1.2 root 234: object = (vm_object_t) kmem_cache_alloc(&vm_object_cache);
1.1.1.5 root 235: if (!object)
236: return 0;
1.1 root 237:
1.1.1.2 root 238: _vm_object_setup(object, size);
1.1 root 239:
240: return object;
241: }
242:
243: vm_object_t vm_object_allocate(
244: vm_size_t size)
245: {
1.1.1.3 root 246: vm_object_t object;
247: ipc_port_t port;
1.1 root 248:
249: object = _vm_object_allocate(size);
1.1.1.5 root 250: if (object == 0)
251: panic("vm_object_allocate");
1.1 root 252: port = ipc_port_alloc_kernel();
253: if (port == IP_NULL)
254: panic("vm_object_allocate");
255: object->pager_name = port;
256: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
257:
258: return object;
259: }
260:
261: /*
262: * vm_object_bootstrap:
263: *
264: * Initialize the VM objects module.
265: */
266: void vm_object_bootstrap(void)
267: {
1.1.1.2 root 268: kmem_cache_init(&vm_object_cache, "vm_object",
1.1.1.5 root 269: sizeof(struct vm_object), 0, NULL, 0);
1.1 root 270:
271: queue_init(&vm_object_cached_list);
272: simple_lock_init(&vm_object_cached_lock_data);
273:
274: /*
275: * Fill in a template object, for quick initialization
276: */
277:
1.1.1.2 root 278: vm_object_template.ref_count = 1;
279: vm_object_template.size = 0;
280: vm_object_template.resident_page_count = 0;
281: vm_object_template.copy = VM_OBJECT_NULL;
282: vm_object_template.shadow = VM_OBJECT_NULL;
283: vm_object_template.shadow_offset = (vm_offset_t) 0;
284:
285: vm_object_template.pager = IP_NULL;
286: vm_object_template.paging_offset = 0;
287: vm_object_template.pager_request = PAGER_REQUEST_NULL;
288: vm_object_template.pager_name = IP_NULL;
289:
290: vm_object_template.pager_created = FALSE;
291: vm_object_template.pager_initialized = FALSE;
292: vm_object_template.pager_ready = FALSE;
1.1 root 293:
1.1.1.2 root 294: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE;
1.1 root 295: /* ignored if temporary, will be reset before
296: * permanent object becomes ready */
1.1.1.2 root 297: vm_object_template.use_shared_copy = FALSE;
298: vm_object_template.shadowed = FALSE;
1.1 root 299:
1.1.1.2 root 300: vm_object_template.absent_count = 0;
301: vm_object_template.all_wanted = 0; /* all bits FALSE */
1.1 root 302:
1.1.1.2 root 303: vm_object_template.paging_in_progress = 0;
1.1.1.5 root 304: vm_object_template.used_for_pageout = FALSE;
1.1.1.2 root 305: vm_object_template.can_persist = FALSE;
1.1.1.5 root 306: vm_object_template.cached = FALSE;
1.1.1.2 root 307: vm_object_template.internal = TRUE;
308: vm_object_template.temporary = TRUE;
309: vm_object_template.alive = TRUE;
310: vm_object_template.lock_in_progress = FALSE;
311: vm_object_template.lock_restart = FALSE;
312: vm_object_template.last_alloc = (vm_offset_t) 0;
1.1 root 313:
314: #if MACH_PAGEMAP
1.1.1.2 root 315: vm_object_template.existence_info = VM_EXTERNAL_NULL;
1.1 root 316: #endif /* MACH_PAGEMAP */
317:
318: /*
319: * Initialize the "kernel object"
320: */
321:
1.1.1.2 root 322: _vm_object_setup(kernel_object,
1.1 root 323: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
324:
325: /*
326: * Initialize the "submap object". Make it as large as the
327: * kernel object so that no limit is imposed on submap sizes.
328: */
329:
1.1.1.2 root 330: _vm_object_setup(vm_submap_object,
1.1 root 331: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
332:
333: #if MACH_PAGEMAP
334: vm_external_module_initialize();
335: #endif /* MACH_PAGEMAP */
336: }
337:
338: void vm_object_init(void)
339: {
340: /*
341: * Finish initializing the kernel object.
342: * The submap object doesn't need a name port.
343: */
344:
345: kernel_object->pager_name = ipc_port_alloc_kernel();
346: ipc_kobject_set(kernel_object->pager_name,
347: (ipc_kobject_t) kernel_object,
348: IKOT_PAGING_NAME);
349: }
350:
351: /*
1.1.1.5 root 352: * Object cache management functions.
353: *
354: * Both the cache and the object must be locked
355: * before calling these functions.
356: */
357:
358: static void vm_object_cache_add(
359: vm_object_t object)
360: {
361: assert(!object->cached);
362: queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list);
363: object->cached = TRUE;
364: }
365:
366: static void vm_object_cache_remove(
367: vm_object_t object)
368: {
369: assert(object->cached);
370: queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list);
371: object->cached = FALSE;
372: }
373:
374: void vm_object_collect(
375: register vm_object_t object)
376: {
377: vm_object_unlock(object);
378:
379: /*
380: * The cache lock must be acquired in the proper order.
381: */
382:
383: vm_object_cache_lock();
384: vm_object_lock(object);
385:
386: /*
387: * If the object was referenced while the lock was
388: * dropped, cancel the termination.
389: */
390:
391: if (!vm_object_collectable(object)) {
392: vm_object_unlock(object);
393: vm_object_cache_unlock();
394: return;
395: }
396:
397: vm_object_cache_remove(object);
398: vm_object_terminate(object);
399: }
400:
401: /*
1.1 root 402: * vm_object_reference:
403: *
404: * Gets another reference to the given object.
405: */
406: void vm_object_reference(
1.1.1.3 root 407: vm_object_t object)
1.1 root 408: {
409: if (object == VM_OBJECT_NULL)
410: return;
411:
412: vm_object_lock(object);
413: assert(object->ref_count > 0);
414: object->ref_count++;
415: vm_object_unlock(object);
416: }
417:
418: /*
419: * vm_object_deallocate:
420: *
421: * Release a reference to the specified object,
422: * gained either through a vm_object_allocate
423: * or a vm_object_reference call. When all references
424: * are gone, storage associated with this object
425: * may be relinquished.
426: *
427: * No object may be locked.
428: */
429: void vm_object_deallocate(
1.1.1.3 root 430: vm_object_t object)
1.1 root 431: {
432: vm_object_t temp;
433:
434: while (object != VM_OBJECT_NULL) {
435:
436: /*
437: * The cache holds a reference (uncounted) to
438: * the object; we must lock it before removing
439: * the object.
440: */
441:
442: vm_object_cache_lock();
443:
444: /*
445: * Lose the reference
446: */
447: vm_object_lock(object);
448: if (--(object->ref_count) > 0) {
449:
450: /*
451: * If there are still references, then
452: * we are done.
453: */
454: vm_object_unlock(object);
455: vm_object_cache_unlock();
456: return;
457: }
458:
459: /*
460: * See whether this object can persist. If so, enter
1.1.1.5 root 461: * it in the cache.
1.1 root 462: */
1.1.1.5 root 463: if (object->can_persist && (object->resident_page_count > 0)) {
464: vm_object_cache_add(object);
1.1 root 465: vm_object_cache_unlock();
466: vm_object_unlock(object);
1.1.1.5 root 467: return;
468: }
1.1 root 469:
1.1.1.5 root 470: if (object->pager_created &&
471: !object->pager_initialized) {
1.1 root 472:
473: /*
1.1.1.5 root 474: * Have to wait for initialization.
475: * Put reference back and retry
476: * when it's initialized.
1.1 root 477: */
478:
1.1.1.5 root 479: object->ref_count++;
480: vm_object_assert_wait(object,
481: VM_OBJECT_EVENT_INITIALIZED, FALSE);
482: vm_object_unlock(object);
483: vm_object_cache_unlock();
484: thread_block((void (*)()) 0);
485: continue;
1.1 root 486: }
487:
488: /*
489: * Take the reference to the shadow object
490: * out of the object to be destroyed.
491: */
492:
493: temp = object->shadow;
494:
495: /*
496: * Destroy the object; the cache lock will
497: * be released in the process.
498: */
499:
500: vm_object_terminate(object);
501:
502: /*
503: * Deallocate the reference to the shadow
504: * by continuing the loop with that object
505: * in place of the original.
506: */
507:
508: object = temp;
509: }
510: }
511:
512: /*
513: * Routine: vm_object_terminate
514: * Purpose:
515: * Free all resources associated with a vm_object.
516: * In/out conditions:
517: * Upon entry, the object and the cache must be locked,
518: * and the object must have no references.
519: *
520: * The shadow object reference is left alone.
521: *
522: * Upon exit, the cache will be unlocked, and the
523: * object will cease to exist.
524: */
525: void vm_object_terminate(
1.1.1.3 root 526: vm_object_t object)
1.1 root 527: {
1.1.1.3 root 528: vm_page_t p;
529: vm_object_t shadow_object;
1.1 root 530:
531: /*
532: * Make sure the object isn't already being terminated
533: */
534:
535: assert(object->alive);
536: object->alive = FALSE;
537:
538: /*
539: * Make sure no one can look us up now.
540: */
541:
542: vm_object_remove(object);
543: vm_object_cache_unlock();
544:
545: /*
546: * Detach the object from its shadow if we are the shadow's
547: * copy.
548: */
549: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
550: vm_object_lock(shadow_object);
551: assert((shadow_object->copy == object) ||
552: (shadow_object->copy == VM_OBJECT_NULL));
553: shadow_object->copy = VM_OBJECT_NULL;
554: vm_object_unlock(shadow_object);
555: }
556:
557: /*
558: * The pageout daemon might be playing with our pages.
559: * Now that the object is dead, it won't touch any more
560: * pages, but some pages might already be on their way out.
561: * Hence, we wait until the active paging activities have ceased.
562: */
563:
564: vm_object_paging_wait(object, FALSE);
565:
566: /*
567: * Clean or free the pages, as appropriate.
568: * It is possible for us to find busy/absent pages,
569: * if some faults on this object were aborted.
570: */
571:
572: if ((object->temporary) || (object->pager == IP_NULL)) {
573: while (!queue_empty(&object->memq)) {
574: p = (vm_page_t) queue_first(&object->memq);
575:
576: VM_PAGE_CHECK(p);
577:
578: VM_PAGE_FREE(p);
579: }
580: } else while (!queue_empty(&object->memq)) {
581: p = (vm_page_t) queue_first(&object->memq);
582:
583: VM_PAGE_CHECK(p);
584:
585: vm_page_lock_queues();
586: VM_PAGE_QUEUES_REMOVE(p);
587: vm_page_unlock_queues();
588:
589: if (p->absent || p->private) {
590:
591: /*
592: * For private pages, VM_PAGE_FREE just
593: * leaves the page structure around for
594: * its owner to clean up. For absent
595: * pages, the structure is returned to
596: * the appropriate pool.
597: */
598:
599: goto free_page;
600: }
601:
602: if (!p->dirty)
603: p->dirty = pmap_is_modified(p->phys_addr);
604:
605: if (p->dirty || p->precious) {
606: p->busy = TRUE;
607: vm_pageout_page(p, FALSE, TRUE); /* flush page */
608: } else {
609: free_page:
610: VM_PAGE_FREE(p);
611: }
612: }
613:
614: assert(object->ref_count == 0);
615: assert(object->paging_in_progress == 0);
1.1.1.5 root 616: assert(!object->cached);
1.1 root 617:
1.1.1.6 ! root 618: if (!object->internal) {
! 619: assert(object->resident_page_count == 0);
! 620:
! 621: vm_page_lock_queues();
! 622: vm_object_external_count--;
! 623: vm_page_unlock_queues();
! 624: }
! 625:
1.1 root 626: /*
627: * Throw away port rights... note that they may
628: * already have been thrown away (by vm_object_destroy
629: * or memory_object_destroy).
630: *
631: * Instead of destroying the control and name ports,
632: * we send all rights off to the memory manager instead,
633: * using memory_object_terminate.
634: */
635:
636: vm_object_unlock(object);
637:
638: if (object->pager != IP_NULL) {
639: /* consumes our rights for pager, pager_request, pager_name */
640: memory_object_release(object->pager,
641: object->pager_request,
642: object->pager_name);
643: } else if (object->pager_name != IP_NULL) {
644: /* consumes our right for pager_name */
645: ipc_port_dealloc_kernel(object->pager_name);
646: }
647:
648: #if MACH_PAGEMAP
649: vm_external_destroy(object->existence_info);
650: #endif /* MACH_PAGEMAP */
651:
652: /*
653: * Free the space for the object.
654: */
655:
1.1.1.2 root 656: kmem_cache_free(&vm_object_cache, (vm_offset_t) object);
1.1 root 657: }
658:
659: /*
660: * Routine: vm_object_pager_wakeup
661: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
662: */
663:
664: void
665: vm_object_pager_wakeup(
666: ipc_port_t pager)
667: {
668: boolean_t someone_waiting;
669:
670: /*
671: * If anyone was waiting for the memory_object_terminate
672: * to be queued, wake them up now.
673: */
674: vm_object_cache_lock();
675: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
676: someone_waiting = (pager->ip_kobject != IKO_NULL);
677: if (ip_active(pager))
678: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
679: vm_object_cache_unlock();
680: if (someone_waiting) {
681: thread_wakeup((event_t) pager);
682: }
683: }
684:
685: /*
686: * Routine: memory_object_release
687: * Purpose: Terminate the pager and release port rights,
688: * just like memory_object_terminate, except
689: * that we wake up anyone blocked in vm_object_enter
690: * waiting for termination message to be queued
691: * before calling memory_object_init.
692: */
693: void memory_object_release(
694: ipc_port_t pager,
695: pager_request_t pager_request,
696: ipc_port_t pager_name)
697: {
698:
699: /*
700: * Keep a reference to pager port;
701: * the terminate might otherwise release all references.
702: */
703: ip_reference(pager);
704:
705: /*
706: * Terminate the pager.
707: */
708: (void) memory_object_terminate(pager, pager_request, pager_name);
709:
710: /*
711: * Wakeup anyone waiting for this terminate
712: */
713: vm_object_pager_wakeup(pager);
714:
715: /*
716: * Release reference to pager port.
717: */
718: ip_release(pager);
719: }
720:
721: /*
722: * Routine: vm_object_abort_activity [internal use only]
723: * Purpose:
724: * Abort paging requests pending on this object.
725: * In/out conditions:
726: * The object is locked on entry and exit.
727: */
728: void vm_object_abort_activity(
729: vm_object_t object)
730: {
731: vm_page_t p;
732: vm_page_t next;
733:
734: /*
735: * Abort all activity that would be waiting
736: * for a result on this memory object.
737: *
738: * We could also choose to destroy all pages
739: * that we have in memory for this object, but
740: * we don't.
741: */
742:
743: p = (vm_page_t) queue_first(&object->memq);
744: while (!queue_end(&object->memq, (queue_entry_t) p)) {
745: next = (vm_page_t) queue_next(&p->listq);
746:
747: /*
748: * If it's being paged in, destroy it.
749: * If an unlock has been requested, start it again.
750: */
751:
752: if (p->busy && p->absent) {
753: VM_PAGE_FREE(p);
754: }
755: else {
756: if (p->unlock_request != VM_PROT_NONE)
757: p->unlock_request = VM_PROT_NONE;
758: PAGE_WAKEUP(p);
759: }
760:
761: p = next;
762: }
763:
764: /*
765: * Wake up threads waiting for the memory object to
766: * become ready.
767: */
768:
769: object->pager_ready = TRUE;
770: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
771: }
772:
773: /*
774: * Routine: memory_object_destroy [user interface]
775: * Purpose:
776: * Shut down a memory object, despite the
777: * presence of address map (or other) references
778: * to the vm_object.
779: * Note:
780: * This routine may be called either from the user interface,
781: * or from port destruction handling (via vm_object_destroy).
782: */
783: kern_return_t memory_object_destroy(
784: vm_object_t object,
785: kern_return_t reason)
786: {
787: ipc_port_t old_object, old_name;
788: pager_request_t old_control;
789:
790: if (object == VM_OBJECT_NULL)
791: return KERN_SUCCESS;
792:
793: /*
794: * Remove the port associations immediately.
795: *
796: * This will prevent the memory manager from further
797: * meddling. [If it wanted to flush data or make
798: * other changes, it should have done so before performing
799: * the destroy call.]
800: */
801:
802: vm_object_cache_lock();
803: vm_object_lock(object);
804: vm_object_remove(object);
805: object->can_persist = FALSE;
806: vm_object_cache_unlock();
807:
808: /*
809: * Rip out the ports from the vm_object now... this
810: * will prevent new memory_object calls from succeeding.
811: */
812:
813: old_object = object->pager;
814: object->pager = IP_NULL;
815:
816: old_control = object->pager_request;
817: object->pager_request = PAGER_REQUEST_NULL;
818:
819: old_name = object->pager_name;
820: object->pager_name = IP_NULL;
821:
822:
823: /*
824: * Wait for existing paging activity (that might
825: * have the old ports) to subside.
826: */
827:
828: vm_object_paging_wait(object, FALSE);
829: vm_object_unlock(object);
830:
831: /*
832: * Shut down the ports now.
833: *
834: * [Paging operations may be proceeding concurrently --
835: * they'll get the null values established above.]
836: */
837:
838: if (old_object != IP_NULL) {
839: /* consumes our rights for object, control, name */
840: memory_object_release(old_object, old_control,
841: old_name);
842: } else if (old_name != IP_NULL) {
843: /* consumes our right for name */
844: ipc_port_dealloc_kernel(object->pager_name);
845: }
846:
847: /*
848: * Lose the reference that was donated for this routine
849: */
850:
851: vm_object_deallocate(object);
852:
853: return KERN_SUCCESS;
854: }
855:
856: /*
857: * Routine: vm_object_pmap_protect
858: *
859: * Purpose:
860: * Reduces the permission for all physical
861: * pages in the specified object range.
862: *
863: * If removing write permission only, it is
864: * sufficient to protect only the pages in
865: * the top-level object; only those pages may
866: * have write permission.
867: *
868: * If removing all access, we must follow the
869: * shadow chain from the top-level object to
870: * remove access to all pages in shadowed objects.
871: *
872: * The object must *not* be locked. The object must
873: * be temporary/internal.
874: *
875: * If pmap is not NULL, this routine assumes that
876: * the only mappings for the pages are in that
877: * pmap.
878: */
879: boolean_t vm_object_pmap_protect_by_page = FALSE;
880:
881: void vm_object_pmap_protect(
1.1.1.3 root 882: vm_object_t object,
883: vm_offset_t offset,
1.1.1.2 root 884: vm_size_t size,
1.1 root 885: pmap_t pmap,
886: vm_offset_t pmap_start,
887: vm_prot_t prot)
888: {
889: if (object == VM_OBJECT_NULL)
890: return;
891:
892: vm_object_lock(object);
893:
894: assert(object->temporary && object->internal);
895:
896: while (TRUE) {
897: if (object->resident_page_count > atop(size) / 2 &&
898: pmap != PMAP_NULL) {
899: vm_object_unlock(object);
900: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
901: return;
902: }
903:
904: {
1.1.1.3 root 905: vm_page_t p;
906: vm_offset_t end;
1.1 root 907:
908: end = offset + size;
909:
910: queue_iterate(&object->memq, p, vm_page_t, listq) {
911: if (!p->fictitious &&
912: (offset <= p->offset) &&
913: (p->offset < end)) {
914: if ((pmap == PMAP_NULL) ||
915: vm_object_pmap_protect_by_page) {
916: pmap_page_protect(p->phys_addr,
917: prot & ~p->page_lock);
918: } else {
919: vm_offset_t start =
920: pmap_start +
921: (p->offset - offset);
922:
923: pmap_protect(pmap,
924: start,
925: start + PAGE_SIZE,
926: prot);
927: }
928: }
929: }
930: }
931:
932: if (prot == VM_PROT_NONE) {
933: /*
934: * Must follow shadow chain to remove access
935: * to pages in shadowed objects.
936: */
1.1.1.3 root 937: vm_object_t next_object;
1.1 root 938:
939: next_object = object->shadow;
940: if (next_object != VM_OBJECT_NULL) {
941: offset += object->shadow_offset;
942: vm_object_lock(next_object);
943: vm_object_unlock(object);
944: object = next_object;
945: }
946: else {
947: /*
948: * End of chain - we are done.
949: */
950: break;
951: }
952: }
953: else {
954: /*
955: * Pages in shadowed objects may never have
956: * write permission - we may stop here.
957: */
958: break;
959: }
960: }
961:
962: vm_object_unlock(object);
963: }
964:
965: /*
966: * vm_object_pmap_remove:
967: *
968: * Removes all physical pages in the specified
969: * object range from all physical maps.
970: *
971: * The object must *not* be locked.
972: */
973: void vm_object_pmap_remove(
1.1.1.3 root 974: vm_object_t object,
975: vm_offset_t start,
976: vm_offset_t end)
1.1 root 977: {
1.1.1.3 root 978: vm_page_t p;
1.1 root 979:
980: if (object == VM_OBJECT_NULL)
981: return;
982:
983: vm_object_lock(object);
984: queue_iterate(&object->memq, p, vm_page_t, listq) {
985: if (!p->fictitious &&
986: (start <= p->offset) &&
987: (p->offset < end))
988: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
989: }
990: vm_object_unlock(object);
991: }
992:
993: /*
994: * Routine: vm_object_copy_slowly
995: *
996: * Description:
997: * Copy the specified range of the source
998: * virtual memory object without using
999: * protection-based optimizations (such
1000: * as copy-on-write). The pages in the
1001: * region are actually copied.
1002: *
1003: * In/out conditions:
1004: * The caller must hold a reference and a lock
1005: * for the source virtual memory object. The source
1006: * object will be returned *unlocked*.
1007: *
1008: * Results:
1009: * If the copy is completed successfully, KERN_SUCCESS is
1010: * returned. If the caller asserted the interruptible
1011: * argument, and an interruption occurred while waiting
1012: * for a user-generated event, MACH_SEND_INTERRUPTED is
1013: * returned. Other values may be returned to indicate
1014: * hard errors during the copy operation.
1015: *
1016: * A new virtual memory object is returned in a
1017: * parameter (_result_object). The contents of this
1018: * new object, starting at a zero offset, are a copy
1019: * of the source memory region. In the event of
1020: * an error, this parameter will contain the value
1021: * VM_OBJECT_NULL.
1022: */
1023: kern_return_t vm_object_copy_slowly(
1024: vm_object_t src_object,
1025: vm_offset_t src_offset,
1026: vm_size_t size,
1027: boolean_t interruptible,
1028: vm_object_t *_result_object) /* OUT */
1029: {
1030: vm_object_t new_object;
1031: vm_offset_t new_offset;
1032:
1033: if (size == 0) {
1034: vm_object_unlock(src_object);
1035: *_result_object = VM_OBJECT_NULL;
1036: return KERN_INVALID_ARGUMENT;
1037: }
1038:
1039: /*
1040: * Prevent destruction of the source object while we copy.
1041: */
1042:
1043: assert(src_object->ref_count > 0);
1044: src_object->ref_count++;
1045: vm_object_unlock(src_object);
1046:
1047: /*
1048: * Create a new object to hold the copied pages.
1049: * A few notes:
1050: * We fill the new object starting at offset 0,
1051: * regardless of the input offset.
1052: * We don't bother to lock the new object within
1053: * this routine, since we have the only reference.
1054: */
1055:
1056: new_object = vm_object_allocate(size);
1057: new_offset = 0;
1058:
1059: assert(size == trunc_page(size)); /* Will the loop terminate? */
1060:
1061: for ( ;
1062: size != 0 ;
1063: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1064: ) {
1065: vm_page_t new_page;
1066: vm_fault_return_t result;
1067:
1068: while ((new_page = vm_page_alloc(new_object, new_offset))
1069: == VM_PAGE_NULL) {
1070: VM_PAGE_WAIT((void (*)()) 0);
1071: }
1072:
1073: do {
1074: vm_prot_t prot = VM_PROT_READ;
1075: vm_page_t _result_page;
1076: vm_page_t top_page;
1077: vm_page_t result_page;
1078:
1079: vm_object_lock(src_object);
1080: src_object->paging_in_progress++;
1081:
1082: result = vm_fault_page(src_object, src_offset,
1083: VM_PROT_READ, FALSE, interruptible,
1084: &prot, &_result_page, &top_page,
1085: FALSE, (void (*)()) 0);
1086:
1087: switch(result) {
1088: case VM_FAULT_SUCCESS:
1089: result_page = _result_page;
1090:
1091: /*
1092: * We don't need to hold the object
1093: * lock -- the busy page will be enough.
1094: * [We don't care about picking up any
1095: * new modifications.]
1096: *
1097: * Copy the page to the new object.
1098: *
1099: * POLICY DECISION:
1100: * If result_page is clean,
1101: * we could steal it instead
1102: * of copying.
1103: */
1104:
1105: vm_object_unlock(result_page->object);
1106: vm_page_copy(result_page, new_page);
1107:
1108: /*
1109: * Let go of both pages (make them
1110: * not busy, perform wakeup, activate).
1111: */
1112:
1113: new_page->busy = FALSE;
1114: new_page->dirty = TRUE;
1115: vm_object_lock(result_page->object);
1116: PAGE_WAKEUP_DONE(result_page);
1117:
1118: vm_page_lock_queues();
1119: if (!result_page->active &&
1120: !result_page->inactive)
1121: vm_page_activate(result_page);
1122: vm_page_activate(new_page);
1123: vm_page_unlock_queues();
1124:
1125: /*
1126: * Release paging references and
1127: * top-level placeholder page, if any.
1128: */
1129:
1130: vm_fault_cleanup(result_page->object,
1131: top_page);
1132:
1133: break;
1134:
1135: case VM_FAULT_RETRY:
1136: break;
1137:
1138: case VM_FAULT_MEMORY_SHORTAGE:
1139: VM_PAGE_WAIT((void (*)()) 0);
1140: break;
1141:
1142: case VM_FAULT_FICTITIOUS_SHORTAGE:
1143: vm_page_more_fictitious();
1144: break;
1145:
1146: case VM_FAULT_INTERRUPTED:
1147: vm_page_free(new_page);
1148: vm_object_deallocate(new_object);
1149: vm_object_deallocate(src_object);
1150: *_result_object = VM_OBJECT_NULL;
1151: return MACH_SEND_INTERRUPTED;
1152:
1153: case VM_FAULT_MEMORY_ERROR:
1154: /*
1155: * A policy choice:
1156: * (a) ignore pages that we can't
1157: * copy
1158: * (b) return the null object if
1159: * any page fails [chosen]
1160: */
1161:
1162: vm_page_free(new_page);
1163: vm_object_deallocate(new_object);
1164: vm_object_deallocate(src_object);
1165: *_result_object = VM_OBJECT_NULL;
1166: return KERN_MEMORY_ERROR;
1167: }
1168: } while (result != VM_FAULT_SUCCESS);
1169: }
1170:
1171: /*
1172: * Lose the extra reference, and return our object.
1173: */
1174:
1175: vm_object_deallocate(src_object);
1176: *_result_object = new_object;
1177: return KERN_SUCCESS;
1178: }
1179:
1180: /*
1181: * Routine: vm_object_copy_temporary
1182: *
1183: * Purpose:
1184: * Copy the specified range of the source virtual
1185: * memory object, if it can be done without blocking.
1186: *
1187: * Results:
1188: * If the copy is successful, the copy is returned in
1189: * the arguments; otherwise, the arguments are not
1190: * affected.
1191: *
1192: * In/out conditions:
1193: * The object should be unlocked on entry and exit.
1194: */
1195:
1196: boolean_t vm_object_copy_temporary(
1197: vm_object_t *_object, /* INOUT */
1198: vm_offset_t *_offset, /* INOUT */
1199: boolean_t *_src_needs_copy, /* OUT */
1200: boolean_t *_dst_needs_copy) /* OUT */
1201: {
1202: vm_object_t object = *_object;
1203:
1204: if (object == VM_OBJECT_NULL) {
1205: *_src_needs_copy = FALSE;
1206: *_dst_needs_copy = FALSE;
1207: return TRUE;
1208: }
1209:
1210: /*
1211: * If the object is temporary, we can perform
1212: * a symmetric copy-on-write without asking.
1213: */
1214:
1215: vm_object_lock(object);
1216: if (object->temporary) {
1217:
1218: /*
1219: * Shared objects use delayed copy
1220: */
1221: if (object->use_shared_copy) {
1222:
1223: /*
1224: * Asymmetric copy strategy. Destination
1225: * must be copied (to allow copy object reuse).
1226: * Source is unaffected.
1227: */
1228: vm_object_unlock(object);
1229: object = vm_object_copy_delayed(object);
1230: *_object = object;
1231: *_src_needs_copy = FALSE;
1232: *_dst_needs_copy = TRUE;
1233: return TRUE;
1234: }
1235:
1236: /*
1237: * Make another reference to the object.
1238: *
1239: * Leave object/offset unchanged.
1240: */
1241:
1242: assert(object->ref_count > 0);
1243: object->ref_count++;
1244: object->shadowed = TRUE;
1245: vm_object_unlock(object);
1246:
1247: /*
1248: * Both source and destination must make
1249: * shadows, and the source must be made
1250: * read-only if not already.
1251: */
1252:
1253: *_src_needs_copy = TRUE;
1254: *_dst_needs_copy = TRUE;
1255: return TRUE;
1256: }
1257:
1258: if (object->pager_ready &&
1259: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1260: /* XXX Do something intelligent (see temporary code above) */
1261: }
1262: vm_object_unlock(object);
1263:
1264: return FALSE;
1265: }
1266:
1267: /*
1268: * Routine: vm_object_copy_call [internal]
1269: *
1270: * Description:
1271: * Copy the specified (src_offset, size) portion
1272: * of the source object (src_object), using the
1273: * user-managed copy algorithm.
1274: *
1275: * In/out conditions:
1276: * The source object must be locked on entry. It
1277: * will be *unlocked* on exit.
1278: *
1279: * Results:
1280: * If the copy is successful, KERN_SUCCESS is returned.
1281: * This routine is interruptible; if a wait for
1282: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1283: * is returned. Other return values indicate hard errors
1284: * in creating the user-managed memory object for the copy.
1285: *
1286: * A new object that represents the copied virtual
1287: * memory is returned in a parameter (*_result_object).
1288: * If the return value indicates an error, this parameter
1289: * is not valid.
1290: */
1291: kern_return_t vm_object_copy_call(
1292: vm_object_t src_object,
1293: vm_offset_t src_offset,
1294: vm_size_t size,
1295: vm_object_t *_result_object) /* OUT */
1296: {
1297: vm_offset_t src_end = src_offset + size;
1298: ipc_port_t new_memory_object;
1299: vm_object_t new_object;
1300: vm_page_t p;
1301:
1302: /*
1303: * Create a memory object port to be associated
1304: * with this new vm_object.
1305: *
1306: * Since the kernel has the only rights to this
1307: * port, we need not hold the cache lock.
1308: *
1309: * Since we have the only object reference, we
1310: * need not be worried about collapse operations.
1311: *
1312: */
1313:
1314: new_memory_object = ipc_port_alloc_kernel();
1.1.1.4 root 1315: if (new_memory_object == IP_NULL)
1316: return KERN_RESOURCE_SHORTAGE;
1317:
1318: /*
1319: * Set the backing object for the new
1320: * temporary object.
1321: */
1322:
1323: assert(src_object->ref_count > 0);
1324: src_object->ref_count++;
1325: vm_object_paging_begin(src_object);
1326: vm_object_unlock(src_object);
1.1 root 1327:
1328: /* we hold a naked receive right for new_memory_object */
1329: (void) ipc_port_make_send(new_memory_object);
1330: /* now we also hold a naked send right for new_memory_object */
1331:
1332: /*
1333: * Let the memory manager know that a copy operation
1334: * is in progress. Note that we're using the old
1335: * memory object's ports (for which we're holding
1336: * a paging reference)... the memory manager cannot
1337: * yet affect the new memory object.
1338: */
1339:
1340: (void) memory_object_copy(src_object->pager,
1341: src_object->pager_request,
1342: src_offset, size,
1343: new_memory_object);
1344: /* no longer hold the naked receive right for new_memory_object */
1345:
1346: vm_object_lock(src_object);
1347: vm_object_paging_end(src_object);
1348:
1349: /*
1350: * Remove write access from all of the pages of
1351: * the old memory object that we can.
1352: */
1353:
1354: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1355: if (!p->fictitious &&
1356: (src_offset <= p->offset) &&
1357: (p->offset < src_end) &&
1358: !(p->page_lock & VM_PROT_WRITE)) {
1359: p->page_lock |= VM_PROT_WRITE;
1360: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1361: }
1362: }
1363:
1364: vm_object_unlock(src_object);
1365:
1366: /*
1367: * Initialize the rest of the paging stuff
1368: */
1369:
1370: new_object = vm_object_enter(new_memory_object, size, FALSE);
1371: new_object->shadow = src_object;
1372: new_object->shadow_offset = src_offset;
1373:
1374: /*
1375: * Drop the reference for new_memory_object taken above.
1376: */
1377:
1378: ipc_port_release_send(new_memory_object);
1379: /* no longer hold the naked send right for new_memory_object */
1380:
1381: *_result_object = new_object;
1382: return KERN_SUCCESS;
1383: }
1384:
1385: /*
1386: * Routine: vm_object_copy_delayed [internal]
1387: *
1388: * Description:
1389: * Copy the specified virtual memory object, using
1390: * the asymmetric copy-on-write algorithm.
1391: *
1392: * In/out conditions:
1393: * The object must be unlocked on entry.
1394: *
1395: * This routine will not block waiting for user-generated
1396: * events. It is not interruptible.
1397: */
1398: vm_object_t vm_object_copy_delayed(
1399: vm_object_t src_object)
1400: {
1401: vm_object_t new_copy;
1402: vm_object_t old_copy;
1403: vm_page_t p;
1404:
1405: /*
1406: * The user-level memory manager wants to see
1407: * all of the changes to this object, but it
1408: * has promised not to make any changes on its own.
1409: *
1410: * Perform an asymmetric copy-on-write, as follows:
1411: * Create a new object, called a "copy object"
1412: * to hold pages modified by the new mapping
1413: * (i.e., the copy, not the original mapping).
1414: * Record the original object as the backing
1415: * object for the copy object. If the
1416: * original mapping does not change a page,
1417: * it may be used read-only by the copy.
1418: * Record the copy object in the original
1419: * object. When the original mapping causes
1420: * a page to be modified, it must be copied
1421: * to a new page that is "pushed" to the
1422: * copy object.
1423: * Mark the new mapping (the copy object)
1424: * copy-on-write. This makes the copy
1425: * object itself read-only, allowing it
1426: * to be reused if the original mapping
1427: * makes no changes, and simplifying the
1428: * synchronization required in the "push"
1429: * operation described above.
1430: *
1.1.1.3 root 1431: * The copy-on-write is said to be asymmetric because
1.1 root 1432: * the original object is *not* marked copy-on-write.
1433: * A copied page is pushed to the copy object, regardless
1434: * which party attempted to modify the page.
1435: *
1436: * Repeated asymmetric copy operations may be done.
1437: * If the original object has not been changed since
1438: * the last copy, its copy object can be reused.
1439: * Otherwise, a new copy object can be inserted
1440: * between the original object and its previous
1441: * copy object. Since any copy object is read-only,
1442: * this cannot affect the contents of the previous copy
1443: * object.
1444: *
1445: * Note that a copy object is higher in the object
1446: * tree than the original object; therefore, use of
1447: * the copy object recorded in the original object
1448: * must be done carefully, to avoid deadlock.
1449: */
1450:
1451: /*
1452: * Allocate a new copy object before locking, even
1453: * though we may not need it later.
1454: */
1455:
1456: new_copy = vm_object_allocate(src_object->size);
1457:
1458: vm_object_lock(src_object);
1459:
1460: /*
1461: * See whether we can reuse the result of a previous
1462: * copy operation.
1463: */
1464: Retry:
1465: old_copy = src_object->copy;
1466: if (old_copy != VM_OBJECT_NULL) {
1467: /*
1468: * Try to get the locks (out of order)
1469: */
1470: if (!vm_object_lock_try(old_copy)) {
1471: vm_object_unlock(src_object);
1472:
1473: simple_lock_pause(); /* wait a bit */
1474:
1475: vm_object_lock(src_object);
1476: goto Retry;
1477: }
1478:
1479: /*
1480: * Determine whether the old copy object has
1481: * been modified.
1482: */
1483:
1484: if (old_copy->resident_page_count == 0 &&
1485: !old_copy->pager_created) {
1486: /*
1487: * It has not been modified.
1488: *
1489: * Return another reference to
1490: * the existing copy-object.
1491: */
1492: assert(old_copy->ref_count > 0);
1493: old_copy->ref_count++;
1494: vm_object_unlock(old_copy);
1495: vm_object_unlock(src_object);
1496:
1497: vm_object_deallocate(new_copy);
1498:
1499: return old_copy;
1500: }
1501:
1502: /*
1503: * The copy-object is always made large enough to
1504: * completely shadow the original object, since
1505: * it may have several users who want to shadow
1506: * the original object at different points.
1507: */
1508:
1509: assert((old_copy->shadow == src_object) &&
1510: (old_copy->shadow_offset == (vm_offset_t) 0));
1511:
1512: /*
1513: * Make the old copy-object shadow the new one.
1514: * It will receive no more pages from the original
1515: * object.
1516: */
1517:
1518: src_object->ref_count--; /* remove ref. from old_copy */
1519: assert(src_object->ref_count > 0);
1520: old_copy->shadow = new_copy;
1521: assert(new_copy->ref_count > 0);
1522: new_copy->ref_count++;
1523: vm_object_unlock(old_copy); /* done with old_copy */
1524: }
1525:
1526: /*
1527: * Point the new copy at the existing object.
1528: */
1529:
1530: new_copy->shadow = src_object;
1531: new_copy->shadow_offset = 0;
1532: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1533: assert(src_object->ref_count > 0);
1534: src_object->ref_count++;
1535: src_object->copy = new_copy;
1536:
1537: /*
1538: * Mark all pages of the existing object copy-on-write.
1539: * This object may have a shadow chain below it, but
1540: * those pages will already be marked copy-on-write.
1541: */
1542:
1543: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1544: if (!p->fictitious)
1545: pmap_page_protect(p->phys_addr,
1546: (VM_PROT_ALL & ~VM_PROT_WRITE &
1547: ~p->page_lock));
1548: }
1549:
1550: vm_object_unlock(src_object);
1551:
1552: return new_copy;
1553: }
1554:
1555: /*
1556: * Routine: vm_object_copy_strategically
1557: *
1558: * Purpose:
1559: * Perform a copy according to the source object's
1560: * declared strategy. This operation may block,
1561: * and may be interrupted.
1562: */
1563: kern_return_t vm_object_copy_strategically(
1564: vm_object_t src_object,
1565: vm_offset_t src_offset,
1566: vm_size_t size,
1567: vm_object_t *dst_object, /* OUT */
1568: vm_offset_t *dst_offset, /* OUT */
1569: boolean_t *dst_needs_copy) /* OUT */
1570: {
1571: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1572: boolean_t interruptible = TRUE; /* XXX */
1573:
1574: assert(src_object != VM_OBJECT_NULL);
1575:
1576: vm_object_lock(src_object);
1577:
1578: /* XXX assert(!src_object->temporary); JSB FIXME */
1579:
1580: /*
1581: * The copy strategy is only valid if the memory manager
1582: * is "ready".
1583: */
1584:
1585: while (!src_object->pager_ready) {
1586: vm_object_wait( src_object,
1587: VM_OBJECT_EVENT_PAGER_READY,
1588: interruptible);
1589: if (interruptible &&
1590: (current_thread()->wait_result != THREAD_AWAKENED)) {
1591: *dst_object = VM_OBJECT_NULL;
1592: *dst_offset = 0;
1593: *dst_needs_copy = FALSE;
1594: return MACH_SEND_INTERRUPTED;
1595: }
1596: vm_object_lock(src_object);
1597: }
1598:
1599: /*
1600: * The object may be temporary (even though it is external).
1601: * If so, do a symmetric copy.
1602: */
1603:
1604: if (src_object->temporary) {
1605: /*
1606: * XXX
1607: * This does not count as intelligent!
1608: * This buys us the object->temporary optimizations,
1609: * but we aren't using a symmetric copy,
1610: * which may confuse the vm code. The correct thing
1611: * to do here is to figure out what to call to get
1612: * a temporary shadowing set up.
1613: */
1614: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1615: }
1616:
1617: /*
1618: * The object is permanent. Use the appropriate copy strategy.
1619: */
1620:
1621: switch (src_object->copy_strategy) {
1622: case MEMORY_OBJECT_COPY_NONE:
1623: if ((result = vm_object_copy_slowly(
1624: src_object,
1625: src_offset,
1626: size,
1627: interruptible,
1628: dst_object))
1629: == KERN_SUCCESS) {
1630: *dst_offset = 0;
1631: *dst_needs_copy = FALSE;
1632: }
1633: break;
1634:
1635: case MEMORY_OBJECT_COPY_CALL:
1636: if ((result = vm_object_copy_call(
1637: src_object,
1638: src_offset,
1639: size,
1640: dst_object))
1641: == KERN_SUCCESS) {
1642: *dst_offset = 0;
1643: *dst_needs_copy = FALSE;
1644: }
1645: break;
1646:
1647: case MEMORY_OBJECT_COPY_DELAY:
1648: vm_object_unlock(src_object);
1649: *dst_object = vm_object_copy_delayed(src_object);
1650: *dst_offset = src_offset;
1651: *dst_needs_copy = TRUE;
1652:
1653: result = KERN_SUCCESS;
1654: break;
1655: }
1656:
1657: return result;
1658: }
1659:
1660: /*
1661: * vm_object_shadow:
1662: *
1663: * Create a new object which is backed by the
1664: * specified existing object range. The source
1665: * object reference is deallocated.
1666: *
1667: * The new object and offset into that object
1668: * are returned in the source parameters.
1669: */
1670:
1671: void vm_object_shadow(
1672: vm_object_t *object, /* IN/OUT */
1673: vm_offset_t *offset, /* IN/OUT */
1674: vm_size_t length)
1675: {
1.1.1.3 root 1676: vm_object_t source;
1677: vm_object_t result;
1.1 root 1678:
1679: source = *object;
1680:
1681: /*
1682: * Allocate a new object with the given length
1683: */
1684:
1685: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1686: panic("vm_object_shadow: no object for shadowing");
1687:
1688: /*
1689: * The new object shadows the source object, adding
1690: * a reference to it. Our caller changes his reference
1691: * to point to the new object, removing a reference to
1692: * the source object. Net result: no change of reference
1693: * count.
1694: */
1695: result->shadow = source;
1696:
1697: /*
1698: * Store the offset into the source object,
1699: * and fix up the offset into the new object.
1700: */
1701:
1702: result->shadow_offset = *offset;
1703:
1704: /*
1705: * Return the new things
1706: */
1707:
1708: *offset = 0;
1709: *object = result;
1710: }
1711:
1712: /*
1713: * The relationship between vm_object structures and
1714: * the memory_object ports requires careful synchronization.
1715: *
1716: * All associations are created by vm_object_enter. All three
1717: * port fields are filled in, as follows:
1718: * pager: the memory_object port itself, supplied by
1719: * the user requesting a mapping (or the kernel,
1720: * when initializing internal objects); the
1721: * kernel simulates holding send rights by keeping
1722: * a port reference;
1723: * pager_request:
1724: * pager_name:
1725: * the memory object control and name ports,
1726: * created by the kernel; the kernel holds
1727: * receive (and ownership) rights to these
1728: * ports, but no other references.
1729: * All of the ports are referenced by their global names.
1730: *
1731: * When initialization is complete, the "initialized" field
1732: * is asserted. Other mappings using a particular memory object,
1733: * and any references to the vm_object gained through the
1734: * port association must wait for this initialization to occur.
1735: *
1736: * In order to allow the memory manager to set attributes before
1737: * requests (notably virtual copy operations, but also data or
1738: * unlock requests) are made, a "ready" attribute is made available.
1739: * Only the memory manager may affect the value of this attribute.
1740: * Its value does not affect critical kernel functions, such as
1741: * internal object initialization or destruction. [Furthermore,
1742: * memory objects created by the kernel are assumed to be ready
1743: * immediately; the default memory manager need not explicitly
1744: * set the "ready" attribute.]
1745: *
1746: * [Both the "initialized" and "ready" attribute wait conditions
1747: * use the "pager" field as the wait event.]
1748: *
1749: * The port associations can be broken down by any of the
1750: * following routines:
1751: * vm_object_terminate:
1752: * No references to the vm_object remain, and
1753: * the object cannot (or will not) be cached.
1754: * This is the normal case, and is done even
1755: * though one of the other cases has already been
1756: * done.
1757: * vm_object_destroy:
1758: * The memory_object port has been destroyed,
1759: * meaning that the kernel cannot flush dirty
1760: * pages or request new data or unlock existing
1761: * data.
1762: * memory_object_destroy:
1763: * The memory manager has requested that the
1764: * kernel relinquish rights to the memory object
1765: * port. [The memory manager may not want to
1766: * destroy the port, but may wish to refuse or
1767: * tear down existing memory mappings.]
1768: * Each routine that breaks an association must break all of
1769: * them at once. At some later time, that routine must clear
1770: * the vm_object port fields and release the port rights.
1771: * [Furthermore, each routine must cope with the simultaneous
1772: * or previous operations of the others.]
1773: *
1774: * In addition to the lock on the object, the vm_object_cache_lock
1775: * governs the port associations. References gained through the
1776: * port association require use of the cache lock.
1777: *
1778: * Because the port fields may be cleared spontaneously, they
1779: * cannot be used to determine whether a memory object has
1780: * ever been associated with a particular vm_object. [This
1781: * knowledge is important to the shadow object mechanism.]
1782: * For this reason, an additional "created" attribute is
1783: * provided.
1784: *
1785: * During various paging operations, the port values found in the
1786: * vm_object must be valid. To prevent these port rights from being
1787: * released, and to prevent the port associations from changing
1788: * (other than being removed, i.e., made null), routines may use
1789: * the vm_object_paging_begin/end routines [actually, macros].
1790: * The implementation uses the "paging_in_progress" and "wanted" fields.
1791: * [Operations that alter the validity of the port values include the
1792: * termination routines and vm_object_collapse.]
1793: */
1794:
1795: vm_object_t vm_object_lookup(
1796: ipc_port_t port)
1797: {
1798: vm_object_t object = VM_OBJECT_NULL;
1799:
1800: if (IP_VALID(port)) {
1801: ip_lock(port);
1802: if (ip_active(port) &&
1803: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1804: vm_object_cache_lock();
1805: object = (vm_object_t) port->ip_kobject;
1806: vm_object_lock(object);
1807:
1808: assert(object->alive);
1809:
1.1.1.5 root 1810: if (object->ref_count == 0)
1811: vm_object_cache_remove(object);
1.1 root 1812:
1813: object->ref_count++;
1814: vm_object_unlock(object);
1815: vm_object_cache_unlock();
1816: }
1817: ip_unlock(port);
1818: }
1819:
1820: return object;
1821: }
1822:
1823: vm_object_t vm_object_lookup_name(
1824: ipc_port_t port)
1825: {
1826: vm_object_t object = VM_OBJECT_NULL;
1827:
1828: if (IP_VALID(port)) {
1829: ip_lock(port);
1830: if (ip_active(port) &&
1831: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1832: vm_object_cache_lock();
1833: object = (vm_object_t) port->ip_kobject;
1834: vm_object_lock(object);
1835:
1836: assert(object->alive);
1837:
1.1.1.5 root 1838: if (object->ref_count == 0)
1839: vm_object_cache_remove(object);
1.1 root 1840:
1841: object->ref_count++;
1842: vm_object_unlock(object);
1843: vm_object_cache_unlock();
1844: }
1845: ip_unlock(port);
1846: }
1847:
1848: return object;
1849: }
1850:
1851: void vm_object_destroy(
1852: ipc_port_t pager)
1853: {
1854: vm_object_t object;
1855: pager_request_t old_request;
1856: ipc_port_t old_name;
1857:
1858: /*
1859: * Perform essentially the same operations as in vm_object_lookup,
1860: * except that this time we look up based on the memory_object
1861: * port, not the control port.
1862: */
1863: vm_object_cache_lock();
1864: if (ip_kotype(pager) != IKOT_PAGER) {
1865: vm_object_cache_unlock();
1866: return;
1867: }
1868:
1869: object = (vm_object_t) pager->ip_kobject;
1870: vm_object_lock(object);
1.1.1.5 root 1871: if (object->ref_count == 0)
1872: vm_object_cache_remove(object);
1.1 root 1873: object->ref_count++;
1874:
1875: object->can_persist = FALSE;
1876:
1877: assert(object->pager == pager);
1878:
1879: /*
1880: * Remove the port associations.
1881: *
1882: * Note that the memory_object itself is dead, so
1883: * we don't bother with it.
1884: */
1885:
1886: object->pager = IP_NULL;
1887: vm_object_remove(object);
1888:
1889: old_request = object->pager_request;
1890: object->pager_request = PAGER_REQUEST_NULL;
1891:
1892: old_name = object->pager_name;
1893: object->pager_name = IP_NULL;
1894:
1895: vm_object_unlock(object);
1896: vm_object_cache_unlock();
1897:
1898: /*
1899: * Clean up the port references. Note that there's no
1900: * point in trying the memory_object_terminate call
1901: * because the memory_object itself is dead.
1902: */
1903:
1904: ipc_port_release_send(pager);
1905: if (old_request != IP_NULL)
1906: ipc_port_dealloc_kernel(old_request);
1907: if (old_name != IP_NULL)
1908: ipc_port_dealloc_kernel(old_name);
1909:
1910: /*
1911: * Restart pending page requests
1912: */
1913:
1914: vm_object_abort_activity(object);
1915:
1916: /*
1917: * Lose the object reference.
1918: */
1919:
1920: vm_object_deallocate(object);
1921: }
1922:
1923: boolean_t vm_object_accept_old_init_protocol = FALSE;
1924:
1925: /*
1926: * Routine: vm_object_enter
1927: * Purpose:
1928: * Find a VM object corresponding to the given
1929: * pager; if no such object exists, create one,
1930: * and initialize the pager.
1931: */
1932: vm_object_t vm_object_enter(
1933: ipc_port_t pager,
1934: vm_size_t size,
1935: boolean_t internal)
1936: {
1937: vm_object_t object;
1938: vm_object_t new_object;
1939: boolean_t must_init;
1940: ipc_kobject_type_t po;
1941:
1942: restart:
1943: if (!IP_VALID(pager))
1944: return vm_object_allocate(size);
1945:
1946: new_object = VM_OBJECT_NULL;
1947: must_init = FALSE;
1948:
1949: /*
1950: * Look for an object associated with this port.
1951: */
1952:
1953: vm_object_cache_lock();
1954: for (;;) {
1955: po = ip_kotype(pager);
1956:
1957: /*
1958: * If a previous object is being terminated,
1959: * we must wait for the termination message
1960: * to be queued.
1961: *
1962: * We set kobject to a non-null value to let the
1963: * terminator know that someone is waiting.
1964: * Among the possibilities is that the port
1965: * could die while we're waiting. Must restart
1966: * instead of continuing the loop.
1967: */
1968:
1969: if (po == IKOT_PAGER_TERMINATING) {
1970: pager->ip_kobject = (ipc_kobject_t) pager;
1971: assert_wait((event_t) pager, FALSE);
1972: vm_object_cache_unlock();
1973: thread_block((void (*)()) 0);
1974: goto restart;
1975: }
1976:
1977: /*
1978: * Bail if there is already a kobject associated
1979: * with the pager port.
1980: */
1981: if (po != IKOT_NONE) {
1982: break;
1983: }
1984:
1985: /*
1986: * We must unlock to create a new object;
1987: * if we do so, we must try the lookup again.
1988: */
1989:
1990: if (new_object == VM_OBJECT_NULL) {
1991: vm_object_cache_unlock();
1992: new_object = vm_object_allocate(size);
1993: vm_object_cache_lock();
1994: } else {
1995: /*
1996: * Lookup failed twice, and we have something
1997: * to insert; set the object.
1998: */
1999:
2000: ipc_kobject_set(pager,
2001: (ipc_kobject_t) new_object,
2002: IKOT_PAGER);
2003: new_object = VM_OBJECT_NULL;
2004: must_init = TRUE;
2005: }
2006: }
2007:
2008: if (internal)
2009: must_init = TRUE;
2010:
2011: /*
2012: * It's only good if it's a VM object!
2013: */
2014:
2015: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2016: : VM_OBJECT_NULL;
2017:
2018: if ((object != VM_OBJECT_NULL) && !must_init) {
2019: vm_object_lock(object);
1.1.1.5 root 2020: if (object->ref_count == 0)
2021: vm_object_cache_remove(object);
1.1 root 2022: object->ref_count++;
2023: vm_object_unlock(object);
2024:
2025: vm_stat.hits++;
2026: }
2027: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2028: ((object->paging_in_progress != 0) && internal));
2029:
2030: vm_stat.lookups++;
2031:
2032: vm_object_cache_unlock();
2033:
2034: /*
2035: * If we raced to create a vm_object but lost, let's
2036: * throw away ours.
2037: */
2038:
2039: if (new_object != VM_OBJECT_NULL)
2040: vm_object_deallocate(new_object);
2041:
2042: if (object == VM_OBJECT_NULL)
2043: return(object);
2044:
2045: if (must_init) {
2046: /*
2047: * Copy the naked send right we were given.
2048: */
2049:
2050: pager = ipc_port_copy_send(pager);
2051: if (!IP_VALID(pager))
2052: panic("vm_object_enter: port died"); /* XXX */
2053:
2054: object->pager_created = TRUE;
2055: object->pager = pager;
2056:
2057: /*
2058: * Allocate request port.
2059: */
2060:
2061: object->pager_request = ipc_port_alloc_kernel();
2062: if (object->pager_request == IP_NULL)
2063: panic("vm_object_enter: pager request alloc");
2064:
2065: ipc_kobject_set(object->pager_request,
2066: (ipc_kobject_t) object,
2067: IKOT_PAGING_REQUEST);
2068:
2069: /*
2070: * Let the pager know we're using it.
2071: */
2072:
2073: if (internal) {
2074: /* acquire a naked send right for the DMM */
2075: ipc_port_t DMM = memory_manager_default_reference();
2076:
2077: /* mark the object internal */
2078: object->internal = TRUE;
2079: assert(object->temporary);
2080:
2081: /* default-pager objects are ready immediately */
2082: object->pager_ready = TRUE;
2083:
2084: /* consumes the naked send right for DMM */
2085: (void) memory_object_create(DMM,
2086: pager,
2087: object->size,
2088: object->pager_request,
2089: object->pager_name,
2090: PAGE_SIZE);
2091: } else {
2092: /* the object is external and not temporary */
2093: object->internal = FALSE;
2094: object->temporary = FALSE;
2095:
1.1.1.6 ! root 2096: assert(object->resident_page_count == 0);
! 2097: vm_object_external_count++;
! 2098:
1.1 root 2099: /* user pager objects are not ready until marked so */
2100: object->pager_ready = FALSE;
2101:
2102: (void) memory_object_init(pager,
2103: object->pager_request,
2104: object->pager_name,
2105: PAGE_SIZE);
2106:
2107: }
2108:
2109: vm_object_lock(object);
2110: object->pager_initialized = TRUE;
2111:
2112: if (vm_object_accept_old_init_protocol)
2113: object->pager_ready = TRUE;
2114:
2115: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2116: } else {
2117: vm_object_lock(object);
2118: }
2119: /*
2120: * [At this point, the object must be locked]
2121: */
2122:
2123: /*
2124: * Wait for the work above to be done by the first
2125: * thread to map this object.
2126: */
2127:
2128: while (!object->pager_initialized) {
2129: vm_object_wait( object,
2130: VM_OBJECT_EVENT_INITIALIZED,
2131: FALSE);
2132: vm_object_lock(object);
2133: }
2134: vm_object_unlock(object);
2135:
2136: return object;
2137: }
2138:
2139: /*
2140: * Routine: vm_object_pager_create
2141: * Purpose:
2142: * Create a memory object for an internal object.
2143: * In/out conditions:
2144: * The object is locked on entry and exit;
2145: * it may be unlocked within this call.
2146: * Limitations:
2147: * Only one thread may be performing a
2148: * vm_object_pager_create on an object at
2149: * a time. Presumably, only the pageout
2150: * daemon will be using this routine.
2151: */
2152: void vm_object_pager_create(
2153: vm_object_t object)
2154: {
2155: ipc_port_t pager;
2156:
2157: if (object->pager_created) {
2158: /*
2159: * Someone else got to it first...
2160: * wait for them to finish initializing
2161: */
2162:
2163: while (!object->pager_initialized) {
2164: vm_object_wait( object,
2165: VM_OBJECT_EVENT_PAGER_READY,
2166: FALSE);
2167: vm_object_lock(object);
2168: }
2169: return;
2170: }
2171:
2172: /*
2173: * Indicate that a memory object has been assigned
2174: * before dropping the lock, to prevent a race.
2175: */
2176:
2177: object->pager_created = TRUE;
2178:
2179: /*
2180: * Prevent collapse or termination by
2181: * holding a paging reference
2182: */
2183:
2184: vm_object_paging_begin(object);
2185: vm_object_unlock(object);
2186:
2187: #if MACH_PAGEMAP
2188: object->existence_info = vm_external_create(
2189: object->size +
2190: object->paging_offset);
2191: assert((object->size + object->paging_offset) >=
2192: object->size);
2193: #endif /* MACH_PAGEMAP */
2194:
2195: /*
2196: * Create the pager, and associate with it
2197: * this object.
2198: *
2199: * Note that we only make the port association
2200: * so that vm_object_enter can properly look up
2201: * the object to complete the initialization...
2202: * we do not expect any user to ever map this
2203: * object.
2204: *
2205: * Since the kernel has the only rights to the
2206: * port, it's safe to install the association
2207: * without holding the cache lock.
2208: */
2209:
2210: pager = ipc_port_alloc_kernel();
2211: if (pager == IP_NULL)
2212: panic("vm_object_pager_create: allocate pager port");
2213:
2214: (void) ipc_port_make_send(pager);
2215: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2216:
2217: /*
2218: * Initialize the rest of the paging stuff
2219: */
2220:
2221: if (vm_object_enter(pager, object->size, TRUE) != object)
2222: panic("vm_object_pager_create: mismatch");
2223:
2224: /*
2225: * Drop the naked send right taken above.
2226: */
2227:
2228: ipc_port_release_send(pager);
2229:
2230: /*
2231: * Release the paging reference
2232: */
2233:
2234: vm_object_lock(object);
2235: vm_object_paging_end(object);
2236: }
2237:
2238: /*
2239: * Routine: vm_object_remove
2240: * Purpose:
2241: * Eliminate the pager/object association
2242: * for this pager.
2243: * Conditions:
2244: * The object cache must be locked.
2245: */
2246: void vm_object_remove(
2247: vm_object_t object)
2248: {
2249: ipc_port_t port;
2250:
2251: if ((port = object->pager) != IP_NULL) {
2252: if (ip_kotype(port) == IKOT_PAGER)
2253: ipc_kobject_set(port, IKO_NULL,
2254: IKOT_PAGER_TERMINATING);
2255: else if (ip_kotype(port) != IKOT_NONE)
2256: panic("vm_object_remove: bad object port");
2257: }
2258: if ((port = object->pager_request) != IP_NULL) {
2259: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2260: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2261: else if (ip_kotype(port) != IKOT_NONE)
2262: panic("vm_object_remove: bad request port");
2263: }
2264: if ((port = object->pager_name) != IP_NULL) {
2265: if (ip_kotype(port) == IKOT_PAGING_NAME)
2266: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2267: else if (ip_kotype(port) != IKOT_NONE)
2268: panic("vm_object_remove: bad name port");
2269: }
2270: }
2271:
2272: /*
2273: * Global variables for vm_object_collapse():
2274: *
2275: * Counts for normal collapses and bypasses.
2276: * Debugging variables, to watch or disable collapse.
2277: */
2278: long object_collapses = 0;
2279: long object_bypasses = 0;
2280:
2281: int vm_object_collapse_debug = 0;
2282: boolean_t vm_object_collapse_allowed = TRUE;
2283: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2284:
2285: /*
2286: * vm_object_collapse:
2287: *
2288: * Collapse an object with the object backing it.
2289: * Pages in the backing object are moved into the
2290: * parent, and the backing object is deallocated.
2291: *
2292: * Requires that the object be locked and the page
2293: * queues be unlocked. May unlock/relock the object,
2294: * so the caller should hold a reference for the object.
2295: */
2296: void vm_object_collapse(
1.1.1.3 root 2297: vm_object_t object)
1.1 root 2298: {
1.1.1.3 root 2299: vm_object_t backing_object;
2300: vm_offset_t backing_offset;
2301: vm_size_t size;
2302: vm_offset_t new_offset;
2303: vm_page_t p, pp;
2304: ipc_port_t old_name_port;
1.1 root 2305:
2306: if (!vm_object_collapse_allowed)
2307: return;
2308:
2309: while (TRUE) {
2310: /*
2311: * Verify that the conditions are right for collapse:
2312: *
2313: * The object exists and no pages in it are currently
2314: * being paged out (or have ever been paged out).
2315: *
2316: * This check is probably overkill -- if a memory
2317: * object has not been created, the fault handler
2318: * shouldn't release the object lock while paging
2319: * is in progress or absent pages exist.
2320: */
2321: if (object == VM_OBJECT_NULL ||
2322: object->pager_created ||
2323: object->paging_in_progress != 0 ||
2324: object->absent_count != 0)
2325: return;
2326:
2327: /*
2328: * There is a backing object, and
2329: */
2330:
2331: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2332: return;
2333:
2334: vm_object_lock(backing_object);
2335: /*
2336: * ...
2337: * The backing object is not read_only,
2338: * and no pages in the backing object are
2339: * currently being paged out.
2340: * The backing object is internal.
2341: *
2342: * XXX It may be sufficient for the backing
2343: * XXX object to be temporary.
2344: */
2345:
2346: if (!backing_object->internal ||
2347: backing_object->paging_in_progress != 0) {
2348: vm_object_unlock(backing_object);
2349: return;
2350: }
2351:
2352: /*
2353: * The backing object can't be a copy-object:
2354: * the shadow_offset for the copy-object must stay
2355: * as 0. Furthermore (for the 'we have all the
2356: * pages' case), if we bypass backing_object and
2357: * just shadow the next object in the chain, old
2358: * pages from that object would then have to be copied
2359: * BOTH into the (former) backing_object and into the
2360: * parent object.
2361: */
2362: if (backing_object->shadow != VM_OBJECT_NULL &&
2363: backing_object->shadow->copy != VM_OBJECT_NULL) {
2364: vm_object_unlock(backing_object);
2365: return;
2366: }
2367:
2368: /*
2369: * We know that we can either collapse the backing
2370: * object (if the parent is the only reference to
2371: * it) or (perhaps) remove the parent's reference
2372: * to it.
2373: */
2374:
2375: backing_offset = object->shadow_offset;
2376: size = object->size;
2377:
2378: /*
2379: * If there is exactly one reference to the backing
2380: * object, we can collapse it into the parent.
2381: */
2382:
2383: if (backing_object->ref_count == 1) {
2384: if (!vm_object_cache_lock_try()) {
2385: vm_object_unlock(backing_object);
2386: return;
2387: }
2388:
2389: /*
2390: * We can collapse the backing object.
2391: *
2392: * Move all in-memory pages from backing_object
2393: * to the parent. Pages that have been paged out
2394: * will be overwritten by any of the parent's
2395: * pages that shadow them.
2396: */
2397:
2398: while (!queue_empty(&backing_object->memq)) {
2399:
2400: p = (vm_page_t)
2401: queue_first(&backing_object->memq);
2402:
2403: new_offset = (p->offset - backing_offset);
2404:
2405: assert(!p->busy || p->absent);
2406:
2407: /*
2408: * If the parent has a page here, or if
2409: * this page falls outside the parent,
2410: * dispose of it.
2411: *
2412: * Otherwise, move it as planned.
2413: */
2414:
2415: if (p->offset < backing_offset ||
2416: new_offset >= size) {
1.1.1.2 root 2417: VM_PAGE_FREE(p);
1.1 root 2418: } else {
2419: pp = vm_page_lookup(object, new_offset);
2420: if (pp != VM_PAGE_NULL && !pp->absent) {
2421: /*
2422: * Parent object has a real page.
2423: * Throw away the backing object's
2424: * page.
2425: */
1.1.1.2 root 2426: VM_PAGE_FREE(p);
1.1 root 2427: }
2428: else {
1.1.1.4 root 2429: assert(pp == VM_PAGE_NULL || !
2430: "vm_object_collapse: bad case");
2431:
1.1 root 2432: /*
2433: * Parent now has no page.
2434: * Move the backing object's page up.
2435: */
2436: vm_page_rename(p, object, new_offset);
2437: }
2438: }
2439: }
2440:
2441: /*
2442: * Move the pager from backing_object to object.
2443: *
2444: * XXX We're only using part of the paging space
2445: * for keeps now... we ought to discard the
2446: * unused portion.
2447: */
2448:
2449: switch (vm_object_collapse_debug) {
2450: case 0:
2451: break;
2452: case 1:
2453: if ((backing_object->pager == IP_NULL) &&
2454: (backing_object->pager_request ==
2455: PAGER_REQUEST_NULL))
2456: break;
2457: /* Fall through to... */
2458:
2459: default:
1.1.1.2 root 2460: printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n",
1.1 root 2461: backing_object, backing_object->pager, backing_object->pager_request,
2462: object);
2463: if (vm_object_collapse_debug > 2)
1.1.1.2 root 2464: SoftDebugger("vm_object_collapse");
1.1 root 2465: }
2466:
2467: object->pager = backing_object->pager;
2468: if (object->pager != IP_NULL)
2469: ipc_kobject_set(object->pager,
2470: (ipc_kobject_t) object,
2471: IKOT_PAGER);
2472: object->pager_initialized = backing_object->pager_initialized;
2473: object->pager_ready = backing_object->pager_ready;
2474: object->pager_created = backing_object->pager_created;
2475:
2476: object->pager_request = backing_object->pager_request;
2477: if (object->pager_request != IP_NULL)
2478: ipc_kobject_set(object->pager_request,
2479: (ipc_kobject_t) object,
2480: IKOT_PAGING_REQUEST);
2481: old_name_port = object->pager_name;
2482: if (old_name_port != IP_NULL)
2483: ipc_kobject_set(old_name_port,
2484: IKO_NULL, IKOT_NONE);
2485: object->pager_name = backing_object->pager_name;
2486: if (object->pager_name != IP_NULL)
2487: ipc_kobject_set(object->pager_name,
2488: (ipc_kobject_t) object,
2489: IKOT_PAGING_NAME);
2490:
2491: vm_object_cache_unlock();
2492:
2493: /*
2494: * If there is no pager, leave paging-offset alone.
2495: */
2496: if (object->pager != IP_NULL)
2497: object->paging_offset =
2498: backing_object->paging_offset +
2499: backing_offset;
2500:
2501: #if MACH_PAGEMAP
2502: assert(object->existence_info == VM_EXTERNAL_NULL);
2503: object->existence_info = backing_object->existence_info;
2504: #endif /* MACH_PAGEMAP */
2505:
2506: /*
2507: * Object now shadows whatever backing_object did.
2508: * Note that the reference to backing_object->shadow
2509: * moves from within backing_object to within object.
2510: */
2511:
2512: object->shadow = backing_object->shadow;
2513: object->shadow_offset += backing_object->shadow_offset;
2514: if (object->shadow != VM_OBJECT_NULL &&
2515: object->shadow->copy != VM_OBJECT_NULL) {
2516: panic("vm_object_collapse: we collapsed a copy-object!");
2517: }
2518: /*
2519: * Discard backing_object.
2520: *
2521: * Since the backing object has no pages, no
2522: * pager left, and no object references within it,
2523: * all that is necessary is to dispose of it.
2524: */
2525:
2526: assert(
2527: (backing_object->ref_count == 1) &&
2528: (backing_object->resident_page_count == 0) &&
2529: (backing_object->paging_in_progress == 0)
2530: );
2531:
2532: assert(backing_object->alive);
1.1.1.5 root 2533: assert(!backing_object->cached);
1.1 root 2534: backing_object->alive = FALSE;
2535: vm_object_unlock(backing_object);
2536:
2537: vm_object_unlock(object);
2538: if (old_name_port != IP_NULL)
2539: ipc_port_dealloc_kernel(old_name_port);
1.1.1.2 root 2540: kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object);
1.1 root 2541: vm_object_lock(object);
2542:
2543: object_collapses++;
2544: }
2545: else {
2546: if (!vm_object_collapse_bypass_allowed) {
2547: vm_object_unlock(backing_object);
2548: return;
2549: }
2550:
2551: /*
2552: * If all of the pages in the backing object are
2553: * shadowed by the parent object, the parent
2554: * object no longer has to shadow the backing
2555: * object; it can shadow the next one in the
2556: * chain.
2557: *
2558: * The backing object must not be paged out - we'd
2559: * have to check all of the paged-out pages, as
2560: * well.
2561: */
2562:
2563: if (backing_object->pager_created) {
2564: vm_object_unlock(backing_object);
2565: return;
2566: }
2567:
2568: /*
2569: * Should have a check for a 'small' number
2570: * of pages here.
2571: */
2572:
2573: queue_iterate(&backing_object->memq, p,
2574: vm_page_t, listq)
2575: {
2576: new_offset = (p->offset - backing_offset);
2577:
2578: /*
2579: * If the parent has a page here, or if
2580: * this page falls outside the parent,
2581: * keep going.
2582: *
2583: * Otherwise, the backing_object must be
2584: * left in the chain.
2585: */
2586:
2587: if (p->offset >= backing_offset &&
2588: new_offset <= size &&
2589: (pp = vm_page_lookup(object, new_offset))
2590: == VM_PAGE_NULL) {
2591: /*
2592: * Page still needed.
2593: * Can't go any further.
2594: */
2595: vm_object_unlock(backing_object);
2596: return;
2597: }
2598: }
2599:
2600: /*
2601: * Make the parent shadow the next object
2602: * in the chain. Deallocating backing_object
2603: * will not remove it, since its reference
2604: * count is at least 2.
2605: */
2606:
2607: vm_object_reference(object->shadow = backing_object->shadow);
2608: object->shadow_offset += backing_object->shadow_offset;
2609:
2610: /*
2611: * Backing object might have had a copy pointer
2612: * to us. If it did, clear it.
2613: */
2614: if (backing_object->copy == object)
2615: backing_object->copy = VM_OBJECT_NULL;
2616:
2617: /*
2618: * Drop the reference count on backing_object.
2619: * Since its ref_count was at least 2, it
2620: * will not vanish; so we don't need to call
2621: * vm_object_deallocate.
2622: */
2623: backing_object->ref_count--;
2624: assert(backing_object->ref_count > 0);
2625: vm_object_unlock(backing_object);
2626:
2627: object_bypasses ++;
2628:
2629: }
2630:
2631: /*
2632: * Try again with this object's new backing object.
2633: */
2634: }
2635: }
2636:
2637: /*
2638: * Routine: vm_object_page_remove: [internal]
2639: * Purpose:
2640: * Removes all physical pages in the specified
2641: * object range from the object's list of pages.
2642: *
2643: * In/out conditions:
2644: * The object must be locked.
2645: */
2646: unsigned int vm_object_page_remove_lookup = 0;
2647: unsigned int vm_object_page_remove_iterate = 0;
2648:
2649: void vm_object_page_remove(
1.1.1.3 root 2650: vm_object_t object,
2651: vm_offset_t start,
2652: vm_offset_t end)
1.1 root 2653: {
1.1.1.3 root 2654: vm_page_t p, next;
1.1 root 2655:
2656: /*
2657: * One and two page removals are most popular.
2658: * The factor of 16 here is somewhat arbitrary.
2659: * It balances vm_object_lookup vs iteration.
2660: */
2661:
1.1.1.5 root 2662: if (atop(end - start) < object->resident_page_count/16) {
1.1 root 2663: vm_object_page_remove_lookup++;
2664:
2665: for (; start < end; start += PAGE_SIZE) {
2666: p = vm_page_lookup(object, start);
2667: if (p != VM_PAGE_NULL) {
2668: if (!p->fictitious)
2669: pmap_page_protect(p->phys_addr,
2670: VM_PROT_NONE);
1.1.1.2 root 2671: VM_PAGE_FREE(p);
1.1 root 2672: }
2673: }
2674: } else {
2675: vm_object_page_remove_iterate++;
2676:
2677: p = (vm_page_t) queue_first(&object->memq);
2678: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2679: next = (vm_page_t) queue_next(&p->listq);
2680: if ((start <= p->offset) && (p->offset < end)) {
2681: if (!p->fictitious)
2682: pmap_page_protect(p->phys_addr,
2683: VM_PROT_NONE);
1.1.1.2 root 2684: VM_PAGE_FREE(p);
1.1 root 2685: }
2686: p = next;
2687: }
2688: }
2689: }
2690:
2691: /*
2692: * Routine: vm_object_coalesce
2693: * Function: Coalesces two objects backing up adjoining
2694: * regions of memory into a single object.
2695: *
2696: * returns TRUE if objects were combined.
2697: *
2698: * NOTE: Only works at the moment if the second object is NULL -
2699: * if it's not, which object do we lock first?
2700: *
2701: * Parameters:
2702: * prev_object First object to coalesce
2703: * prev_offset Offset into prev_object
2704: * next_object Second object into coalesce
2705: * next_offset Offset into next_object
2706: *
2707: * prev_size Size of reference to prev_object
2708: * next_size Size of reference to next_object
2709: *
2710: * Conditions:
2711: * The object must *not* be locked.
2712: */
2713:
2714: boolean_t vm_object_coalesce(
1.1.1.3 root 2715: vm_object_t prev_object,
1.1 root 2716: vm_object_t next_object,
2717: vm_offset_t prev_offset,
2718: vm_offset_t next_offset,
2719: vm_size_t prev_size,
2720: vm_size_t next_size)
2721: {
2722: vm_size_t newsize;
2723:
2724: if (next_object != VM_OBJECT_NULL) {
2725: return FALSE;
2726: }
2727:
2728: if (prev_object == VM_OBJECT_NULL) {
2729: return TRUE;
2730: }
2731:
2732: vm_object_lock(prev_object);
2733:
2734: /*
2735: * Try to collapse the object first
2736: */
2737: vm_object_collapse(prev_object);
2738:
2739: /*
2740: * Can't coalesce if pages not mapped to
2741: * prev_entry may be in use anyway:
2742: * . more than one reference
2743: * . paged out
2744: * . shadows another object
2745: * . has a copy elsewhere
2746: * . paging references (pages might be in page-list)
2747: */
2748:
2749: if ((prev_object->ref_count > 1) ||
2750: prev_object->pager_created ||
1.1.1.5 root 2751: prev_object->used_for_pageout ||
1.1 root 2752: (prev_object->shadow != VM_OBJECT_NULL) ||
2753: (prev_object->copy != VM_OBJECT_NULL) ||
2754: (prev_object->paging_in_progress != 0)) {
2755: vm_object_unlock(prev_object);
2756: return FALSE;
2757: }
2758:
2759: /*
2760: * Remove any pages that may still be in the object from
2761: * a previous deallocation.
2762: */
2763:
2764: vm_object_page_remove(prev_object,
2765: prev_offset + prev_size,
2766: prev_offset + prev_size + next_size);
2767:
2768: /*
2769: * Extend the object if necessary.
2770: */
2771: newsize = prev_offset + prev_size + next_size;
2772: if (newsize > prev_object->size)
2773: prev_object->size = newsize;
2774:
2775: vm_object_unlock(prev_object);
2776: return TRUE;
2777: }
2778:
2779: vm_object_t vm_object_request_object(
2780: ipc_port_t p)
2781: {
2782: return vm_object_lookup(p);
2783: }
2784:
2785: /*
2786: * Routine: vm_object_name
2787: * Purpose:
2788: * Returns a naked send right to the "name" port associated
2789: * with this object.
2790: */
2791: ipc_port_t vm_object_name(
2792: vm_object_t object)
2793: {
2794: ipc_port_t p;
2795:
2796: if (object == VM_OBJECT_NULL)
2797: return IP_NULL;
2798:
2799: vm_object_lock(object);
2800:
2801: while (object->shadow != VM_OBJECT_NULL) {
2802: vm_object_t new_object = object->shadow;
2803: vm_object_lock(new_object);
2804: vm_object_unlock(object);
2805: object = new_object;
2806: }
2807:
2808: p = object->pager_name;
2809: if (p != IP_NULL)
2810: p = ipc_port_make_send(p);
2811: vm_object_unlock(object);
2812:
2813: return p;
2814: }
2815:
2816: /*
2817: * Attach a set of physical pages to an object, so that they can
2818: * be mapped by mapping the object. Typically used to map IO memory.
2819: *
2820: * The mapping function and its private data are used to obtain the
2821: * physical addresses for each page to be mapped.
2822: */
2823: void
2824: vm_object_page_map(
2825: vm_object_t object,
2826: vm_offset_t offset,
2827: vm_size_t size,
2828: vm_offset_t (*map_fn)(void *, vm_offset_t),
2829: void * map_fn_data) /* private to map_fn */
2830: {
2831: int num_pages;
2832: int i;
2833: vm_page_t m;
2834: vm_page_t old_page;
2835: vm_offset_t addr;
2836:
2837: num_pages = atop(size);
2838:
2839: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2840:
2841: addr = (*map_fn)(map_fn_data, offset);
2842:
2843: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
2844: vm_page_more_fictitious();
2845:
2846: vm_object_lock(object);
2847: if ((old_page = vm_page_lookup(object, offset))
2848: != VM_PAGE_NULL)
2849: {
1.1.1.2 root 2850: VM_PAGE_FREE(old_page);
1.1 root 2851: }
2852:
1.1.1.5 root 2853: vm_page_init(m);
2854: m->phys_addr = addr;
1.1 root 2855: m->private = TRUE; /* don`t free page */
2856: m->wire_count = 1;
2857: vm_page_lock_queues();
2858: vm_page_insert(m, object, offset);
2859: vm_page_unlock_queues();
2860:
2861: PAGE_WAKEUP_DONE(m);
2862: vm_object_unlock(object);
2863: }
2864: }
2865:
2866:
2867: #if MACH_KDB
1.1.1.2 root 2868: #include <vm/vm_print.h>
1.1 root 2869: #define printf kdbprintf
2870:
2871: boolean_t vm_object_print_pages = FALSE;
2872:
2873: /*
2874: * vm_object_print: [ debug ]
2875: */
2876: void vm_object_print(
2877: vm_object_t object)
2878: {
1.1.1.3 root 2879: vm_page_t p;
1.1 root 2880:
1.1.1.3 root 2881: int count;
1.1 root 2882:
2883: if (object == VM_OBJECT_NULL)
2884: return;
2885:
1.1.1.5 root 2886: iprintf("Object 0x%X: size=0x%X, %d references",
2887: (vm_offset_t) object, (vm_offset_t) object->size,
2888: object->ref_count);
2889: printf("\n");
2890: iprintf("%lu resident pages,", object->resident_page_count);
1.1 root 2891: printf(" %d absent pages,", object->absent_count);
2892: printf(" %d paging ops\n", object->paging_in_progress);
1.1.1.5 root 2893: indent += 1;
1.1 root 2894: iprintf("memory object=0x%X (offset=0x%X),",
2895: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
2896: printf("control=0x%X, name=0x%X\n",
2897: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
2898: iprintf("%s%s",
2899: object->pager_ready ? " ready" : "",
2900: object->pager_created ? " created" : "");
2901: printf("%s,%s ",
2902: object->pager_initialized ? "" : "uninitialized",
2903: object->temporary ? "temporary" : "permanent");
2904: printf("%s%s,",
2905: object->internal ? "internal" : "external",
2906: object->can_persist ? " cacheable" : "");
2907: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
2908: iprintf("shadow=0x%X (offset=0x%X),",
2909: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
2910: printf("copy=0x%X\n", (vm_offset_t) object->copy);
2911:
1.1.1.5 root 2912: indent += 1;
1.1 root 2913:
2914: if (vm_object_print_pages) {
2915: count = 0;
2916: p = (vm_page_t) queue_first(&object->memq);
2917: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2918: if (count == 0) iprintf("memory:=");
2919: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
2920: else printf(",");
2921: count++;
2922:
2923: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
2924: p = (vm_page_t) queue_next(&p->listq);
2925: }
2926: if (count != 0)
2927: printf("\n");
2928: }
1.1.1.5 root 2929: indent -= 2;
1.1 root 2930: }
2931:
2932: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.