![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
1.1.1.2 ! root 36: #include <kern/printf.h>
! 37: #include <string.h>
1.1 root 38:
39: #include <mach/memory_object.h>
1.1.1.2 ! root 40: #include <vm/memory_object_default.user.h>
! 41: #include <vm/memory_object_user.user.h>
! 42: #include <machine/vm_param.h>
1.1 root 43: #include <ipc/ipc_port.h>
44: #include <ipc/ipc_space.h>
45: #include <kern/assert.h>
1.1.1.2 ! root 46: #include <kern/debug.h>
1.1 root 47: #include <kern/lock.h>
48: #include <kern/queue.h>
49: #include <kern/xpr.h>
1.1.1.2 ! root 50: #include <kern/slab.h>
1.1 root 51: #include <vm/memory_object.h>
52: #include <vm/vm_fault.h>
53: #include <vm/vm_map.h>
54: #include <vm/vm_object.h>
55: #include <vm/vm_page.h>
56: #include <vm/vm_pageout.h>
57:
1.1.1.2 ! root 58: #if MACH_KDB
! 59: #include <ddb/db_output.h>
! 60: #endif /* MACH_KDB */
! 61:
1.1 root 62:
63: void memory_object_release(
64: ipc_port_t pager,
65: pager_request_t pager_request,
66: ipc_port_t pager_name); /* forward */
67:
68: void vm_object_deactivate_pages(vm_object_t);
69:
70: /*
71: * Virtual memory objects maintain the actual data
72: * associated with allocated virtual memory. A given
73: * page of memory exists within exactly one object.
74: *
75: * An object is only deallocated when all "references"
76: * are given up. Only one "reference" to a given
77: * region of an object should be writeable.
78: *
79: * Associated with each object is a list of all resident
80: * memory pages belonging to that object; this list is
81: * maintained by the "vm_page" module, but locked by the object's
82: * lock.
83: *
84: * Each object also records the memory object port
85: * that is used by the kernel to request and write
86: * back data (the memory object port, field "pager"),
87: * and the ports provided to the memory manager, the server that
88: * manages that data, to return data and control its
89: * use (the memory object control port, field "pager_request")
90: * and for naming (the memory object name port, field "pager_name").
91: *
92: * Virtual memory objects are allocated to provide
93: * zero-filled memory (vm_allocate) or map a user-defined
94: * memory object into a virtual address space (vm_map).
95: *
96: * Virtual memory objects that refer to a user-defined
97: * memory object are called "permanent", because all changes
98: * made in virtual memory are reflected back to the
99: * memory manager, which may then store it permanently.
100: * Other virtual memory objects are called "temporary",
101: * meaning that changes need be written back only when
102: * necessary to reclaim pages, and that storage associated
103: * with the object can be discarded once it is no longer
104: * mapped.
105: *
106: * A permanent memory object may be mapped into more
107: * than one virtual address space. Moreover, two threads
108: * may attempt to make the first mapping of a memory
109: * object concurrently. Only one thread is allowed to
110: * complete this mapping; all others wait for the
111: * "pager_initialized" field is asserted, indicating
112: * that the first thread has initialized all of the
113: * necessary fields in the virtual memory object structure.
114: *
115: * The kernel relies on a *default memory manager* to
116: * provide backing storage for the zero-filled virtual
117: * memory objects. The memory object ports associated
118: * with these temporary virtual memory objects are only
119: * generated and passed to the default memory manager
120: * when it becomes necessary. Virtual memory objects
121: * that depend on the default memory manager are called
122: * "internal". The "pager_created" field is provided to
123: * indicate whether these ports have ever been allocated.
124: *
125: * The kernel may also create virtual memory objects to
126: * hold changed pages after a copy-on-write operation.
127: * In this case, the virtual memory object (and its
128: * backing storage -- its memory object) only contain
129: * those pages that have been changed. The "shadow"
130: * field refers to the virtual memory object that contains
131: * the remainder of the contents. The "shadow_offset"
132: * field indicates where in the "shadow" these contents begin.
133: * The "copy" field refers to a virtual memory object
134: * to which changed pages must be copied before changing
135: * this object, in order to implement another form
136: * of copy-on-write optimization.
137: *
138: * The virtual memory object structure also records
139: * the attributes associated with its memory object.
140: * The "pager_ready", "can_persist" and "copy_strategy"
141: * fields represent those attributes. The "cached_list"
142: * field is used in the implementation of the persistence
143: * attribute.
144: *
145: * ZZZ Continue this comment.
146: */
147:
1.1.1.2 ! root 148: struct kmem_cache vm_object_cache; /* vm backing store cache */
1.1 root 149:
150: /*
151: * All wired-down kernel memory belongs to a single virtual
152: * memory object (kernel_object) to avoid wasting data structures.
153: */
1.1.1.2 ! root 154: static struct vm_object kernel_object_store;
! 155: vm_object_t kernel_object = &kernel_object_store;
1.1 root 156:
157: /*
158: * Virtual memory objects that are not referenced by
159: * any address maps, but that are allowed to persist
160: * (an attribute specified by the associated memory manager),
161: * are kept in a queue (vm_object_cached_list).
162: *
163: * When an object from this queue is referenced again,
164: * for example to make another address space mapping,
165: * it must be removed from the queue. That is, the
166: * queue contains *only* objects with zero references.
167: *
168: * The kernel may choose to terminate objects from this
169: * queue in order to reclaim storage. The current policy
170: * is to permit a fixed maximum number of unreferenced
171: * objects (vm_object_cached_max).
172: *
173: * A simple lock (accessed by routines
174: * vm_object_cache_{lock,lock_try,unlock}) governs the
175: * object cache. It must be held when objects are
176: * added to or removed from the cache (in vm_object_terminate).
177: * The routines that acquire a reference to a virtual
178: * memory object based on one of the memory object ports
179: * must also lock the cache.
180: *
181: * Ideally, the object cache should be more isolated
182: * from the reference mechanism, so that the lock need
183: * not be held to make simple references.
184: */
185: queue_head_t vm_object_cached_list;
186: int vm_object_cached_count;
1.1.1.2 ! root 187: int vm_object_cached_max = 4000; /* may be patched*/
1.1 root 188:
189: decl_simple_lock_data(,vm_object_cached_lock_data)
190:
191: #define vm_object_cache_lock() \
192: simple_lock(&vm_object_cached_lock_data)
193: #define vm_object_cache_lock_try() \
194: simple_lock_try(&vm_object_cached_lock_data)
195: #define vm_object_cache_unlock() \
196: simple_unlock(&vm_object_cached_lock_data)
197:
198: /*
1.1.1.2 ! root 199: * Number of physical pages referenced by cached objects.
! 200: * This counter is protected by its own lock to work around
! 201: * lock ordering issues.
! 202: */
! 203: int vm_object_cached_pages;
! 204:
! 205: decl_simple_lock_data(,vm_object_cached_pages_lock_data)
! 206:
! 207: /*
1.1 root 208: * Virtual memory objects are initialized from
209: * a template (see vm_object_allocate).
210: *
211: * When adding a new field to the virtual memory
212: * object structure, be sure to add initialization
213: * (see vm_object_init).
214: */
1.1.1.2 ! root 215: struct vm_object vm_object_template;
1.1 root 216:
217: /*
218: * vm_object_allocate:
219: *
220: * Returns a new object with the given size.
221: */
222:
1.1.1.2 ! root 223: static void _vm_object_setup(
! 224: vm_object_t object,
! 225: vm_size_t size)
! 226: {
! 227: *object = vm_object_template;
! 228: queue_init(&object->memq);
! 229: vm_object_lock_init(object);
! 230: object->size = size;
! 231: }
! 232:
1.1 root 233: vm_object_t _vm_object_allocate(
234: vm_size_t size)
235: {
236: register vm_object_t object;
237:
1.1.1.2 ! root 238: object = (vm_object_t) kmem_cache_alloc(&vm_object_cache);
1.1 root 239:
1.1.1.2 ! root 240: _vm_object_setup(object, size);
1.1 root 241:
242: return object;
243: }
244:
245: vm_object_t vm_object_allocate(
246: vm_size_t size)
247: {
248: register vm_object_t object;
249: register ipc_port_t port;
250:
251: object = _vm_object_allocate(size);
252: port = ipc_port_alloc_kernel();
253: if (port == IP_NULL)
254: panic("vm_object_allocate");
255: object->pager_name = port;
256: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
257:
258: return object;
259: }
260:
261: /*
262: * vm_object_bootstrap:
263: *
264: * Initialize the VM objects module.
265: */
266: void vm_object_bootstrap(void)
267: {
1.1.1.2 ! root 268: kmem_cache_init(&vm_object_cache, "vm_object",
! 269: sizeof(struct vm_object), 0, NULL, NULL, NULL, 0);
1.1 root 270:
271: queue_init(&vm_object_cached_list);
272: simple_lock_init(&vm_object_cached_lock_data);
273:
274: /*
275: * Fill in a template object, for quick initialization
276: */
277:
1.1.1.2 ! root 278: vm_object_template.ref_count = 1;
! 279: vm_object_template.size = 0;
! 280: vm_object_template.resident_page_count = 0;
! 281: vm_object_template.copy = VM_OBJECT_NULL;
! 282: vm_object_template.shadow = VM_OBJECT_NULL;
! 283: vm_object_template.shadow_offset = (vm_offset_t) 0;
! 284:
! 285: vm_object_template.pager = IP_NULL;
! 286: vm_object_template.paging_offset = 0;
! 287: vm_object_template.pager_request = PAGER_REQUEST_NULL;
! 288: vm_object_template.pager_name = IP_NULL;
! 289:
! 290: vm_object_template.pager_created = FALSE;
! 291: vm_object_template.pager_initialized = FALSE;
! 292: vm_object_template.pager_ready = FALSE;
1.1 root 293:
1.1.1.2 ! root 294: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE;
1.1 root 295: /* ignored if temporary, will be reset before
296: * permanent object becomes ready */
1.1.1.2 ! root 297: vm_object_template.use_shared_copy = FALSE;
! 298: vm_object_template.shadowed = FALSE;
1.1 root 299:
1.1.1.2 ! root 300: vm_object_template.absent_count = 0;
! 301: vm_object_template.all_wanted = 0; /* all bits FALSE */
1.1 root 302:
1.1.1.2 ! root 303: vm_object_template.paging_in_progress = 0;
! 304: vm_object_template.can_persist = FALSE;
! 305: vm_object_template.internal = TRUE;
! 306: vm_object_template.temporary = TRUE;
! 307: vm_object_template.alive = TRUE;
! 308: vm_object_template.lock_in_progress = FALSE;
! 309: vm_object_template.lock_restart = FALSE;
! 310: vm_object_template.use_old_pageout = TRUE; /* XXX change later */
! 311: vm_object_template.last_alloc = (vm_offset_t) 0;
1.1 root 312:
313: #if MACH_PAGEMAP
1.1.1.2 ! root 314: vm_object_template.existence_info = VM_EXTERNAL_NULL;
1.1 root 315: #endif /* MACH_PAGEMAP */
316:
317: /*
318: * Initialize the "kernel object"
319: */
320:
1.1.1.2 ! root 321: _vm_object_setup(kernel_object,
1.1 root 322: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
323:
324: /*
325: * Initialize the "submap object". Make it as large as the
326: * kernel object so that no limit is imposed on submap sizes.
327: */
328:
1.1.1.2 ! root 329: _vm_object_setup(vm_submap_object,
1.1 root 330: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
331:
332: #if MACH_PAGEMAP
333: vm_external_module_initialize();
334: #endif /* MACH_PAGEMAP */
335: }
336:
337: void vm_object_init(void)
338: {
339: /*
340: * Finish initializing the kernel object.
341: * The submap object doesn't need a name port.
342: */
343:
344: kernel_object->pager_name = ipc_port_alloc_kernel();
345: ipc_kobject_set(kernel_object->pager_name,
346: (ipc_kobject_t) kernel_object,
347: IKOT_PAGING_NAME);
348: }
349:
350: /*
351: * vm_object_reference:
352: *
353: * Gets another reference to the given object.
354: */
355: void vm_object_reference(
356: register vm_object_t object)
357: {
358: if (object == VM_OBJECT_NULL)
359: return;
360:
361: vm_object_lock(object);
362: assert(object->ref_count > 0);
363: object->ref_count++;
364: vm_object_unlock(object);
365: }
366:
367: /*
368: * vm_object_deallocate:
369: *
370: * Release a reference to the specified object,
371: * gained either through a vm_object_allocate
372: * or a vm_object_reference call. When all references
373: * are gone, storage associated with this object
374: * may be relinquished.
375: *
376: * No object may be locked.
377: */
378: void vm_object_deallocate(
379: register vm_object_t object)
380: {
381: vm_object_t temp;
382:
383: while (object != VM_OBJECT_NULL) {
384:
385: /*
386: * The cache holds a reference (uncounted) to
387: * the object; we must lock it before removing
388: * the object.
389: */
390:
391: vm_object_cache_lock();
392:
393: /*
394: * Lose the reference
395: */
396: vm_object_lock(object);
397: if (--(object->ref_count) > 0) {
398:
399: /*
400: * If there are still references, then
401: * we are done.
402: */
403: vm_object_unlock(object);
404: vm_object_cache_unlock();
405: return;
406: }
407:
408: /*
409: * See whether this object can persist. If so, enter
410: * it in the cache, then deactivate all of its
411: * pages.
412: */
413: if (object->can_persist) {
414: boolean_t overflow;
415:
416: /*
417: * Enter the object onto the queue
418: * of "cached" objects. Remember whether
419: * we've caused the queue to overflow,
420: * as a hint.
421: */
422:
423: queue_enter(&vm_object_cached_list, object,
424: vm_object_t, cached_list);
425: overflow = (++vm_object_cached_count > vm_object_cached_max);
1.1.1.2 ! root 426: vm_object_cached_pages_update(object->resident_page_count);
1.1 root 427: vm_object_cache_unlock();
428:
429: vm_object_deactivate_pages(object);
430: vm_object_unlock(object);
431:
432: /*
433: * If we didn't overflow, or if the queue has
434: * been reduced back to below the specified
435: * minimum, then quit.
436: */
437: if (!overflow)
438: return;
439:
440: while (TRUE) {
441: vm_object_cache_lock();
442: if (vm_object_cached_count <=
443: vm_object_cached_max) {
444: vm_object_cache_unlock();
445: return;
446: }
447:
448: /*
449: * If we must trim down the queue, take
450: * the first object, and proceed to
451: * terminate it instead of the original
452: * object. Have to wait for pager init.
453: * if it's in progress.
454: */
455: object= (vm_object_t)
456: queue_first(&vm_object_cached_list);
457: vm_object_lock(object);
458:
459: if (!(object->pager_created &&
460: !object->pager_initialized)) {
461:
462: /*
463: * Ok to terminate, hang on to lock.
464: */
465: break;
466: }
467:
468: vm_object_assert_wait(object,
469: VM_OBJECT_EVENT_INITIALIZED, FALSE);
470: vm_object_unlock(object);
471: vm_object_cache_unlock();
472: thread_block((void (*)()) 0);
473:
474: /*
475: * Continue loop to check if cache still
476: * needs to be trimmed.
477: */
478: }
479:
480: /*
481: * Actually remove object from cache.
482: */
483:
484: queue_remove(&vm_object_cached_list, object,
485: vm_object_t, cached_list);
486: vm_object_cached_count--;
487:
488: assert(object->ref_count == 0);
489: }
490: else {
491: if (object->pager_created &&
492: !object->pager_initialized) {
493:
494: /*
495: * Have to wait for initialization.
496: * Put reference back and retry
497: * when it's initialized.
498: */
499: object->ref_count++;
500: vm_object_assert_wait(object,
501: VM_OBJECT_EVENT_INITIALIZED, FALSE);
502: vm_object_unlock(object);
503: vm_object_cache_unlock();
504: thread_block((void (*)()) 0);
505: continue;
506: }
507: }
508:
509: /*
510: * Take the reference to the shadow object
511: * out of the object to be destroyed.
512: */
513:
514: temp = object->shadow;
515:
516: /*
517: * Destroy the object; the cache lock will
518: * be released in the process.
519: */
520:
521: vm_object_terminate(object);
522:
523: /*
524: * Deallocate the reference to the shadow
525: * by continuing the loop with that object
526: * in place of the original.
527: */
528:
529: object = temp;
530: }
531: }
532:
533: boolean_t vm_object_terminate_remove_all = FALSE;
534:
535: /*
536: * Routine: vm_object_terminate
537: * Purpose:
538: * Free all resources associated with a vm_object.
539: * In/out conditions:
540: * Upon entry, the object and the cache must be locked,
541: * and the object must have no references.
542: *
543: * The shadow object reference is left alone.
544: *
545: * Upon exit, the cache will be unlocked, and the
546: * object will cease to exist.
547: */
548: void vm_object_terminate(
549: register vm_object_t object)
550: {
551: register vm_page_t p;
552: vm_object_t shadow_object;
553:
554: /*
555: * Make sure the object isn't already being terminated
556: */
557:
558: assert(object->alive);
559: object->alive = FALSE;
560:
561: /*
562: * Make sure no one can look us up now.
563: */
564:
565: vm_object_remove(object);
566: vm_object_cache_unlock();
567:
568: /*
569: * Detach the object from its shadow if we are the shadow's
570: * copy.
571: */
572: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
573: vm_object_lock(shadow_object);
574: assert((shadow_object->copy == object) ||
575: (shadow_object->copy == VM_OBJECT_NULL));
576: shadow_object->copy = VM_OBJECT_NULL;
577: vm_object_unlock(shadow_object);
578: }
579:
580: /*
581: * The pageout daemon might be playing with our pages.
582: * Now that the object is dead, it won't touch any more
583: * pages, but some pages might already be on their way out.
584: * Hence, we wait until the active paging activities have ceased.
585: */
586:
587: vm_object_paging_wait(object, FALSE);
588:
589: /*
590: * Clean or free the pages, as appropriate.
591: * It is possible for us to find busy/absent pages,
592: * if some faults on this object were aborted.
593: */
594:
595: if ((object->temporary) || (object->pager == IP_NULL)) {
596: while (!queue_empty(&object->memq)) {
597: p = (vm_page_t) queue_first(&object->memq);
598:
599: VM_PAGE_CHECK(p);
600:
601: if (p->busy && !p->absent)
602: panic("vm_object_terminate.2 0x%x 0x%x",
603: object, p);
604:
605: VM_PAGE_FREE(p);
606: }
607: } else while (!queue_empty(&object->memq)) {
608: p = (vm_page_t) queue_first(&object->memq);
609:
610: VM_PAGE_CHECK(p);
611:
612: if (p->busy && !p->absent)
613: panic("vm_object_terminate.3 0x%x 0x%x", object, p);
614:
615: vm_page_lock_queues();
616: VM_PAGE_QUEUES_REMOVE(p);
617: vm_page_unlock_queues();
618:
619: if (p->absent || p->private) {
620:
621: /*
622: * For private pages, VM_PAGE_FREE just
623: * leaves the page structure around for
624: * its owner to clean up. For absent
625: * pages, the structure is returned to
626: * the appropriate pool.
627: */
628:
629: goto free_page;
630: }
631:
632: if (p->fictitious)
633: panic("vm_object_terminate.4 0x%x 0x%x", object, p);
634:
635: if (!p->dirty)
636: p->dirty = pmap_is_modified(p->phys_addr);
637:
638: if (p->dirty || p->precious) {
639: p->busy = TRUE;
640: vm_pageout_page(p, FALSE, TRUE); /* flush page */
641: } else {
642: free_page:
643: VM_PAGE_FREE(p);
644: }
645: }
646:
647: assert(object->ref_count == 0);
648: assert(object->paging_in_progress == 0);
649:
650: /*
651: * Throw away port rights... note that they may
652: * already have been thrown away (by vm_object_destroy
653: * or memory_object_destroy).
654: *
655: * Instead of destroying the control and name ports,
656: * we send all rights off to the memory manager instead,
657: * using memory_object_terminate.
658: */
659:
660: vm_object_unlock(object);
661:
662: if (object->pager != IP_NULL) {
663: /* consumes our rights for pager, pager_request, pager_name */
664: memory_object_release(object->pager,
665: object->pager_request,
666: object->pager_name);
667: } else if (object->pager_name != IP_NULL) {
668: /* consumes our right for pager_name */
669: ipc_port_dealloc_kernel(object->pager_name);
670: }
671:
672: #if MACH_PAGEMAP
673: vm_external_destroy(object->existence_info);
674: #endif /* MACH_PAGEMAP */
675:
676: /*
677: * Free the space for the object.
678: */
679:
1.1.1.2 ! root 680: kmem_cache_free(&vm_object_cache, (vm_offset_t) object);
1.1 root 681: }
682:
683: /*
684: * Routine: vm_object_pager_wakeup
685: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
686: */
687:
688: void
689: vm_object_pager_wakeup(
690: ipc_port_t pager)
691: {
692: boolean_t someone_waiting;
693:
694: /*
695: * If anyone was waiting for the memory_object_terminate
696: * to be queued, wake them up now.
697: */
698: vm_object_cache_lock();
699: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
700: someone_waiting = (pager->ip_kobject != IKO_NULL);
701: if (ip_active(pager))
702: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
703: vm_object_cache_unlock();
704: if (someone_waiting) {
705: thread_wakeup((event_t) pager);
706: }
707: }
708:
709: /*
710: * Routine: memory_object_release
711: * Purpose: Terminate the pager and release port rights,
712: * just like memory_object_terminate, except
713: * that we wake up anyone blocked in vm_object_enter
714: * waiting for termination message to be queued
715: * before calling memory_object_init.
716: */
717: void memory_object_release(
718: ipc_port_t pager,
719: pager_request_t pager_request,
720: ipc_port_t pager_name)
721: {
722:
723: /*
724: * Keep a reference to pager port;
725: * the terminate might otherwise release all references.
726: */
727: ip_reference(pager);
728:
729: /*
730: * Terminate the pager.
731: */
732: (void) memory_object_terminate(pager, pager_request, pager_name);
733:
734: /*
735: * Wakeup anyone waiting for this terminate
736: */
737: vm_object_pager_wakeup(pager);
738:
739: /*
740: * Release reference to pager port.
741: */
742: ip_release(pager);
743: }
744:
745: /*
746: * Routine: vm_object_abort_activity [internal use only]
747: * Purpose:
748: * Abort paging requests pending on this object.
749: * In/out conditions:
750: * The object is locked on entry and exit.
751: */
752: void vm_object_abort_activity(
753: vm_object_t object)
754: {
755: register
756: vm_page_t p;
757: vm_page_t next;
758:
759: /*
760: * Abort all activity that would be waiting
761: * for a result on this memory object.
762: *
763: * We could also choose to destroy all pages
764: * that we have in memory for this object, but
765: * we don't.
766: */
767:
768: p = (vm_page_t) queue_first(&object->memq);
769: while (!queue_end(&object->memq, (queue_entry_t) p)) {
770: next = (vm_page_t) queue_next(&p->listq);
771:
772: /*
773: * If it's being paged in, destroy it.
774: * If an unlock has been requested, start it again.
775: */
776:
777: if (p->busy && p->absent) {
778: VM_PAGE_FREE(p);
779: }
780: else {
781: if (p->unlock_request != VM_PROT_NONE)
782: p->unlock_request = VM_PROT_NONE;
783: PAGE_WAKEUP(p);
784: }
785:
786: p = next;
787: }
788:
789: /*
790: * Wake up threads waiting for the memory object to
791: * become ready.
792: */
793:
794: object->pager_ready = TRUE;
795: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
796: }
797:
798: /*
799: * Routine: memory_object_destroy [user interface]
800: * Purpose:
801: * Shut down a memory object, despite the
802: * presence of address map (or other) references
803: * to the vm_object.
804: * Note:
805: * This routine may be called either from the user interface,
806: * or from port destruction handling (via vm_object_destroy).
807: */
808: kern_return_t memory_object_destroy(
809: register
810: vm_object_t object,
811: kern_return_t reason)
812: {
813: ipc_port_t old_object, old_name;
814: pager_request_t old_control;
815:
816: #ifdef lint
817: reason++;
818: #endif /* lint */
819:
820: if (object == VM_OBJECT_NULL)
821: return KERN_SUCCESS;
822:
823: /*
824: * Remove the port associations immediately.
825: *
826: * This will prevent the memory manager from further
827: * meddling. [If it wanted to flush data or make
828: * other changes, it should have done so before performing
829: * the destroy call.]
830: */
831:
832: vm_object_cache_lock();
833: vm_object_lock(object);
834: vm_object_remove(object);
835: object->can_persist = FALSE;
836: vm_object_cache_unlock();
837:
838: /*
839: * Rip out the ports from the vm_object now... this
840: * will prevent new memory_object calls from succeeding.
841: */
842:
843: old_object = object->pager;
844: object->pager = IP_NULL;
845:
846: old_control = object->pager_request;
847: object->pager_request = PAGER_REQUEST_NULL;
848:
849: old_name = object->pager_name;
850: object->pager_name = IP_NULL;
851:
852:
853: /*
854: * Wait for existing paging activity (that might
855: * have the old ports) to subside.
856: */
857:
858: vm_object_paging_wait(object, FALSE);
859: vm_object_unlock(object);
860:
861: /*
862: * Shut down the ports now.
863: *
864: * [Paging operations may be proceeding concurrently --
865: * they'll get the null values established above.]
866: */
867:
868: if (old_object != IP_NULL) {
869: /* consumes our rights for object, control, name */
870: memory_object_release(old_object, old_control,
871: old_name);
872: } else if (old_name != IP_NULL) {
873: /* consumes our right for name */
874: ipc_port_dealloc_kernel(object->pager_name);
875: }
876:
877: /*
878: * Lose the reference that was donated for this routine
879: */
880:
881: vm_object_deallocate(object);
882:
883: return KERN_SUCCESS;
884: }
885:
886: /*
887: * vm_object_deactivate_pages
888: *
889: * Deactivate all pages in the specified object. (Keep its pages
890: * in memory even though it is no longer referenced.)
891: *
892: * The object must be locked.
893: */
894: void vm_object_deactivate_pages(
895: register vm_object_t object)
896: {
897: register vm_page_t p;
898:
899: queue_iterate(&object->memq, p, vm_page_t, listq) {
900: vm_page_lock_queues();
901: if (!p->busy)
902: vm_page_deactivate(p);
903: vm_page_unlock_queues();
904: }
905: }
906:
907:
908: /*
909: * Routine: vm_object_pmap_protect
910: *
911: * Purpose:
912: * Reduces the permission for all physical
913: * pages in the specified object range.
914: *
915: * If removing write permission only, it is
916: * sufficient to protect only the pages in
917: * the top-level object; only those pages may
918: * have write permission.
919: *
920: * If removing all access, we must follow the
921: * shadow chain from the top-level object to
922: * remove access to all pages in shadowed objects.
923: *
924: * The object must *not* be locked. The object must
925: * be temporary/internal.
926: *
927: * If pmap is not NULL, this routine assumes that
928: * the only mappings for the pages are in that
929: * pmap.
930: */
931: boolean_t vm_object_pmap_protect_by_page = FALSE;
932:
933: void vm_object_pmap_protect(
934: register vm_object_t object,
935: register vm_offset_t offset,
1.1.1.2 ! root 936: vm_size_t size,
1.1 root 937: pmap_t pmap,
938: vm_offset_t pmap_start,
939: vm_prot_t prot)
940: {
941: if (object == VM_OBJECT_NULL)
942: return;
943:
944: vm_object_lock(object);
945:
946: assert(object->temporary && object->internal);
947:
948: while (TRUE) {
949: if (object->resident_page_count > atop(size) / 2 &&
950: pmap != PMAP_NULL) {
951: vm_object_unlock(object);
952: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
953: return;
954: }
955:
956: {
957: register vm_page_t p;
958: register vm_offset_t end;
959:
960: end = offset + size;
961:
962: queue_iterate(&object->memq, p, vm_page_t, listq) {
963: if (!p->fictitious &&
964: (offset <= p->offset) &&
965: (p->offset < end)) {
966: if ((pmap == PMAP_NULL) ||
967: vm_object_pmap_protect_by_page) {
968: pmap_page_protect(p->phys_addr,
969: prot & ~p->page_lock);
970: } else {
971: vm_offset_t start =
972: pmap_start +
973: (p->offset - offset);
974:
975: pmap_protect(pmap,
976: start,
977: start + PAGE_SIZE,
978: prot);
979: }
980: }
981: }
982: }
983:
984: if (prot == VM_PROT_NONE) {
985: /*
986: * Must follow shadow chain to remove access
987: * to pages in shadowed objects.
988: */
989: register vm_object_t next_object;
990:
991: next_object = object->shadow;
992: if (next_object != VM_OBJECT_NULL) {
993: offset += object->shadow_offset;
994: vm_object_lock(next_object);
995: vm_object_unlock(object);
996: object = next_object;
997: }
998: else {
999: /*
1000: * End of chain - we are done.
1001: */
1002: break;
1003: }
1004: }
1005: else {
1006: /*
1007: * Pages in shadowed objects may never have
1008: * write permission - we may stop here.
1009: */
1010: break;
1011: }
1012: }
1013:
1014: vm_object_unlock(object);
1015: }
1016:
1017: /*
1018: * vm_object_pmap_remove:
1019: *
1020: * Removes all physical pages in the specified
1021: * object range from all physical maps.
1022: *
1023: * The object must *not* be locked.
1024: */
1025: void vm_object_pmap_remove(
1026: register vm_object_t object,
1027: register vm_offset_t start,
1028: register vm_offset_t end)
1029: {
1030: register vm_page_t p;
1031:
1032: if (object == VM_OBJECT_NULL)
1033: return;
1034:
1035: vm_object_lock(object);
1036: queue_iterate(&object->memq, p, vm_page_t, listq) {
1037: if (!p->fictitious &&
1038: (start <= p->offset) &&
1039: (p->offset < end))
1040: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
1041: }
1042: vm_object_unlock(object);
1043: }
1044:
1045: /*
1046: * Routine: vm_object_copy_slowly
1047: *
1048: * Description:
1049: * Copy the specified range of the source
1050: * virtual memory object without using
1051: * protection-based optimizations (such
1052: * as copy-on-write). The pages in the
1053: * region are actually copied.
1054: *
1055: * In/out conditions:
1056: * The caller must hold a reference and a lock
1057: * for the source virtual memory object. The source
1058: * object will be returned *unlocked*.
1059: *
1060: * Results:
1061: * If the copy is completed successfully, KERN_SUCCESS is
1062: * returned. If the caller asserted the interruptible
1063: * argument, and an interruption occurred while waiting
1064: * for a user-generated event, MACH_SEND_INTERRUPTED is
1065: * returned. Other values may be returned to indicate
1066: * hard errors during the copy operation.
1067: *
1068: * A new virtual memory object is returned in a
1069: * parameter (_result_object). The contents of this
1070: * new object, starting at a zero offset, are a copy
1071: * of the source memory region. In the event of
1072: * an error, this parameter will contain the value
1073: * VM_OBJECT_NULL.
1074: */
1075: kern_return_t vm_object_copy_slowly(
1076: register
1077: vm_object_t src_object,
1078: vm_offset_t src_offset,
1079: vm_size_t size,
1080: boolean_t interruptible,
1081: vm_object_t *_result_object) /* OUT */
1082: {
1083: vm_object_t new_object;
1084: vm_offset_t new_offset;
1085:
1086: if (size == 0) {
1087: vm_object_unlock(src_object);
1088: *_result_object = VM_OBJECT_NULL;
1089: return KERN_INVALID_ARGUMENT;
1090: }
1091:
1092: /*
1093: * Prevent destruction of the source object while we copy.
1094: */
1095:
1096: assert(src_object->ref_count > 0);
1097: src_object->ref_count++;
1098: vm_object_unlock(src_object);
1099:
1100: /*
1101: * Create a new object to hold the copied pages.
1102: * A few notes:
1103: * We fill the new object starting at offset 0,
1104: * regardless of the input offset.
1105: * We don't bother to lock the new object within
1106: * this routine, since we have the only reference.
1107: */
1108:
1109: new_object = vm_object_allocate(size);
1110: new_offset = 0;
1111:
1112: assert(size == trunc_page(size)); /* Will the loop terminate? */
1113:
1114: for ( ;
1115: size != 0 ;
1116: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1117: ) {
1118: vm_page_t new_page;
1119: vm_fault_return_t result;
1120:
1121: while ((new_page = vm_page_alloc(new_object, new_offset))
1122: == VM_PAGE_NULL) {
1123: VM_PAGE_WAIT((void (*)()) 0);
1124: }
1125:
1126: do {
1127: vm_prot_t prot = VM_PROT_READ;
1128: vm_page_t _result_page;
1129: vm_page_t top_page;
1130: register
1131: vm_page_t result_page;
1132:
1133: vm_object_lock(src_object);
1134: src_object->paging_in_progress++;
1135:
1136: result = vm_fault_page(src_object, src_offset,
1137: VM_PROT_READ, FALSE, interruptible,
1138: &prot, &_result_page, &top_page,
1139: FALSE, (void (*)()) 0);
1140:
1141: switch(result) {
1142: case VM_FAULT_SUCCESS:
1143: result_page = _result_page;
1144:
1145: /*
1146: * We don't need to hold the object
1147: * lock -- the busy page will be enough.
1148: * [We don't care about picking up any
1149: * new modifications.]
1150: *
1151: * Copy the page to the new object.
1152: *
1153: * POLICY DECISION:
1154: * If result_page is clean,
1155: * we could steal it instead
1156: * of copying.
1157: */
1158:
1159: vm_object_unlock(result_page->object);
1160: vm_page_copy(result_page, new_page);
1161:
1162: /*
1163: * Let go of both pages (make them
1164: * not busy, perform wakeup, activate).
1165: */
1166:
1167: new_page->busy = FALSE;
1168: new_page->dirty = TRUE;
1169: vm_object_lock(result_page->object);
1170: PAGE_WAKEUP_DONE(result_page);
1171:
1172: vm_page_lock_queues();
1173: if (!result_page->active &&
1174: !result_page->inactive)
1175: vm_page_activate(result_page);
1176: vm_page_activate(new_page);
1177: vm_page_unlock_queues();
1178:
1179: /*
1180: * Release paging references and
1181: * top-level placeholder page, if any.
1182: */
1183:
1184: vm_fault_cleanup(result_page->object,
1185: top_page);
1186:
1187: break;
1188:
1189: case VM_FAULT_RETRY:
1190: break;
1191:
1192: case VM_FAULT_MEMORY_SHORTAGE:
1193: VM_PAGE_WAIT((void (*)()) 0);
1194: break;
1195:
1196: case VM_FAULT_FICTITIOUS_SHORTAGE:
1197: vm_page_more_fictitious();
1198: break;
1199:
1200: case VM_FAULT_INTERRUPTED:
1201: vm_page_free(new_page);
1202: vm_object_deallocate(new_object);
1203: vm_object_deallocate(src_object);
1204: *_result_object = VM_OBJECT_NULL;
1205: return MACH_SEND_INTERRUPTED;
1206:
1207: case VM_FAULT_MEMORY_ERROR:
1208: /*
1209: * A policy choice:
1210: * (a) ignore pages that we can't
1211: * copy
1212: * (b) return the null object if
1213: * any page fails [chosen]
1214: */
1215:
1216: vm_page_free(new_page);
1217: vm_object_deallocate(new_object);
1218: vm_object_deallocate(src_object);
1219: *_result_object = VM_OBJECT_NULL;
1220: return KERN_MEMORY_ERROR;
1221: }
1222: } while (result != VM_FAULT_SUCCESS);
1223: }
1224:
1225: /*
1226: * Lose the extra reference, and return our object.
1227: */
1228:
1229: vm_object_deallocate(src_object);
1230: *_result_object = new_object;
1231: return KERN_SUCCESS;
1232: }
1233:
1234: /*
1235: * Routine: vm_object_copy_temporary
1236: *
1237: * Purpose:
1238: * Copy the specified range of the source virtual
1239: * memory object, if it can be done without blocking.
1240: *
1241: * Results:
1242: * If the copy is successful, the copy is returned in
1243: * the arguments; otherwise, the arguments are not
1244: * affected.
1245: *
1246: * In/out conditions:
1247: * The object should be unlocked on entry and exit.
1248: */
1249:
1250: vm_object_t vm_object_copy_delayed(); /* forward declaration */
1251:
1252: boolean_t vm_object_copy_temporary(
1253: vm_object_t *_object, /* INOUT */
1254: vm_offset_t *_offset, /* INOUT */
1255: boolean_t *_src_needs_copy, /* OUT */
1256: boolean_t *_dst_needs_copy) /* OUT */
1257: {
1258: vm_object_t object = *_object;
1259:
1260: #ifdef lint
1261: ++*_offset;
1262: #endif /* lint */
1263:
1264: if (object == VM_OBJECT_NULL) {
1265: *_src_needs_copy = FALSE;
1266: *_dst_needs_copy = FALSE;
1267: return TRUE;
1268: }
1269:
1270: /*
1271: * If the object is temporary, we can perform
1272: * a symmetric copy-on-write without asking.
1273: */
1274:
1275: vm_object_lock(object);
1276: if (object->temporary) {
1277:
1278: /*
1279: * Shared objects use delayed copy
1280: */
1281: if (object->use_shared_copy) {
1282:
1283: /*
1284: * Asymmetric copy strategy. Destination
1285: * must be copied (to allow copy object reuse).
1286: * Source is unaffected.
1287: */
1288: vm_object_unlock(object);
1289: object = vm_object_copy_delayed(object);
1290: *_object = object;
1291: *_src_needs_copy = FALSE;
1292: *_dst_needs_copy = TRUE;
1293: return TRUE;
1294: }
1295:
1296: /*
1297: * Make another reference to the object.
1298: *
1299: * Leave object/offset unchanged.
1300: */
1301:
1302: assert(object->ref_count > 0);
1303: object->ref_count++;
1304: object->shadowed = TRUE;
1305: vm_object_unlock(object);
1306:
1307: /*
1308: * Both source and destination must make
1309: * shadows, and the source must be made
1310: * read-only if not already.
1311: */
1312:
1313: *_src_needs_copy = TRUE;
1314: *_dst_needs_copy = TRUE;
1315: return TRUE;
1316: }
1317:
1318: if (object->pager_ready &&
1319: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1320: /* XXX Do something intelligent (see temporary code above) */
1321: }
1322: vm_object_unlock(object);
1323:
1324: return FALSE;
1325: }
1326:
1327: /*
1328: * Routine: vm_object_copy_call [internal]
1329: *
1330: * Description:
1331: * Copy the specified (src_offset, size) portion
1332: * of the source object (src_object), using the
1333: * user-managed copy algorithm.
1334: *
1335: * In/out conditions:
1336: * The source object must be locked on entry. It
1337: * will be *unlocked* on exit.
1338: *
1339: * Results:
1340: * If the copy is successful, KERN_SUCCESS is returned.
1341: * This routine is interruptible; if a wait for
1342: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1343: * is returned. Other return values indicate hard errors
1344: * in creating the user-managed memory object for the copy.
1345: *
1346: * A new object that represents the copied virtual
1347: * memory is returned in a parameter (*_result_object).
1348: * If the return value indicates an error, this parameter
1349: * is not valid.
1350: */
1351: kern_return_t vm_object_copy_call(
1352: vm_object_t src_object,
1353: vm_offset_t src_offset,
1354: vm_size_t size,
1355: vm_object_t *_result_object) /* OUT */
1356: {
1357: vm_offset_t src_end = src_offset + size;
1358: ipc_port_t new_memory_object;
1359: vm_object_t new_object;
1360: vm_page_t p;
1361:
1362: /*
1363: * Set the backing object for the new
1364: * temporary object.
1365: */
1366:
1367: assert(src_object->ref_count > 0);
1368: src_object->ref_count++;
1369: vm_object_paging_begin(src_object);
1370: vm_object_unlock(src_object);
1371:
1372: /*
1373: * Create a memory object port to be associated
1374: * with this new vm_object.
1375: *
1376: * Since the kernel has the only rights to this
1377: * port, we need not hold the cache lock.
1378: *
1379: * Since we have the only object reference, we
1380: * need not be worried about collapse operations.
1381: *
1382: */
1383:
1384: new_memory_object = ipc_port_alloc_kernel();
1385: if (new_memory_object == IP_NULL) {
1386: panic("vm_object_copy_call: allocate memory object port");
1387: /* XXX Shouldn't panic here. */
1388: }
1389:
1390: /* we hold a naked receive right for new_memory_object */
1391: (void) ipc_port_make_send(new_memory_object);
1392: /* now we also hold a naked send right for new_memory_object */
1393:
1394: /*
1395: * Let the memory manager know that a copy operation
1396: * is in progress. Note that we're using the old
1397: * memory object's ports (for which we're holding
1398: * a paging reference)... the memory manager cannot
1399: * yet affect the new memory object.
1400: */
1401:
1402: (void) memory_object_copy(src_object->pager,
1403: src_object->pager_request,
1404: src_offset, size,
1405: new_memory_object);
1406: /* no longer hold the naked receive right for new_memory_object */
1407:
1408: vm_object_lock(src_object);
1409: vm_object_paging_end(src_object);
1410:
1411: /*
1412: * Remove write access from all of the pages of
1413: * the old memory object that we can.
1414: */
1415:
1416: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1417: if (!p->fictitious &&
1418: (src_offset <= p->offset) &&
1419: (p->offset < src_end) &&
1420: !(p->page_lock & VM_PROT_WRITE)) {
1421: p->page_lock |= VM_PROT_WRITE;
1422: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1423: }
1424: }
1425:
1426: vm_object_unlock(src_object);
1427:
1428: /*
1429: * Initialize the rest of the paging stuff
1430: */
1431:
1432: new_object = vm_object_enter(new_memory_object, size, FALSE);
1433: new_object->shadow = src_object;
1434: new_object->shadow_offset = src_offset;
1435:
1436: /*
1437: * Drop the reference for new_memory_object taken above.
1438: */
1439:
1440: ipc_port_release_send(new_memory_object);
1441: /* no longer hold the naked send right for new_memory_object */
1442:
1443: *_result_object = new_object;
1444: return KERN_SUCCESS;
1445: }
1446:
1447: /*
1448: * Routine: vm_object_copy_delayed [internal]
1449: *
1450: * Description:
1451: * Copy the specified virtual memory object, using
1452: * the asymmetric copy-on-write algorithm.
1453: *
1454: * In/out conditions:
1455: * The object must be unlocked on entry.
1456: *
1457: * This routine will not block waiting for user-generated
1458: * events. It is not interruptible.
1459: */
1460: vm_object_t vm_object_copy_delayed(
1461: vm_object_t src_object)
1462: {
1463: vm_object_t new_copy;
1464: vm_object_t old_copy;
1465: vm_page_t p;
1466:
1467: /*
1468: * The user-level memory manager wants to see
1469: * all of the changes to this object, but it
1470: * has promised not to make any changes on its own.
1471: *
1472: * Perform an asymmetric copy-on-write, as follows:
1473: * Create a new object, called a "copy object"
1474: * to hold pages modified by the new mapping
1475: * (i.e., the copy, not the original mapping).
1476: * Record the original object as the backing
1477: * object for the copy object. If the
1478: * original mapping does not change a page,
1479: * it may be used read-only by the copy.
1480: * Record the copy object in the original
1481: * object. When the original mapping causes
1482: * a page to be modified, it must be copied
1483: * to a new page that is "pushed" to the
1484: * copy object.
1485: * Mark the new mapping (the copy object)
1486: * copy-on-write. This makes the copy
1487: * object itself read-only, allowing it
1488: * to be reused if the original mapping
1489: * makes no changes, and simplifying the
1490: * synchronization required in the "push"
1491: * operation described above.
1492: *
1493: * The copy-on-write is said to be assymetric because
1494: * the original object is *not* marked copy-on-write.
1495: * A copied page is pushed to the copy object, regardless
1496: * which party attempted to modify the page.
1497: *
1498: * Repeated asymmetric copy operations may be done.
1499: * If the original object has not been changed since
1500: * the last copy, its copy object can be reused.
1501: * Otherwise, a new copy object can be inserted
1502: * between the original object and its previous
1503: * copy object. Since any copy object is read-only,
1504: * this cannot affect the contents of the previous copy
1505: * object.
1506: *
1507: * Note that a copy object is higher in the object
1508: * tree than the original object; therefore, use of
1509: * the copy object recorded in the original object
1510: * must be done carefully, to avoid deadlock.
1511: */
1512:
1513: /*
1514: * Allocate a new copy object before locking, even
1515: * though we may not need it later.
1516: */
1517:
1518: new_copy = vm_object_allocate(src_object->size);
1519:
1520: vm_object_lock(src_object);
1521:
1522: /*
1523: * See whether we can reuse the result of a previous
1524: * copy operation.
1525: */
1526: Retry:
1527: old_copy = src_object->copy;
1528: if (old_copy != VM_OBJECT_NULL) {
1529: /*
1530: * Try to get the locks (out of order)
1531: */
1532: if (!vm_object_lock_try(old_copy)) {
1533: vm_object_unlock(src_object);
1534:
1535: simple_lock_pause(); /* wait a bit */
1536:
1537: vm_object_lock(src_object);
1538: goto Retry;
1539: }
1540:
1541: /*
1542: * Determine whether the old copy object has
1543: * been modified.
1544: */
1545:
1546: if (old_copy->resident_page_count == 0 &&
1547: !old_copy->pager_created) {
1548: /*
1549: * It has not been modified.
1550: *
1551: * Return another reference to
1552: * the existing copy-object.
1553: */
1554: assert(old_copy->ref_count > 0);
1555: old_copy->ref_count++;
1556: vm_object_unlock(old_copy);
1557: vm_object_unlock(src_object);
1558:
1559: vm_object_deallocate(new_copy);
1560:
1561: return old_copy;
1562: }
1563:
1564: /*
1565: * The copy-object is always made large enough to
1566: * completely shadow the original object, since
1567: * it may have several users who want to shadow
1568: * the original object at different points.
1569: */
1570:
1571: assert((old_copy->shadow == src_object) &&
1572: (old_copy->shadow_offset == (vm_offset_t) 0));
1573:
1574: /*
1575: * Make the old copy-object shadow the new one.
1576: * It will receive no more pages from the original
1577: * object.
1578: */
1579:
1580: src_object->ref_count--; /* remove ref. from old_copy */
1581: assert(src_object->ref_count > 0);
1582: old_copy->shadow = new_copy;
1583: assert(new_copy->ref_count > 0);
1584: new_copy->ref_count++;
1585: vm_object_unlock(old_copy); /* done with old_copy */
1586: }
1587:
1588: /*
1589: * Point the new copy at the existing object.
1590: */
1591:
1592: new_copy->shadow = src_object;
1593: new_copy->shadow_offset = 0;
1594: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1595: assert(src_object->ref_count > 0);
1596: src_object->ref_count++;
1597: src_object->copy = new_copy;
1598:
1599: /*
1600: * Mark all pages of the existing object copy-on-write.
1601: * This object may have a shadow chain below it, but
1602: * those pages will already be marked copy-on-write.
1603: */
1604:
1605: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1606: if (!p->fictitious)
1607: pmap_page_protect(p->phys_addr,
1608: (VM_PROT_ALL & ~VM_PROT_WRITE &
1609: ~p->page_lock));
1610: }
1611:
1612: vm_object_unlock(src_object);
1613:
1614: return new_copy;
1615: }
1616:
1617: /*
1618: * Routine: vm_object_copy_strategically
1619: *
1620: * Purpose:
1621: * Perform a copy according to the source object's
1622: * declared strategy. This operation may block,
1623: * and may be interrupted.
1624: */
1625: kern_return_t vm_object_copy_strategically(
1626: register
1627: vm_object_t src_object,
1628: vm_offset_t src_offset,
1629: vm_size_t size,
1630: vm_object_t *dst_object, /* OUT */
1631: vm_offset_t *dst_offset, /* OUT */
1632: boolean_t *dst_needs_copy) /* OUT */
1633: {
1634: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1635: boolean_t interruptible = TRUE; /* XXX */
1636:
1637: assert(src_object != VM_OBJECT_NULL);
1638:
1639: vm_object_lock(src_object);
1640:
1641: /* XXX assert(!src_object->temporary); JSB FIXME */
1642:
1643: /*
1644: * The copy strategy is only valid if the memory manager
1645: * is "ready".
1646: */
1647:
1648: while (!src_object->pager_ready) {
1649: vm_object_wait( src_object,
1650: VM_OBJECT_EVENT_PAGER_READY,
1651: interruptible);
1652: if (interruptible &&
1653: (current_thread()->wait_result != THREAD_AWAKENED)) {
1654: *dst_object = VM_OBJECT_NULL;
1655: *dst_offset = 0;
1656: *dst_needs_copy = FALSE;
1657: return MACH_SEND_INTERRUPTED;
1658: }
1659: vm_object_lock(src_object);
1660: }
1661:
1662: /*
1663: * The object may be temporary (even though it is external).
1664: * If so, do a symmetric copy.
1665: */
1666:
1667: if (src_object->temporary) {
1668: /*
1669: * XXX
1670: * This does not count as intelligent!
1671: * This buys us the object->temporary optimizations,
1672: * but we aren't using a symmetric copy,
1673: * which may confuse the vm code. The correct thing
1674: * to do here is to figure out what to call to get
1675: * a temporary shadowing set up.
1676: */
1677: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1678: }
1679:
1680: /*
1681: * The object is permanent. Use the appropriate copy strategy.
1682: */
1683:
1684: switch (src_object->copy_strategy) {
1685: case MEMORY_OBJECT_COPY_NONE:
1686: if ((result = vm_object_copy_slowly(
1687: src_object,
1688: src_offset,
1689: size,
1690: interruptible,
1691: dst_object))
1692: == KERN_SUCCESS) {
1693: *dst_offset = 0;
1694: *dst_needs_copy = FALSE;
1695: }
1696: break;
1697:
1698: case MEMORY_OBJECT_COPY_CALL:
1699: if ((result = vm_object_copy_call(
1700: src_object,
1701: src_offset,
1702: size,
1703: dst_object))
1704: == KERN_SUCCESS) {
1705: *dst_offset = 0;
1706: *dst_needs_copy = FALSE;
1707: }
1708: break;
1709:
1710: case MEMORY_OBJECT_COPY_DELAY:
1711: vm_object_unlock(src_object);
1712: *dst_object = vm_object_copy_delayed(src_object);
1713: *dst_offset = src_offset;
1714: *dst_needs_copy = TRUE;
1715:
1716: result = KERN_SUCCESS;
1717: break;
1718: }
1719:
1720: return result;
1721: }
1722:
1723: /*
1724: * vm_object_shadow:
1725: *
1726: * Create a new object which is backed by the
1727: * specified existing object range. The source
1728: * object reference is deallocated.
1729: *
1730: * The new object and offset into that object
1731: * are returned in the source parameters.
1732: */
1733:
1734: void vm_object_shadow(
1735: vm_object_t *object, /* IN/OUT */
1736: vm_offset_t *offset, /* IN/OUT */
1737: vm_size_t length)
1738: {
1739: register vm_object_t source;
1740: register vm_object_t result;
1741:
1742: source = *object;
1743:
1744: /*
1745: * Allocate a new object with the given length
1746: */
1747:
1748: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1749: panic("vm_object_shadow: no object for shadowing");
1750:
1751: /*
1752: * The new object shadows the source object, adding
1753: * a reference to it. Our caller changes his reference
1754: * to point to the new object, removing a reference to
1755: * the source object. Net result: no change of reference
1756: * count.
1757: */
1758: result->shadow = source;
1759:
1760: /*
1761: * Store the offset into the source object,
1762: * and fix up the offset into the new object.
1763: */
1764:
1765: result->shadow_offset = *offset;
1766:
1767: /*
1768: * Return the new things
1769: */
1770:
1771: *offset = 0;
1772: *object = result;
1773: }
1774:
1775: /*
1776: * The relationship between vm_object structures and
1777: * the memory_object ports requires careful synchronization.
1778: *
1779: * All associations are created by vm_object_enter. All three
1780: * port fields are filled in, as follows:
1781: * pager: the memory_object port itself, supplied by
1782: * the user requesting a mapping (or the kernel,
1783: * when initializing internal objects); the
1784: * kernel simulates holding send rights by keeping
1785: * a port reference;
1786: * pager_request:
1787: * pager_name:
1788: * the memory object control and name ports,
1789: * created by the kernel; the kernel holds
1790: * receive (and ownership) rights to these
1791: * ports, but no other references.
1792: * All of the ports are referenced by their global names.
1793: *
1794: * When initialization is complete, the "initialized" field
1795: * is asserted. Other mappings using a particular memory object,
1796: * and any references to the vm_object gained through the
1797: * port association must wait for this initialization to occur.
1798: *
1799: * In order to allow the memory manager to set attributes before
1800: * requests (notably virtual copy operations, but also data or
1801: * unlock requests) are made, a "ready" attribute is made available.
1802: * Only the memory manager may affect the value of this attribute.
1803: * Its value does not affect critical kernel functions, such as
1804: * internal object initialization or destruction. [Furthermore,
1805: * memory objects created by the kernel are assumed to be ready
1806: * immediately; the default memory manager need not explicitly
1807: * set the "ready" attribute.]
1808: *
1809: * [Both the "initialized" and "ready" attribute wait conditions
1810: * use the "pager" field as the wait event.]
1811: *
1812: * The port associations can be broken down by any of the
1813: * following routines:
1814: * vm_object_terminate:
1815: * No references to the vm_object remain, and
1816: * the object cannot (or will not) be cached.
1817: * This is the normal case, and is done even
1818: * though one of the other cases has already been
1819: * done.
1820: * vm_object_destroy:
1821: * The memory_object port has been destroyed,
1822: * meaning that the kernel cannot flush dirty
1823: * pages or request new data or unlock existing
1824: * data.
1825: * memory_object_destroy:
1826: * The memory manager has requested that the
1827: * kernel relinquish rights to the memory object
1828: * port. [The memory manager may not want to
1829: * destroy the port, but may wish to refuse or
1830: * tear down existing memory mappings.]
1831: * Each routine that breaks an association must break all of
1832: * them at once. At some later time, that routine must clear
1833: * the vm_object port fields and release the port rights.
1834: * [Furthermore, each routine must cope with the simultaneous
1835: * or previous operations of the others.]
1836: *
1837: * In addition to the lock on the object, the vm_object_cache_lock
1838: * governs the port associations. References gained through the
1839: * port association require use of the cache lock.
1840: *
1841: * Because the port fields may be cleared spontaneously, they
1842: * cannot be used to determine whether a memory object has
1843: * ever been associated with a particular vm_object. [This
1844: * knowledge is important to the shadow object mechanism.]
1845: * For this reason, an additional "created" attribute is
1846: * provided.
1847: *
1848: * During various paging operations, the port values found in the
1849: * vm_object must be valid. To prevent these port rights from being
1850: * released, and to prevent the port associations from changing
1851: * (other than being removed, i.e., made null), routines may use
1852: * the vm_object_paging_begin/end routines [actually, macros].
1853: * The implementation uses the "paging_in_progress" and "wanted" fields.
1854: * [Operations that alter the validity of the port values include the
1855: * termination routines and vm_object_collapse.]
1856: */
1857:
1858: vm_object_t vm_object_lookup(
1859: ipc_port_t port)
1860: {
1861: vm_object_t object = VM_OBJECT_NULL;
1862:
1863: if (IP_VALID(port)) {
1864: ip_lock(port);
1865: if (ip_active(port) &&
1866: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1867: vm_object_cache_lock();
1868: object = (vm_object_t) port->ip_kobject;
1869: vm_object_lock(object);
1870:
1871: assert(object->alive);
1872:
1873: if (object->ref_count == 0) {
1874: queue_remove(&vm_object_cached_list, object,
1875: vm_object_t, cached_list);
1876: vm_object_cached_count--;
1.1.1.2 ! root 1877: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1878: }
1879:
1880: object->ref_count++;
1881: vm_object_unlock(object);
1882: vm_object_cache_unlock();
1883: }
1884: ip_unlock(port);
1885: }
1886:
1887: return object;
1888: }
1889:
1890: vm_object_t vm_object_lookup_name(
1891: ipc_port_t port)
1892: {
1893: vm_object_t object = VM_OBJECT_NULL;
1894:
1895: if (IP_VALID(port)) {
1896: ip_lock(port);
1897: if (ip_active(port) &&
1898: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1899: vm_object_cache_lock();
1900: object = (vm_object_t) port->ip_kobject;
1901: vm_object_lock(object);
1902:
1903: assert(object->alive);
1904:
1905: if (object->ref_count == 0) {
1906: queue_remove(&vm_object_cached_list, object,
1907: vm_object_t, cached_list);
1908: vm_object_cached_count--;
1.1.1.2 ! root 1909: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1910: }
1911:
1912: object->ref_count++;
1913: vm_object_unlock(object);
1914: vm_object_cache_unlock();
1915: }
1916: ip_unlock(port);
1917: }
1918:
1919: return object;
1920: }
1921:
1922: void vm_object_destroy(
1923: ipc_port_t pager)
1924: {
1925: vm_object_t object;
1926: pager_request_t old_request;
1927: ipc_port_t old_name;
1928:
1929: /*
1930: * Perform essentially the same operations as in vm_object_lookup,
1931: * except that this time we look up based on the memory_object
1932: * port, not the control port.
1933: */
1934: vm_object_cache_lock();
1935: if (ip_kotype(pager) != IKOT_PAGER) {
1936: vm_object_cache_unlock();
1937: return;
1938: }
1939:
1940: object = (vm_object_t) pager->ip_kobject;
1941: vm_object_lock(object);
1942: if (object->ref_count == 0) {
1943: queue_remove(&vm_object_cached_list, object,
1944: vm_object_t, cached_list);
1945: vm_object_cached_count--;
1.1.1.2 ! root 1946: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1947: }
1948: object->ref_count++;
1949:
1950: object->can_persist = FALSE;
1951:
1952: assert(object->pager == pager);
1953:
1954: /*
1955: * Remove the port associations.
1956: *
1957: * Note that the memory_object itself is dead, so
1958: * we don't bother with it.
1959: */
1960:
1961: object->pager = IP_NULL;
1962: vm_object_remove(object);
1963:
1964: old_request = object->pager_request;
1965: object->pager_request = PAGER_REQUEST_NULL;
1966:
1967: old_name = object->pager_name;
1968: object->pager_name = IP_NULL;
1969:
1970: vm_object_unlock(object);
1971: vm_object_cache_unlock();
1972:
1973: /*
1974: * Clean up the port references. Note that there's no
1975: * point in trying the memory_object_terminate call
1976: * because the memory_object itself is dead.
1977: */
1978:
1979: ipc_port_release_send(pager);
1980: if (old_request != IP_NULL)
1981: ipc_port_dealloc_kernel(old_request);
1982: if (old_name != IP_NULL)
1983: ipc_port_dealloc_kernel(old_name);
1984:
1985: /*
1986: * Restart pending page requests
1987: */
1988:
1989: vm_object_abort_activity(object);
1990:
1991: /*
1992: * Lose the object reference.
1993: */
1994:
1995: vm_object_deallocate(object);
1996: }
1997:
1998: boolean_t vm_object_accept_old_init_protocol = FALSE;
1999:
2000: /*
2001: * Routine: vm_object_enter
2002: * Purpose:
2003: * Find a VM object corresponding to the given
2004: * pager; if no such object exists, create one,
2005: * and initialize the pager.
2006: */
2007: vm_object_t vm_object_enter(
2008: ipc_port_t pager,
2009: vm_size_t size,
2010: boolean_t internal)
2011: {
2012: register
2013: vm_object_t object;
2014: vm_object_t new_object;
2015: boolean_t must_init;
2016: ipc_kobject_type_t po;
2017:
2018: restart:
2019: if (!IP_VALID(pager))
2020: return vm_object_allocate(size);
2021:
2022: new_object = VM_OBJECT_NULL;
2023: must_init = FALSE;
2024:
2025: /*
2026: * Look for an object associated with this port.
2027: */
2028:
2029: vm_object_cache_lock();
2030: for (;;) {
2031: po = ip_kotype(pager);
2032:
2033: /*
2034: * If a previous object is being terminated,
2035: * we must wait for the termination message
2036: * to be queued.
2037: *
2038: * We set kobject to a non-null value to let the
2039: * terminator know that someone is waiting.
2040: * Among the possibilities is that the port
2041: * could die while we're waiting. Must restart
2042: * instead of continuing the loop.
2043: */
2044:
2045: if (po == IKOT_PAGER_TERMINATING) {
2046: pager->ip_kobject = (ipc_kobject_t) pager;
2047: assert_wait((event_t) pager, FALSE);
2048: vm_object_cache_unlock();
2049: thread_block((void (*)()) 0);
2050: goto restart;
2051: }
2052:
2053: /*
2054: * Bail if there is already a kobject associated
2055: * with the pager port.
2056: */
2057: if (po != IKOT_NONE) {
2058: break;
2059: }
2060:
2061: /*
2062: * We must unlock to create a new object;
2063: * if we do so, we must try the lookup again.
2064: */
2065:
2066: if (new_object == VM_OBJECT_NULL) {
2067: vm_object_cache_unlock();
2068: new_object = vm_object_allocate(size);
2069: vm_object_cache_lock();
2070: } else {
2071: /*
2072: * Lookup failed twice, and we have something
2073: * to insert; set the object.
2074: */
2075:
2076: ipc_kobject_set(pager,
2077: (ipc_kobject_t) new_object,
2078: IKOT_PAGER);
2079: new_object = VM_OBJECT_NULL;
2080: must_init = TRUE;
2081: }
2082: }
2083:
2084: if (internal)
2085: must_init = TRUE;
2086:
2087: /*
2088: * It's only good if it's a VM object!
2089: */
2090:
2091: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2092: : VM_OBJECT_NULL;
2093:
2094: if ((object != VM_OBJECT_NULL) && !must_init) {
2095: vm_object_lock(object);
2096: if (object->ref_count == 0) {
2097: queue_remove(&vm_object_cached_list, object,
2098: vm_object_t, cached_list);
2099: vm_object_cached_count--;
1.1.1.2 ! root 2100: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 2101: }
2102: object->ref_count++;
2103: vm_object_unlock(object);
2104:
2105: vm_stat.hits++;
2106: }
2107: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2108: ((object->paging_in_progress != 0) && internal));
2109:
2110: vm_stat.lookups++;
2111:
2112: vm_object_cache_unlock();
2113:
2114: /*
2115: * If we raced to create a vm_object but lost, let's
2116: * throw away ours.
2117: */
2118:
2119: if (new_object != VM_OBJECT_NULL)
2120: vm_object_deallocate(new_object);
2121:
2122: if (object == VM_OBJECT_NULL)
2123: return(object);
2124:
2125: if (must_init) {
2126: /*
2127: * Copy the naked send right we were given.
2128: */
2129:
2130: pager = ipc_port_copy_send(pager);
2131: if (!IP_VALID(pager))
2132: panic("vm_object_enter: port died"); /* XXX */
2133:
2134: object->pager_created = TRUE;
2135: object->pager = pager;
2136:
2137: /*
2138: * Allocate request port.
2139: */
2140:
2141: object->pager_request = ipc_port_alloc_kernel();
2142: if (object->pager_request == IP_NULL)
2143: panic("vm_object_enter: pager request alloc");
2144:
2145: ipc_kobject_set(object->pager_request,
2146: (ipc_kobject_t) object,
2147: IKOT_PAGING_REQUEST);
2148:
2149: /*
2150: * Let the pager know we're using it.
2151: */
2152:
2153: if (internal) {
2154: /* acquire a naked send right for the DMM */
2155: ipc_port_t DMM = memory_manager_default_reference();
2156:
2157: /* mark the object internal */
2158: object->internal = TRUE;
2159: assert(object->temporary);
2160:
2161: /* default-pager objects are ready immediately */
2162: object->pager_ready = TRUE;
2163:
2164: /* consumes the naked send right for DMM */
2165: (void) memory_object_create(DMM,
2166: pager,
2167: object->size,
2168: object->pager_request,
2169: object->pager_name,
2170: PAGE_SIZE);
2171: } else {
2172: /* the object is external and not temporary */
2173: object->internal = FALSE;
2174: object->temporary = FALSE;
2175:
2176: /* user pager objects are not ready until marked so */
2177: object->pager_ready = FALSE;
2178:
2179: (void) memory_object_init(pager,
2180: object->pager_request,
2181: object->pager_name,
2182: PAGE_SIZE);
2183:
2184: }
2185:
2186: vm_object_lock(object);
2187: object->pager_initialized = TRUE;
2188:
2189: if (vm_object_accept_old_init_protocol)
2190: object->pager_ready = TRUE;
2191:
2192: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2193: } else {
2194: vm_object_lock(object);
2195: }
2196: /*
2197: * [At this point, the object must be locked]
2198: */
2199:
2200: /*
2201: * Wait for the work above to be done by the first
2202: * thread to map this object.
2203: */
2204:
2205: while (!object->pager_initialized) {
2206: vm_object_wait( object,
2207: VM_OBJECT_EVENT_INITIALIZED,
2208: FALSE);
2209: vm_object_lock(object);
2210: }
2211: vm_object_unlock(object);
2212:
2213: return object;
2214: }
2215:
2216: /*
2217: * Routine: vm_object_pager_create
2218: * Purpose:
2219: * Create a memory object for an internal object.
2220: * In/out conditions:
2221: * The object is locked on entry and exit;
2222: * it may be unlocked within this call.
2223: * Limitations:
2224: * Only one thread may be performing a
2225: * vm_object_pager_create on an object at
2226: * a time. Presumably, only the pageout
2227: * daemon will be using this routine.
2228: */
2229: void vm_object_pager_create(
2230: register
2231: vm_object_t object)
2232: {
2233: ipc_port_t pager;
2234:
2235: if (object->pager_created) {
2236: /*
2237: * Someone else got to it first...
2238: * wait for them to finish initializing
2239: */
2240:
2241: while (!object->pager_initialized) {
2242: vm_object_wait( object,
2243: VM_OBJECT_EVENT_PAGER_READY,
2244: FALSE);
2245: vm_object_lock(object);
2246: }
2247: return;
2248: }
2249:
2250: /*
2251: * Indicate that a memory object has been assigned
2252: * before dropping the lock, to prevent a race.
2253: */
2254:
2255: object->pager_created = TRUE;
2256:
2257: /*
2258: * Prevent collapse or termination by
2259: * holding a paging reference
2260: */
2261:
2262: vm_object_paging_begin(object);
2263: vm_object_unlock(object);
2264:
2265: #if MACH_PAGEMAP
2266: object->existence_info = vm_external_create(
2267: object->size +
2268: object->paging_offset);
2269: assert((object->size + object->paging_offset) >=
2270: object->size);
2271: #endif /* MACH_PAGEMAP */
2272:
2273: /*
2274: * Create the pager, and associate with it
2275: * this object.
2276: *
2277: * Note that we only make the port association
2278: * so that vm_object_enter can properly look up
2279: * the object to complete the initialization...
2280: * we do not expect any user to ever map this
2281: * object.
2282: *
2283: * Since the kernel has the only rights to the
2284: * port, it's safe to install the association
2285: * without holding the cache lock.
2286: */
2287:
2288: pager = ipc_port_alloc_kernel();
2289: if (pager == IP_NULL)
2290: panic("vm_object_pager_create: allocate pager port");
2291:
2292: (void) ipc_port_make_send(pager);
2293: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2294:
2295: /*
2296: * Initialize the rest of the paging stuff
2297: */
2298:
2299: if (vm_object_enter(pager, object->size, TRUE) != object)
2300: panic("vm_object_pager_create: mismatch");
2301:
2302: /*
2303: * Drop the naked send right taken above.
2304: */
2305:
2306: ipc_port_release_send(pager);
2307:
2308: /*
2309: * Release the paging reference
2310: */
2311:
2312: vm_object_lock(object);
2313: vm_object_paging_end(object);
2314: }
2315:
2316: /*
2317: * Routine: vm_object_remove
2318: * Purpose:
2319: * Eliminate the pager/object association
2320: * for this pager.
2321: * Conditions:
2322: * The object cache must be locked.
2323: */
2324: void vm_object_remove(
2325: vm_object_t object)
2326: {
2327: ipc_port_t port;
2328:
2329: if ((port = object->pager) != IP_NULL) {
2330: if (ip_kotype(port) == IKOT_PAGER)
2331: ipc_kobject_set(port, IKO_NULL,
2332: IKOT_PAGER_TERMINATING);
2333: else if (ip_kotype(port) != IKOT_NONE)
2334: panic("vm_object_remove: bad object port");
2335: }
2336: if ((port = object->pager_request) != IP_NULL) {
2337: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2338: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2339: else if (ip_kotype(port) != IKOT_NONE)
2340: panic("vm_object_remove: bad request port");
2341: }
2342: if ((port = object->pager_name) != IP_NULL) {
2343: if (ip_kotype(port) == IKOT_PAGING_NAME)
2344: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2345: else if (ip_kotype(port) != IKOT_NONE)
2346: panic("vm_object_remove: bad name port");
2347: }
2348: }
2349:
2350: /*
2351: * Global variables for vm_object_collapse():
2352: *
2353: * Counts for normal collapses and bypasses.
2354: * Debugging variables, to watch or disable collapse.
2355: */
2356: long object_collapses = 0;
2357: long object_bypasses = 0;
2358:
2359: int vm_object_collapse_debug = 0;
2360: boolean_t vm_object_collapse_allowed = TRUE;
2361: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2362:
2363: /*
2364: * vm_object_collapse:
2365: *
2366: * Collapse an object with the object backing it.
2367: * Pages in the backing object are moved into the
2368: * parent, and the backing object is deallocated.
2369: *
2370: * Requires that the object be locked and the page
2371: * queues be unlocked. May unlock/relock the object,
2372: * so the caller should hold a reference for the object.
2373: */
2374: void vm_object_collapse(
2375: register vm_object_t object)
2376: {
2377: register vm_object_t backing_object;
2378: register vm_offset_t backing_offset;
2379: register vm_size_t size;
2380: register vm_offset_t new_offset;
2381: register vm_page_t p, pp;
2382: ipc_port_t old_name_port;
2383:
2384: if (!vm_object_collapse_allowed)
2385: return;
2386:
2387: while (TRUE) {
2388: /*
2389: * Verify that the conditions are right for collapse:
2390: *
2391: * The object exists and no pages in it are currently
2392: * being paged out (or have ever been paged out).
2393: *
2394: * This check is probably overkill -- if a memory
2395: * object has not been created, the fault handler
2396: * shouldn't release the object lock while paging
2397: * is in progress or absent pages exist.
2398: */
2399: if (object == VM_OBJECT_NULL ||
2400: object->pager_created ||
2401: object->paging_in_progress != 0 ||
2402: object->absent_count != 0)
2403: return;
2404:
2405: /*
2406: * There is a backing object, and
2407: */
2408:
2409: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2410: return;
2411:
2412: vm_object_lock(backing_object);
2413: /*
2414: * ...
2415: * The backing object is not read_only,
2416: * and no pages in the backing object are
2417: * currently being paged out.
2418: * The backing object is internal.
2419: *
2420: * XXX It may be sufficient for the backing
2421: * XXX object to be temporary.
2422: */
2423:
2424: if (!backing_object->internal ||
2425: backing_object->paging_in_progress != 0) {
2426: vm_object_unlock(backing_object);
2427: return;
2428: }
2429:
2430: /*
2431: * The backing object can't be a copy-object:
2432: * the shadow_offset for the copy-object must stay
2433: * as 0. Furthermore (for the 'we have all the
2434: * pages' case), if we bypass backing_object and
2435: * just shadow the next object in the chain, old
2436: * pages from that object would then have to be copied
2437: * BOTH into the (former) backing_object and into the
2438: * parent object.
2439: */
2440: if (backing_object->shadow != VM_OBJECT_NULL &&
2441: backing_object->shadow->copy != VM_OBJECT_NULL) {
2442: vm_object_unlock(backing_object);
2443: return;
2444: }
2445:
2446: /*
2447: * We know that we can either collapse the backing
2448: * object (if the parent is the only reference to
2449: * it) or (perhaps) remove the parent's reference
2450: * to it.
2451: */
2452:
2453: backing_offset = object->shadow_offset;
2454: size = object->size;
2455:
2456: /*
2457: * If there is exactly one reference to the backing
2458: * object, we can collapse it into the parent.
2459: */
2460:
2461: if (backing_object->ref_count == 1) {
2462: if (!vm_object_cache_lock_try()) {
2463: vm_object_unlock(backing_object);
2464: return;
2465: }
2466:
2467: /*
2468: * We can collapse the backing object.
2469: *
2470: * Move all in-memory pages from backing_object
2471: * to the parent. Pages that have been paged out
2472: * will be overwritten by any of the parent's
2473: * pages that shadow them.
2474: */
2475:
2476: while (!queue_empty(&backing_object->memq)) {
2477:
2478: p = (vm_page_t)
2479: queue_first(&backing_object->memq);
2480:
2481: new_offset = (p->offset - backing_offset);
2482:
2483: assert(!p->busy || p->absent);
2484:
2485: /*
2486: * If the parent has a page here, or if
2487: * this page falls outside the parent,
2488: * dispose of it.
2489: *
2490: * Otherwise, move it as planned.
2491: */
2492:
2493: if (p->offset < backing_offset ||
2494: new_offset >= size) {
1.1.1.2 ! root 2495: VM_PAGE_FREE(p);
1.1 root 2496: } else {
2497: pp = vm_page_lookup(object, new_offset);
2498: if (pp != VM_PAGE_NULL && !pp->absent) {
2499: /*
2500: * Parent object has a real page.
2501: * Throw away the backing object's
2502: * page.
2503: */
1.1.1.2 ! root 2504: VM_PAGE_FREE(p);
1.1 root 2505: }
2506: else {
2507: if (pp != VM_PAGE_NULL) {
2508: /*
2509: * Parent has an absent page...
2510: * it's not being paged in, so
2511: * it must really be missing from
2512: * the parent.
2513: *
2514: * Throw out the absent page...
2515: * any faults looking for that
2516: * page will restart with the new
2517: * one.
2518: */
2519:
2520: /*
2521: * This should never happen -- the
2522: * parent cannot have ever had an
2523: * external memory object, and thus
2524: * cannot have absent pages.
2525: */
2526: panic("vm_object_collapse: bad case");
2527:
1.1.1.2 ! root 2528: VM_PAGE_FREE(pp);
1.1 root 2529:
2530: /*
2531: * Fall through to move the backing
2532: * object's page up.
2533: */
2534: }
2535: /*
2536: * Parent now has no page.
2537: * Move the backing object's page up.
2538: */
2539: vm_page_rename(p, object, new_offset);
2540: }
2541: }
2542: }
2543:
2544: /*
2545: * Move the pager from backing_object to object.
2546: *
2547: * XXX We're only using part of the paging space
2548: * for keeps now... we ought to discard the
2549: * unused portion.
2550: */
2551:
2552: switch (vm_object_collapse_debug) {
2553: case 0:
2554: break;
2555: case 1:
2556: if ((backing_object->pager == IP_NULL) &&
2557: (backing_object->pager_request ==
2558: PAGER_REQUEST_NULL))
2559: break;
2560: /* Fall through to... */
2561:
2562: default:
1.1.1.2 ! root 2563: printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n",
1.1 root 2564: backing_object, backing_object->pager, backing_object->pager_request,
2565: object);
2566: if (vm_object_collapse_debug > 2)
1.1.1.2 ! root 2567: SoftDebugger("vm_object_collapse");
1.1 root 2568: }
2569:
2570: object->pager = backing_object->pager;
2571: if (object->pager != IP_NULL)
2572: ipc_kobject_set(object->pager,
2573: (ipc_kobject_t) object,
2574: IKOT_PAGER);
2575: object->pager_initialized = backing_object->pager_initialized;
2576: object->pager_ready = backing_object->pager_ready;
2577: object->pager_created = backing_object->pager_created;
2578:
2579: object->pager_request = backing_object->pager_request;
2580: if (object->pager_request != IP_NULL)
2581: ipc_kobject_set(object->pager_request,
2582: (ipc_kobject_t) object,
2583: IKOT_PAGING_REQUEST);
2584: old_name_port = object->pager_name;
2585: if (old_name_port != IP_NULL)
2586: ipc_kobject_set(old_name_port,
2587: IKO_NULL, IKOT_NONE);
2588: object->pager_name = backing_object->pager_name;
2589: if (object->pager_name != IP_NULL)
2590: ipc_kobject_set(object->pager_name,
2591: (ipc_kobject_t) object,
2592: IKOT_PAGING_NAME);
2593:
2594: vm_object_cache_unlock();
2595:
2596: /*
2597: * If there is no pager, leave paging-offset alone.
2598: */
2599: if (object->pager != IP_NULL)
2600: object->paging_offset =
2601: backing_object->paging_offset +
2602: backing_offset;
2603:
2604: #if MACH_PAGEMAP
2605: assert(object->existence_info == VM_EXTERNAL_NULL);
2606: object->existence_info = backing_object->existence_info;
2607: #endif /* MACH_PAGEMAP */
2608:
2609: /*
2610: * Object now shadows whatever backing_object did.
2611: * Note that the reference to backing_object->shadow
2612: * moves from within backing_object to within object.
2613: */
2614:
2615: object->shadow = backing_object->shadow;
2616: object->shadow_offset += backing_object->shadow_offset;
2617: if (object->shadow != VM_OBJECT_NULL &&
2618: object->shadow->copy != VM_OBJECT_NULL) {
2619: panic("vm_object_collapse: we collapsed a copy-object!");
2620: }
2621: /*
2622: * Discard backing_object.
2623: *
2624: * Since the backing object has no pages, no
2625: * pager left, and no object references within it,
2626: * all that is necessary is to dispose of it.
2627: */
2628:
2629: assert(
2630: (backing_object->ref_count == 1) &&
2631: (backing_object->resident_page_count == 0) &&
2632: (backing_object->paging_in_progress == 0)
2633: );
2634:
2635: assert(backing_object->alive);
2636: backing_object->alive = FALSE;
2637: vm_object_unlock(backing_object);
2638:
2639: vm_object_unlock(object);
2640: if (old_name_port != IP_NULL)
2641: ipc_port_dealloc_kernel(old_name_port);
1.1.1.2 ! root 2642: kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object);
1.1 root 2643: vm_object_lock(object);
2644:
2645: object_collapses++;
2646: }
2647: else {
2648: if (!vm_object_collapse_bypass_allowed) {
2649: vm_object_unlock(backing_object);
2650: return;
2651: }
2652:
2653: /*
2654: * If all of the pages in the backing object are
2655: * shadowed by the parent object, the parent
2656: * object no longer has to shadow the backing
2657: * object; it can shadow the next one in the
2658: * chain.
2659: *
2660: * The backing object must not be paged out - we'd
2661: * have to check all of the paged-out pages, as
2662: * well.
2663: */
2664:
2665: if (backing_object->pager_created) {
2666: vm_object_unlock(backing_object);
2667: return;
2668: }
2669:
2670: /*
2671: * Should have a check for a 'small' number
2672: * of pages here.
2673: */
2674:
2675: queue_iterate(&backing_object->memq, p,
2676: vm_page_t, listq)
2677: {
2678: new_offset = (p->offset - backing_offset);
2679:
2680: /*
2681: * If the parent has a page here, or if
2682: * this page falls outside the parent,
2683: * keep going.
2684: *
2685: * Otherwise, the backing_object must be
2686: * left in the chain.
2687: */
2688:
2689: if (p->offset >= backing_offset &&
2690: new_offset <= size &&
2691: (pp = vm_page_lookup(object, new_offset))
2692: == VM_PAGE_NULL) {
2693: /*
2694: * Page still needed.
2695: * Can't go any further.
2696: */
2697: vm_object_unlock(backing_object);
2698: return;
2699: }
2700: }
2701:
2702: /*
2703: * Make the parent shadow the next object
2704: * in the chain. Deallocating backing_object
2705: * will not remove it, since its reference
2706: * count is at least 2.
2707: */
2708:
2709: vm_object_reference(object->shadow = backing_object->shadow);
2710: object->shadow_offset += backing_object->shadow_offset;
2711:
2712: /*
2713: * Backing object might have had a copy pointer
2714: * to us. If it did, clear it.
2715: */
2716: if (backing_object->copy == object)
2717: backing_object->copy = VM_OBJECT_NULL;
2718:
2719: /*
2720: * Drop the reference count on backing_object.
2721: * Since its ref_count was at least 2, it
2722: * will not vanish; so we don't need to call
2723: * vm_object_deallocate.
2724: */
2725: backing_object->ref_count--;
2726: assert(backing_object->ref_count > 0);
2727: vm_object_unlock(backing_object);
2728:
2729: object_bypasses ++;
2730:
2731: }
2732:
2733: /*
2734: * Try again with this object's new backing object.
2735: */
2736: }
2737: }
2738:
2739: /*
2740: * Routine: vm_object_page_remove: [internal]
2741: * Purpose:
2742: * Removes all physical pages in the specified
2743: * object range from the object's list of pages.
2744: *
2745: * In/out conditions:
2746: * The object must be locked.
2747: */
2748: unsigned int vm_object_page_remove_lookup = 0;
2749: unsigned int vm_object_page_remove_iterate = 0;
2750:
2751: void vm_object_page_remove(
2752: register vm_object_t object,
2753: register vm_offset_t start,
2754: register vm_offset_t end)
2755: {
2756: register vm_page_t p, next;
2757:
2758: /*
2759: * One and two page removals are most popular.
2760: * The factor of 16 here is somewhat arbitrary.
2761: * It balances vm_object_lookup vs iteration.
2762: */
2763:
2764: if (atop(end - start) < (unsigned)object->resident_page_count/16) {
2765: vm_object_page_remove_lookup++;
2766:
2767: for (; start < end; start += PAGE_SIZE) {
2768: p = vm_page_lookup(object, start);
2769: if (p != VM_PAGE_NULL) {
2770: if (!p->fictitious)
2771: pmap_page_protect(p->phys_addr,
2772: VM_PROT_NONE);
1.1.1.2 ! root 2773: VM_PAGE_FREE(p);
1.1 root 2774: }
2775: }
2776: } else {
2777: vm_object_page_remove_iterate++;
2778:
2779: p = (vm_page_t) queue_first(&object->memq);
2780: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2781: next = (vm_page_t) queue_next(&p->listq);
2782: if ((start <= p->offset) && (p->offset < end)) {
2783: if (!p->fictitious)
2784: pmap_page_protect(p->phys_addr,
2785: VM_PROT_NONE);
1.1.1.2 ! root 2786: VM_PAGE_FREE(p);
1.1 root 2787: }
2788: p = next;
2789: }
2790: }
2791: }
2792:
2793: /*
2794: * Routine: vm_object_coalesce
2795: * Function: Coalesces two objects backing up adjoining
2796: * regions of memory into a single object.
2797: *
2798: * returns TRUE if objects were combined.
2799: *
2800: * NOTE: Only works at the moment if the second object is NULL -
2801: * if it's not, which object do we lock first?
2802: *
2803: * Parameters:
2804: * prev_object First object to coalesce
2805: * prev_offset Offset into prev_object
2806: * next_object Second object into coalesce
2807: * next_offset Offset into next_object
2808: *
2809: * prev_size Size of reference to prev_object
2810: * next_size Size of reference to next_object
2811: *
2812: * Conditions:
2813: * The object must *not* be locked.
2814: */
2815:
2816: boolean_t vm_object_coalesce(
2817: register vm_object_t prev_object,
2818: vm_object_t next_object,
2819: vm_offset_t prev_offset,
2820: vm_offset_t next_offset,
2821: vm_size_t prev_size,
2822: vm_size_t next_size)
2823: {
2824: vm_size_t newsize;
2825:
2826: #ifdef lint
2827: next_offset++;
2828: #endif /* lint */
2829:
2830: if (next_object != VM_OBJECT_NULL) {
2831: return FALSE;
2832: }
2833:
2834: if (prev_object == VM_OBJECT_NULL) {
2835: return TRUE;
2836: }
2837:
2838: vm_object_lock(prev_object);
2839:
2840: /*
2841: * Try to collapse the object first
2842: */
2843: vm_object_collapse(prev_object);
2844:
2845: /*
2846: * Can't coalesce if pages not mapped to
2847: * prev_entry may be in use anyway:
2848: * . more than one reference
2849: * . paged out
2850: * . shadows another object
2851: * . has a copy elsewhere
2852: * . paging references (pages might be in page-list)
2853: */
2854:
2855: if ((prev_object->ref_count > 1) ||
2856: prev_object->pager_created ||
2857: (prev_object->shadow != VM_OBJECT_NULL) ||
2858: (prev_object->copy != VM_OBJECT_NULL) ||
2859: (prev_object->paging_in_progress != 0)) {
2860: vm_object_unlock(prev_object);
2861: return FALSE;
2862: }
2863:
2864: /*
2865: * Remove any pages that may still be in the object from
2866: * a previous deallocation.
2867: */
2868:
2869: vm_object_page_remove(prev_object,
2870: prev_offset + prev_size,
2871: prev_offset + prev_size + next_size);
2872:
2873: /*
2874: * Extend the object if necessary.
2875: */
2876: newsize = prev_offset + prev_size + next_size;
2877: if (newsize > prev_object->size)
2878: prev_object->size = newsize;
2879:
2880: vm_object_unlock(prev_object);
2881: return TRUE;
2882: }
2883:
2884: vm_object_t vm_object_request_object(
2885: ipc_port_t p)
2886: {
2887: return vm_object_lookup(p);
2888: }
2889:
2890: /*
2891: * Routine: vm_object_name
2892: * Purpose:
2893: * Returns a naked send right to the "name" port associated
2894: * with this object.
2895: */
2896: ipc_port_t vm_object_name(
2897: vm_object_t object)
2898: {
2899: ipc_port_t p;
2900:
2901: if (object == VM_OBJECT_NULL)
2902: return IP_NULL;
2903:
2904: vm_object_lock(object);
2905:
2906: while (object->shadow != VM_OBJECT_NULL) {
2907: vm_object_t new_object = object->shadow;
2908: vm_object_lock(new_object);
2909: vm_object_unlock(object);
2910: object = new_object;
2911: }
2912:
2913: p = object->pager_name;
2914: if (p != IP_NULL)
2915: p = ipc_port_make_send(p);
2916: vm_object_unlock(object);
2917:
2918: return p;
2919: }
2920:
2921: /*
2922: * Attach a set of physical pages to an object, so that they can
2923: * be mapped by mapping the object. Typically used to map IO memory.
2924: *
2925: * The mapping function and its private data are used to obtain the
2926: * physical addresses for each page to be mapped.
2927: */
2928: void
2929: vm_object_page_map(
2930: vm_object_t object,
2931: vm_offset_t offset,
2932: vm_size_t size,
2933: vm_offset_t (*map_fn)(void *, vm_offset_t),
2934: void * map_fn_data) /* private to map_fn */
2935: {
2936: int num_pages;
2937: int i;
2938: vm_page_t m;
2939: vm_page_t old_page;
2940: vm_offset_t addr;
2941:
2942: num_pages = atop(size);
2943:
2944: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2945:
2946: addr = (*map_fn)(map_fn_data, offset);
2947:
2948: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
2949: vm_page_more_fictitious();
2950:
2951: vm_object_lock(object);
2952: if ((old_page = vm_page_lookup(object, offset))
2953: != VM_PAGE_NULL)
2954: {
1.1.1.2 ! root 2955: VM_PAGE_FREE(old_page);
1.1 root 2956: }
2957:
2958: vm_page_init(m, addr);
2959: m->private = TRUE; /* don`t free page */
2960: m->wire_count = 1;
2961: vm_page_lock_queues();
2962: vm_page_insert(m, object, offset);
2963: vm_page_unlock_queues();
2964:
2965: PAGE_WAKEUP_DONE(m);
2966: vm_object_unlock(object);
2967: }
2968: }
2969:
2970:
2971: #if MACH_KDB
1.1.1.2 ! root 2972: #include <vm/vm_print.h>
1.1 root 2973: #define printf kdbprintf
2974:
2975: boolean_t vm_object_print_pages = FALSE;
2976:
2977: /*
2978: * vm_object_print: [ debug ]
2979: */
2980: void vm_object_print(
2981: vm_object_t object)
2982: {
2983: register vm_page_t p;
2984:
2985: register int count;
2986:
2987: if (object == VM_OBJECT_NULL)
2988: return;
2989:
2990: iprintf("Object 0x%X: size=0x%X",
2991: (vm_offset_t) object, (vm_offset_t) object->size);
2992: printf(", %d references, %d resident pages,", object->ref_count,
2993: object->resident_page_count);
2994: printf(" %d absent pages,", object->absent_count);
2995: printf(" %d paging ops\n", object->paging_in_progress);
2996: indent += 2;
2997: iprintf("memory object=0x%X (offset=0x%X),",
2998: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
2999: printf("control=0x%X, name=0x%X\n",
3000: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
3001: iprintf("%s%s",
3002: object->pager_ready ? " ready" : "",
3003: object->pager_created ? " created" : "");
3004: printf("%s,%s ",
3005: object->pager_initialized ? "" : "uninitialized",
3006: object->temporary ? "temporary" : "permanent");
3007: printf("%s%s,",
3008: object->internal ? "internal" : "external",
3009: object->can_persist ? " cacheable" : "");
3010: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
3011: iprintf("shadow=0x%X (offset=0x%X),",
3012: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
3013: printf("copy=0x%X\n", (vm_offset_t) object->copy);
3014:
3015: indent += 2;
3016:
3017: if (vm_object_print_pages) {
3018: count = 0;
3019: p = (vm_page_t) queue_first(&object->memq);
3020: while (!queue_end(&object->memq, (queue_entry_t) p)) {
3021: if (count == 0) iprintf("memory:=");
3022: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
3023: else printf(",");
3024: count++;
3025:
3026: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
3027: p = (vm_page_t) queue_next(&p->listq);
3028: }
3029: if (count != 0)
3030: printf("\n");
3031: }
3032: indent -= 4;
3033: }
3034:
3035: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.