![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
1.1.1.2 root 36: #include <kern/printf.h>
37: #include <string.h>
1.1 root 38:
39: #include <mach/memory_object.h>
1.1.1.2 root 40: #include <vm/memory_object_default.user.h>
41: #include <vm/memory_object_user.user.h>
42: #include <machine/vm_param.h>
1.1 root 43: #include <ipc/ipc_port.h>
44: #include <ipc/ipc_space.h>
45: #include <kern/assert.h>
1.1.1.2 root 46: #include <kern/debug.h>
1.1 root 47: #include <kern/lock.h>
48: #include <kern/queue.h>
49: #include <kern/xpr.h>
1.1.1.2 root 50: #include <kern/slab.h>
1.1 root 51: #include <vm/memory_object.h>
52: #include <vm/vm_fault.h>
53: #include <vm/vm_map.h>
54: #include <vm/vm_object.h>
55: #include <vm/vm_page.h>
56: #include <vm/vm_pageout.h>
57:
1.1.1.2 root 58: #if MACH_KDB
59: #include <ddb/db_output.h>
60: #endif /* MACH_KDB */
61:
1.1 root 62: /*
63: * Virtual memory objects maintain the actual data
64: * associated with allocated virtual memory. A given
65: * page of memory exists within exactly one object.
66: *
67: * An object is only deallocated when all "references"
68: * are given up. Only one "reference" to a given
69: * region of an object should be writeable.
70: *
71: * Associated with each object is a list of all resident
72: * memory pages belonging to that object; this list is
73: * maintained by the "vm_page" module, but locked by the object's
74: * lock.
75: *
76: * Each object also records the memory object port
77: * that is used by the kernel to request and write
78: * back data (the memory object port, field "pager"),
79: * and the ports provided to the memory manager, the server that
80: * manages that data, to return data and control its
81: * use (the memory object control port, field "pager_request")
82: * and for naming (the memory object name port, field "pager_name").
83: *
84: * Virtual memory objects are allocated to provide
85: * zero-filled memory (vm_allocate) or map a user-defined
86: * memory object into a virtual address space (vm_map).
87: *
88: * Virtual memory objects that refer to a user-defined
89: * memory object are called "permanent", because all changes
90: * made in virtual memory are reflected back to the
91: * memory manager, which may then store it permanently.
92: * Other virtual memory objects are called "temporary",
93: * meaning that changes need be written back only when
94: * necessary to reclaim pages, and that storage associated
95: * with the object can be discarded once it is no longer
96: * mapped.
97: *
98: * A permanent memory object may be mapped into more
99: * than one virtual address space. Moreover, two threads
100: * may attempt to make the first mapping of a memory
101: * object concurrently. Only one thread is allowed to
102: * complete this mapping; all others wait for the
103: * "pager_initialized" field is asserted, indicating
104: * that the first thread has initialized all of the
105: * necessary fields in the virtual memory object structure.
106: *
107: * The kernel relies on a *default memory manager* to
108: * provide backing storage for the zero-filled virtual
109: * memory objects. The memory object ports associated
110: * with these temporary virtual memory objects are only
111: * generated and passed to the default memory manager
112: * when it becomes necessary. Virtual memory objects
113: * that depend on the default memory manager are called
114: * "internal". The "pager_created" field is provided to
115: * indicate whether these ports have ever been allocated.
116: *
117: * The kernel may also create virtual memory objects to
118: * hold changed pages after a copy-on-write operation.
119: * In this case, the virtual memory object (and its
120: * backing storage -- its memory object) only contain
121: * those pages that have been changed. The "shadow"
122: * field refers to the virtual memory object that contains
123: * the remainder of the contents. The "shadow_offset"
124: * field indicates where in the "shadow" these contents begin.
125: * The "copy" field refers to a virtual memory object
126: * to which changed pages must be copied before changing
127: * this object, in order to implement another form
128: * of copy-on-write optimization.
129: *
130: * The virtual memory object structure also records
131: * the attributes associated with its memory object.
132: * The "pager_ready", "can_persist" and "copy_strategy"
133: * fields represent those attributes. The "cached_list"
134: * field is used in the implementation of the persistence
135: * attribute.
136: *
137: * ZZZ Continue this comment.
138: */
139:
1.1.1.2 root 140: struct kmem_cache vm_object_cache; /* vm backing store cache */
1.1 root 141:
142: /*
143: * All wired-down kernel memory belongs to a single virtual
144: * memory object (kernel_object) to avoid wasting data structures.
145: */
1.1.1.2 root 146: static struct vm_object kernel_object_store;
147: vm_object_t kernel_object = &kernel_object_store;
1.1 root 148:
149: /*
150: * Virtual memory objects that are not referenced by
151: * any address maps, but that are allowed to persist
152: * (an attribute specified by the associated memory manager),
153: * are kept in a queue (vm_object_cached_list).
154: *
155: * When an object from this queue is referenced again,
156: * for example to make another address space mapping,
157: * it must be removed from the queue. That is, the
158: * queue contains *only* objects with zero references.
159: *
160: * The kernel may choose to terminate objects from this
161: * queue in order to reclaim storage. The current policy
162: * is to permit a fixed maximum number of unreferenced
163: * objects (vm_object_cached_max).
164: *
165: * A simple lock (accessed by routines
166: * vm_object_cache_{lock,lock_try,unlock}) governs the
167: * object cache. It must be held when objects are
168: * added to or removed from the cache (in vm_object_terminate).
169: * The routines that acquire a reference to a virtual
170: * memory object based on one of the memory object ports
171: * must also lock the cache.
172: *
173: * Ideally, the object cache should be more isolated
174: * from the reference mechanism, so that the lock need
175: * not be held to make simple references.
176: */
177: queue_head_t vm_object_cached_list;
178: int vm_object_cached_count;
1.1.1.2 root 179: int vm_object_cached_max = 4000; /* may be patched*/
1.1 root 180:
181: decl_simple_lock_data(,vm_object_cached_lock_data)
182:
183: #define vm_object_cache_lock() \
184: simple_lock(&vm_object_cached_lock_data)
185: #define vm_object_cache_lock_try() \
186: simple_lock_try(&vm_object_cached_lock_data)
187: #define vm_object_cache_unlock() \
188: simple_unlock(&vm_object_cached_lock_data)
189:
190: /*
1.1.1.2 root 191: * Number of physical pages referenced by cached objects.
192: * This counter is protected by its own lock to work around
193: * lock ordering issues.
194: */
195: int vm_object_cached_pages;
196:
197: decl_simple_lock_data(,vm_object_cached_pages_lock_data)
198:
199: /*
1.1 root 200: * Virtual memory objects are initialized from
201: * a template (see vm_object_allocate).
202: *
203: * When adding a new field to the virtual memory
204: * object structure, be sure to add initialization
205: * (see vm_object_init).
206: */
1.1.1.2 root 207: struct vm_object vm_object_template;
1.1 root 208:
209: /*
210: * vm_object_allocate:
211: *
212: * Returns a new object with the given size.
213: */
214:
1.1.1.2 root 215: static void _vm_object_setup(
216: vm_object_t object,
217: vm_size_t size)
218: {
219: *object = vm_object_template;
220: queue_init(&object->memq);
221: vm_object_lock_init(object);
222: object->size = size;
223: }
224:
1.1 root 225: vm_object_t _vm_object_allocate(
226: vm_size_t size)
227: {
1.1.1.3 ! root 228: vm_object_t object;
1.1 root 229:
1.1.1.2 root 230: object = (vm_object_t) kmem_cache_alloc(&vm_object_cache);
1.1 root 231:
1.1.1.2 root 232: _vm_object_setup(object, size);
1.1 root 233:
234: return object;
235: }
236:
237: vm_object_t vm_object_allocate(
238: vm_size_t size)
239: {
1.1.1.3 ! root 240: vm_object_t object;
! 241: ipc_port_t port;
1.1 root 242:
243: object = _vm_object_allocate(size);
244: port = ipc_port_alloc_kernel();
245: if (port == IP_NULL)
246: panic("vm_object_allocate");
247: object->pager_name = port;
248: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
249:
250: return object;
251: }
252:
253: /*
254: * vm_object_bootstrap:
255: *
256: * Initialize the VM objects module.
257: */
258: void vm_object_bootstrap(void)
259: {
1.1.1.2 root 260: kmem_cache_init(&vm_object_cache, "vm_object",
261: sizeof(struct vm_object), 0, NULL, NULL, NULL, 0);
1.1 root 262:
263: queue_init(&vm_object_cached_list);
264: simple_lock_init(&vm_object_cached_lock_data);
265:
266: /*
267: * Fill in a template object, for quick initialization
268: */
269:
1.1.1.2 root 270: vm_object_template.ref_count = 1;
271: vm_object_template.size = 0;
272: vm_object_template.resident_page_count = 0;
273: vm_object_template.copy = VM_OBJECT_NULL;
274: vm_object_template.shadow = VM_OBJECT_NULL;
275: vm_object_template.shadow_offset = (vm_offset_t) 0;
276:
277: vm_object_template.pager = IP_NULL;
278: vm_object_template.paging_offset = 0;
279: vm_object_template.pager_request = PAGER_REQUEST_NULL;
280: vm_object_template.pager_name = IP_NULL;
281:
282: vm_object_template.pager_created = FALSE;
283: vm_object_template.pager_initialized = FALSE;
284: vm_object_template.pager_ready = FALSE;
1.1 root 285:
1.1.1.2 root 286: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE;
1.1 root 287: /* ignored if temporary, will be reset before
288: * permanent object becomes ready */
1.1.1.2 root 289: vm_object_template.use_shared_copy = FALSE;
290: vm_object_template.shadowed = FALSE;
1.1 root 291:
1.1.1.2 root 292: vm_object_template.absent_count = 0;
293: vm_object_template.all_wanted = 0; /* all bits FALSE */
1.1 root 294:
1.1.1.2 root 295: vm_object_template.paging_in_progress = 0;
296: vm_object_template.can_persist = FALSE;
297: vm_object_template.internal = TRUE;
298: vm_object_template.temporary = TRUE;
299: vm_object_template.alive = TRUE;
300: vm_object_template.lock_in_progress = FALSE;
301: vm_object_template.lock_restart = FALSE;
302: vm_object_template.use_old_pageout = TRUE; /* XXX change later */
303: vm_object_template.last_alloc = (vm_offset_t) 0;
1.1 root 304:
305: #if MACH_PAGEMAP
1.1.1.2 root 306: vm_object_template.existence_info = VM_EXTERNAL_NULL;
1.1 root 307: #endif /* MACH_PAGEMAP */
308:
309: /*
310: * Initialize the "kernel object"
311: */
312:
1.1.1.2 root 313: _vm_object_setup(kernel_object,
1.1 root 314: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
315:
316: /*
317: * Initialize the "submap object". Make it as large as the
318: * kernel object so that no limit is imposed on submap sizes.
319: */
320:
1.1.1.2 root 321: _vm_object_setup(vm_submap_object,
1.1 root 322: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
323:
324: #if MACH_PAGEMAP
325: vm_external_module_initialize();
326: #endif /* MACH_PAGEMAP */
327: }
328:
329: void vm_object_init(void)
330: {
331: /*
332: * Finish initializing the kernel object.
333: * The submap object doesn't need a name port.
334: */
335:
336: kernel_object->pager_name = ipc_port_alloc_kernel();
337: ipc_kobject_set(kernel_object->pager_name,
338: (ipc_kobject_t) kernel_object,
339: IKOT_PAGING_NAME);
340: }
341:
342: /*
343: * vm_object_reference:
344: *
345: * Gets another reference to the given object.
346: */
347: void vm_object_reference(
1.1.1.3 ! root 348: vm_object_t object)
1.1 root 349: {
350: if (object == VM_OBJECT_NULL)
351: return;
352:
353: vm_object_lock(object);
354: assert(object->ref_count > 0);
355: object->ref_count++;
356: vm_object_unlock(object);
357: }
358:
359: /*
360: * vm_object_deallocate:
361: *
362: * Release a reference to the specified object,
363: * gained either through a vm_object_allocate
364: * or a vm_object_reference call. When all references
365: * are gone, storage associated with this object
366: * may be relinquished.
367: *
368: * No object may be locked.
369: */
370: void vm_object_deallocate(
1.1.1.3 ! root 371: vm_object_t object)
1.1 root 372: {
373: vm_object_t temp;
374:
375: while (object != VM_OBJECT_NULL) {
376:
377: /*
378: * The cache holds a reference (uncounted) to
379: * the object; we must lock it before removing
380: * the object.
381: */
382:
383: vm_object_cache_lock();
384:
385: /*
386: * Lose the reference
387: */
388: vm_object_lock(object);
389: if (--(object->ref_count) > 0) {
390:
391: /*
392: * If there are still references, then
393: * we are done.
394: */
395: vm_object_unlock(object);
396: vm_object_cache_unlock();
397: return;
398: }
399:
400: /*
401: * See whether this object can persist. If so, enter
402: * it in the cache, then deactivate all of its
403: * pages.
404: */
405: if (object->can_persist) {
406: boolean_t overflow;
407:
408: /*
409: * Enter the object onto the queue
410: * of "cached" objects. Remember whether
411: * we've caused the queue to overflow,
412: * as a hint.
413: */
414:
415: queue_enter(&vm_object_cached_list, object,
416: vm_object_t, cached_list);
417: overflow = (++vm_object_cached_count > vm_object_cached_max);
1.1.1.2 root 418: vm_object_cached_pages_update(object->resident_page_count);
1.1 root 419: vm_object_cache_unlock();
420:
421: vm_object_deactivate_pages(object);
422: vm_object_unlock(object);
423:
424: /*
425: * If we didn't overflow, or if the queue has
426: * been reduced back to below the specified
427: * minimum, then quit.
428: */
429: if (!overflow)
430: return;
431:
432: while (TRUE) {
433: vm_object_cache_lock();
434: if (vm_object_cached_count <=
435: vm_object_cached_max) {
436: vm_object_cache_unlock();
437: return;
438: }
439:
440: /*
441: * If we must trim down the queue, take
442: * the first object, and proceed to
443: * terminate it instead of the original
444: * object. Have to wait for pager init.
445: * if it's in progress.
446: */
447: object= (vm_object_t)
448: queue_first(&vm_object_cached_list);
449: vm_object_lock(object);
450:
451: if (!(object->pager_created &&
452: !object->pager_initialized)) {
453:
454: /*
455: * Ok to terminate, hang on to lock.
456: */
457: break;
458: }
459:
460: vm_object_assert_wait(object,
461: VM_OBJECT_EVENT_INITIALIZED, FALSE);
462: vm_object_unlock(object);
463: vm_object_cache_unlock();
464: thread_block((void (*)()) 0);
465:
466: /*
467: * Continue loop to check if cache still
468: * needs to be trimmed.
469: */
470: }
471:
472: /*
473: * Actually remove object from cache.
474: */
475:
476: queue_remove(&vm_object_cached_list, object,
477: vm_object_t, cached_list);
478: vm_object_cached_count--;
479:
480: assert(object->ref_count == 0);
481: }
482: else {
483: if (object->pager_created &&
484: !object->pager_initialized) {
485:
486: /*
487: * Have to wait for initialization.
488: * Put reference back and retry
489: * when it's initialized.
490: */
491: object->ref_count++;
492: vm_object_assert_wait(object,
493: VM_OBJECT_EVENT_INITIALIZED, FALSE);
494: vm_object_unlock(object);
495: vm_object_cache_unlock();
496: thread_block((void (*)()) 0);
497: continue;
498: }
499: }
500:
501: /*
502: * Take the reference to the shadow object
503: * out of the object to be destroyed.
504: */
505:
506: temp = object->shadow;
507:
508: /*
509: * Destroy the object; the cache lock will
510: * be released in the process.
511: */
512:
513: vm_object_terminate(object);
514:
515: /*
516: * Deallocate the reference to the shadow
517: * by continuing the loop with that object
518: * in place of the original.
519: */
520:
521: object = temp;
522: }
523: }
524:
525: /*
526: * Routine: vm_object_terminate
527: * Purpose:
528: * Free all resources associated with a vm_object.
529: * In/out conditions:
530: * Upon entry, the object and the cache must be locked,
531: * and the object must have no references.
532: *
533: * The shadow object reference is left alone.
534: *
535: * Upon exit, the cache will be unlocked, and the
536: * object will cease to exist.
537: */
538: void vm_object_terminate(
1.1.1.3 ! root 539: vm_object_t object)
1.1 root 540: {
1.1.1.3 ! root 541: vm_page_t p;
! 542: vm_object_t shadow_object;
1.1 root 543:
544: /*
545: * Make sure the object isn't already being terminated
546: */
547:
548: assert(object->alive);
549: object->alive = FALSE;
550:
551: /*
552: * Make sure no one can look us up now.
553: */
554:
555: vm_object_remove(object);
556: vm_object_cache_unlock();
557:
558: /*
559: * Detach the object from its shadow if we are the shadow's
560: * copy.
561: */
562: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
563: vm_object_lock(shadow_object);
564: assert((shadow_object->copy == object) ||
565: (shadow_object->copy == VM_OBJECT_NULL));
566: shadow_object->copy = VM_OBJECT_NULL;
567: vm_object_unlock(shadow_object);
568: }
569:
570: /*
571: * The pageout daemon might be playing with our pages.
572: * Now that the object is dead, it won't touch any more
573: * pages, but some pages might already be on their way out.
574: * Hence, we wait until the active paging activities have ceased.
575: */
576:
577: vm_object_paging_wait(object, FALSE);
578:
579: /*
580: * Clean or free the pages, as appropriate.
581: * It is possible for us to find busy/absent pages,
582: * if some faults on this object were aborted.
583: */
584:
585: if ((object->temporary) || (object->pager == IP_NULL)) {
586: while (!queue_empty(&object->memq)) {
587: p = (vm_page_t) queue_first(&object->memq);
588:
589: VM_PAGE_CHECK(p);
590:
591: if (p->busy && !p->absent)
592: panic("vm_object_terminate.2 0x%x 0x%x",
593: object, p);
594:
595: VM_PAGE_FREE(p);
596: }
597: } else while (!queue_empty(&object->memq)) {
598: p = (vm_page_t) queue_first(&object->memq);
599:
600: VM_PAGE_CHECK(p);
601:
602: if (p->busy && !p->absent)
603: panic("vm_object_terminate.3 0x%x 0x%x", object, p);
604:
605: vm_page_lock_queues();
606: VM_PAGE_QUEUES_REMOVE(p);
607: vm_page_unlock_queues();
608:
609: if (p->absent || p->private) {
610:
611: /*
612: * For private pages, VM_PAGE_FREE just
613: * leaves the page structure around for
614: * its owner to clean up. For absent
615: * pages, the structure is returned to
616: * the appropriate pool.
617: */
618:
619: goto free_page;
620: }
621:
622: if (p->fictitious)
623: panic("vm_object_terminate.4 0x%x 0x%x", object, p);
624:
625: if (!p->dirty)
626: p->dirty = pmap_is_modified(p->phys_addr);
627:
628: if (p->dirty || p->precious) {
629: p->busy = TRUE;
630: vm_pageout_page(p, FALSE, TRUE); /* flush page */
631: } else {
632: free_page:
633: VM_PAGE_FREE(p);
634: }
635: }
636:
637: assert(object->ref_count == 0);
638: assert(object->paging_in_progress == 0);
639:
640: /*
641: * Throw away port rights... note that they may
642: * already have been thrown away (by vm_object_destroy
643: * or memory_object_destroy).
644: *
645: * Instead of destroying the control and name ports,
646: * we send all rights off to the memory manager instead,
647: * using memory_object_terminate.
648: */
649:
650: vm_object_unlock(object);
651:
652: if (object->pager != IP_NULL) {
653: /* consumes our rights for pager, pager_request, pager_name */
654: memory_object_release(object->pager,
655: object->pager_request,
656: object->pager_name);
657: } else if (object->pager_name != IP_NULL) {
658: /* consumes our right for pager_name */
659: ipc_port_dealloc_kernel(object->pager_name);
660: }
661:
662: #if MACH_PAGEMAP
663: vm_external_destroy(object->existence_info);
664: #endif /* MACH_PAGEMAP */
665:
666: /*
667: * Free the space for the object.
668: */
669:
1.1.1.2 root 670: kmem_cache_free(&vm_object_cache, (vm_offset_t) object);
1.1 root 671: }
672:
673: /*
674: * Routine: vm_object_pager_wakeup
675: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
676: */
677:
678: void
679: vm_object_pager_wakeup(
680: ipc_port_t pager)
681: {
682: boolean_t someone_waiting;
683:
684: /*
685: * If anyone was waiting for the memory_object_terminate
686: * to be queued, wake them up now.
687: */
688: vm_object_cache_lock();
689: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
690: someone_waiting = (pager->ip_kobject != IKO_NULL);
691: if (ip_active(pager))
692: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
693: vm_object_cache_unlock();
694: if (someone_waiting) {
695: thread_wakeup((event_t) pager);
696: }
697: }
698:
699: /*
700: * Routine: memory_object_release
701: * Purpose: Terminate the pager and release port rights,
702: * just like memory_object_terminate, except
703: * that we wake up anyone blocked in vm_object_enter
704: * waiting for termination message to be queued
705: * before calling memory_object_init.
706: */
707: void memory_object_release(
708: ipc_port_t pager,
709: pager_request_t pager_request,
710: ipc_port_t pager_name)
711: {
712:
713: /*
714: * Keep a reference to pager port;
715: * the terminate might otherwise release all references.
716: */
717: ip_reference(pager);
718:
719: /*
720: * Terminate the pager.
721: */
722: (void) memory_object_terminate(pager, pager_request, pager_name);
723:
724: /*
725: * Wakeup anyone waiting for this terminate
726: */
727: vm_object_pager_wakeup(pager);
728:
729: /*
730: * Release reference to pager port.
731: */
732: ip_release(pager);
733: }
734:
735: /*
736: * Routine: vm_object_abort_activity [internal use only]
737: * Purpose:
738: * Abort paging requests pending on this object.
739: * In/out conditions:
740: * The object is locked on entry and exit.
741: */
742: void vm_object_abort_activity(
743: vm_object_t object)
744: {
745: vm_page_t p;
746: vm_page_t next;
747:
748: /*
749: * Abort all activity that would be waiting
750: * for a result on this memory object.
751: *
752: * We could also choose to destroy all pages
753: * that we have in memory for this object, but
754: * we don't.
755: */
756:
757: p = (vm_page_t) queue_first(&object->memq);
758: while (!queue_end(&object->memq, (queue_entry_t) p)) {
759: next = (vm_page_t) queue_next(&p->listq);
760:
761: /*
762: * If it's being paged in, destroy it.
763: * If an unlock has been requested, start it again.
764: */
765:
766: if (p->busy && p->absent) {
767: VM_PAGE_FREE(p);
768: }
769: else {
770: if (p->unlock_request != VM_PROT_NONE)
771: p->unlock_request = VM_PROT_NONE;
772: PAGE_WAKEUP(p);
773: }
774:
775: p = next;
776: }
777:
778: /*
779: * Wake up threads waiting for the memory object to
780: * become ready.
781: */
782:
783: object->pager_ready = TRUE;
784: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
785: }
786:
787: /*
788: * Routine: memory_object_destroy [user interface]
789: * Purpose:
790: * Shut down a memory object, despite the
791: * presence of address map (or other) references
792: * to the vm_object.
793: * Note:
794: * This routine may be called either from the user interface,
795: * or from port destruction handling (via vm_object_destroy).
796: */
797: kern_return_t memory_object_destroy(
798: vm_object_t object,
799: kern_return_t reason)
800: {
801: ipc_port_t old_object, old_name;
802: pager_request_t old_control;
803:
804: if (object == VM_OBJECT_NULL)
805: return KERN_SUCCESS;
806:
807: /*
808: * Remove the port associations immediately.
809: *
810: * This will prevent the memory manager from further
811: * meddling. [If it wanted to flush data or make
812: * other changes, it should have done so before performing
813: * the destroy call.]
814: */
815:
816: vm_object_cache_lock();
817: vm_object_lock(object);
818: vm_object_remove(object);
819: object->can_persist = FALSE;
820: vm_object_cache_unlock();
821:
822: /*
823: * Rip out the ports from the vm_object now... this
824: * will prevent new memory_object calls from succeeding.
825: */
826:
827: old_object = object->pager;
828: object->pager = IP_NULL;
829:
830: old_control = object->pager_request;
831: object->pager_request = PAGER_REQUEST_NULL;
832:
833: old_name = object->pager_name;
834: object->pager_name = IP_NULL;
835:
836:
837: /*
838: * Wait for existing paging activity (that might
839: * have the old ports) to subside.
840: */
841:
842: vm_object_paging_wait(object, FALSE);
843: vm_object_unlock(object);
844:
845: /*
846: * Shut down the ports now.
847: *
848: * [Paging operations may be proceeding concurrently --
849: * they'll get the null values established above.]
850: */
851:
852: if (old_object != IP_NULL) {
853: /* consumes our rights for object, control, name */
854: memory_object_release(old_object, old_control,
855: old_name);
856: } else if (old_name != IP_NULL) {
857: /* consumes our right for name */
858: ipc_port_dealloc_kernel(object->pager_name);
859: }
860:
861: /*
862: * Lose the reference that was donated for this routine
863: */
864:
865: vm_object_deallocate(object);
866:
867: return KERN_SUCCESS;
868: }
869:
870: /*
871: * vm_object_deactivate_pages
872: *
873: * Deactivate all pages in the specified object. (Keep its pages
874: * in memory even though it is no longer referenced.)
875: *
876: * The object must be locked.
877: */
878: void vm_object_deactivate_pages(
1.1.1.3 ! root 879: vm_object_t object)
1.1 root 880: {
1.1.1.3 ! root 881: vm_page_t p;
1.1 root 882:
883: queue_iterate(&object->memq, p, vm_page_t, listq) {
884: vm_page_lock_queues();
885: if (!p->busy)
886: vm_page_deactivate(p);
887: vm_page_unlock_queues();
888: }
889: }
890:
891:
892: /*
893: * Routine: vm_object_pmap_protect
894: *
895: * Purpose:
896: * Reduces the permission for all physical
897: * pages in the specified object range.
898: *
899: * If removing write permission only, it is
900: * sufficient to protect only the pages in
901: * the top-level object; only those pages may
902: * have write permission.
903: *
904: * If removing all access, we must follow the
905: * shadow chain from the top-level object to
906: * remove access to all pages in shadowed objects.
907: *
908: * The object must *not* be locked. The object must
909: * be temporary/internal.
910: *
911: * If pmap is not NULL, this routine assumes that
912: * the only mappings for the pages are in that
913: * pmap.
914: */
915: boolean_t vm_object_pmap_protect_by_page = FALSE;
916:
917: void vm_object_pmap_protect(
1.1.1.3 ! root 918: vm_object_t object,
! 919: vm_offset_t offset,
1.1.1.2 root 920: vm_size_t size,
1.1 root 921: pmap_t pmap,
922: vm_offset_t pmap_start,
923: vm_prot_t prot)
924: {
925: if (object == VM_OBJECT_NULL)
926: return;
927:
928: vm_object_lock(object);
929:
930: assert(object->temporary && object->internal);
931:
932: while (TRUE) {
933: if (object->resident_page_count > atop(size) / 2 &&
934: pmap != PMAP_NULL) {
935: vm_object_unlock(object);
936: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
937: return;
938: }
939:
940: {
1.1.1.3 ! root 941: vm_page_t p;
! 942: vm_offset_t end;
1.1 root 943:
944: end = offset + size;
945:
946: queue_iterate(&object->memq, p, vm_page_t, listq) {
947: if (!p->fictitious &&
948: (offset <= p->offset) &&
949: (p->offset < end)) {
950: if ((pmap == PMAP_NULL) ||
951: vm_object_pmap_protect_by_page) {
952: pmap_page_protect(p->phys_addr,
953: prot & ~p->page_lock);
954: } else {
955: vm_offset_t start =
956: pmap_start +
957: (p->offset - offset);
958:
959: pmap_protect(pmap,
960: start,
961: start + PAGE_SIZE,
962: prot);
963: }
964: }
965: }
966: }
967:
968: if (prot == VM_PROT_NONE) {
969: /*
970: * Must follow shadow chain to remove access
971: * to pages in shadowed objects.
972: */
1.1.1.3 ! root 973: vm_object_t next_object;
1.1 root 974:
975: next_object = object->shadow;
976: if (next_object != VM_OBJECT_NULL) {
977: offset += object->shadow_offset;
978: vm_object_lock(next_object);
979: vm_object_unlock(object);
980: object = next_object;
981: }
982: else {
983: /*
984: * End of chain - we are done.
985: */
986: break;
987: }
988: }
989: else {
990: /*
991: * Pages in shadowed objects may never have
992: * write permission - we may stop here.
993: */
994: break;
995: }
996: }
997:
998: vm_object_unlock(object);
999: }
1000:
1001: /*
1002: * vm_object_pmap_remove:
1003: *
1004: * Removes all physical pages in the specified
1005: * object range from all physical maps.
1006: *
1007: * The object must *not* be locked.
1008: */
1009: void vm_object_pmap_remove(
1.1.1.3 ! root 1010: vm_object_t object,
! 1011: vm_offset_t start,
! 1012: vm_offset_t end)
1.1 root 1013: {
1.1.1.3 ! root 1014: vm_page_t p;
1.1 root 1015:
1016: if (object == VM_OBJECT_NULL)
1017: return;
1018:
1019: vm_object_lock(object);
1020: queue_iterate(&object->memq, p, vm_page_t, listq) {
1021: if (!p->fictitious &&
1022: (start <= p->offset) &&
1023: (p->offset < end))
1024: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
1025: }
1026: vm_object_unlock(object);
1027: }
1028:
1029: /*
1030: * Routine: vm_object_copy_slowly
1031: *
1032: * Description:
1033: * Copy the specified range of the source
1034: * virtual memory object without using
1035: * protection-based optimizations (such
1036: * as copy-on-write). The pages in the
1037: * region are actually copied.
1038: *
1039: * In/out conditions:
1040: * The caller must hold a reference and a lock
1041: * for the source virtual memory object. The source
1042: * object will be returned *unlocked*.
1043: *
1044: * Results:
1045: * If the copy is completed successfully, KERN_SUCCESS is
1046: * returned. If the caller asserted the interruptible
1047: * argument, and an interruption occurred while waiting
1048: * for a user-generated event, MACH_SEND_INTERRUPTED is
1049: * returned. Other values may be returned to indicate
1050: * hard errors during the copy operation.
1051: *
1052: * A new virtual memory object is returned in a
1053: * parameter (_result_object). The contents of this
1054: * new object, starting at a zero offset, are a copy
1055: * of the source memory region. In the event of
1056: * an error, this parameter will contain the value
1057: * VM_OBJECT_NULL.
1058: */
1059: kern_return_t vm_object_copy_slowly(
1060: vm_object_t src_object,
1061: vm_offset_t src_offset,
1062: vm_size_t size,
1063: boolean_t interruptible,
1064: vm_object_t *_result_object) /* OUT */
1065: {
1066: vm_object_t new_object;
1067: vm_offset_t new_offset;
1068:
1069: if (size == 0) {
1070: vm_object_unlock(src_object);
1071: *_result_object = VM_OBJECT_NULL;
1072: return KERN_INVALID_ARGUMENT;
1073: }
1074:
1075: /*
1076: * Prevent destruction of the source object while we copy.
1077: */
1078:
1079: assert(src_object->ref_count > 0);
1080: src_object->ref_count++;
1081: vm_object_unlock(src_object);
1082:
1083: /*
1084: * Create a new object to hold the copied pages.
1085: * A few notes:
1086: * We fill the new object starting at offset 0,
1087: * regardless of the input offset.
1088: * We don't bother to lock the new object within
1089: * this routine, since we have the only reference.
1090: */
1091:
1092: new_object = vm_object_allocate(size);
1093: new_offset = 0;
1094:
1095: assert(size == trunc_page(size)); /* Will the loop terminate? */
1096:
1097: for ( ;
1098: size != 0 ;
1099: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1100: ) {
1101: vm_page_t new_page;
1102: vm_fault_return_t result;
1103:
1104: while ((new_page = vm_page_alloc(new_object, new_offset))
1105: == VM_PAGE_NULL) {
1106: VM_PAGE_WAIT((void (*)()) 0);
1107: }
1108:
1109: do {
1110: vm_prot_t prot = VM_PROT_READ;
1111: vm_page_t _result_page;
1112: vm_page_t top_page;
1113: vm_page_t result_page;
1114:
1115: vm_object_lock(src_object);
1116: src_object->paging_in_progress++;
1117:
1118: result = vm_fault_page(src_object, src_offset,
1119: VM_PROT_READ, FALSE, interruptible,
1120: &prot, &_result_page, &top_page,
1121: FALSE, (void (*)()) 0);
1122:
1123: switch(result) {
1124: case VM_FAULT_SUCCESS:
1125: result_page = _result_page;
1126:
1127: /*
1128: * We don't need to hold the object
1129: * lock -- the busy page will be enough.
1130: * [We don't care about picking up any
1131: * new modifications.]
1132: *
1133: * Copy the page to the new object.
1134: *
1135: * POLICY DECISION:
1136: * If result_page is clean,
1137: * we could steal it instead
1138: * of copying.
1139: */
1140:
1141: vm_object_unlock(result_page->object);
1142: vm_page_copy(result_page, new_page);
1143:
1144: /*
1145: * Let go of both pages (make them
1146: * not busy, perform wakeup, activate).
1147: */
1148:
1149: new_page->busy = FALSE;
1150: new_page->dirty = TRUE;
1151: vm_object_lock(result_page->object);
1152: PAGE_WAKEUP_DONE(result_page);
1153:
1154: vm_page_lock_queues();
1155: if (!result_page->active &&
1156: !result_page->inactive)
1157: vm_page_activate(result_page);
1158: vm_page_activate(new_page);
1159: vm_page_unlock_queues();
1160:
1161: /*
1162: * Release paging references and
1163: * top-level placeholder page, if any.
1164: */
1165:
1166: vm_fault_cleanup(result_page->object,
1167: top_page);
1168:
1169: break;
1170:
1171: case VM_FAULT_RETRY:
1172: break;
1173:
1174: case VM_FAULT_MEMORY_SHORTAGE:
1175: VM_PAGE_WAIT((void (*)()) 0);
1176: break;
1177:
1178: case VM_FAULT_FICTITIOUS_SHORTAGE:
1179: vm_page_more_fictitious();
1180: break;
1181:
1182: case VM_FAULT_INTERRUPTED:
1183: vm_page_free(new_page);
1184: vm_object_deallocate(new_object);
1185: vm_object_deallocate(src_object);
1186: *_result_object = VM_OBJECT_NULL;
1187: return MACH_SEND_INTERRUPTED;
1188:
1189: case VM_FAULT_MEMORY_ERROR:
1190: /*
1191: * A policy choice:
1192: * (a) ignore pages that we can't
1193: * copy
1194: * (b) return the null object if
1195: * any page fails [chosen]
1196: */
1197:
1198: vm_page_free(new_page);
1199: vm_object_deallocate(new_object);
1200: vm_object_deallocate(src_object);
1201: *_result_object = VM_OBJECT_NULL;
1202: return KERN_MEMORY_ERROR;
1203: }
1204: } while (result != VM_FAULT_SUCCESS);
1205: }
1206:
1207: /*
1208: * Lose the extra reference, and return our object.
1209: */
1210:
1211: vm_object_deallocate(src_object);
1212: *_result_object = new_object;
1213: return KERN_SUCCESS;
1214: }
1215:
1216: /*
1217: * Routine: vm_object_copy_temporary
1218: *
1219: * Purpose:
1220: * Copy the specified range of the source virtual
1221: * memory object, if it can be done without blocking.
1222: *
1223: * Results:
1224: * If the copy is successful, the copy is returned in
1225: * the arguments; otherwise, the arguments are not
1226: * affected.
1227: *
1228: * In/out conditions:
1229: * The object should be unlocked on entry and exit.
1230: */
1231:
1232: boolean_t vm_object_copy_temporary(
1233: vm_object_t *_object, /* INOUT */
1234: vm_offset_t *_offset, /* INOUT */
1235: boolean_t *_src_needs_copy, /* OUT */
1236: boolean_t *_dst_needs_copy) /* OUT */
1237: {
1238: vm_object_t object = *_object;
1239:
1240: if (object == VM_OBJECT_NULL) {
1241: *_src_needs_copy = FALSE;
1242: *_dst_needs_copy = FALSE;
1243: return TRUE;
1244: }
1245:
1246: /*
1247: * If the object is temporary, we can perform
1248: * a symmetric copy-on-write without asking.
1249: */
1250:
1251: vm_object_lock(object);
1252: if (object->temporary) {
1253:
1254: /*
1255: * Shared objects use delayed copy
1256: */
1257: if (object->use_shared_copy) {
1258:
1259: /*
1260: * Asymmetric copy strategy. Destination
1261: * must be copied (to allow copy object reuse).
1262: * Source is unaffected.
1263: */
1264: vm_object_unlock(object);
1265: object = vm_object_copy_delayed(object);
1266: *_object = object;
1267: *_src_needs_copy = FALSE;
1268: *_dst_needs_copy = TRUE;
1269: return TRUE;
1270: }
1271:
1272: /*
1273: * Make another reference to the object.
1274: *
1275: * Leave object/offset unchanged.
1276: */
1277:
1278: assert(object->ref_count > 0);
1279: object->ref_count++;
1280: object->shadowed = TRUE;
1281: vm_object_unlock(object);
1282:
1283: /*
1284: * Both source and destination must make
1285: * shadows, and the source must be made
1286: * read-only if not already.
1287: */
1288:
1289: *_src_needs_copy = TRUE;
1290: *_dst_needs_copy = TRUE;
1291: return TRUE;
1292: }
1293:
1294: if (object->pager_ready &&
1295: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1296: /* XXX Do something intelligent (see temporary code above) */
1297: }
1298: vm_object_unlock(object);
1299:
1300: return FALSE;
1301: }
1302:
1303: /*
1304: * Routine: vm_object_copy_call [internal]
1305: *
1306: * Description:
1307: * Copy the specified (src_offset, size) portion
1308: * of the source object (src_object), using the
1309: * user-managed copy algorithm.
1310: *
1311: * In/out conditions:
1312: * The source object must be locked on entry. It
1313: * will be *unlocked* on exit.
1314: *
1315: * Results:
1316: * If the copy is successful, KERN_SUCCESS is returned.
1317: * This routine is interruptible; if a wait for
1318: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1319: * is returned. Other return values indicate hard errors
1320: * in creating the user-managed memory object for the copy.
1321: *
1322: * A new object that represents the copied virtual
1323: * memory is returned in a parameter (*_result_object).
1324: * If the return value indicates an error, this parameter
1325: * is not valid.
1326: */
1327: kern_return_t vm_object_copy_call(
1328: vm_object_t src_object,
1329: vm_offset_t src_offset,
1330: vm_size_t size,
1331: vm_object_t *_result_object) /* OUT */
1332: {
1333: vm_offset_t src_end = src_offset + size;
1334: ipc_port_t new_memory_object;
1335: vm_object_t new_object;
1336: vm_page_t p;
1337:
1338: /*
1339: * Set the backing object for the new
1340: * temporary object.
1341: */
1342:
1343: assert(src_object->ref_count > 0);
1344: src_object->ref_count++;
1345: vm_object_paging_begin(src_object);
1346: vm_object_unlock(src_object);
1347:
1348: /*
1349: * Create a memory object port to be associated
1350: * with this new vm_object.
1351: *
1352: * Since the kernel has the only rights to this
1353: * port, we need not hold the cache lock.
1354: *
1355: * Since we have the only object reference, we
1356: * need not be worried about collapse operations.
1357: *
1358: */
1359:
1360: new_memory_object = ipc_port_alloc_kernel();
1361: if (new_memory_object == IP_NULL) {
1362: panic("vm_object_copy_call: allocate memory object port");
1363: /* XXX Shouldn't panic here. */
1364: }
1365:
1366: /* we hold a naked receive right for new_memory_object */
1367: (void) ipc_port_make_send(new_memory_object);
1368: /* now we also hold a naked send right for new_memory_object */
1369:
1370: /*
1371: * Let the memory manager know that a copy operation
1372: * is in progress. Note that we're using the old
1373: * memory object's ports (for which we're holding
1374: * a paging reference)... the memory manager cannot
1375: * yet affect the new memory object.
1376: */
1377:
1378: (void) memory_object_copy(src_object->pager,
1379: src_object->pager_request,
1380: src_offset, size,
1381: new_memory_object);
1382: /* no longer hold the naked receive right for new_memory_object */
1383:
1384: vm_object_lock(src_object);
1385: vm_object_paging_end(src_object);
1386:
1387: /*
1388: * Remove write access from all of the pages of
1389: * the old memory object that we can.
1390: */
1391:
1392: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1393: if (!p->fictitious &&
1394: (src_offset <= p->offset) &&
1395: (p->offset < src_end) &&
1396: !(p->page_lock & VM_PROT_WRITE)) {
1397: p->page_lock |= VM_PROT_WRITE;
1398: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1399: }
1400: }
1401:
1402: vm_object_unlock(src_object);
1403:
1404: /*
1405: * Initialize the rest of the paging stuff
1406: */
1407:
1408: new_object = vm_object_enter(new_memory_object, size, FALSE);
1409: new_object->shadow = src_object;
1410: new_object->shadow_offset = src_offset;
1411:
1412: /*
1413: * Drop the reference for new_memory_object taken above.
1414: */
1415:
1416: ipc_port_release_send(new_memory_object);
1417: /* no longer hold the naked send right for new_memory_object */
1418:
1419: *_result_object = new_object;
1420: return KERN_SUCCESS;
1421: }
1422:
1423: /*
1424: * Routine: vm_object_copy_delayed [internal]
1425: *
1426: * Description:
1427: * Copy the specified virtual memory object, using
1428: * the asymmetric copy-on-write algorithm.
1429: *
1430: * In/out conditions:
1431: * The object must be unlocked on entry.
1432: *
1433: * This routine will not block waiting for user-generated
1434: * events. It is not interruptible.
1435: */
1436: vm_object_t vm_object_copy_delayed(
1437: vm_object_t src_object)
1438: {
1439: vm_object_t new_copy;
1440: vm_object_t old_copy;
1441: vm_page_t p;
1442:
1443: /*
1444: * The user-level memory manager wants to see
1445: * all of the changes to this object, but it
1446: * has promised not to make any changes on its own.
1447: *
1448: * Perform an asymmetric copy-on-write, as follows:
1449: * Create a new object, called a "copy object"
1450: * to hold pages modified by the new mapping
1451: * (i.e., the copy, not the original mapping).
1452: * Record the original object as the backing
1453: * object for the copy object. If the
1454: * original mapping does not change a page,
1455: * it may be used read-only by the copy.
1456: * Record the copy object in the original
1457: * object. When the original mapping causes
1458: * a page to be modified, it must be copied
1459: * to a new page that is "pushed" to the
1460: * copy object.
1461: * Mark the new mapping (the copy object)
1462: * copy-on-write. This makes the copy
1463: * object itself read-only, allowing it
1464: * to be reused if the original mapping
1465: * makes no changes, and simplifying the
1466: * synchronization required in the "push"
1467: * operation described above.
1468: *
1.1.1.3 ! root 1469: * The copy-on-write is said to be asymmetric because
1.1 root 1470: * the original object is *not* marked copy-on-write.
1471: * A copied page is pushed to the copy object, regardless
1472: * which party attempted to modify the page.
1473: *
1474: * Repeated asymmetric copy operations may be done.
1475: * If the original object has not been changed since
1476: * the last copy, its copy object can be reused.
1477: * Otherwise, a new copy object can be inserted
1478: * between the original object and its previous
1479: * copy object. Since any copy object is read-only,
1480: * this cannot affect the contents of the previous copy
1481: * object.
1482: *
1483: * Note that a copy object is higher in the object
1484: * tree than the original object; therefore, use of
1485: * the copy object recorded in the original object
1486: * must be done carefully, to avoid deadlock.
1487: */
1488:
1489: /*
1490: * Allocate a new copy object before locking, even
1491: * though we may not need it later.
1492: */
1493:
1494: new_copy = vm_object_allocate(src_object->size);
1495:
1496: vm_object_lock(src_object);
1497:
1498: /*
1499: * See whether we can reuse the result of a previous
1500: * copy operation.
1501: */
1502: Retry:
1503: old_copy = src_object->copy;
1504: if (old_copy != VM_OBJECT_NULL) {
1505: /*
1506: * Try to get the locks (out of order)
1507: */
1508: if (!vm_object_lock_try(old_copy)) {
1509: vm_object_unlock(src_object);
1510:
1511: simple_lock_pause(); /* wait a bit */
1512:
1513: vm_object_lock(src_object);
1514: goto Retry;
1515: }
1516:
1517: /*
1518: * Determine whether the old copy object has
1519: * been modified.
1520: */
1521:
1522: if (old_copy->resident_page_count == 0 &&
1523: !old_copy->pager_created) {
1524: /*
1525: * It has not been modified.
1526: *
1527: * Return another reference to
1528: * the existing copy-object.
1529: */
1530: assert(old_copy->ref_count > 0);
1531: old_copy->ref_count++;
1532: vm_object_unlock(old_copy);
1533: vm_object_unlock(src_object);
1534:
1535: vm_object_deallocate(new_copy);
1536:
1537: return old_copy;
1538: }
1539:
1540: /*
1541: * The copy-object is always made large enough to
1542: * completely shadow the original object, since
1543: * it may have several users who want to shadow
1544: * the original object at different points.
1545: */
1546:
1547: assert((old_copy->shadow == src_object) &&
1548: (old_copy->shadow_offset == (vm_offset_t) 0));
1549:
1550: /*
1551: * Make the old copy-object shadow the new one.
1552: * It will receive no more pages from the original
1553: * object.
1554: */
1555:
1556: src_object->ref_count--; /* remove ref. from old_copy */
1557: assert(src_object->ref_count > 0);
1558: old_copy->shadow = new_copy;
1559: assert(new_copy->ref_count > 0);
1560: new_copy->ref_count++;
1561: vm_object_unlock(old_copy); /* done with old_copy */
1562: }
1563:
1564: /*
1565: * Point the new copy at the existing object.
1566: */
1567:
1568: new_copy->shadow = src_object;
1569: new_copy->shadow_offset = 0;
1570: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1571: assert(src_object->ref_count > 0);
1572: src_object->ref_count++;
1573: src_object->copy = new_copy;
1574:
1575: /*
1576: * Mark all pages of the existing object copy-on-write.
1577: * This object may have a shadow chain below it, but
1578: * those pages will already be marked copy-on-write.
1579: */
1580:
1581: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1582: if (!p->fictitious)
1583: pmap_page_protect(p->phys_addr,
1584: (VM_PROT_ALL & ~VM_PROT_WRITE &
1585: ~p->page_lock));
1586: }
1587:
1588: vm_object_unlock(src_object);
1589:
1590: return new_copy;
1591: }
1592:
1593: /*
1594: * Routine: vm_object_copy_strategically
1595: *
1596: * Purpose:
1597: * Perform a copy according to the source object's
1598: * declared strategy. This operation may block,
1599: * and may be interrupted.
1600: */
1601: kern_return_t vm_object_copy_strategically(
1602: vm_object_t src_object,
1603: vm_offset_t src_offset,
1604: vm_size_t size,
1605: vm_object_t *dst_object, /* OUT */
1606: vm_offset_t *dst_offset, /* OUT */
1607: boolean_t *dst_needs_copy) /* OUT */
1608: {
1609: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1610: boolean_t interruptible = TRUE; /* XXX */
1611:
1612: assert(src_object != VM_OBJECT_NULL);
1613:
1614: vm_object_lock(src_object);
1615:
1616: /* XXX assert(!src_object->temporary); JSB FIXME */
1617:
1618: /*
1619: * The copy strategy is only valid if the memory manager
1620: * is "ready".
1621: */
1622:
1623: while (!src_object->pager_ready) {
1624: vm_object_wait( src_object,
1625: VM_OBJECT_EVENT_PAGER_READY,
1626: interruptible);
1627: if (interruptible &&
1628: (current_thread()->wait_result != THREAD_AWAKENED)) {
1629: *dst_object = VM_OBJECT_NULL;
1630: *dst_offset = 0;
1631: *dst_needs_copy = FALSE;
1632: return MACH_SEND_INTERRUPTED;
1633: }
1634: vm_object_lock(src_object);
1635: }
1636:
1637: /*
1638: * The object may be temporary (even though it is external).
1639: * If so, do a symmetric copy.
1640: */
1641:
1642: if (src_object->temporary) {
1643: /*
1644: * XXX
1645: * This does not count as intelligent!
1646: * This buys us the object->temporary optimizations,
1647: * but we aren't using a symmetric copy,
1648: * which may confuse the vm code. The correct thing
1649: * to do here is to figure out what to call to get
1650: * a temporary shadowing set up.
1651: */
1652: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1653: }
1654:
1655: /*
1656: * The object is permanent. Use the appropriate copy strategy.
1657: */
1658:
1659: switch (src_object->copy_strategy) {
1660: case MEMORY_OBJECT_COPY_NONE:
1661: if ((result = vm_object_copy_slowly(
1662: src_object,
1663: src_offset,
1664: size,
1665: interruptible,
1666: dst_object))
1667: == KERN_SUCCESS) {
1668: *dst_offset = 0;
1669: *dst_needs_copy = FALSE;
1670: }
1671: break;
1672:
1673: case MEMORY_OBJECT_COPY_CALL:
1674: if ((result = vm_object_copy_call(
1675: src_object,
1676: src_offset,
1677: size,
1678: dst_object))
1679: == KERN_SUCCESS) {
1680: *dst_offset = 0;
1681: *dst_needs_copy = FALSE;
1682: }
1683: break;
1684:
1685: case MEMORY_OBJECT_COPY_DELAY:
1686: vm_object_unlock(src_object);
1687: *dst_object = vm_object_copy_delayed(src_object);
1688: *dst_offset = src_offset;
1689: *dst_needs_copy = TRUE;
1690:
1691: result = KERN_SUCCESS;
1692: break;
1693: }
1694:
1695: return result;
1696: }
1697:
1698: /*
1699: * vm_object_shadow:
1700: *
1701: * Create a new object which is backed by the
1702: * specified existing object range. The source
1703: * object reference is deallocated.
1704: *
1705: * The new object and offset into that object
1706: * are returned in the source parameters.
1707: */
1708:
1709: void vm_object_shadow(
1710: vm_object_t *object, /* IN/OUT */
1711: vm_offset_t *offset, /* IN/OUT */
1712: vm_size_t length)
1713: {
1.1.1.3 ! root 1714: vm_object_t source;
! 1715: vm_object_t result;
1.1 root 1716:
1717: source = *object;
1718:
1719: /*
1720: * Allocate a new object with the given length
1721: */
1722:
1723: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1724: panic("vm_object_shadow: no object for shadowing");
1725:
1726: /*
1727: * The new object shadows the source object, adding
1728: * a reference to it. Our caller changes his reference
1729: * to point to the new object, removing a reference to
1730: * the source object. Net result: no change of reference
1731: * count.
1732: */
1733: result->shadow = source;
1734:
1735: /*
1736: * Store the offset into the source object,
1737: * and fix up the offset into the new object.
1738: */
1739:
1740: result->shadow_offset = *offset;
1741:
1742: /*
1743: * Return the new things
1744: */
1745:
1746: *offset = 0;
1747: *object = result;
1748: }
1749:
1750: /*
1751: * The relationship between vm_object structures and
1752: * the memory_object ports requires careful synchronization.
1753: *
1754: * All associations are created by vm_object_enter. All three
1755: * port fields are filled in, as follows:
1756: * pager: the memory_object port itself, supplied by
1757: * the user requesting a mapping (or the kernel,
1758: * when initializing internal objects); the
1759: * kernel simulates holding send rights by keeping
1760: * a port reference;
1761: * pager_request:
1762: * pager_name:
1763: * the memory object control and name ports,
1764: * created by the kernel; the kernel holds
1765: * receive (and ownership) rights to these
1766: * ports, but no other references.
1767: * All of the ports are referenced by their global names.
1768: *
1769: * When initialization is complete, the "initialized" field
1770: * is asserted. Other mappings using a particular memory object,
1771: * and any references to the vm_object gained through the
1772: * port association must wait for this initialization to occur.
1773: *
1774: * In order to allow the memory manager to set attributes before
1775: * requests (notably virtual copy operations, but also data or
1776: * unlock requests) are made, a "ready" attribute is made available.
1777: * Only the memory manager may affect the value of this attribute.
1778: * Its value does not affect critical kernel functions, such as
1779: * internal object initialization or destruction. [Furthermore,
1780: * memory objects created by the kernel are assumed to be ready
1781: * immediately; the default memory manager need not explicitly
1782: * set the "ready" attribute.]
1783: *
1784: * [Both the "initialized" and "ready" attribute wait conditions
1785: * use the "pager" field as the wait event.]
1786: *
1787: * The port associations can be broken down by any of the
1788: * following routines:
1789: * vm_object_terminate:
1790: * No references to the vm_object remain, and
1791: * the object cannot (or will not) be cached.
1792: * This is the normal case, and is done even
1793: * though one of the other cases has already been
1794: * done.
1795: * vm_object_destroy:
1796: * The memory_object port has been destroyed,
1797: * meaning that the kernel cannot flush dirty
1798: * pages or request new data or unlock existing
1799: * data.
1800: * memory_object_destroy:
1801: * The memory manager has requested that the
1802: * kernel relinquish rights to the memory object
1803: * port. [The memory manager may not want to
1804: * destroy the port, but may wish to refuse or
1805: * tear down existing memory mappings.]
1806: * Each routine that breaks an association must break all of
1807: * them at once. At some later time, that routine must clear
1808: * the vm_object port fields and release the port rights.
1809: * [Furthermore, each routine must cope with the simultaneous
1810: * or previous operations of the others.]
1811: *
1812: * In addition to the lock on the object, the vm_object_cache_lock
1813: * governs the port associations. References gained through the
1814: * port association require use of the cache lock.
1815: *
1816: * Because the port fields may be cleared spontaneously, they
1817: * cannot be used to determine whether a memory object has
1818: * ever been associated with a particular vm_object. [This
1819: * knowledge is important to the shadow object mechanism.]
1820: * For this reason, an additional "created" attribute is
1821: * provided.
1822: *
1823: * During various paging operations, the port values found in the
1824: * vm_object must be valid. To prevent these port rights from being
1825: * released, and to prevent the port associations from changing
1826: * (other than being removed, i.e., made null), routines may use
1827: * the vm_object_paging_begin/end routines [actually, macros].
1828: * The implementation uses the "paging_in_progress" and "wanted" fields.
1829: * [Operations that alter the validity of the port values include the
1830: * termination routines and vm_object_collapse.]
1831: */
1832:
1833: vm_object_t vm_object_lookup(
1834: ipc_port_t port)
1835: {
1836: vm_object_t object = VM_OBJECT_NULL;
1837:
1838: if (IP_VALID(port)) {
1839: ip_lock(port);
1840: if (ip_active(port) &&
1841: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1842: vm_object_cache_lock();
1843: object = (vm_object_t) port->ip_kobject;
1844: vm_object_lock(object);
1845:
1846: assert(object->alive);
1847:
1848: if (object->ref_count == 0) {
1849: queue_remove(&vm_object_cached_list, object,
1850: vm_object_t, cached_list);
1851: vm_object_cached_count--;
1.1.1.2 root 1852: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1853: }
1854:
1855: object->ref_count++;
1856: vm_object_unlock(object);
1857: vm_object_cache_unlock();
1858: }
1859: ip_unlock(port);
1860: }
1861:
1862: return object;
1863: }
1864:
1865: vm_object_t vm_object_lookup_name(
1866: ipc_port_t port)
1867: {
1868: vm_object_t object = VM_OBJECT_NULL;
1869:
1870: if (IP_VALID(port)) {
1871: ip_lock(port);
1872: if (ip_active(port) &&
1873: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1874: vm_object_cache_lock();
1875: object = (vm_object_t) port->ip_kobject;
1876: vm_object_lock(object);
1877:
1878: assert(object->alive);
1879:
1880: if (object->ref_count == 0) {
1881: queue_remove(&vm_object_cached_list, object,
1882: vm_object_t, cached_list);
1883: vm_object_cached_count--;
1.1.1.2 root 1884: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1885: }
1886:
1887: object->ref_count++;
1888: vm_object_unlock(object);
1889: vm_object_cache_unlock();
1890: }
1891: ip_unlock(port);
1892: }
1893:
1894: return object;
1895: }
1896:
1897: void vm_object_destroy(
1898: ipc_port_t pager)
1899: {
1900: vm_object_t object;
1901: pager_request_t old_request;
1902: ipc_port_t old_name;
1903:
1904: /*
1905: * Perform essentially the same operations as in vm_object_lookup,
1906: * except that this time we look up based on the memory_object
1907: * port, not the control port.
1908: */
1909: vm_object_cache_lock();
1910: if (ip_kotype(pager) != IKOT_PAGER) {
1911: vm_object_cache_unlock();
1912: return;
1913: }
1914:
1915: object = (vm_object_t) pager->ip_kobject;
1916: vm_object_lock(object);
1917: if (object->ref_count == 0) {
1918: queue_remove(&vm_object_cached_list, object,
1919: vm_object_t, cached_list);
1920: vm_object_cached_count--;
1.1.1.2 root 1921: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1922: }
1923: object->ref_count++;
1924:
1925: object->can_persist = FALSE;
1926:
1927: assert(object->pager == pager);
1928:
1929: /*
1930: * Remove the port associations.
1931: *
1932: * Note that the memory_object itself is dead, so
1933: * we don't bother with it.
1934: */
1935:
1936: object->pager = IP_NULL;
1937: vm_object_remove(object);
1938:
1939: old_request = object->pager_request;
1940: object->pager_request = PAGER_REQUEST_NULL;
1941:
1942: old_name = object->pager_name;
1943: object->pager_name = IP_NULL;
1944:
1945: vm_object_unlock(object);
1946: vm_object_cache_unlock();
1947:
1948: /*
1949: * Clean up the port references. Note that there's no
1950: * point in trying the memory_object_terminate call
1951: * because the memory_object itself is dead.
1952: */
1953:
1954: ipc_port_release_send(pager);
1955: if (old_request != IP_NULL)
1956: ipc_port_dealloc_kernel(old_request);
1957: if (old_name != IP_NULL)
1958: ipc_port_dealloc_kernel(old_name);
1959:
1960: /*
1961: * Restart pending page requests
1962: */
1963:
1964: vm_object_abort_activity(object);
1965:
1966: /*
1967: * Lose the object reference.
1968: */
1969:
1970: vm_object_deallocate(object);
1971: }
1972:
1973: boolean_t vm_object_accept_old_init_protocol = FALSE;
1974:
1975: /*
1976: * Routine: vm_object_enter
1977: * Purpose:
1978: * Find a VM object corresponding to the given
1979: * pager; if no such object exists, create one,
1980: * and initialize the pager.
1981: */
1982: vm_object_t vm_object_enter(
1983: ipc_port_t pager,
1984: vm_size_t size,
1985: boolean_t internal)
1986: {
1987: vm_object_t object;
1988: vm_object_t new_object;
1989: boolean_t must_init;
1990: ipc_kobject_type_t po;
1991:
1992: restart:
1993: if (!IP_VALID(pager))
1994: return vm_object_allocate(size);
1995:
1996: new_object = VM_OBJECT_NULL;
1997: must_init = FALSE;
1998:
1999: /*
2000: * Look for an object associated with this port.
2001: */
2002:
2003: vm_object_cache_lock();
2004: for (;;) {
2005: po = ip_kotype(pager);
2006:
2007: /*
2008: * If a previous object is being terminated,
2009: * we must wait for the termination message
2010: * to be queued.
2011: *
2012: * We set kobject to a non-null value to let the
2013: * terminator know that someone is waiting.
2014: * Among the possibilities is that the port
2015: * could die while we're waiting. Must restart
2016: * instead of continuing the loop.
2017: */
2018:
2019: if (po == IKOT_PAGER_TERMINATING) {
2020: pager->ip_kobject = (ipc_kobject_t) pager;
2021: assert_wait((event_t) pager, FALSE);
2022: vm_object_cache_unlock();
2023: thread_block((void (*)()) 0);
2024: goto restart;
2025: }
2026:
2027: /*
2028: * Bail if there is already a kobject associated
2029: * with the pager port.
2030: */
2031: if (po != IKOT_NONE) {
2032: break;
2033: }
2034:
2035: /*
2036: * We must unlock to create a new object;
2037: * if we do so, we must try the lookup again.
2038: */
2039:
2040: if (new_object == VM_OBJECT_NULL) {
2041: vm_object_cache_unlock();
2042: new_object = vm_object_allocate(size);
2043: vm_object_cache_lock();
2044: } else {
2045: /*
2046: * Lookup failed twice, and we have something
2047: * to insert; set the object.
2048: */
2049:
2050: ipc_kobject_set(pager,
2051: (ipc_kobject_t) new_object,
2052: IKOT_PAGER);
2053: new_object = VM_OBJECT_NULL;
2054: must_init = TRUE;
2055: }
2056: }
2057:
2058: if (internal)
2059: must_init = TRUE;
2060:
2061: /*
2062: * It's only good if it's a VM object!
2063: */
2064:
2065: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2066: : VM_OBJECT_NULL;
2067:
2068: if ((object != VM_OBJECT_NULL) && !must_init) {
2069: vm_object_lock(object);
2070: if (object->ref_count == 0) {
2071: queue_remove(&vm_object_cached_list, object,
2072: vm_object_t, cached_list);
2073: vm_object_cached_count--;
1.1.1.2 root 2074: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 2075: }
2076: object->ref_count++;
2077: vm_object_unlock(object);
2078:
2079: vm_stat.hits++;
2080: }
2081: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2082: ((object->paging_in_progress != 0) && internal));
2083:
2084: vm_stat.lookups++;
2085:
2086: vm_object_cache_unlock();
2087:
2088: /*
2089: * If we raced to create a vm_object but lost, let's
2090: * throw away ours.
2091: */
2092:
2093: if (new_object != VM_OBJECT_NULL)
2094: vm_object_deallocate(new_object);
2095:
2096: if (object == VM_OBJECT_NULL)
2097: return(object);
2098:
2099: if (must_init) {
2100: /*
2101: * Copy the naked send right we were given.
2102: */
2103:
2104: pager = ipc_port_copy_send(pager);
2105: if (!IP_VALID(pager))
2106: panic("vm_object_enter: port died"); /* XXX */
2107:
2108: object->pager_created = TRUE;
2109: object->pager = pager;
2110:
2111: /*
2112: * Allocate request port.
2113: */
2114:
2115: object->pager_request = ipc_port_alloc_kernel();
2116: if (object->pager_request == IP_NULL)
2117: panic("vm_object_enter: pager request alloc");
2118:
2119: ipc_kobject_set(object->pager_request,
2120: (ipc_kobject_t) object,
2121: IKOT_PAGING_REQUEST);
2122:
2123: /*
2124: * Let the pager know we're using it.
2125: */
2126:
2127: if (internal) {
2128: /* acquire a naked send right for the DMM */
2129: ipc_port_t DMM = memory_manager_default_reference();
2130:
2131: /* mark the object internal */
2132: object->internal = TRUE;
2133: assert(object->temporary);
2134:
2135: /* default-pager objects are ready immediately */
2136: object->pager_ready = TRUE;
2137:
2138: /* consumes the naked send right for DMM */
2139: (void) memory_object_create(DMM,
2140: pager,
2141: object->size,
2142: object->pager_request,
2143: object->pager_name,
2144: PAGE_SIZE);
2145: } else {
2146: /* the object is external and not temporary */
2147: object->internal = FALSE;
2148: object->temporary = FALSE;
2149:
2150: /* user pager objects are not ready until marked so */
2151: object->pager_ready = FALSE;
2152:
2153: (void) memory_object_init(pager,
2154: object->pager_request,
2155: object->pager_name,
2156: PAGE_SIZE);
2157:
2158: }
2159:
2160: vm_object_lock(object);
2161: object->pager_initialized = TRUE;
2162:
2163: if (vm_object_accept_old_init_protocol)
2164: object->pager_ready = TRUE;
2165:
2166: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2167: } else {
2168: vm_object_lock(object);
2169: }
2170: /*
2171: * [At this point, the object must be locked]
2172: */
2173:
2174: /*
2175: * Wait for the work above to be done by the first
2176: * thread to map this object.
2177: */
2178:
2179: while (!object->pager_initialized) {
2180: vm_object_wait( object,
2181: VM_OBJECT_EVENT_INITIALIZED,
2182: FALSE);
2183: vm_object_lock(object);
2184: }
2185: vm_object_unlock(object);
2186:
2187: return object;
2188: }
2189:
2190: /*
2191: * Routine: vm_object_pager_create
2192: * Purpose:
2193: * Create a memory object for an internal object.
2194: * In/out conditions:
2195: * The object is locked on entry and exit;
2196: * it may be unlocked within this call.
2197: * Limitations:
2198: * Only one thread may be performing a
2199: * vm_object_pager_create on an object at
2200: * a time. Presumably, only the pageout
2201: * daemon will be using this routine.
2202: */
2203: void vm_object_pager_create(
2204: vm_object_t object)
2205: {
2206: ipc_port_t pager;
2207:
2208: if (object->pager_created) {
2209: /*
2210: * Someone else got to it first...
2211: * wait for them to finish initializing
2212: */
2213:
2214: while (!object->pager_initialized) {
2215: vm_object_wait( object,
2216: VM_OBJECT_EVENT_PAGER_READY,
2217: FALSE);
2218: vm_object_lock(object);
2219: }
2220: return;
2221: }
2222:
2223: /*
2224: * Indicate that a memory object has been assigned
2225: * before dropping the lock, to prevent a race.
2226: */
2227:
2228: object->pager_created = TRUE;
2229:
2230: /*
2231: * Prevent collapse or termination by
2232: * holding a paging reference
2233: */
2234:
2235: vm_object_paging_begin(object);
2236: vm_object_unlock(object);
2237:
2238: #if MACH_PAGEMAP
2239: object->existence_info = vm_external_create(
2240: object->size +
2241: object->paging_offset);
2242: assert((object->size + object->paging_offset) >=
2243: object->size);
2244: #endif /* MACH_PAGEMAP */
2245:
2246: /*
2247: * Create the pager, and associate with it
2248: * this object.
2249: *
2250: * Note that we only make the port association
2251: * so that vm_object_enter can properly look up
2252: * the object to complete the initialization...
2253: * we do not expect any user to ever map this
2254: * object.
2255: *
2256: * Since the kernel has the only rights to the
2257: * port, it's safe to install the association
2258: * without holding the cache lock.
2259: */
2260:
2261: pager = ipc_port_alloc_kernel();
2262: if (pager == IP_NULL)
2263: panic("vm_object_pager_create: allocate pager port");
2264:
2265: (void) ipc_port_make_send(pager);
2266: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2267:
2268: /*
2269: * Initialize the rest of the paging stuff
2270: */
2271:
2272: if (vm_object_enter(pager, object->size, TRUE) != object)
2273: panic("vm_object_pager_create: mismatch");
2274:
2275: /*
2276: * Drop the naked send right taken above.
2277: */
2278:
2279: ipc_port_release_send(pager);
2280:
2281: /*
2282: * Release the paging reference
2283: */
2284:
2285: vm_object_lock(object);
2286: vm_object_paging_end(object);
2287: }
2288:
2289: /*
2290: * Routine: vm_object_remove
2291: * Purpose:
2292: * Eliminate the pager/object association
2293: * for this pager.
2294: * Conditions:
2295: * The object cache must be locked.
2296: */
2297: void vm_object_remove(
2298: vm_object_t object)
2299: {
2300: ipc_port_t port;
2301:
2302: if ((port = object->pager) != IP_NULL) {
2303: if (ip_kotype(port) == IKOT_PAGER)
2304: ipc_kobject_set(port, IKO_NULL,
2305: IKOT_PAGER_TERMINATING);
2306: else if (ip_kotype(port) != IKOT_NONE)
2307: panic("vm_object_remove: bad object port");
2308: }
2309: if ((port = object->pager_request) != IP_NULL) {
2310: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2311: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2312: else if (ip_kotype(port) != IKOT_NONE)
2313: panic("vm_object_remove: bad request port");
2314: }
2315: if ((port = object->pager_name) != IP_NULL) {
2316: if (ip_kotype(port) == IKOT_PAGING_NAME)
2317: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2318: else if (ip_kotype(port) != IKOT_NONE)
2319: panic("vm_object_remove: bad name port");
2320: }
2321: }
2322:
2323: /*
2324: * Global variables for vm_object_collapse():
2325: *
2326: * Counts for normal collapses and bypasses.
2327: * Debugging variables, to watch or disable collapse.
2328: */
2329: long object_collapses = 0;
2330: long object_bypasses = 0;
2331:
2332: int vm_object_collapse_debug = 0;
2333: boolean_t vm_object_collapse_allowed = TRUE;
2334: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2335:
2336: /*
2337: * vm_object_collapse:
2338: *
2339: * Collapse an object with the object backing it.
2340: * Pages in the backing object are moved into the
2341: * parent, and the backing object is deallocated.
2342: *
2343: * Requires that the object be locked and the page
2344: * queues be unlocked. May unlock/relock the object,
2345: * so the caller should hold a reference for the object.
2346: */
2347: void vm_object_collapse(
1.1.1.3 ! root 2348: vm_object_t object)
1.1 root 2349: {
1.1.1.3 ! root 2350: vm_object_t backing_object;
! 2351: vm_offset_t backing_offset;
! 2352: vm_size_t size;
! 2353: vm_offset_t new_offset;
! 2354: vm_page_t p, pp;
! 2355: ipc_port_t old_name_port;
1.1 root 2356:
2357: if (!vm_object_collapse_allowed)
2358: return;
2359:
2360: while (TRUE) {
2361: /*
2362: * Verify that the conditions are right for collapse:
2363: *
2364: * The object exists and no pages in it are currently
2365: * being paged out (or have ever been paged out).
2366: *
2367: * This check is probably overkill -- if a memory
2368: * object has not been created, the fault handler
2369: * shouldn't release the object lock while paging
2370: * is in progress or absent pages exist.
2371: */
2372: if (object == VM_OBJECT_NULL ||
2373: object->pager_created ||
2374: object->paging_in_progress != 0 ||
2375: object->absent_count != 0)
2376: return;
2377:
2378: /*
2379: * There is a backing object, and
2380: */
2381:
2382: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2383: return;
2384:
2385: vm_object_lock(backing_object);
2386: /*
2387: * ...
2388: * The backing object is not read_only,
2389: * and no pages in the backing object are
2390: * currently being paged out.
2391: * The backing object is internal.
2392: *
2393: * XXX It may be sufficient for the backing
2394: * XXX object to be temporary.
2395: */
2396:
2397: if (!backing_object->internal ||
2398: backing_object->paging_in_progress != 0) {
2399: vm_object_unlock(backing_object);
2400: return;
2401: }
2402:
2403: /*
2404: * The backing object can't be a copy-object:
2405: * the shadow_offset for the copy-object must stay
2406: * as 0. Furthermore (for the 'we have all the
2407: * pages' case), if we bypass backing_object and
2408: * just shadow the next object in the chain, old
2409: * pages from that object would then have to be copied
2410: * BOTH into the (former) backing_object and into the
2411: * parent object.
2412: */
2413: if (backing_object->shadow != VM_OBJECT_NULL &&
2414: backing_object->shadow->copy != VM_OBJECT_NULL) {
2415: vm_object_unlock(backing_object);
2416: return;
2417: }
2418:
2419: /*
2420: * We know that we can either collapse the backing
2421: * object (if the parent is the only reference to
2422: * it) or (perhaps) remove the parent's reference
2423: * to it.
2424: */
2425:
2426: backing_offset = object->shadow_offset;
2427: size = object->size;
2428:
2429: /*
2430: * If there is exactly one reference to the backing
2431: * object, we can collapse it into the parent.
2432: */
2433:
2434: if (backing_object->ref_count == 1) {
2435: if (!vm_object_cache_lock_try()) {
2436: vm_object_unlock(backing_object);
2437: return;
2438: }
2439:
2440: /*
2441: * We can collapse the backing object.
2442: *
2443: * Move all in-memory pages from backing_object
2444: * to the parent. Pages that have been paged out
2445: * will be overwritten by any of the parent's
2446: * pages that shadow them.
2447: */
2448:
2449: while (!queue_empty(&backing_object->memq)) {
2450:
2451: p = (vm_page_t)
2452: queue_first(&backing_object->memq);
2453:
2454: new_offset = (p->offset - backing_offset);
2455:
2456: assert(!p->busy || p->absent);
2457:
2458: /*
2459: * If the parent has a page here, or if
2460: * this page falls outside the parent,
2461: * dispose of it.
2462: *
2463: * Otherwise, move it as planned.
2464: */
2465:
2466: if (p->offset < backing_offset ||
2467: new_offset >= size) {
1.1.1.2 root 2468: VM_PAGE_FREE(p);
1.1 root 2469: } else {
2470: pp = vm_page_lookup(object, new_offset);
2471: if (pp != VM_PAGE_NULL && !pp->absent) {
2472: /*
2473: * Parent object has a real page.
2474: * Throw away the backing object's
2475: * page.
2476: */
1.1.1.2 root 2477: VM_PAGE_FREE(p);
1.1 root 2478: }
2479: else {
2480: if (pp != VM_PAGE_NULL) {
2481: /*
2482: * Parent has an absent page...
2483: * it's not being paged in, so
2484: * it must really be missing from
2485: * the parent.
2486: *
2487: * Throw out the absent page...
2488: * any faults looking for that
2489: * page will restart with the new
2490: * one.
2491: */
2492:
2493: /*
2494: * This should never happen -- the
2495: * parent cannot have ever had an
2496: * external memory object, and thus
2497: * cannot have absent pages.
2498: */
2499: panic("vm_object_collapse: bad case");
2500:
1.1.1.2 root 2501: VM_PAGE_FREE(pp);
1.1 root 2502:
2503: /*
2504: * Fall through to move the backing
2505: * object's page up.
2506: */
2507: }
2508: /*
2509: * Parent now has no page.
2510: * Move the backing object's page up.
2511: */
2512: vm_page_rename(p, object, new_offset);
2513: }
2514: }
2515: }
2516:
2517: /*
2518: * Move the pager from backing_object to object.
2519: *
2520: * XXX We're only using part of the paging space
2521: * for keeps now... we ought to discard the
2522: * unused portion.
2523: */
2524:
2525: switch (vm_object_collapse_debug) {
2526: case 0:
2527: break;
2528: case 1:
2529: if ((backing_object->pager == IP_NULL) &&
2530: (backing_object->pager_request ==
2531: PAGER_REQUEST_NULL))
2532: break;
2533: /* Fall through to... */
2534:
2535: default:
1.1.1.2 root 2536: printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n",
1.1 root 2537: backing_object, backing_object->pager, backing_object->pager_request,
2538: object);
2539: if (vm_object_collapse_debug > 2)
1.1.1.2 root 2540: SoftDebugger("vm_object_collapse");
1.1 root 2541: }
2542:
2543: object->pager = backing_object->pager;
2544: if (object->pager != IP_NULL)
2545: ipc_kobject_set(object->pager,
2546: (ipc_kobject_t) object,
2547: IKOT_PAGER);
2548: object->pager_initialized = backing_object->pager_initialized;
2549: object->pager_ready = backing_object->pager_ready;
2550: object->pager_created = backing_object->pager_created;
2551:
2552: object->pager_request = backing_object->pager_request;
2553: if (object->pager_request != IP_NULL)
2554: ipc_kobject_set(object->pager_request,
2555: (ipc_kobject_t) object,
2556: IKOT_PAGING_REQUEST);
2557: old_name_port = object->pager_name;
2558: if (old_name_port != IP_NULL)
2559: ipc_kobject_set(old_name_port,
2560: IKO_NULL, IKOT_NONE);
2561: object->pager_name = backing_object->pager_name;
2562: if (object->pager_name != IP_NULL)
2563: ipc_kobject_set(object->pager_name,
2564: (ipc_kobject_t) object,
2565: IKOT_PAGING_NAME);
2566:
2567: vm_object_cache_unlock();
2568:
2569: /*
2570: * If there is no pager, leave paging-offset alone.
2571: */
2572: if (object->pager != IP_NULL)
2573: object->paging_offset =
2574: backing_object->paging_offset +
2575: backing_offset;
2576:
2577: #if MACH_PAGEMAP
2578: assert(object->existence_info == VM_EXTERNAL_NULL);
2579: object->existence_info = backing_object->existence_info;
2580: #endif /* MACH_PAGEMAP */
2581:
2582: /*
2583: * Object now shadows whatever backing_object did.
2584: * Note that the reference to backing_object->shadow
2585: * moves from within backing_object to within object.
2586: */
2587:
2588: object->shadow = backing_object->shadow;
2589: object->shadow_offset += backing_object->shadow_offset;
2590: if (object->shadow != VM_OBJECT_NULL &&
2591: object->shadow->copy != VM_OBJECT_NULL) {
2592: panic("vm_object_collapse: we collapsed a copy-object!");
2593: }
2594: /*
2595: * Discard backing_object.
2596: *
2597: * Since the backing object has no pages, no
2598: * pager left, and no object references within it,
2599: * all that is necessary is to dispose of it.
2600: */
2601:
2602: assert(
2603: (backing_object->ref_count == 1) &&
2604: (backing_object->resident_page_count == 0) &&
2605: (backing_object->paging_in_progress == 0)
2606: );
2607:
2608: assert(backing_object->alive);
2609: backing_object->alive = FALSE;
2610: vm_object_unlock(backing_object);
2611:
2612: vm_object_unlock(object);
2613: if (old_name_port != IP_NULL)
2614: ipc_port_dealloc_kernel(old_name_port);
1.1.1.2 root 2615: kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object);
1.1 root 2616: vm_object_lock(object);
2617:
2618: object_collapses++;
2619: }
2620: else {
2621: if (!vm_object_collapse_bypass_allowed) {
2622: vm_object_unlock(backing_object);
2623: return;
2624: }
2625:
2626: /*
2627: * If all of the pages in the backing object are
2628: * shadowed by the parent object, the parent
2629: * object no longer has to shadow the backing
2630: * object; it can shadow the next one in the
2631: * chain.
2632: *
2633: * The backing object must not be paged out - we'd
2634: * have to check all of the paged-out pages, as
2635: * well.
2636: */
2637:
2638: if (backing_object->pager_created) {
2639: vm_object_unlock(backing_object);
2640: return;
2641: }
2642:
2643: /*
2644: * Should have a check for a 'small' number
2645: * of pages here.
2646: */
2647:
2648: queue_iterate(&backing_object->memq, p,
2649: vm_page_t, listq)
2650: {
2651: new_offset = (p->offset - backing_offset);
2652:
2653: /*
2654: * If the parent has a page here, or if
2655: * this page falls outside the parent,
2656: * keep going.
2657: *
2658: * Otherwise, the backing_object must be
2659: * left in the chain.
2660: */
2661:
2662: if (p->offset >= backing_offset &&
2663: new_offset <= size &&
2664: (pp = vm_page_lookup(object, new_offset))
2665: == VM_PAGE_NULL) {
2666: /*
2667: * Page still needed.
2668: * Can't go any further.
2669: */
2670: vm_object_unlock(backing_object);
2671: return;
2672: }
2673: }
2674:
2675: /*
2676: * Make the parent shadow the next object
2677: * in the chain. Deallocating backing_object
2678: * will not remove it, since its reference
2679: * count is at least 2.
2680: */
2681:
2682: vm_object_reference(object->shadow = backing_object->shadow);
2683: object->shadow_offset += backing_object->shadow_offset;
2684:
2685: /*
2686: * Backing object might have had a copy pointer
2687: * to us. If it did, clear it.
2688: */
2689: if (backing_object->copy == object)
2690: backing_object->copy = VM_OBJECT_NULL;
2691:
2692: /*
2693: * Drop the reference count on backing_object.
2694: * Since its ref_count was at least 2, it
2695: * will not vanish; so we don't need to call
2696: * vm_object_deallocate.
2697: */
2698: backing_object->ref_count--;
2699: assert(backing_object->ref_count > 0);
2700: vm_object_unlock(backing_object);
2701:
2702: object_bypasses ++;
2703:
2704: }
2705:
2706: /*
2707: * Try again with this object's new backing object.
2708: */
2709: }
2710: }
2711:
2712: /*
2713: * Routine: vm_object_page_remove: [internal]
2714: * Purpose:
2715: * Removes all physical pages in the specified
2716: * object range from the object's list of pages.
2717: *
2718: * In/out conditions:
2719: * The object must be locked.
2720: */
2721: unsigned int vm_object_page_remove_lookup = 0;
2722: unsigned int vm_object_page_remove_iterate = 0;
2723:
2724: void vm_object_page_remove(
1.1.1.3 ! root 2725: vm_object_t object,
! 2726: vm_offset_t start,
! 2727: vm_offset_t end)
1.1 root 2728: {
1.1.1.3 ! root 2729: vm_page_t p, next;
1.1 root 2730:
2731: /*
2732: * One and two page removals are most popular.
2733: * The factor of 16 here is somewhat arbitrary.
2734: * It balances vm_object_lookup vs iteration.
2735: */
2736:
2737: if (atop(end - start) < (unsigned)object->resident_page_count/16) {
2738: vm_object_page_remove_lookup++;
2739:
2740: for (; start < end; start += PAGE_SIZE) {
2741: p = vm_page_lookup(object, start);
2742: if (p != VM_PAGE_NULL) {
2743: if (!p->fictitious)
2744: pmap_page_protect(p->phys_addr,
2745: VM_PROT_NONE);
1.1.1.2 root 2746: VM_PAGE_FREE(p);
1.1 root 2747: }
2748: }
2749: } else {
2750: vm_object_page_remove_iterate++;
2751:
2752: p = (vm_page_t) queue_first(&object->memq);
2753: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2754: next = (vm_page_t) queue_next(&p->listq);
2755: if ((start <= p->offset) && (p->offset < end)) {
2756: if (!p->fictitious)
2757: pmap_page_protect(p->phys_addr,
2758: VM_PROT_NONE);
1.1.1.2 root 2759: VM_PAGE_FREE(p);
1.1 root 2760: }
2761: p = next;
2762: }
2763: }
2764: }
2765:
2766: /*
2767: * Routine: vm_object_coalesce
2768: * Function: Coalesces two objects backing up adjoining
2769: * regions of memory into a single object.
2770: *
2771: * returns TRUE if objects were combined.
2772: *
2773: * NOTE: Only works at the moment if the second object is NULL -
2774: * if it's not, which object do we lock first?
2775: *
2776: * Parameters:
2777: * prev_object First object to coalesce
2778: * prev_offset Offset into prev_object
2779: * next_object Second object into coalesce
2780: * next_offset Offset into next_object
2781: *
2782: * prev_size Size of reference to prev_object
2783: * next_size Size of reference to next_object
2784: *
2785: * Conditions:
2786: * The object must *not* be locked.
2787: */
2788:
2789: boolean_t vm_object_coalesce(
1.1.1.3 ! root 2790: vm_object_t prev_object,
1.1 root 2791: vm_object_t next_object,
2792: vm_offset_t prev_offset,
2793: vm_offset_t next_offset,
2794: vm_size_t prev_size,
2795: vm_size_t next_size)
2796: {
2797: vm_size_t newsize;
2798:
2799: if (next_object != VM_OBJECT_NULL) {
2800: return FALSE;
2801: }
2802:
2803: if (prev_object == VM_OBJECT_NULL) {
2804: return TRUE;
2805: }
2806:
2807: vm_object_lock(prev_object);
2808:
2809: /*
2810: * Try to collapse the object first
2811: */
2812: vm_object_collapse(prev_object);
2813:
2814: /*
2815: * Can't coalesce if pages not mapped to
2816: * prev_entry may be in use anyway:
2817: * . more than one reference
2818: * . paged out
2819: * . shadows another object
2820: * . has a copy elsewhere
2821: * . paging references (pages might be in page-list)
2822: */
2823:
2824: if ((prev_object->ref_count > 1) ||
2825: prev_object->pager_created ||
2826: (prev_object->shadow != VM_OBJECT_NULL) ||
2827: (prev_object->copy != VM_OBJECT_NULL) ||
2828: (prev_object->paging_in_progress != 0)) {
2829: vm_object_unlock(prev_object);
2830: return FALSE;
2831: }
2832:
2833: /*
2834: * Remove any pages that may still be in the object from
2835: * a previous deallocation.
2836: */
2837:
2838: vm_object_page_remove(prev_object,
2839: prev_offset + prev_size,
2840: prev_offset + prev_size + next_size);
2841:
2842: /*
2843: * Extend the object if necessary.
2844: */
2845: newsize = prev_offset + prev_size + next_size;
2846: if (newsize > prev_object->size)
2847: prev_object->size = newsize;
2848:
2849: vm_object_unlock(prev_object);
2850: return TRUE;
2851: }
2852:
2853: vm_object_t vm_object_request_object(
2854: ipc_port_t p)
2855: {
2856: return vm_object_lookup(p);
2857: }
2858:
2859: /*
2860: * Routine: vm_object_name
2861: * Purpose:
2862: * Returns a naked send right to the "name" port associated
2863: * with this object.
2864: */
2865: ipc_port_t vm_object_name(
2866: vm_object_t object)
2867: {
2868: ipc_port_t p;
2869:
2870: if (object == VM_OBJECT_NULL)
2871: return IP_NULL;
2872:
2873: vm_object_lock(object);
2874:
2875: while (object->shadow != VM_OBJECT_NULL) {
2876: vm_object_t new_object = object->shadow;
2877: vm_object_lock(new_object);
2878: vm_object_unlock(object);
2879: object = new_object;
2880: }
2881:
2882: p = object->pager_name;
2883: if (p != IP_NULL)
2884: p = ipc_port_make_send(p);
2885: vm_object_unlock(object);
2886:
2887: return p;
2888: }
2889:
2890: /*
2891: * Attach a set of physical pages to an object, so that they can
2892: * be mapped by mapping the object. Typically used to map IO memory.
2893: *
2894: * The mapping function and its private data are used to obtain the
2895: * physical addresses for each page to be mapped.
2896: */
2897: void
2898: vm_object_page_map(
2899: vm_object_t object,
2900: vm_offset_t offset,
2901: vm_size_t size,
2902: vm_offset_t (*map_fn)(void *, vm_offset_t),
2903: void * map_fn_data) /* private to map_fn */
2904: {
2905: int num_pages;
2906: int i;
2907: vm_page_t m;
2908: vm_page_t old_page;
2909: vm_offset_t addr;
2910:
2911: num_pages = atop(size);
2912:
2913: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2914:
2915: addr = (*map_fn)(map_fn_data, offset);
2916:
2917: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
2918: vm_page_more_fictitious();
2919:
2920: vm_object_lock(object);
2921: if ((old_page = vm_page_lookup(object, offset))
2922: != VM_PAGE_NULL)
2923: {
1.1.1.2 root 2924: VM_PAGE_FREE(old_page);
1.1 root 2925: }
2926:
2927: vm_page_init(m, addr);
2928: m->private = TRUE; /* don`t free page */
2929: m->wire_count = 1;
2930: vm_page_lock_queues();
2931: vm_page_insert(m, object, offset);
2932: vm_page_unlock_queues();
2933:
2934: PAGE_WAKEUP_DONE(m);
2935: vm_object_unlock(object);
2936: }
2937: }
2938:
2939:
2940: #if MACH_KDB
1.1.1.2 root 2941: #include <vm/vm_print.h>
1.1 root 2942: #define printf kdbprintf
2943:
2944: boolean_t vm_object_print_pages = FALSE;
2945:
2946: /*
2947: * vm_object_print: [ debug ]
2948: */
2949: void vm_object_print(
2950: vm_object_t object)
2951: {
1.1.1.3 ! root 2952: vm_page_t p;
1.1 root 2953:
1.1.1.3 ! root 2954: int count;
1.1 root 2955:
2956: if (object == VM_OBJECT_NULL)
2957: return;
2958:
2959: iprintf("Object 0x%X: size=0x%X",
2960: (vm_offset_t) object, (vm_offset_t) object->size);
2961: printf(", %d references, %d resident pages,", object->ref_count,
2962: object->resident_page_count);
2963: printf(" %d absent pages,", object->absent_count);
2964: printf(" %d paging ops\n", object->paging_in_progress);
2965: indent += 2;
2966: iprintf("memory object=0x%X (offset=0x%X),",
2967: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
2968: printf("control=0x%X, name=0x%X\n",
2969: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
2970: iprintf("%s%s",
2971: object->pager_ready ? " ready" : "",
2972: object->pager_created ? " created" : "");
2973: printf("%s,%s ",
2974: object->pager_initialized ? "" : "uninitialized",
2975: object->temporary ? "temporary" : "permanent");
2976: printf("%s%s,",
2977: object->internal ? "internal" : "external",
2978: object->can_persist ? " cacheable" : "");
2979: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
2980: iprintf("shadow=0x%X (offset=0x%X),",
2981: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
2982: printf("copy=0x%X\n", (vm_offset_t) object->copy);
2983:
2984: indent += 2;
2985:
2986: if (vm_object_print_pages) {
2987: count = 0;
2988: p = (vm_page_t) queue_first(&object->memq);
2989: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2990: if (count == 0) iprintf("memory:=");
2991: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
2992: else printf(",");
2993: count++;
2994:
2995: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
2996: p = (vm_page_t) queue_next(&p->listq);
2997: }
2998: if (count != 0)
2999: printf("\n");
3000: }
3001: indent -= 4;
3002: }
3003:
3004: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.