![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
1.1.1.2 root 36: #include <kern/printf.h>
37: #include <string.h>
1.1 root 38:
39: #include <mach/memory_object.h>
1.1.1.2 root 40: #include <vm/memory_object_default.user.h>
41: #include <vm/memory_object_user.user.h>
42: #include <machine/vm_param.h>
1.1 root 43: #include <ipc/ipc_port.h>
44: #include <ipc/ipc_space.h>
45: #include <kern/assert.h>
1.1.1.2 root 46: #include <kern/debug.h>
1.1 root 47: #include <kern/lock.h>
48: #include <kern/queue.h>
49: #include <kern/xpr.h>
1.1.1.2 root 50: #include <kern/slab.h>
1.1 root 51: #include <vm/memory_object.h>
52: #include <vm/vm_fault.h>
53: #include <vm/vm_map.h>
54: #include <vm/vm_object.h>
55: #include <vm/vm_page.h>
56: #include <vm/vm_pageout.h>
57:
1.1.1.2 root 58: #if MACH_KDB
59: #include <ddb/db_output.h>
60: #endif /* MACH_KDB */
61:
1.1 root 62: /*
63: * Virtual memory objects maintain the actual data
64: * associated with allocated virtual memory. A given
65: * page of memory exists within exactly one object.
66: *
67: * An object is only deallocated when all "references"
68: * are given up. Only one "reference" to a given
69: * region of an object should be writeable.
70: *
71: * Associated with each object is a list of all resident
72: * memory pages belonging to that object; this list is
73: * maintained by the "vm_page" module, but locked by the object's
74: * lock.
75: *
76: * Each object also records the memory object port
77: * that is used by the kernel to request and write
78: * back data (the memory object port, field "pager"),
79: * and the ports provided to the memory manager, the server that
80: * manages that data, to return data and control its
81: * use (the memory object control port, field "pager_request")
82: * and for naming (the memory object name port, field "pager_name").
83: *
84: * Virtual memory objects are allocated to provide
85: * zero-filled memory (vm_allocate) or map a user-defined
86: * memory object into a virtual address space (vm_map).
87: *
88: * Virtual memory objects that refer to a user-defined
89: * memory object are called "permanent", because all changes
90: * made in virtual memory are reflected back to the
91: * memory manager, which may then store it permanently.
92: * Other virtual memory objects are called "temporary",
93: * meaning that changes need be written back only when
94: * necessary to reclaim pages, and that storage associated
95: * with the object can be discarded once it is no longer
96: * mapped.
97: *
98: * A permanent memory object may be mapped into more
99: * than one virtual address space. Moreover, two threads
100: * may attempt to make the first mapping of a memory
101: * object concurrently. Only one thread is allowed to
102: * complete this mapping; all others wait for the
103: * "pager_initialized" field is asserted, indicating
104: * that the first thread has initialized all of the
105: * necessary fields in the virtual memory object structure.
106: *
107: * The kernel relies on a *default memory manager* to
108: * provide backing storage for the zero-filled virtual
109: * memory objects. The memory object ports associated
110: * with these temporary virtual memory objects are only
111: * generated and passed to the default memory manager
112: * when it becomes necessary. Virtual memory objects
113: * that depend on the default memory manager are called
114: * "internal". The "pager_created" field is provided to
115: * indicate whether these ports have ever been allocated.
116: *
117: * The kernel may also create virtual memory objects to
118: * hold changed pages after a copy-on-write operation.
119: * In this case, the virtual memory object (and its
120: * backing storage -- its memory object) only contain
121: * those pages that have been changed. The "shadow"
122: * field refers to the virtual memory object that contains
123: * the remainder of the contents. The "shadow_offset"
124: * field indicates where in the "shadow" these contents begin.
125: * The "copy" field refers to a virtual memory object
126: * to which changed pages must be copied before changing
127: * this object, in order to implement another form
128: * of copy-on-write optimization.
129: *
130: * The virtual memory object structure also records
131: * the attributes associated with its memory object.
132: * The "pager_ready", "can_persist" and "copy_strategy"
133: * fields represent those attributes. The "cached_list"
134: * field is used in the implementation of the persistence
135: * attribute.
136: *
137: * ZZZ Continue this comment.
138: */
139:
1.1.1.2 root 140: struct kmem_cache vm_object_cache; /* vm backing store cache */
1.1 root 141:
142: /*
143: * All wired-down kernel memory belongs to a single virtual
144: * memory object (kernel_object) to avoid wasting data structures.
145: */
1.1.1.2 root 146: static struct vm_object kernel_object_store;
147: vm_object_t kernel_object = &kernel_object_store;
1.1 root 148:
149: /*
150: * Virtual memory objects that are not referenced by
151: * any address maps, but that are allowed to persist
152: * (an attribute specified by the associated memory manager),
153: * are kept in a queue (vm_object_cached_list).
154: *
155: * When an object from this queue is referenced again,
156: * for example to make another address space mapping,
157: * it must be removed from the queue. That is, the
158: * queue contains *only* objects with zero references.
159: *
160: * The kernel may choose to terminate objects from this
161: * queue in order to reclaim storage. The current policy
162: * is to permit a fixed maximum number of unreferenced
163: * objects (vm_object_cached_max).
164: *
165: * A simple lock (accessed by routines
166: * vm_object_cache_{lock,lock_try,unlock}) governs the
167: * object cache. It must be held when objects are
168: * added to or removed from the cache (in vm_object_terminate).
169: * The routines that acquire a reference to a virtual
170: * memory object based on one of the memory object ports
171: * must also lock the cache.
172: *
173: * Ideally, the object cache should be more isolated
174: * from the reference mechanism, so that the lock need
175: * not be held to make simple references.
176: */
177: queue_head_t vm_object_cached_list;
178: int vm_object_cached_count;
1.1.1.2 root 179: int vm_object_cached_max = 4000; /* may be patched*/
1.1 root 180:
181: decl_simple_lock_data(,vm_object_cached_lock_data)
182:
183: #define vm_object_cache_lock() \
184: simple_lock(&vm_object_cached_lock_data)
185: #define vm_object_cache_lock_try() \
186: simple_lock_try(&vm_object_cached_lock_data)
187: #define vm_object_cache_unlock() \
188: simple_unlock(&vm_object_cached_lock_data)
189:
190: /*
1.1.1.2 root 191: * Number of physical pages referenced by cached objects.
192: * This counter is protected by its own lock to work around
193: * lock ordering issues.
194: */
195: int vm_object_cached_pages;
196:
197: decl_simple_lock_data(,vm_object_cached_pages_lock_data)
198:
199: /*
1.1 root 200: * Virtual memory objects are initialized from
201: * a template (see vm_object_allocate).
202: *
203: * When adding a new field to the virtual memory
204: * object structure, be sure to add initialization
205: * (see vm_object_init).
206: */
1.1.1.2 root 207: struct vm_object vm_object_template;
1.1 root 208:
209: /*
210: * vm_object_allocate:
211: *
212: * Returns a new object with the given size.
213: */
214:
1.1.1.2 root 215: static void _vm_object_setup(
216: vm_object_t object,
217: vm_size_t size)
218: {
219: *object = vm_object_template;
220: queue_init(&object->memq);
221: vm_object_lock_init(object);
222: object->size = size;
223: }
224:
1.1 root 225: vm_object_t _vm_object_allocate(
226: vm_size_t size)
227: {
1.1.1.3 root 228: vm_object_t object;
1.1 root 229:
1.1.1.2 root 230: object = (vm_object_t) kmem_cache_alloc(&vm_object_cache);
1.1 root 231:
1.1.1.2 root 232: _vm_object_setup(object, size);
1.1 root 233:
234: return object;
235: }
236:
237: vm_object_t vm_object_allocate(
238: vm_size_t size)
239: {
1.1.1.3 root 240: vm_object_t object;
241: ipc_port_t port;
1.1 root 242:
243: object = _vm_object_allocate(size);
244: port = ipc_port_alloc_kernel();
245: if (port == IP_NULL)
246: panic("vm_object_allocate");
247: object->pager_name = port;
248: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
249:
250: return object;
251: }
252:
253: /*
254: * vm_object_bootstrap:
255: *
256: * Initialize the VM objects module.
257: */
258: void vm_object_bootstrap(void)
259: {
1.1.1.2 root 260: kmem_cache_init(&vm_object_cache, "vm_object",
261: sizeof(struct vm_object), 0, NULL, NULL, NULL, 0);
1.1 root 262:
263: queue_init(&vm_object_cached_list);
264: simple_lock_init(&vm_object_cached_lock_data);
265:
266: /*
267: * Fill in a template object, for quick initialization
268: */
269:
1.1.1.2 root 270: vm_object_template.ref_count = 1;
271: vm_object_template.size = 0;
272: vm_object_template.resident_page_count = 0;
273: vm_object_template.copy = VM_OBJECT_NULL;
274: vm_object_template.shadow = VM_OBJECT_NULL;
275: vm_object_template.shadow_offset = (vm_offset_t) 0;
276:
277: vm_object_template.pager = IP_NULL;
278: vm_object_template.paging_offset = 0;
279: vm_object_template.pager_request = PAGER_REQUEST_NULL;
280: vm_object_template.pager_name = IP_NULL;
281:
282: vm_object_template.pager_created = FALSE;
283: vm_object_template.pager_initialized = FALSE;
284: vm_object_template.pager_ready = FALSE;
1.1 root 285:
1.1.1.2 root 286: vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_NONE;
1.1 root 287: /* ignored if temporary, will be reset before
288: * permanent object becomes ready */
1.1.1.2 root 289: vm_object_template.use_shared_copy = FALSE;
290: vm_object_template.shadowed = FALSE;
1.1 root 291:
1.1.1.2 root 292: vm_object_template.absent_count = 0;
293: vm_object_template.all_wanted = 0; /* all bits FALSE */
1.1 root 294:
1.1.1.2 root 295: vm_object_template.paging_in_progress = 0;
296: vm_object_template.can_persist = FALSE;
297: vm_object_template.internal = TRUE;
298: vm_object_template.temporary = TRUE;
299: vm_object_template.alive = TRUE;
300: vm_object_template.lock_in_progress = FALSE;
301: vm_object_template.lock_restart = FALSE;
302: vm_object_template.use_old_pageout = TRUE; /* XXX change later */
303: vm_object_template.last_alloc = (vm_offset_t) 0;
1.1 root 304:
305: #if MACH_PAGEMAP
1.1.1.2 root 306: vm_object_template.existence_info = VM_EXTERNAL_NULL;
1.1 root 307: #endif /* MACH_PAGEMAP */
308:
309: /*
310: * Initialize the "kernel object"
311: */
312:
1.1.1.2 root 313: _vm_object_setup(kernel_object,
1.1 root 314: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
315:
316: /*
317: * Initialize the "submap object". Make it as large as the
318: * kernel object so that no limit is imposed on submap sizes.
319: */
320:
1.1.1.2 root 321: _vm_object_setup(vm_submap_object,
1.1 root 322: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
323:
324: #if MACH_PAGEMAP
325: vm_external_module_initialize();
326: #endif /* MACH_PAGEMAP */
327: }
328:
329: void vm_object_init(void)
330: {
331: /*
332: * Finish initializing the kernel object.
333: * The submap object doesn't need a name port.
334: */
335:
336: kernel_object->pager_name = ipc_port_alloc_kernel();
337: ipc_kobject_set(kernel_object->pager_name,
338: (ipc_kobject_t) kernel_object,
339: IKOT_PAGING_NAME);
340: }
341:
342: /*
343: * vm_object_reference:
344: *
345: * Gets another reference to the given object.
346: */
347: void vm_object_reference(
1.1.1.3 root 348: vm_object_t object)
1.1 root 349: {
350: if (object == VM_OBJECT_NULL)
351: return;
352:
353: vm_object_lock(object);
354: assert(object->ref_count > 0);
355: object->ref_count++;
356: vm_object_unlock(object);
357: }
358:
359: /*
360: * vm_object_deallocate:
361: *
362: * Release a reference to the specified object,
363: * gained either through a vm_object_allocate
364: * or a vm_object_reference call. When all references
365: * are gone, storage associated with this object
366: * may be relinquished.
367: *
368: * No object may be locked.
369: */
370: void vm_object_deallocate(
1.1.1.3 root 371: vm_object_t object)
1.1 root 372: {
373: vm_object_t temp;
374:
375: while (object != VM_OBJECT_NULL) {
376:
377: /*
378: * The cache holds a reference (uncounted) to
379: * the object; we must lock it before removing
380: * the object.
381: */
382:
383: vm_object_cache_lock();
384:
385: /*
386: * Lose the reference
387: */
388: vm_object_lock(object);
389: if (--(object->ref_count) > 0) {
390:
391: /*
392: * If there are still references, then
393: * we are done.
394: */
395: vm_object_unlock(object);
396: vm_object_cache_unlock();
397: return;
398: }
399:
400: /*
401: * See whether this object can persist. If so, enter
402: * it in the cache, then deactivate all of its
403: * pages.
404: */
405: if (object->can_persist) {
406: boolean_t overflow;
407:
408: /*
409: * Enter the object onto the queue
410: * of "cached" objects. Remember whether
411: * we've caused the queue to overflow,
412: * as a hint.
413: */
414:
415: queue_enter(&vm_object_cached_list, object,
416: vm_object_t, cached_list);
417: overflow = (++vm_object_cached_count > vm_object_cached_max);
1.1.1.2 root 418: vm_object_cached_pages_update(object->resident_page_count);
1.1 root 419: vm_object_cache_unlock();
420:
421: vm_object_deactivate_pages(object);
422: vm_object_unlock(object);
423:
424: /*
425: * If we didn't overflow, or if the queue has
426: * been reduced back to below the specified
427: * minimum, then quit.
428: */
429: if (!overflow)
430: return;
431:
432: while (TRUE) {
433: vm_object_cache_lock();
434: if (vm_object_cached_count <=
435: vm_object_cached_max) {
436: vm_object_cache_unlock();
437: return;
438: }
439:
440: /*
441: * If we must trim down the queue, take
442: * the first object, and proceed to
443: * terminate it instead of the original
444: * object. Have to wait for pager init.
445: * if it's in progress.
446: */
447: object= (vm_object_t)
448: queue_first(&vm_object_cached_list);
449: vm_object_lock(object);
450:
451: if (!(object->pager_created &&
452: !object->pager_initialized)) {
453:
454: /*
455: * Ok to terminate, hang on to lock.
456: */
457: break;
458: }
459:
460: vm_object_assert_wait(object,
461: VM_OBJECT_EVENT_INITIALIZED, FALSE);
462: vm_object_unlock(object);
463: vm_object_cache_unlock();
464: thread_block((void (*)()) 0);
465:
466: /*
467: * Continue loop to check if cache still
468: * needs to be trimmed.
469: */
470: }
471:
472: /*
473: * Actually remove object from cache.
474: */
475:
476: queue_remove(&vm_object_cached_list, object,
477: vm_object_t, cached_list);
478: vm_object_cached_count--;
479:
480: assert(object->ref_count == 0);
481: }
482: else {
483: if (object->pager_created &&
484: !object->pager_initialized) {
485:
486: /*
487: * Have to wait for initialization.
488: * Put reference back and retry
489: * when it's initialized.
490: */
491: object->ref_count++;
492: vm_object_assert_wait(object,
493: VM_OBJECT_EVENT_INITIALIZED, FALSE);
494: vm_object_unlock(object);
495: vm_object_cache_unlock();
496: thread_block((void (*)()) 0);
497: continue;
498: }
499: }
500:
501: /*
502: * Take the reference to the shadow object
503: * out of the object to be destroyed.
504: */
505:
506: temp = object->shadow;
507:
508: /*
509: * Destroy the object; the cache lock will
510: * be released in the process.
511: */
512:
513: vm_object_terminate(object);
514:
515: /*
516: * Deallocate the reference to the shadow
517: * by continuing the loop with that object
518: * in place of the original.
519: */
520:
521: object = temp;
522: }
523: }
524:
525: /*
526: * Routine: vm_object_terminate
527: * Purpose:
528: * Free all resources associated with a vm_object.
529: * In/out conditions:
530: * Upon entry, the object and the cache must be locked,
531: * and the object must have no references.
532: *
533: * The shadow object reference is left alone.
534: *
535: * Upon exit, the cache will be unlocked, and the
536: * object will cease to exist.
537: */
538: void vm_object_terminate(
1.1.1.3 root 539: vm_object_t object)
1.1 root 540: {
1.1.1.3 root 541: vm_page_t p;
542: vm_object_t shadow_object;
1.1 root 543:
544: /*
545: * Make sure the object isn't already being terminated
546: */
547:
548: assert(object->alive);
549: object->alive = FALSE;
550:
551: /*
552: * Make sure no one can look us up now.
553: */
554:
555: vm_object_remove(object);
556: vm_object_cache_unlock();
557:
558: /*
559: * Detach the object from its shadow if we are the shadow's
560: * copy.
561: */
562: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
563: vm_object_lock(shadow_object);
564: assert((shadow_object->copy == object) ||
565: (shadow_object->copy == VM_OBJECT_NULL));
566: shadow_object->copy = VM_OBJECT_NULL;
567: vm_object_unlock(shadow_object);
568: }
569:
570: /*
571: * The pageout daemon might be playing with our pages.
572: * Now that the object is dead, it won't touch any more
573: * pages, but some pages might already be on their way out.
574: * Hence, we wait until the active paging activities have ceased.
575: */
576:
577: vm_object_paging_wait(object, FALSE);
578:
579: /*
580: * Clean or free the pages, as appropriate.
581: * It is possible for us to find busy/absent pages,
582: * if some faults on this object were aborted.
583: */
584:
585: if ((object->temporary) || (object->pager == IP_NULL)) {
586: while (!queue_empty(&object->memq)) {
587: p = (vm_page_t) queue_first(&object->memq);
588:
589: VM_PAGE_CHECK(p);
590:
591: VM_PAGE_FREE(p);
592: }
593: } else while (!queue_empty(&object->memq)) {
594: p = (vm_page_t) queue_first(&object->memq);
595:
596: VM_PAGE_CHECK(p);
597:
598: vm_page_lock_queues();
599: VM_PAGE_QUEUES_REMOVE(p);
600: vm_page_unlock_queues();
601:
602: if (p->absent || p->private) {
603:
604: /*
605: * For private pages, VM_PAGE_FREE just
606: * leaves the page structure around for
607: * its owner to clean up. For absent
608: * pages, the structure is returned to
609: * the appropriate pool.
610: */
611:
612: goto free_page;
613: }
614:
615: if (!p->dirty)
616: p->dirty = pmap_is_modified(p->phys_addr);
617:
618: if (p->dirty || p->precious) {
619: p->busy = TRUE;
620: vm_pageout_page(p, FALSE, TRUE); /* flush page */
621: } else {
622: free_page:
623: VM_PAGE_FREE(p);
624: }
625: }
626:
627: assert(object->ref_count == 0);
628: assert(object->paging_in_progress == 0);
629:
630: /*
631: * Throw away port rights... note that they may
632: * already have been thrown away (by vm_object_destroy
633: * or memory_object_destroy).
634: *
635: * Instead of destroying the control and name ports,
636: * we send all rights off to the memory manager instead,
637: * using memory_object_terminate.
638: */
639:
640: vm_object_unlock(object);
641:
642: if (object->pager != IP_NULL) {
643: /* consumes our rights for pager, pager_request, pager_name */
644: memory_object_release(object->pager,
645: object->pager_request,
646: object->pager_name);
647: } else if (object->pager_name != IP_NULL) {
648: /* consumes our right for pager_name */
649: ipc_port_dealloc_kernel(object->pager_name);
650: }
651:
652: #if MACH_PAGEMAP
653: vm_external_destroy(object->existence_info);
654: #endif /* MACH_PAGEMAP */
655:
656: /*
657: * Free the space for the object.
658: */
659:
1.1.1.2 root 660: kmem_cache_free(&vm_object_cache, (vm_offset_t) object);
1.1 root 661: }
662:
663: /*
664: * Routine: vm_object_pager_wakeup
665: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
666: */
667:
668: void
669: vm_object_pager_wakeup(
670: ipc_port_t pager)
671: {
672: boolean_t someone_waiting;
673:
674: /*
675: * If anyone was waiting for the memory_object_terminate
676: * to be queued, wake them up now.
677: */
678: vm_object_cache_lock();
679: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
680: someone_waiting = (pager->ip_kobject != IKO_NULL);
681: if (ip_active(pager))
682: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
683: vm_object_cache_unlock();
684: if (someone_waiting) {
685: thread_wakeup((event_t) pager);
686: }
687: }
688:
689: /*
690: * Routine: memory_object_release
691: * Purpose: Terminate the pager and release port rights,
692: * just like memory_object_terminate, except
693: * that we wake up anyone blocked in vm_object_enter
694: * waiting for termination message to be queued
695: * before calling memory_object_init.
696: */
697: void memory_object_release(
698: ipc_port_t pager,
699: pager_request_t pager_request,
700: ipc_port_t pager_name)
701: {
702:
703: /*
704: * Keep a reference to pager port;
705: * the terminate might otherwise release all references.
706: */
707: ip_reference(pager);
708:
709: /*
710: * Terminate the pager.
711: */
712: (void) memory_object_terminate(pager, pager_request, pager_name);
713:
714: /*
715: * Wakeup anyone waiting for this terminate
716: */
717: vm_object_pager_wakeup(pager);
718:
719: /*
720: * Release reference to pager port.
721: */
722: ip_release(pager);
723: }
724:
725: /*
726: * Routine: vm_object_abort_activity [internal use only]
727: * Purpose:
728: * Abort paging requests pending on this object.
729: * In/out conditions:
730: * The object is locked on entry and exit.
731: */
732: void vm_object_abort_activity(
733: vm_object_t object)
734: {
735: vm_page_t p;
736: vm_page_t next;
737:
738: /*
739: * Abort all activity that would be waiting
740: * for a result on this memory object.
741: *
742: * We could also choose to destroy all pages
743: * that we have in memory for this object, but
744: * we don't.
745: */
746:
747: p = (vm_page_t) queue_first(&object->memq);
748: while (!queue_end(&object->memq, (queue_entry_t) p)) {
749: next = (vm_page_t) queue_next(&p->listq);
750:
751: /*
752: * If it's being paged in, destroy it.
753: * If an unlock has been requested, start it again.
754: */
755:
756: if (p->busy && p->absent) {
757: VM_PAGE_FREE(p);
758: }
759: else {
760: if (p->unlock_request != VM_PROT_NONE)
761: p->unlock_request = VM_PROT_NONE;
762: PAGE_WAKEUP(p);
763: }
764:
765: p = next;
766: }
767:
768: /*
769: * Wake up threads waiting for the memory object to
770: * become ready.
771: */
772:
773: object->pager_ready = TRUE;
774: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
775: }
776:
777: /*
778: * Routine: memory_object_destroy [user interface]
779: * Purpose:
780: * Shut down a memory object, despite the
781: * presence of address map (or other) references
782: * to the vm_object.
783: * Note:
784: * This routine may be called either from the user interface,
785: * or from port destruction handling (via vm_object_destroy).
786: */
787: kern_return_t memory_object_destroy(
788: vm_object_t object,
789: kern_return_t reason)
790: {
791: ipc_port_t old_object, old_name;
792: pager_request_t old_control;
793:
794: if (object == VM_OBJECT_NULL)
795: return KERN_SUCCESS;
796:
797: /*
798: * Remove the port associations immediately.
799: *
800: * This will prevent the memory manager from further
801: * meddling. [If it wanted to flush data or make
802: * other changes, it should have done so before performing
803: * the destroy call.]
804: */
805:
806: vm_object_cache_lock();
807: vm_object_lock(object);
808: vm_object_remove(object);
809: object->can_persist = FALSE;
810: vm_object_cache_unlock();
811:
812: /*
813: * Rip out the ports from the vm_object now... this
814: * will prevent new memory_object calls from succeeding.
815: */
816:
817: old_object = object->pager;
818: object->pager = IP_NULL;
819:
820: old_control = object->pager_request;
821: object->pager_request = PAGER_REQUEST_NULL;
822:
823: old_name = object->pager_name;
824: object->pager_name = IP_NULL;
825:
826:
827: /*
828: * Wait for existing paging activity (that might
829: * have the old ports) to subside.
830: */
831:
832: vm_object_paging_wait(object, FALSE);
833: vm_object_unlock(object);
834:
835: /*
836: * Shut down the ports now.
837: *
838: * [Paging operations may be proceeding concurrently --
839: * they'll get the null values established above.]
840: */
841:
842: if (old_object != IP_NULL) {
843: /* consumes our rights for object, control, name */
844: memory_object_release(old_object, old_control,
845: old_name);
846: } else if (old_name != IP_NULL) {
847: /* consumes our right for name */
848: ipc_port_dealloc_kernel(object->pager_name);
849: }
850:
851: /*
852: * Lose the reference that was donated for this routine
853: */
854:
855: vm_object_deallocate(object);
856:
857: return KERN_SUCCESS;
858: }
859:
860: /*
861: * vm_object_deactivate_pages
862: *
863: * Deactivate all pages in the specified object. (Keep its pages
864: * in memory even though it is no longer referenced.)
865: *
866: * The object must be locked.
867: */
868: void vm_object_deactivate_pages(
1.1.1.3 root 869: vm_object_t object)
1.1 root 870: {
1.1.1.3 root 871: vm_page_t p;
1.1 root 872:
873: queue_iterate(&object->memq, p, vm_page_t, listq) {
874: vm_page_lock_queues();
875: if (!p->busy)
876: vm_page_deactivate(p);
877: vm_page_unlock_queues();
878: }
879: }
880:
881:
882: /*
883: * Routine: vm_object_pmap_protect
884: *
885: * Purpose:
886: * Reduces the permission for all physical
887: * pages in the specified object range.
888: *
889: * If removing write permission only, it is
890: * sufficient to protect only the pages in
891: * the top-level object; only those pages may
892: * have write permission.
893: *
894: * If removing all access, we must follow the
895: * shadow chain from the top-level object to
896: * remove access to all pages in shadowed objects.
897: *
898: * The object must *not* be locked. The object must
899: * be temporary/internal.
900: *
901: * If pmap is not NULL, this routine assumes that
902: * the only mappings for the pages are in that
903: * pmap.
904: */
905: boolean_t vm_object_pmap_protect_by_page = FALSE;
906:
907: void vm_object_pmap_protect(
1.1.1.3 root 908: vm_object_t object,
909: vm_offset_t offset,
1.1.1.2 root 910: vm_size_t size,
1.1 root 911: pmap_t pmap,
912: vm_offset_t pmap_start,
913: vm_prot_t prot)
914: {
915: if (object == VM_OBJECT_NULL)
916: return;
917:
918: vm_object_lock(object);
919:
920: assert(object->temporary && object->internal);
921:
922: while (TRUE) {
923: if (object->resident_page_count > atop(size) / 2 &&
924: pmap != PMAP_NULL) {
925: vm_object_unlock(object);
926: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
927: return;
928: }
929:
930: {
1.1.1.3 root 931: vm_page_t p;
932: vm_offset_t end;
1.1 root 933:
934: end = offset + size;
935:
936: queue_iterate(&object->memq, p, vm_page_t, listq) {
937: if (!p->fictitious &&
938: (offset <= p->offset) &&
939: (p->offset < end)) {
940: if ((pmap == PMAP_NULL) ||
941: vm_object_pmap_protect_by_page) {
942: pmap_page_protect(p->phys_addr,
943: prot & ~p->page_lock);
944: } else {
945: vm_offset_t start =
946: pmap_start +
947: (p->offset - offset);
948:
949: pmap_protect(pmap,
950: start,
951: start + PAGE_SIZE,
952: prot);
953: }
954: }
955: }
956: }
957:
958: if (prot == VM_PROT_NONE) {
959: /*
960: * Must follow shadow chain to remove access
961: * to pages in shadowed objects.
962: */
1.1.1.3 root 963: vm_object_t next_object;
1.1 root 964:
965: next_object = object->shadow;
966: if (next_object != VM_OBJECT_NULL) {
967: offset += object->shadow_offset;
968: vm_object_lock(next_object);
969: vm_object_unlock(object);
970: object = next_object;
971: }
972: else {
973: /*
974: * End of chain - we are done.
975: */
976: break;
977: }
978: }
979: else {
980: /*
981: * Pages in shadowed objects may never have
982: * write permission - we may stop here.
983: */
984: break;
985: }
986: }
987:
988: vm_object_unlock(object);
989: }
990:
991: /*
992: * vm_object_pmap_remove:
993: *
994: * Removes all physical pages in the specified
995: * object range from all physical maps.
996: *
997: * The object must *not* be locked.
998: */
999: void vm_object_pmap_remove(
1.1.1.3 root 1000: vm_object_t object,
1001: vm_offset_t start,
1002: vm_offset_t end)
1.1 root 1003: {
1.1.1.3 root 1004: vm_page_t p;
1.1 root 1005:
1006: if (object == VM_OBJECT_NULL)
1007: return;
1008:
1009: vm_object_lock(object);
1010: queue_iterate(&object->memq, p, vm_page_t, listq) {
1011: if (!p->fictitious &&
1012: (start <= p->offset) &&
1013: (p->offset < end))
1014: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
1015: }
1016: vm_object_unlock(object);
1017: }
1018:
1019: /*
1020: * Routine: vm_object_copy_slowly
1021: *
1022: * Description:
1023: * Copy the specified range of the source
1024: * virtual memory object without using
1025: * protection-based optimizations (such
1026: * as copy-on-write). The pages in the
1027: * region are actually copied.
1028: *
1029: * In/out conditions:
1030: * The caller must hold a reference and a lock
1031: * for the source virtual memory object. The source
1032: * object will be returned *unlocked*.
1033: *
1034: * Results:
1035: * If the copy is completed successfully, KERN_SUCCESS is
1036: * returned. If the caller asserted the interruptible
1037: * argument, and an interruption occurred while waiting
1038: * for a user-generated event, MACH_SEND_INTERRUPTED is
1039: * returned. Other values may be returned to indicate
1040: * hard errors during the copy operation.
1041: *
1042: * A new virtual memory object is returned in a
1043: * parameter (_result_object). The contents of this
1044: * new object, starting at a zero offset, are a copy
1045: * of the source memory region. In the event of
1046: * an error, this parameter will contain the value
1047: * VM_OBJECT_NULL.
1048: */
1049: kern_return_t vm_object_copy_slowly(
1050: vm_object_t src_object,
1051: vm_offset_t src_offset,
1052: vm_size_t size,
1053: boolean_t interruptible,
1054: vm_object_t *_result_object) /* OUT */
1055: {
1056: vm_object_t new_object;
1057: vm_offset_t new_offset;
1058:
1059: if (size == 0) {
1060: vm_object_unlock(src_object);
1061: *_result_object = VM_OBJECT_NULL;
1062: return KERN_INVALID_ARGUMENT;
1063: }
1064:
1065: /*
1066: * Prevent destruction of the source object while we copy.
1067: */
1068:
1069: assert(src_object->ref_count > 0);
1070: src_object->ref_count++;
1071: vm_object_unlock(src_object);
1072:
1073: /*
1074: * Create a new object to hold the copied pages.
1075: * A few notes:
1076: * We fill the new object starting at offset 0,
1077: * regardless of the input offset.
1078: * We don't bother to lock the new object within
1079: * this routine, since we have the only reference.
1080: */
1081:
1082: new_object = vm_object_allocate(size);
1083: new_offset = 0;
1084:
1085: assert(size == trunc_page(size)); /* Will the loop terminate? */
1086:
1087: for ( ;
1088: size != 0 ;
1089: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1090: ) {
1091: vm_page_t new_page;
1092: vm_fault_return_t result;
1093:
1094: while ((new_page = vm_page_alloc(new_object, new_offset))
1095: == VM_PAGE_NULL) {
1096: VM_PAGE_WAIT((void (*)()) 0);
1097: }
1098:
1099: do {
1100: vm_prot_t prot = VM_PROT_READ;
1101: vm_page_t _result_page;
1102: vm_page_t top_page;
1103: vm_page_t result_page;
1104:
1105: vm_object_lock(src_object);
1106: src_object->paging_in_progress++;
1107:
1108: result = vm_fault_page(src_object, src_offset,
1109: VM_PROT_READ, FALSE, interruptible,
1110: &prot, &_result_page, &top_page,
1111: FALSE, (void (*)()) 0);
1112:
1113: switch(result) {
1114: case VM_FAULT_SUCCESS:
1115: result_page = _result_page;
1116:
1117: /*
1118: * We don't need to hold the object
1119: * lock -- the busy page will be enough.
1120: * [We don't care about picking up any
1121: * new modifications.]
1122: *
1123: * Copy the page to the new object.
1124: *
1125: * POLICY DECISION:
1126: * If result_page is clean,
1127: * we could steal it instead
1128: * of copying.
1129: */
1130:
1131: vm_object_unlock(result_page->object);
1132: vm_page_copy(result_page, new_page);
1133:
1134: /*
1135: * Let go of both pages (make them
1136: * not busy, perform wakeup, activate).
1137: */
1138:
1139: new_page->busy = FALSE;
1140: new_page->dirty = TRUE;
1141: vm_object_lock(result_page->object);
1142: PAGE_WAKEUP_DONE(result_page);
1143:
1144: vm_page_lock_queues();
1145: if (!result_page->active &&
1146: !result_page->inactive)
1147: vm_page_activate(result_page);
1148: vm_page_activate(new_page);
1149: vm_page_unlock_queues();
1150:
1151: /*
1152: * Release paging references and
1153: * top-level placeholder page, if any.
1154: */
1155:
1156: vm_fault_cleanup(result_page->object,
1157: top_page);
1158:
1159: break;
1160:
1161: case VM_FAULT_RETRY:
1162: break;
1163:
1164: case VM_FAULT_MEMORY_SHORTAGE:
1165: VM_PAGE_WAIT((void (*)()) 0);
1166: break;
1167:
1168: case VM_FAULT_FICTITIOUS_SHORTAGE:
1169: vm_page_more_fictitious();
1170: break;
1171:
1172: case VM_FAULT_INTERRUPTED:
1173: vm_page_free(new_page);
1174: vm_object_deallocate(new_object);
1175: vm_object_deallocate(src_object);
1176: *_result_object = VM_OBJECT_NULL;
1177: return MACH_SEND_INTERRUPTED;
1178:
1179: case VM_FAULT_MEMORY_ERROR:
1180: /*
1181: * A policy choice:
1182: * (a) ignore pages that we can't
1183: * copy
1184: * (b) return the null object if
1185: * any page fails [chosen]
1186: */
1187:
1188: vm_page_free(new_page);
1189: vm_object_deallocate(new_object);
1190: vm_object_deallocate(src_object);
1191: *_result_object = VM_OBJECT_NULL;
1192: return KERN_MEMORY_ERROR;
1193: }
1194: } while (result != VM_FAULT_SUCCESS);
1195: }
1196:
1197: /*
1198: * Lose the extra reference, and return our object.
1199: */
1200:
1201: vm_object_deallocate(src_object);
1202: *_result_object = new_object;
1203: return KERN_SUCCESS;
1204: }
1205:
1206: /*
1207: * Routine: vm_object_copy_temporary
1208: *
1209: * Purpose:
1210: * Copy the specified range of the source virtual
1211: * memory object, if it can be done without blocking.
1212: *
1213: * Results:
1214: * If the copy is successful, the copy is returned in
1215: * the arguments; otherwise, the arguments are not
1216: * affected.
1217: *
1218: * In/out conditions:
1219: * The object should be unlocked on entry and exit.
1220: */
1221:
1222: boolean_t vm_object_copy_temporary(
1223: vm_object_t *_object, /* INOUT */
1224: vm_offset_t *_offset, /* INOUT */
1225: boolean_t *_src_needs_copy, /* OUT */
1226: boolean_t *_dst_needs_copy) /* OUT */
1227: {
1228: vm_object_t object = *_object;
1229:
1230: if (object == VM_OBJECT_NULL) {
1231: *_src_needs_copy = FALSE;
1232: *_dst_needs_copy = FALSE;
1233: return TRUE;
1234: }
1235:
1236: /*
1237: * If the object is temporary, we can perform
1238: * a symmetric copy-on-write without asking.
1239: */
1240:
1241: vm_object_lock(object);
1242: if (object->temporary) {
1243:
1244: /*
1245: * Shared objects use delayed copy
1246: */
1247: if (object->use_shared_copy) {
1248:
1249: /*
1250: * Asymmetric copy strategy. Destination
1251: * must be copied (to allow copy object reuse).
1252: * Source is unaffected.
1253: */
1254: vm_object_unlock(object);
1255: object = vm_object_copy_delayed(object);
1256: *_object = object;
1257: *_src_needs_copy = FALSE;
1258: *_dst_needs_copy = TRUE;
1259: return TRUE;
1260: }
1261:
1262: /*
1263: * Make another reference to the object.
1264: *
1265: * Leave object/offset unchanged.
1266: */
1267:
1268: assert(object->ref_count > 0);
1269: object->ref_count++;
1270: object->shadowed = TRUE;
1271: vm_object_unlock(object);
1272:
1273: /*
1274: * Both source and destination must make
1275: * shadows, and the source must be made
1276: * read-only if not already.
1277: */
1278:
1279: *_src_needs_copy = TRUE;
1280: *_dst_needs_copy = TRUE;
1281: return TRUE;
1282: }
1283:
1284: if (object->pager_ready &&
1285: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1286: /* XXX Do something intelligent (see temporary code above) */
1287: }
1288: vm_object_unlock(object);
1289:
1290: return FALSE;
1291: }
1292:
1293: /*
1294: * Routine: vm_object_copy_call [internal]
1295: *
1296: * Description:
1297: * Copy the specified (src_offset, size) portion
1298: * of the source object (src_object), using the
1299: * user-managed copy algorithm.
1300: *
1301: * In/out conditions:
1302: * The source object must be locked on entry. It
1303: * will be *unlocked* on exit.
1304: *
1305: * Results:
1306: * If the copy is successful, KERN_SUCCESS is returned.
1307: * This routine is interruptible; if a wait for
1308: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1309: * is returned. Other return values indicate hard errors
1310: * in creating the user-managed memory object for the copy.
1311: *
1312: * A new object that represents the copied virtual
1313: * memory is returned in a parameter (*_result_object).
1314: * If the return value indicates an error, this parameter
1315: * is not valid.
1316: */
1317: kern_return_t vm_object_copy_call(
1318: vm_object_t src_object,
1319: vm_offset_t src_offset,
1320: vm_size_t size,
1321: vm_object_t *_result_object) /* OUT */
1322: {
1323: vm_offset_t src_end = src_offset + size;
1324: ipc_port_t new_memory_object;
1325: vm_object_t new_object;
1326: vm_page_t p;
1327:
1328: /*
1329: * Create a memory object port to be associated
1330: * with this new vm_object.
1331: *
1332: * Since the kernel has the only rights to this
1333: * port, we need not hold the cache lock.
1334: *
1335: * Since we have the only object reference, we
1336: * need not be worried about collapse operations.
1337: *
1338: */
1339:
1340: new_memory_object = ipc_port_alloc_kernel();
1.1.1.4 ! root 1341: if (new_memory_object == IP_NULL)
! 1342: return KERN_RESOURCE_SHORTAGE;
! 1343:
! 1344: /*
! 1345: * Set the backing object for the new
! 1346: * temporary object.
! 1347: */
! 1348:
! 1349: assert(src_object->ref_count > 0);
! 1350: src_object->ref_count++;
! 1351: vm_object_paging_begin(src_object);
! 1352: vm_object_unlock(src_object);
1.1 root 1353:
1354: /* we hold a naked receive right for new_memory_object */
1355: (void) ipc_port_make_send(new_memory_object);
1356: /* now we also hold a naked send right for new_memory_object */
1357:
1358: /*
1359: * Let the memory manager know that a copy operation
1360: * is in progress. Note that we're using the old
1361: * memory object's ports (for which we're holding
1362: * a paging reference)... the memory manager cannot
1363: * yet affect the new memory object.
1364: */
1365:
1366: (void) memory_object_copy(src_object->pager,
1367: src_object->pager_request,
1368: src_offset, size,
1369: new_memory_object);
1370: /* no longer hold the naked receive right for new_memory_object */
1371:
1372: vm_object_lock(src_object);
1373: vm_object_paging_end(src_object);
1374:
1375: /*
1376: * Remove write access from all of the pages of
1377: * the old memory object that we can.
1378: */
1379:
1380: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1381: if (!p->fictitious &&
1382: (src_offset <= p->offset) &&
1383: (p->offset < src_end) &&
1384: !(p->page_lock & VM_PROT_WRITE)) {
1385: p->page_lock |= VM_PROT_WRITE;
1386: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1387: }
1388: }
1389:
1390: vm_object_unlock(src_object);
1391:
1392: /*
1393: * Initialize the rest of the paging stuff
1394: */
1395:
1396: new_object = vm_object_enter(new_memory_object, size, FALSE);
1397: new_object->shadow = src_object;
1398: new_object->shadow_offset = src_offset;
1399:
1400: /*
1401: * Drop the reference for new_memory_object taken above.
1402: */
1403:
1404: ipc_port_release_send(new_memory_object);
1405: /* no longer hold the naked send right for new_memory_object */
1406:
1407: *_result_object = new_object;
1408: return KERN_SUCCESS;
1409: }
1410:
1411: /*
1412: * Routine: vm_object_copy_delayed [internal]
1413: *
1414: * Description:
1415: * Copy the specified virtual memory object, using
1416: * the asymmetric copy-on-write algorithm.
1417: *
1418: * In/out conditions:
1419: * The object must be unlocked on entry.
1420: *
1421: * This routine will not block waiting for user-generated
1422: * events. It is not interruptible.
1423: */
1424: vm_object_t vm_object_copy_delayed(
1425: vm_object_t src_object)
1426: {
1427: vm_object_t new_copy;
1428: vm_object_t old_copy;
1429: vm_page_t p;
1430:
1431: /*
1432: * The user-level memory manager wants to see
1433: * all of the changes to this object, but it
1434: * has promised not to make any changes on its own.
1435: *
1436: * Perform an asymmetric copy-on-write, as follows:
1437: * Create a new object, called a "copy object"
1438: * to hold pages modified by the new mapping
1439: * (i.e., the copy, not the original mapping).
1440: * Record the original object as the backing
1441: * object for the copy object. If the
1442: * original mapping does not change a page,
1443: * it may be used read-only by the copy.
1444: * Record the copy object in the original
1445: * object. When the original mapping causes
1446: * a page to be modified, it must be copied
1447: * to a new page that is "pushed" to the
1448: * copy object.
1449: * Mark the new mapping (the copy object)
1450: * copy-on-write. This makes the copy
1451: * object itself read-only, allowing it
1452: * to be reused if the original mapping
1453: * makes no changes, and simplifying the
1454: * synchronization required in the "push"
1455: * operation described above.
1456: *
1.1.1.3 root 1457: * The copy-on-write is said to be asymmetric because
1.1 root 1458: * the original object is *not* marked copy-on-write.
1459: * A copied page is pushed to the copy object, regardless
1460: * which party attempted to modify the page.
1461: *
1462: * Repeated asymmetric copy operations may be done.
1463: * If the original object has not been changed since
1464: * the last copy, its copy object can be reused.
1465: * Otherwise, a new copy object can be inserted
1466: * between the original object and its previous
1467: * copy object. Since any copy object is read-only,
1468: * this cannot affect the contents of the previous copy
1469: * object.
1470: *
1471: * Note that a copy object is higher in the object
1472: * tree than the original object; therefore, use of
1473: * the copy object recorded in the original object
1474: * must be done carefully, to avoid deadlock.
1475: */
1476:
1477: /*
1478: * Allocate a new copy object before locking, even
1479: * though we may not need it later.
1480: */
1481:
1482: new_copy = vm_object_allocate(src_object->size);
1483:
1484: vm_object_lock(src_object);
1485:
1486: /*
1487: * See whether we can reuse the result of a previous
1488: * copy operation.
1489: */
1490: Retry:
1491: old_copy = src_object->copy;
1492: if (old_copy != VM_OBJECT_NULL) {
1493: /*
1494: * Try to get the locks (out of order)
1495: */
1496: if (!vm_object_lock_try(old_copy)) {
1497: vm_object_unlock(src_object);
1498:
1499: simple_lock_pause(); /* wait a bit */
1500:
1501: vm_object_lock(src_object);
1502: goto Retry;
1503: }
1504:
1505: /*
1506: * Determine whether the old copy object has
1507: * been modified.
1508: */
1509:
1510: if (old_copy->resident_page_count == 0 &&
1511: !old_copy->pager_created) {
1512: /*
1513: * It has not been modified.
1514: *
1515: * Return another reference to
1516: * the existing copy-object.
1517: */
1518: assert(old_copy->ref_count > 0);
1519: old_copy->ref_count++;
1520: vm_object_unlock(old_copy);
1521: vm_object_unlock(src_object);
1522:
1523: vm_object_deallocate(new_copy);
1524:
1525: return old_copy;
1526: }
1527:
1528: /*
1529: * The copy-object is always made large enough to
1530: * completely shadow the original object, since
1531: * it may have several users who want to shadow
1532: * the original object at different points.
1533: */
1534:
1535: assert((old_copy->shadow == src_object) &&
1536: (old_copy->shadow_offset == (vm_offset_t) 0));
1537:
1538: /*
1539: * Make the old copy-object shadow the new one.
1540: * It will receive no more pages from the original
1541: * object.
1542: */
1543:
1544: src_object->ref_count--; /* remove ref. from old_copy */
1545: assert(src_object->ref_count > 0);
1546: old_copy->shadow = new_copy;
1547: assert(new_copy->ref_count > 0);
1548: new_copy->ref_count++;
1549: vm_object_unlock(old_copy); /* done with old_copy */
1550: }
1551:
1552: /*
1553: * Point the new copy at the existing object.
1554: */
1555:
1556: new_copy->shadow = src_object;
1557: new_copy->shadow_offset = 0;
1558: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1559: assert(src_object->ref_count > 0);
1560: src_object->ref_count++;
1561: src_object->copy = new_copy;
1562:
1563: /*
1564: * Mark all pages of the existing object copy-on-write.
1565: * This object may have a shadow chain below it, but
1566: * those pages will already be marked copy-on-write.
1567: */
1568:
1569: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1570: if (!p->fictitious)
1571: pmap_page_protect(p->phys_addr,
1572: (VM_PROT_ALL & ~VM_PROT_WRITE &
1573: ~p->page_lock));
1574: }
1575:
1576: vm_object_unlock(src_object);
1577:
1578: return new_copy;
1579: }
1580:
1581: /*
1582: * Routine: vm_object_copy_strategically
1583: *
1584: * Purpose:
1585: * Perform a copy according to the source object's
1586: * declared strategy. This operation may block,
1587: * and may be interrupted.
1588: */
1589: kern_return_t vm_object_copy_strategically(
1590: vm_object_t src_object,
1591: vm_offset_t src_offset,
1592: vm_size_t size,
1593: vm_object_t *dst_object, /* OUT */
1594: vm_offset_t *dst_offset, /* OUT */
1595: boolean_t *dst_needs_copy) /* OUT */
1596: {
1597: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1598: boolean_t interruptible = TRUE; /* XXX */
1599:
1600: assert(src_object != VM_OBJECT_NULL);
1601:
1602: vm_object_lock(src_object);
1603:
1604: /* XXX assert(!src_object->temporary); JSB FIXME */
1605:
1606: /*
1607: * The copy strategy is only valid if the memory manager
1608: * is "ready".
1609: */
1610:
1611: while (!src_object->pager_ready) {
1612: vm_object_wait( src_object,
1613: VM_OBJECT_EVENT_PAGER_READY,
1614: interruptible);
1615: if (interruptible &&
1616: (current_thread()->wait_result != THREAD_AWAKENED)) {
1617: *dst_object = VM_OBJECT_NULL;
1618: *dst_offset = 0;
1619: *dst_needs_copy = FALSE;
1620: return MACH_SEND_INTERRUPTED;
1621: }
1622: vm_object_lock(src_object);
1623: }
1624:
1625: /*
1626: * The object may be temporary (even though it is external).
1627: * If so, do a symmetric copy.
1628: */
1629:
1630: if (src_object->temporary) {
1631: /*
1632: * XXX
1633: * This does not count as intelligent!
1634: * This buys us the object->temporary optimizations,
1635: * but we aren't using a symmetric copy,
1636: * which may confuse the vm code. The correct thing
1637: * to do here is to figure out what to call to get
1638: * a temporary shadowing set up.
1639: */
1640: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1641: }
1642:
1643: /*
1644: * The object is permanent. Use the appropriate copy strategy.
1645: */
1646:
1647: switch (src_object->copy_strategy) {
1648: case MEMORY_OBJECT_COPY_NONE:
1649: if ((result = vm_object_copy_slowly(
1650: src_object,
1651: src_offset,
1652: size,
1653: interruptible,
1654: dst_object))
1655: == KERN_SUCCESS) {
1656: *dst_offset = 0;
1657: *dst_needs_copy = FALSE;
1658: }
1659: break;
1660:
1661: case MEMORY_OBJECT_COPY_CALL:
1662: if ((result = vm_object_copy_call(
1663: src_object,
1664: src_offset,
1665: size,
1666: dst_object))
1667: == KERN_SUCCESS) {
1668: *dst_offset = 0;
1669: *dst_needs_copy = FALSE;
1670: }
1671: break;
1672:
1673: case MEMORY_OBJECT_COPY_DELAY:
1674: vm_object_unlock(src_object);
1675: *dst_object = vm_object_copy_delayed(src_object);
1676: *dst_offset = src_offset;
1677: *dst_needs_copy = TRUE;
1678:
1679: result = KERN_SUCCESS;
1680: break;
1681: }
1682:
1683: return result;
1684: }
1685:
1686: /*
1687: * vm_object_shadow:
1688: *
1689: * Create a new object which is backed by the
1690: * specified existing object range. The source
1691: * object reference is deallocated.
1692: *
1693: * The new object and offset into that object
1694: * are returned in the source parameters.
1695: */
1696:
1697: void vm_object_shadow(
1698: vm_object_t *object, /* IN/OUT */
1699: vm_offset_t *offset, /* IN/OUT */
1700: vm_size_t length)
1701: {
1.1.1.3 root 1702: vm_object_t source;
1703: vm_object_t result;
1.1 root 1704:
1705: source = *object;
1706:
1707: /*
1708: * Allocate a new object with the given length
1709: */
1710:
1711: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1712: panic("vm_object_shadow: no object for shadowing");
1713:
1714: /*
1715: * The new object shadows the source object, adding
1716: * a reference to it. Our caller changes his reference
1717: * to point to the new object, removing a reference to
1718: * the source object. Net result: no change of reference
1719: * count.
1720: */
1721: result->shadow = source;
1722:
1723: /*
1724: * Store the offset into the source object,
1725: * and fix up the offset into the new object.
1726: */
1727:
1728: result->shadow_offset = *offset;
1729:
1730: /*
1731: * Return the new things
1732: */
1733:
1734: *offset = 0;
1735: *object = result;
1736: }
1737:
1738: /*
1739: * The relationship between vm_object structures and
1740: * the memory_object ports requires careful synchronization.
1741: *
1742: * All associations are created by vm_object_enter. All three
1743: * port fields are filled in, as follows:
1744: * pager: the memory_object port itself, supplied by
1745: * the user requesting a mapping (or the kernel,
1746: * when initializing internal objects); the
1747: * kernel simulates holding send rights by keeping
1748: * a port reference;
1749: * pager_request:
1750: * pager_name:
1751: * the memory object control and name ports,
1752: * created by the kernel; the kernel holds
1753: * receive (and ownership) rights to these
1754: * ports, but no other references.
1755: * All of the ports are referenced by their global names.
1756: *
1757: * When initialization is complete, the "initialized" field
1758: * is asserted. Other mappings using a particular memory object,
1759: * and any references to the vm_object gained through the
1760: * port association must wait for this initialization to occur.
1761: *
1762: * In order to allow the memory manager to set attributes before
1763: * requests (notably virtual copy operations, but also data or
1764: * unlock requests) are made, a "ready" attribute is made available.
1765: * Only the memory manager may affect the value of this attribute.
1766: * Its value does not affect critical kernel functions, such as
1767: * internal object initialization or destruction. [Furthermore,
1768: * memory objects created by the kernel are assumed to be ready
1769: * immediately; the default memory manager need not explicitly
1770: * set the "ready" attribute.]
1771: *
1772: * [Both the "initialized" and "ready" attribute wait conditions
1773: * use the "pager" field as the wait event.]
1774: *
1775: * The port associations can be broken down by any of the
1776: * following routines:
1777: * vm_object_terminate:
1778: * No references to the vm_object remain, and
1779: * the object cannot (or will not) be cached.
1780: * This is the normal case, and is done even
1781: * though one of the other cases has already been
1782: * done.
1783: * vm_object_destroy:
1784: * The memory_object port has been destroyed,
1785: * meaning that the kernel cannot flush dirty
1786: * pages or request new data or unlock existing
1787: * data.
1788: * memory_object_destroy:
1789: * The memory manager has requested that the
1790: * kernel relinquish rights to the memory object
1791: * port. [The memory manager may not want to
1792: * destroy the port, but may wish to refuse or
1793: * tear down existing memory mappings.]
1794: * Each routine that breaks an association must break all of
1795: * them at once. At some later time, that routine must clear
1796: * the vm_object port fields and release the port rights.
1797: * [Furthermore, each routine must cope with the simultaneous
1798: * or previous operations of the others.]
1799: *
1800: * In addition to the lock on the object, the vm_object_cache_lock
1801: * governs the port associations. References gained through the
1802: * port association require use of the cache lock.
1803: *
1804: * Because the port fields may be cleared spontaneously, they
1805: * cannot be used to determine whether a memory object has
1806: * ever been associated with a particular vm_object. [This
1807: * knowledge is important to the shadow object mechanism.]
1808: * For this reason, an additional "created" attribute is
1809: * provided.
1810: *
1811: * During various paging operations, the port values found in the
1812: * vm_object must be valid. To prevent these port rights from being
1813: * released, and to prevent the port associations from changing
1814: * (other than being removed, i.e., made null), routines may use
1815: * the vm_object_paging_begin/end routines [actually, macros].
1816: * The implementation uses the "paging_in_progress" and "wanted" fields.
1817: * [Operations that alter the validity of the port values include the
1818: * termination routines and vm_object_collapse.]
1819: */
1820:
1821: vm_object_t vm_object_lookup(
1822: ipc_port_t port)
1823: {
1824: vm_object_t object = VM_OBJECT_NULL;
1825:
1826: if (IP_VALID(port)) {
1827: ip_lock(port);
1828: if (ip_active(port) &&
1829: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1830: vm_object_cache_lock();
1831: object = (vm_object_t) port->ip_kobject;
1832: vm_object_lock(object);
1833:
1834: assert(object->alive);
1835:
1836: if (object->ref_count == 0) {
1837: queue_remove(&vm_object_cached_list, object,
1838: vm_object_t, cached_list);
1839: vm_object_cached_count--;
1.1.1.2 root 1840: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1841: }
1842:
1843: object->ref_count++;
1844: vm_object_unlock(object);
1845: vm_object_cache_unlock();
1846: }
1847: ip_unlock(port);
1848: }
1849:
1850: return object;
1851: }
1852:
1853: vm_object_t vm_object_lookup_name(
1854: ipc_port_t port)
1855: {
1856: vm_object_t object = VM_OBJECT_NULL;
1857:
1858: if (IP_VALID(port)) {
1859: ip_lock(port);
1860: if (ip_active(port) &&
1861: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1862: vm_object_cache_lock();
1863: object = (vm_object_t) port->ip_kobject;
1864: vm_object_lock(object);
1865:
1866: assert(object->alive);
1867:
1868: if (object->ref_count == 0) {
1869: queue_remove(&vm_object_cached_list, object,
1870: vm_object_t, cached_list);
1871: vm_object_cached_count--;
1.1.1.2 root 1872: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1873: }
1874:
1875: object->ref_count++;
1876: vm_object_unlock(object);
1877: vm_object_cache_unlock();
1878: }
1879: ip_unlock(port);
1880: }
1881:
1882: return object;
1883: }
1884:
1885: void vm_object_destroy(
1886: ipc_port_t pager)
1887: {
1888: vm_object_t object;
1889: pager_request_t old_request;
1890: ipc_port_t old_name;
1891:
1892: /*
1893: * Perform essentially the same operations as in vm_object_lookup,
1894: * except that this time we look up based on the memory_object
1895: * port, not the control port.
1896: */
1897: vm_object_cache_lock();
1898: if (ip_kotype(pager) != IKOT_PAGER) {
1899: vm_object_cache_unlock();
1900: return;
1901: }
1902:
1903: object = (vm_object_t) pager->ip_kobject;
1904: vm_object_lock(object);
1905: if (object->ref_count == 0) {
1906: queue_remove(&vm_object_cached_list, object,
1907: vm_object_t, cached_list);
1908: vm_object_cached_count--;
1.1.1.2 root 1909: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 1910: }
1911: object->ref_count++;
1912:
1913: object->can_persist = FALSE;
1914:
1915: assert(object->pager == pager);
1916:
1917: /*
1918: * Remove the port associations.
1919: *
1920: * Note that the memory_object itself is dead, so
1921: * we don't bother with it.
1922: */
1923:
1924: object->pager = IP_NULL;
1925: vm_object_remove(object);
1926:
1927: old_request = object->pager_request;
1928: object->pager_request = PAGER_REQUEST_NULL;
1929:
1930: old_name = object->pager_name;
1931: object->pager_name = IP_NULL;
1932:
1933: vm_object_unlock(object);
1934: vm_object_cache_unlock();
1935:
1936: /*
1937: * Clean up the port references. Note that there's no
1938: * point in trying the memory_object_terminate call
1939: * because the memory_object itself is dead.
1940: */
1941:
1942: ipc_port_release_send(pager);
1943: if (old_request != IP_NULL)
1944: ipc_port_dealloc_kernel(old_request);
1945: if (old_name != IP_NULL)
1946: ipc_port_dealloc_kernel(old_name);
1947:
1948: /*
1949: * Restart pending page requests
1950: */
1951:
1952: vm_object_abort_activity(object);
1953:
1954: /*
1955: * Lose the object reference.
1956: */
1957:
1958: vm_object_deallocate(object);
1959: }
1960:
1961: boolean_t vm_object_accept_old_init_protocol = FALSE;
1962:
1963: /*
1964: * Routine: vm_object_enter
1965: * Purpose:
1966: * Find a VM object corresponding to the given
1967: * pager; if no such object exists, create one,
1968: * and initialize the pager.
1969: */
1970: vm_object_t vm_object_enter(
1971: ipc_port_t pager,
1972: vm_size_t size,
1973: boolean_t internal)
1974: {
1975: vm_object_t object;
1976: vm_object_t new_object;
1977: boolean_t must_init;
1978: ipc_kobject_type_t po;
1979:
1980: restart:
1981: if (!IP_VALID(pager))
1982: return vm_object_allocate(size);
1983:
1984: new_object = VM_OBJECT_NULL;
1985: must_init = FALSE;
1986:
1987: /*
1988: * Look for an object associated with this port.
1989: */
1990:
1991: vm_object_cache_lock();
1992: for (;;) {
1993: po = ip_kotype(pager);
1994:
1995: /*
1996: * If a previous object is being terminated,
1997: * we must wait for the termination message
1998: * to be queued.
1999: *
2000: * We set kobject to a non-null value to let the
2001: * terminator know that someone is waiting.
2002: * Among the possibilities is that the port
2003: * could die while we're waiting. Must restart
2004: * instead of continuing the loop.
2005: */
2006:
2007: if (po == IKOT_PAGER_TERMINATING) {
2008: pager->ip_kobject = (ipc_kobject_t) pager;
2009: assert_wait((event_t) pager, FALSE);
2010: vm_object_cache_unlock();
2011: thread_block((void (*)()) 0);
2012: goto restart;
2013: }
2014:
2015: /*
2016: * Bail if there is already a kobject associated
2017: * with the pager port.
2018: */
2019: if (po != IKOT_NONE) {
2020: break;
2021: }
2022:
2023: /*
2024: * We must unlock to create a new object;
2025: * if we do so, we must try the lookup again.
2026: */
2027:
2028: if (new_object == VM_OBJECT_NULL) {
2029: vm_object_cache_unlock();
2030: new_object = vm_object_allocate(size);
2031: vm_object_cache_lock();
2032: } else {
2033: /*
2034: * Lookup failed twice, and we have something
2035: * to insert; set the object.
2036: */
2037:
2038: ipc_kobject_set(pager,
2039: (ipc_kobject_t) new_object,
2040: IKOT_PAGER);
2041: new_object = VM_OBJECT_NULL;
2042: must_init = TRUE;
2043: }
2044: }
2045:
2046: if (internal)
2047: must_init = TRUE;
2048:
2049: /*
2050: * It's only good if it's a VM object!
2051: */
2052:
2053: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2054: : VM_OBJECT_NULL;
2055:
2056: if ((object != VM_OBJECT_NULL) && !must_init) {
2057: vm_object_lock(object);
2058: if (object->ref_count == 0) {
2059: queue_remove(&vm_object_cached_list, object,
2060: vm_object_t, cached_list);
2061: vm_object_cached_count--;
1.1.1.2 root 2062: vm_object_cached_pages_update(-object->resident_page_count);
1.1 root 2063: }
2064: object->ref_count++;
2065: vm_object_unlock(object);
2066:
2067: vm_stat.hits++;
2068: }
2069: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2070: ((object->paging_in_progress != 0) && internal));
2071:
2072: vm_stat.lookups++;
2073:
2074: vm_object_cache_unlock();
2075:
2076: /*
2077: * If we raced to create a vm_object but lost, let's
2078: * throw away ours.
2079: */
2080:
2081: if (new_object != VM_OBJECT_NULL)
2082: vm_object_deallocate(new_object);
2083:
2084: if (object == VM_OBJECT_NULL)
2085: return(object);
2086:
2087: if (must_init) {
2088: /*
2089: * Copy the naked send right we were given.
2090: */
2091:
2092: pager = ipc_port_copy_send(pager);
2093: if (!IP_VALID(pager))
2094: panic("vm_object_enter: port died"); /* XXX */
2095:
2096: object->pager_created = TRUE;
2097: object->pager = pager;
2098:
2099: /*
2100: * Allocate request port.
2101: */
2102:
2103: object->pager_request = ipc_port_alloc_kernel();
2104: if (object->pager_request == IP_NULL)
2105: panic("vm_object_enter: pager request alloc");
2106:
2107: ipc_kobject_set(object->pager_request,
2108: (ipc_kobject_t) object,
2109: IKOT_PAGING_REQUEST);
2110:
2111: /*
2112: * Let the pager know we're using it.
2113: */
2114:
2115: if (internal) {
2116: /* acquire a naked send right for the DMM */
2117: ipc_port_t DMM = memory_manager_default_reference();
2118:
2119: /* mark the object internal */
2120: object->internal = TRUE;
2121: assert(object->temporary);
2122:
2123: /* default-pager objects are ready immediately */
2124: object->pager_ready = TRUE;
2125:
2126: /* consumes the naked send right for DMM */
2127: (void) memory_object_create(DMM,
2128: pager,
2129: object->size,
2130: object->pager_request,
2131: object->pager_name,
2132: PAGE_SIZE);
2133: } else {
2134: /* the object is external and not temporary */
2135: object->internal = FALSE;
2136: object->temporary = FALSE;
2137:
2138: /* user pager objects are not ready until marked so */
2139: object->pager_ready = FALSE;
2140:
2141: (void) memory_object_init(pager,
2142: object->pager_request,
2143: object->pager_name,
2144: PAGE_SIZE);
2145:
2146: }
2147:
2148: vm_object_lock(object);
2149: object->pager_initialized = TRUE;
2150:
2151: if (vm_object_accept_old_init_protocol)
2152: object->pager_ready = TRUE;
2153:
2154: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2155: } else {
2156: vm_object_lock(object);
2157: }
2158: /*
2159: * [At this point, the object must be locked]
2160: */
2161:
2162: /*
2163: * Wait for the work above to be done by the first
2164: * thread to map this object.
2165: */
2166:
2167: while (!object->pager_initialized) {
2168: vm_object_wait( object,
2169: VM_OBJECT_EVENT_INITIALIZED,
2170: FALSE);
2171: vm_object_lock(object);
2172: }
2173: vm_object_unlock(object);
2174:
2175: return object;
2176: }
2177:
2178: /*
2179: * Routine: vm_object_pager_create
2180: * Purpose:
2181: * Create a memory object for an internal object.
2182: * In/out conditions:
2183: * The object is locked on entry and exit;
2184: * it may be unlocked within this call.
2185: * Limitations:
2186: * Only one thread may be performing a
2187: * vm_object_pager_create on an object at
2188: * a time. Presumably, only the pageout
2189: * daemon will be using this routine.
2190: */
2191: void vm_object_pager_create(
2192: vm_object_t object)
2193: {
2194: ipc_port_t pager;
2195:
2196: if (object->pager_created) {
2197: /*
2198: * Someone else got to it first...
2199: * wait for them to finish initializing
2200: */
2201:
2202: while (!object->pager_initialized) {
2203: vm_object_wait( object,
2204: VM_OBJECT_EVENT_PAGER_READY,
2205: FALSE);
2206: vm_object_lock(object);
2207: }
2208: return;
2209: }
2210:
2211: /*
2212: * Indicate that a memory object has been assigned
2213: * before dropping the lock, to prevent a race.
2214: */
2215:
2216: object->pager_created = TRUE;
2217:
2218: /*
2219: * Prevent collapse or termination by
2220: * holding a paging reference
2221: */
2222:
2223: vm_object_paging_begin(object);
2224: vm_object_unlock(object);
2225:
2226: #if MACH_PAGEMAP
2227: object->existence_info = vm_external_create(
2228: object->size +
2229: object->paging_offset);
2230: assert((object->size + object->paging_offset) >=
2231: object->size);
2232: #endif /* MACH_PAGEMAP */
2233:
2234: /*
2235: * Create the pager, and associate with it
2236: * this object.
2237: *
2238: * Note that we only make the port association
2239: * so that vm_object_enter can properly look up
2240: * the object to complete the initialization...
2241: * we do not expect any user to ever map this
2242: * object.
2243: *
2244: * Since the kernel has the only rights to the
2245: * port, it's safe to install the association
2246: * without holding the cache lock.
2247: */
2248:
2249: pager = ipc_port_alloc_kernel();
2250: if (pager == IP_NULL)
2251: panic("vm_object_pager_create: allocate pager port");
2252:
2253: (void) ipc_port_make_send(pager);
2254: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2255:
2256: /*
2257: * Initialize the rest of the paging stuff
2258: */
2259:
2260: if (vm_object_enter(pager, object->size, TRUE) != object)
2261: panic("vm_object_pager_create: mismatch");
2262:
2263: /*
2264: * Drop the naked send right taken above.
2265: */
2266:
2267: ipc_port_release_send(pager);
2268:
2269: /*
2270: * Release the paging reference
2271: */
2272:
2273: vm_object_lock(object);
2274: vm_object_paging_end(object);
2275: }
2276:
2277: /*
2278: * Routine: vm_object_remove
2279: * Purpose:
2280: * Eliminate the pager/object association
2281: * for this pager.
2282: * Conditions:
2283: * The object cache must be locked.
2284: */
2285: void vm_object_remove(
2286: vm_object_t object)
2287: {
2288: ipc_port_t port;
2289:
2290: if ((port = object->pager) != IP_NULL) {
2291: if (ip_kotype(port) == IKOT_PAGER)
2292: ipc_kobject_set(port, IKO_NULL,
2293: IKOT_PAGER_TERMINATING);
2294: else if (ip_kotype(port) != IKOT_NONE)
2295: panic("vm_object_remove: bad object port");
2296: }
2297: if ((port = object->pager_request) != IP_NULL) {
2298: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2299: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2300: else if (ip_kotype(port) != IKOT_NONE)
2301: panic("vm_object_remove: bad request port");
2302: }
2303: if ((port = object->pager_name) != IP_NULL) {
2304: if (ip_kotype(port) == IKOT_PAGING_NAME)
2305: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2306: else if (ip_kotype(port) != IKOT_NONE)
2307: panic("vm_object_remove: bad name port");
2308: }
2309: }
2310:
2311: /*
2312: * Global variables for vm_object_collapse():
2313: *
2314: * Counts for normal collapses and bypasses.
2315: * Debugging variables, to watch or disable collapse.
2316: */
2317: long object_collapses = 0;
2318: long object_bypasses = 0;
2319:
2320: int vm_object_collapse_debug = 0;
2321: boolean_t vm_object_collapse_allowed = TRUE;
2322: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2323:
2324: /*
2325: * vm_object_collapse:
2326: *
2327: * Collapse an object with the object backing it.
2328: * Pages in the backing object are moved into the
2329: * parent, and the backing object is deallocated.
2330: *
2331: * Requires that the object be locked and the page
2332: * queues be unlocked. May unlock/relock the object,
2333: * so the caller should hold a reference for the object.
2334: */
2335: void vm_object_collapse(
1.1.1.3 root 2336: vm_object_t object)
1.1 root 2337: {
1.1.1.3 root 2338: vm_object_t backing_object;
2339: vm_offset_t backing_offset;
2340: vm_size_t size;
2341: vm_offset_t new_offset;
2342: vm_page_t p, pp;
2343: ipc_port_t old_name_port;
1.1 root 2344:
2345: if (!vm_object_collapse_allowed)
2346: return;
2347:
2348: while (TRUE) {
2349: /*
2350: * Verify that the conditions are right for collapse:
2351: *
2352: * The object exists and no pages in it are currently
2353: * being paged out (or have ever been paged out).
2354: *
2355: * This check is probably overkill -- if a memory
2356: * object has not been created, the fault handler
2357: * shouldn't release the object lock while paging
2358: * is in progress or absent pages exist.
2359: */
2360: if (object == VM_OBJECT_NULL ||
2361: object->pager_created ||
2362: object->paging_in_progress != 0 ||
2363: object->absent_count != 0)
2364: return;
2365:
2366: /*
2367: * There is a backing object, and
2368: */
2369:
2370: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2371: return;
2372:
2373: vm_object_lock(backing_object);
2374: /*
2375: * ...
2376: * The backing object is not read_only,
2377: * and no pages in the backing object are
2378: * currently being paged out.
2379: * The backing object is internal.
2380: *
2381: * XXX It may be sufficient for the backing
2382: * XXX object to be temporary.
2383: */
2384:
2385: if (!backing_object->internal ||
2386: backing_object->paging_in_progress != 0) {
2387: vm_object_unlock(backing_object);
2388: return;
2389: }
2390:
2391: /*
2392: * The backing object can't be a copy-object:
2393: * the shadow_offset for the copy-object must stay
2394: * as 0. Furthermore (for the 'we have all the
2395: * pages' case), if we bypass backing_object and
2396: * just shadow the next object in the chain, old
2397: * pages from that object would then have to be copied
2398: * BOTH into the (former) backing_object and into the
2399: * parent object.
2400: */
2401: if (backing_object->shadow != VM_OBJECT_NULL &&
2402: backing_object->shadow->copy != VM_OBJECT_NULL) {
2403: vm_object_unlock(backing_object);
2404: return;
2405: }
2406:
2407: /*
2408: * We know that we can either collapse the backing
2409: * object (if the parent is the only reference to
2410: * it) or (perhaps) remove the parent's reference
2411: * to it.
2412: */
2413:
2414: backing_offset = object->shadow_offset;
2415: size = object->size;
2416:
2417: /*
2418: * If there is exactly one reference to the backing
2419: * object, we can collapse it into the parent.
2420: */
2421:
2422: if (backing_object->ref_count == 1) {
2423: if (!vm_object_cache_lock_try()) {
2424: vm_object_unlock(backing_object);
2425: return;
2426: }
2427:
2428: /*
2429: * We can collapse the backing object.
2430: *
2431: * Move all in-memory pages from backing_object
2432: * to the parent. Pages that have been paged out
2433: * will be overwritten by any of the parent's
2434: * pages that shadow them.
2435: */
2436:
2437: while (!queue_empty(&backing_object->memq)) {
2438:
2439: p = (vm_page_t)
2440: queue_first(&backing_object->memq);
2441:
2442: new_offset = (p->offset - backing_offset);
2443:
2444: assert(!p->busy || p->absent);
2445:
2446: /*
2447: * If the parent has a page here, or if
2448: * this page falls outside the parent,
2449: * dispose of it.
2450: *
2451: * Otherwise, move it as planned.
2452: */
2453:
2454: if (p->offset < backing_offset ||
2455: new_offset >= size) {
1.1.1.2 root 2456: VM_PAGE_FREE(p);
1.1 root 2457: } else {
2458: pp = vm_page_lookup(object, new_offset);
2459: if (pp != VM_PAGE_NULL && !pp->absent) {
2460: /*
2461: * Parent object has a real page.
2462: * Throw away the backing object's
2463: * page.
2464: */
1.1.1.2 root 2465: VM_PAGE_FREE(p);
1.1 root 2466: }
2467: else {
1.1.1.4 ! root 2468: assert(pp == VM_PAGE_NULL || !
! 2469: "vm_object_collapse: bad case");
! 2470:
1.1 root 2471: /*
2472: * Parent now has no page.
2473: * Move the backing object's page up.
2474: */
2475: vm_page_rename(p, object, new_offset);
2476: }
2477: }
2478: }
2479:
2480: /*
2481: * Move the pager from backing_object to object.
2482: *
2483: * XXX We're only using part of the paging space
2484: * for keeps now... we ought to discard the
2485: * unused portion.
2486: */
2487:
2488: switch (vm_object_collapse_debug) {
2489: case 0:
2490: break;
2491: case 1:
2492: if ((backing_object->pager == IP_NULL) &&
2493: (backing_object->pager_request ==
2494: PAGER_REQUEST_NULL))
2495: break;
2496: /* Fall through to... */
2497:
2498: default:
1.1.1.2 root 2499: printf("vm_object_collapse: %p (pager %p, request %p) up to %p\n",
1.1 root 2500: backing_object, backing_object->pager, backing_object->pager_request,
2501: object);
2502: if (vm_object_collapse_debug > 2)
1.1.1.2 root 2503: SoftDebugger("vm_object_collapse");
1.1 root 2504: }
2505:
2506: object->pager = backing_object->pager;
2507: if (object->pager != IP_NULL)
2508: ipc_kobject_set(object->pager,
2509: (ipc_kobject_t) object,
2510: IKOT_PAGER);
2511: object->pager_initialized = backing_object->pager_initialized;
2512: object->pager_ready = backing_object->pager_ready;
2513: object->pager_created = backing_object->pager_created;
2514:
2515: object->pager_request = backing_object->pager_request;
2516: if (object->pager_request != IP_NULL)
2517: ipc_kobject_set(object->pager_request,
2518: (ipc_kobject_t) object,
2519: IKOT_PAGING_REQUEST);
2520: old_name_port = object->pager_name;
2521: if (old_name_port != IP_NULL)
2522: ipc_kobject_set(old_name_port,
2523: IKO_NULL, IKOT_NONE);
2524: object->pager_name = backing_object->pager_name;
2525: if (object->pager_name != IP_NULL)
2526: ipc_kobject_set(object->pager_name,
2527: (ipc_kobject_t) object,
2528: IKOT_PAGING_NAME);
2529:
2530: vm_object_cache_unlock();
2531:
2532: /*
2533: * If there is no pager, leave paging-offset alone.
2534: */
2535: if (object->pager != IP_NULL)
2536: object->paging_offset =
2537: backing_object->paging_offset +
2538: backing_offset;
2539:
2540: #if MACH_PAGEMAP
2541: assert(object->existence_info == VM_EXTERNAL_NULL);
2542: object->existence_info = backing_object->existence_info;
2543: #endif /* MACH_PAGEMAP */
2544:
2545: /*
2546: * Object now shadows whatever backing_object did.
2547: * Note that the reference to backing_object->shadow
2548: * moves from within backing_object to within object.
2549: */
2550:
2551: object->shadow = backing_object->shadow;
2552: object->shadow_offset += backing_object->shadow_offset;
2553: if (object->shadow != VM_OBJECT_NULL &&
2554: object->shadow->copy != VM_OBJECT_NULL) {
2555: panic("vm_object_collapse: we collapsed a copy-object!");
2556: }
2557: /*
2558: * Discard backing_object.
2559: *
2560: * Since the backing object has no pages, no
2561: * pager left, and no object references within it,
2562: * all that is necessary is to dispose of it.
2563: */
2564:
2565: assert(
2566: (backing_object->ref_count == 1) &&
2567: (backing_object->resident_page_count == 0) &&
2568: (backing_object->paging_in_progress == 0)
2569: );
2570:
2571: assert(backing_object->alive);
2572: backing_object->alive = FALSE;
2573: vm_object_unlock(backing_object);
2574:
2575: vm_object_unlock(object);
2576: if (old_name_port != IP_NULL)
2577: ipc_port_dealloc_kernel(old_name_port);
1.1.1.2 root 2578: kmem_cache_free(&vm_object_cache, (vm_offset_t) backing_object);
1.1 root 2579: vm_object_lock(object);
2580:
2581: object_collapses++;
2582: }
2583: else {
2584: if (!vm_object_collapse_bypass_allowed) {
2585: vm_object_unlock(backing_object);
2586: return;
2587: }
2588:
2589: /*
2590: * If all of the pages in the backing object are
2591: * shadowed by the parent object, the parent
2592: * object no longer has to shadow the backing
2593: * object; it can shadow the next one in the
2594: * chain.
2595: *
2596: * The backing object must not be paged out - we'd
2597: * have to check all of the paged-out pages, as
2598: * well.
2599: */
2600:
2601: if (backing_object->pager_created) {
2602: vm_object_unlock(backing_object);
2603: return;
2604: }
2605:
2606: /*
2607: * Should have a check for a 'small' number
2608: * of pages here.
2609: */
2610:
2611: queue_iterate(&backing_object->memq, p,
2612: vm_page_t, listq)
2613: {
2614: new_offset = (p->offset - backing_offset);
2615:
2616: /*
2617: * If the parent has a page here, or if
2618: * this page falls outside the parent,
2619: * keep going.
2620: *
2621: * Otherwise, the backing_object must be
2622: * left in the chain.
2623: */
2624:
2625: if (p->offset >= backing_offset &&
2626: new_offset <= size &&
2627: (pp = vm_page_lookup(object, new_offset))
2628: == VM_PAGE_NULL) {
2629: /*
2630: * Page still needed.
2631: * Can't go any further.
2632: */
2633: vm_object_unlock(backing_object);
2634: return;
2635: }
2636: }
2637:
2638: /*
2639: * Make the parent shadow the next object
2640: * in the chain. Deallocating backing_object
2641: * will not remove it, since its reference
2642: * count is at least 2.
2643: */
2644:
2645: vm_object_reference(object->shadow = backing_object->shadow);
2646: object->shadow_offset += backing_object->shadow_offset;
2647:
2648: /*
2649: * Backing object might have had a copy pointer
2650: * to us. If it did, clear it.
2651: */
2652: if (backing_object->copy == object)
2653: backing_object->copy = VM_OBJECT_NULL;
2654:
2655: /*
2656: * Drop the reference count on backing_object.
2657: * Since its ref_count was at least 2, it
2658: * will not vanish; so we don't need to call
2659: * vm_object_deallocate.
2660: */
2661: backing_object->ref_count--;
2662: assert(backing_object->ref_count > 0);
2663: vm_object_unlock(backing_object);
2664:
2665: object_bypasses ++;
2666:
2667: }
2668:
2669: /*
2670: * Try again with this object's new backing object.
2671: */
2672: }
2673: }
2674:
2675: /*
2676: * Routine: vm_object_page_remove: [internal]
2677: * Purpose:
2678: * Removes all physical pages in the specified
2679: * object range from the object's list of pages.
2680: *
2681: * In/out conditions:
2682: * The object must be locked.
2683: */
2684: unsigned int vm_object_page_remove_lookup = 0;
2685: unsigned int vm_object_page_remove_iterate = 0;
2686:
2687: void vm_object_page_remove(
1.1.1.3 root 2688: vm_object_t object,
2689: vm_offset_t start,
2690: vm_offset_t end)
1.1 root 2691: {
1.1.1.3 root 2692: vm_page_t p, next;
1.1 root 2693:
2694: /*
2695: * One and two page removals are most popular.
2696: * The factor of 16 here is somewhat arbitrary.
2697: * It balances vm_object_lookup vs iteration.
2698: */
2699:
2700: if (atop(end - start) < (unsigned)object->resident_page_count/16) {
2701: vm_object_page_remove_lookup++;
2702:
2703: for (; start < end; start += PAGE_SIZE) {
2704: p = vm_page_lookup(object, start);
2705: if (p != VM_PAGE_NULL) {
2706: if (!p->fictitious)
2707: pmap_page_protect(p->phys_addr,
2708: VM_PROT_NONE);
1.1.1.2 root 2709: VM_PAGE_FREE(p);
1.1 root 2710: }
2711: }
2712: } else {
2713: vm_object_page_remove_iterate++;
2714:
2715: p = (vm_page_t) queue_first(&object->memq);
2716: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2717: next = (vm_page_t) queue_next(&p->listq);
2718: if ((start <= p->offset) && (p->offset < end)) {
2719: if (!p->fictitious)
2720: pmap_page_protect(p->phys_addr,
2721: VM_PROT_NONE);
1.1.1.2 root 2722: VM_PAGE_FREE(p);
1.1 root 2723: }
2724: p = next;
2725: }
2726: }
2727: }
2728:
2729: /*
2730: * Routine: vm_object_coalesce
2731: * Function: Coalesces two objects backing up adjoining
2732: * regions of memory into a single object.
2733: *
2734: * returns TRUE if objects were combined.
2735: *
2736: * NOTE: Only works at the moment if the second object is NULL -
2737: * if it's not, which object do we lock first?
2738: *
2739: * Parameters:
2740: * prev_object First object to coalesce
2741: * prev_offset Offset into prev_object
2742: * next_object Second object into coalesce
2743: * next_offset Offset into next_object
2744: *
2745: * prev_size Size of reference to prev_object
2746: * next_size Size of reference to next_object
2747: *
2748: * Conditions:
2749: * The object must *not* be locked.
2750: */
2751:
2752: boolean_t vm_object_coalesce(
1.1.1.3 root 2753: vm_object_t prev_object,
1.1 root 2754: vm_object_t next_object,
2755: vm_offset_t prev_offset,
2756: vm_offset_t next_offset,
2757: vm_size_t prev_size,
2758: vm_size_t next_size)
2759: {
2760: vm_size_t newsize;
2761:
2762: if (next_object != VM_OBJECT_NULL) {
2763: return FALSE;
2764: }
2765:
2766: if (prev_object == VM_OBJECT_NULL) {
2767: return TRUE;
2768: }
2769:
2770: vm_object_lock(prev_object);
2771:
2772: /*
2773: * Try to collapse the object first
2774: */
2775: vm_object_collapse(prev_object);
2776:
2777: /*
2778: * Can't coalesce if pages not mapped to
2779: * prev_entry may be in use anyway:
2780: * . more than one reference
2781: * . paged out
2782: * . shadows another object
2783: * . has a copy elsewhere
2784: * . paging references (pages might be in page-list)
2785: */
2786:
2787: if ((prev_object->ref_count > 1) ||
2788: prev_object->pager_created ||
2789: (prev_object->shadow != VM_OBJECT_NULL) ||
2790: (prev_object->copy != VM_OBJECT_NULL) ||
2791: (prev_object->paging_in_progress != 0)) {
2792: vm_object_unlock(prev_object);
2793: return FALSE;
2794: }
2795:
2796: /*
2797: * Remove any pages that may still be in the object from
2798: * a previous deallocation.
2799: */
2800:
2801: vm_object_page_remove(prev_object,
2802: prev_offset + prev_size,
2803: prev_offset + prev_size + next_size);
2804:
2805: /*
2806: * Extend the object if necessary.
2807: */
2808: newsize = prev_offset + prev_size + next_size;
2809: if (newsize > prev_object->size)
2810: prev_object->size = newsize;
2811:
2812: vm_object_unlock(prev_object);
2813: return TRUE;
2814: }
2815:
2816: vm_object_t vm_object_request_object(
2817: ipc_port_t p)
2818: {
2819: return vm_object_lookup(p);
2820: }
2821:
2822: /*
2823: * Routine: vm_object_name
2824: * Purpose:
2825: * Returns a naked send right to the "name" port associated
2826: * with this object.
2827: */
2828: ipc_port_t vm_object_name(
2829: vm_object_t object)
2830: {
2831: ipc_port_t p;
2832:
2833: if (object == VM_OBJECT_NULL)
2834: return IP_NULL;
2835:
2836: vm_object_lock(object);
2837:
2838: while (object->shadow != VM_OBJECT_NULL) {
2839: vm_object_t new_object = object->shadow;
2840: vm_object_lock(new_object);
2841: vm_object_unlock(object);
2842: object = new_object;
2843: }
2844:
2845: p = object->pager_name;
2846: if (p != IP_NULL)
2847: p = ipc_port_make_send(p);
2848: vm_object_unlock(object);
2849:
2850: return p;
2851: }
2852:
2853: /*
2854: * Attach a set of physical pages to an object, so that they can
2855: * be mapped by mapping the object. Typically used to map IO memory.
2856: *
2857: * The mapping function and its private data are used to obtain the
2858: * physical addresses for each page to be mapped.
2859: */
2860: void
2861: vm_object_page_map(
2862: vm_object_t object,
2863: vm_offset_t offset,
2864: vm_size_t size,
2865: vm_offset_t (*map_fn)(void *, vm_offset_t),
2866: void * map_fn_data) /* private to map_fn */
2867: {
2868: int num_pages;
2869: int i;
2870: vm_page_t m;
2871: vm_page_t old_page;
2872: vm_offset_t addr;
2873:
2874: num_pages = atop(size);
2875:
2876: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2877:
2878: addr = (*map_fn)(map_fn_data, offset);
2879:
2880: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
2881: vm_page_more_fictitious();
2882:
2883: vm_object_lock(object);
2884: if ((old_page = vm_page_lookup(object, offset))
2885: != VM_PAGE_NULL)
2886: {
1.1.1.2 root 2887: VM_PAGE_FREE(old_page);
1.1 root 2888: }
2889:
2890: vm_page_init(m, addr);
2891: m->private = TRUE; /* don`t free page */
2892: m->wire_count = 1;
2893: vm_page_lock_queues();
2894: vm_page_insert(m, object, offset);
2895: vm_page_unlock_queues();
2896:
2897: PAGE_WAKEUP_DONE(m);
2898: vm_object_unlock(object);
2899: }
2900: }
2901:
2902:
2903: #if MACH_KDB
1.1.1.2 root 2904: #include <vm/vm_print.h>
1.1 root 2905: #define printf kdbprintf
2906:
2907: boolean_t vm_object_print_pages = FALSE;
2908:
2909: /*
2910: * vm_object_print: [ debug ]
2911: */
2912: void vm_object_print(
2913: vm_object_t object)
2914: {
1.1.1.3 root 2915: vm_page_t p;
1.1 root 2916:
1.1.1.3 root 2917: int count;
1.1 root 2918:
2919: if (object == VM_OBJECT_NULL)
2920: return;
2921:
2922: iprintf("Object 0x%X: size=0x%X",
2923: (vm_offset_t) object, (vm_offset_t) object->size);
2924: printf(", %d references, %d resident pages,", object->ref_count,
2925: object->resident_page_count);
2926: printf(" %d absent pages,", object->absent_count);
2927: printf(" %d paging ops\n", object->paging_in_progress);
2928: indent += 2;
2929: iprintf("memory object=0x%X (offset=0x%X),",
2930: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
2931: printf("control=0x%X, name=0x%X\n",
2932: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
2933: iprintf("%s%s",
2934: object->pager_ready ? " ready" : "",
2935: object->pager_created ? " created" : "");
2936: printf("%s,%s ",
2937: object->pager_initialized ? "" : "uninitialized",
2938: object->temporary ? "temporary" : "permanent");
2939: printf("%s%s,",
2940: object->internal ? "internal" : "external",
2941: object->can_persist ? " cacheable" : "");
2942: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
2943: iprintf("shadow=0x%X (offset=0x%X),",
2944: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
2945: printf("copy=0x%X\n", (vm_offset_t) object->copy);
2946:
2947: indent += 2;
2948:
2949: if (vm_object_print_pages) {
2950: count = 0;
2951: p = (vm_page_t) queue_first(&object->memq);
2952: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2953: if (count == 0) iprintf("memory:=");
2954: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
2955: else printf(",");
2956: count++;
2957:
2958: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
2959: p = (vm_page_t) queue_next(&p->listq);
2960: }
2961: if (count != 0)
2962: printf("\n");
2963: }
2964: indent -= 4;
2965: }
2966:
2967: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.