![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_object.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_object.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Virtual memory object module.
34: */
35:
36: #include <norma_vm.h>
37: #include <mach_pagemap.h>
38:
39: #if NORMA_VM
40: #include <norma/xmm_server_rename.h>
41: #endif /* NORMA_VM */
42:
43: #include <mach/memory_object.h>
44: #include "memory_object_default.h"
45: #include "memory_object_user.h"
46: #include "vm_param.h"
47: #include <ipc/ipc_port.h>
48: #include <ipc/ipc_space.h>
49: #include <kern/assert.h>
50: #include <kern/lock.h>
51: #include <kern/queue.h>
52: #include <kern/zalloc.h>
53: #include <kern/debug.h>
54: #include <vm/memory_object.h>
55: #include <vm/vm_fault.h>
56: #include <vm/vm_map.h>
57: #include <vm/vm_object.h>
58: #include <vm/vm_page.h>
59: #include <vm/vm_pageout.h>
60:
61:
62: void memory_object_release(
63: ipc_port_t pager,
64: pager_request_t pager_request,
65: ipc_port_t pager_name); /* forward */
66:
67: void vm_object_deactivate_pages(vm_object_t);
68:
69: /*
70: * Virtual memory objects maintain the actual data
71: * associated with allocated virtual memory. A given
72: * page of memory exists within exactly one object.
73: *
74: * An object is only deallocated when all "references"
75: * are given up. Only one "reference" to a given
76: * region of an object should be writeable.
77: *
78: * Associated with each object is a list of all resident
79: * memory pages belonging to that object; this list is
80: * maintained by the "vm_page" module, but locked by the object's
81: * lock.
82: *
83: * Each object also records the memory object port
84: * that is used by the kernel to request and write
85: * back data (the memory object port, field "pager"),
86: * and the ports provided to the memory manager, the server that
87: * manages that data, to return data and control its
88: * use (the memory object control port, field "pager_request")
89: * and for naming (the memory object name port, field "pager_name").
90: *
91: * Virtual memory objects are allocated to provide
92: * zero-filled memory (vm_allocate) or map a user-defined
93: * memory object into a virtual address space (vm_map).
94: *
95: * Virtual memory objects that refer to a user-defined
96: * memory object are called "permanent", because all changes
97: * made in virtual memory are reflected back to the
98: * memory manager, which may then store it permanently.
99: * Other virtual memory objects are called "temporary",
100: * meaning that changes need be written back only when
101: * necessary to reclaim pages, and that storage associated
102: * with the object can be discarded once it is no longer
103: * mapped.
104: *
105: * A permanent memory object may be mapped into more
106: * than one virtual address space. Moreover, two threads
107: * may attempt to make the first mapping of a memory
108: * object concurrently. Only one thread is allowed to
109: * complete this mapping; all others wait for the
110: * "pager_initialized" field is asserted, indicating
111: * that the first thread has initialized all of the
112: * necessary fields in the virtual memory object structure.
113: *
114: * The kernel relies on a *default memory manager* to
115: * provide backing storage for the zero-filled virtual
116: * memory objects. The memory object ports associated
117: * with these temporary virtual memory objects are only
118: * generated and passed to the default memory manager
119: * when it becomes necessary. Virtual memory objects
120: * that depend on the default memory manager are called
121: * "internal". The "pager_created" field is provided to
122: * indicate whether these ports have ever been allocated.
123: *
124: * The kernel may also create virtual memory objects to
125: * hold changed pages after a copy-on-write operation.
126: * In this case, the virtual memory object (and its
127: * backing storage -- its memory object) only contain
128: * those pages that have been changed. The "shadow"
129: * field refers to the virtual memory object that contains
130: * the remainder of the contents. The "shadow_offset"
131: * field indicates where in the "shadow" these contents begin.
132: * The "copy" field refers to a virtual memory object
133: * to which changed pages must be copied before changing
134: * this object, in order to implement another form
135: * of copy-on-write optimization.
136: *
137: * The virtual memory object structure also records
138: * the attributes associated with its memory object.
139: * The "pager_ready", "can_persist" and "copy_strategy"
140: * fields represent those attributes. The "cached_list"
141: * field is used in the implementation of the persistence
142: * attribute.
143: *
144: * ZZZ Continue this comment.
145: */
146:
147: zone_t vm_object_zone; /* vm backing store zone */
148:
149: /*
150: * All wired-down kernel memory belongs to a single virtual
151: * memory object (kernel_object) to avoid wasting data structures.
152: */
153: vm_object_t kernel_object;
154:
155: /*
156: * Virtual memory objects that are not referenced by
157: * any address maps, but that are allowed to persist
158: * (an attribute specified by the associated memory manager),
159: * are kept in a queue (vm_object_cached_list).
160: *
161: * When an object from this queue is referenced again,
162: * for example to make another address space mapping,
163: * it must be removed from the queue. That is, the
164: * queue contains *only* objects with zero references.
165: *
166: * The kernel may choose to terminate objects from this
167: * queue in order to reclaim storage. The current policy
168: * is to permit a fixed maximum number of unreferenced
169: * objects (vm_object_cached_max).
170: *
171: * A simple lock (accessed by routines
172: * vm_object_cache_{lock,lock_try,unlock}) governs the
173: * object cache. It must be held when objects are
174: * added to or removed from the cache (in vm_object_terminate).
175: * The routines that acquire a reference to a virtual
176: * memory object based on one of the memory object ports
177: * must also lock the cache.
178: *
179: * Ideally, the object cache should be more isolated
180: * from the reference mechanism, so that the lock need
181: * not be held to make simple references.
182: */
183: queue_head_t vm_object_cached_list;
184: int vm_object_cached_count;
185: int vm_object_cached_max = 200; /* may be patched*/
186:
187: decl_simple_lock_data(,vm_object_cached_lock_data)
188:
189: #define vm_object_cache_lock() \
190: simple_lock(&vm_object_cached_lock_data)
191: #define vm_object_cache_lock_try() \
192: simple_lock_try(&vm_object_cached_lock_data)
193: #define vm_object_cache_unlock() \
194: simple_unlock(&vm_object_cached_lock_data)
195:
196: /*
197: * Virtual memory objects are initialized from
198: * a template (see vm_object_allocate).
199: *
200: * When adding a new field to the virtual memory
201: * object structure, be sure to add initialization
202: * (see vm_object_init).
203: */
204: vm_object_t vm_object_template;
205:
206: /*
207: * vm_object_allocate:
208: *
209: * Returns a new object with the given size.
210: */
211:
212: vm_object_t _vm_object_allocate(
213: vm_size_t size)
214: {
215: register vm_object_t object;
216:
217: object = (vm_object_t) zalloc(vm_object_zone);
218:
219: *object = *vm_object_template;
220: queue_init(&object->memq);
221: vm_object_lock_init(object);
222: object->size = size;
223:
224: return object;
225: }
226:
227: vm_object_t vm_object_allocate(
228: vm_size_t size)
229: {
230: register vm_object_t object;
231: register ipc_port_t port;
232:
233: object = _vm_object_allocate(size);
234: #if !NORMA_VM
235: port = ipc_port_alloc_kernel();
236: if (port == IP_NULL)
237: panic("vm_object_allocate");
238: object->pager_name = port;
239: ipc_kobject_set(port, (ipc_kobject_t) object, IKOT_PAGING_NAME);
240: #endif /* !NORMA_VM */
241:
242: return object;
243: }
244:
245: /*
246: * vm_object_bootstrap:
247: *
248: * Initialize the VM objects module.
249: */
250: void vm_object_bootstrap(void)
251: {
252: vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
253: round_page(512*1024),
254: round_page(12*1024),
255: 0, "objects");
256:
257: queue_init(&vm_object_cached_list);
258: simple_lock_init(&vm_object_cached_lock_data);
259:
260: /*
261: * Fill in a template object, for quick initialization
262: */
263:
264: vm_object_template = (vm_object_t) zalloc(vm_object_zone);
265: bzero((char *) vm_object_template, sizeof *vm_object_template);
266:
267: vm_object_template->ref_count = 1;
268: vm_object_template->size = 0;
269: vm_object_template->resident_page_count = 0;
270: vm_object_template->copy = VM_OBJECT_NULL;
271: vm_object_template->shadow = VM_OBJECT_NULL;
272: vm_object_template->shadow_offset = (vm_offset_t) 0;
273:
274: vm_object_template->pager = IP_NULL;
275: vm_object_template->paging_offset = 0;
276: vm_object_template->pager_request = PAGER_REQUEST_NULL;
277: vm_object_template->pager_name = IP_NULL;
278:
279: vm_object_template->pager_created = FALSE;
280: vm_object_template->pager_initialized = FALSE;
281: vm_object_template->pager_ready = FALSE;
282:
283: vm_object_template->copy_strategy = MEMORY_OBJECT_COPY_NONE;
284: /* ignored if temporary, will be reset before
285: * permanent object becomes ready */
286: vm_object_template->use_shared_copy = FALSE;
287: vm_object_template->shadowed = FALSE;
288:
289: vm_object_template->absent_count = 0;
290: vm_object_template->all_wanted = 0; /* all bits FALSE */
291:
292: vm_object_template->paging_in_progress = 0;
293: vm_object_template->can_persist = FALSE;
294: vm_object_template->internal = TRUE;
295: vm_object_template->temporary = TRUE;
296: vm_object_template->alive = TRUE;
297: vm_object_template->lock_in_progress = FALSE;
298: vm_object_template->lock_restart = FALSE;
299: vm_object_template->use_old_pageout = TRUE; /* XXX change later */
300: vm_object_template->last_alloc = (vm_offset_t) 0;
301:
302: #if MACH_PAGEMAP
303: vm_object_template->existence_info = VM_EXTERNAL_NULL;
304: #endif /* MACH_PAGEMAP */
305:
306: /*
307: * Initialize the "kernel object"
308: */
309:
310: kernel_object = _vm_object_allocate(
311: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
312:
313: /*
314: * Initialize the "submap object". Make it as large as the
315: * kernel object so that no limit is imposed on submap sizes.
316: */
317:
318: vm_submap_object = _vm_object_allocate(
319: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
320:
321: #if MACH_PAGEMAP
322: vm_external_module_initialize();
323: #endif /* MACH_PAGEMAP */
324: }
325:
326: void vm_object_init(void)
327: {
328: #if !NORMA_VM
329: /*
330: * Finish initializing the kernel object.
331: * The submap object doesn't need a name port.
332: */
333:
334: kernel_object->pager_name = ipc_port_alloc_kernel();
335: ipc_kobject_set(kernel_object->pager_name,
336: (ipc_kobject_t) kernel_object,
337: IKOT_PAGING_NAME);
338: #endif /* !NORMA_VM */
339: }
340:
341: /*
342: * vm_object_reference:
343: *
344: * Gets another reference to the given object.
345: */
346: void vm_object_reference(
347: register vm_object_t object)
348: {
349: if (object == VM_OBJECT_NULL)
350: return;
351:
352: vm_object_lock(object);
353: assert(object->ref_count > 0);
354: object->ref_count++;
355: vm_object_unlock(object);
356: }
357:
358: /*
359: * vm_object_deallocate:
360: *
361: * Release a reference to the specified object,
362: * gained either through a vm_object_allocate
363: * or a vm_object_reference call. When all references
364: * are gone, storage associated with this object
365: * may be relinquished.
366: *
367: * No object may be locked.
368: */
369: void vm_object_deallocate(
370: register vm_object_t object)
371: {
372: vm_object_t temp;
373:
374: while (object != VM_OBJECT_NULL) {
375:
376: /*
377: * The cache holds a reference (uncounted) to
378: * the object; we must lock it before removing
379: * the object.
380: */
381:
382: vm_object_cache_lock();
383:
384: /*
385: * Lose the reference
386: */
387: vm_object_lock(object);
388: if (--(object->ref_count) > 0) {
389:
390: /*
391: * If there are still references, then
392: * we are done.
393: */
394: vm_object_unlock(object);
395: vm_object_cache_unlock();
396: return;
397: }
398:
399: /*
400: * See whether this object can persist. If so, enter
401: * it in the cache, then deactivate all of its
402: * pages.
403: */
404: if (object->can_persist) {
405: boolean_t overflow;
406:
407: /*
408: * Enter the object onto the queue
409: * of "cached" objects. Remember whether
410: * we've caused the queue to overflow,
411: * as a hint.
412: */
413:
414: queue_enter(&vm_object_cached_list, object,
415: vm_object_t, cached_list);
416: overflow = (++vm_object_cached_count > vm_object_cached_max);
417: vm_object_cache_unlock();
418:
419: vm_object_deactivate_pages(object);
420: vm_object_unlock(object);
421:
422: /*
423: * If we didn't overflow, or if the queue has
424: * been reduced back to below the specified
425: * minimum, then quit.
426: */
427: if (!overflow)
428: return;
429:
430: while (TRUE) {
431: vm_object_cache_lock();
432: if (vm_object_cached_count <=
433: vm_object_cached_max) {
434: vm_object_cache_unlock();
435: return;
436: }
437:
438: /*
439: * If we must trim down the queue, take
440: * the first object, and proceed to
441: * terminate it instead of the original
442: * object. Have to wait for pager init.
443: * if it's in progress.
444: */
445: object= (vm_object_t)
446: queue_first(&vm_object_cached_list);
447: vm_object_lock(object);
448:
449: if (!(object->pager_created &&
450: !object->pager_initialized)) {
451:
452: /*
453: * Ok to terminate, hang on to lock.
454: */
455: break;
456: }
457:
458: vm_object_assert_wait(object,
459: VM_OBJECT_EVENT_INITIALIZED, FALSE);
460: vm_object_unlock(object);
461: vm_object_cache_unlock();
462: thread_block((void (*)()) 0);
463:
464: /*
465: * Continue loop to check if cache still
466: * needs to be trimmed.
467: */
468: }
469:
470: /*
471: * Actually remove object from cache.
472: */
473:
474: queue_remove(&vm_object_cached_list, object,
475: vm_object_t, cached_list);
476: vm_object_cached_count--;
477:
478: assert(object->ref_count == 0);
479: }
480: else {
481: if (object->pager_created &&
482: !object->pager_initialized) {
483:
484: /*
485: * Have to wait for initialization.
486: * Put reference back and retry
487: * when it's initialized.
488: */
489: object->ref_count++;
490: vm_object_assert_wait(object,
491: VM_OBJECT_EVENT_INITIALIZED, FALSE);
492: vm_object_unlock(object);
493: vm_object_cache_unlock();
494: thread_block((void (*)()) 0);
495: continue;
496: }
497: }
498:
499: /*
500: * Take the reference to the shadow object
501: * out of the object to be destroyed.
502: */
503:
504: temp = object->shadow;
505:
506: /*
507: * Destroy the object; the cache lock will
508: * be released in the process.
509: */
510:
511: vm_object_terminate(object);
512:
513: /*
514: * Deallocate the reference to the shadow
515: * by continuing the loop with that object
516: * in place of the original.
517: */
518:
519: object = temp;
520: }
521: }
522:
523: boolean_t vm_object_terminate_remove_all = FALSE;
524:
525: /*
526: * Routine: vm_object_terminate
527: * Purpose:
528: * Free all resources associated with a vm_object.
529: * In/out conditions:
530: * Upon entry, the object and the cache must be locked,
531: * and the object must have no references.
532: *
533: * The shadow object reference is left alone.
534: *
535: * Upon exit, the cache will be unlocked, and the
536: * object will cease to exist.
537: */
538: void vm_object_terminate(
539: register vm_object_t object)
540: {
541: register vm_page_t p;
542: vm_object_t shadow_object;
543:
544: /*
545: * Make sure the object isn't already being terminated
546: */
547:
548: assert(object->alive);
549: object->alive = FALSE;
550:
551: /*
552: * Make sure no one can look us up now.
553: */
554:
555: vm_object_remove(object);
556: vm_object_cache_unlock();
557:
558: /*
559: * Detach the object from its shadow if we are the shadow's
560: * copy.
561: */
562: if ((shadow_object = object->shadow) != VM_OBJECT_NULL) {
563: vm_object_lock(shadow_object);
564: assert((shadow_object->copy == object) ||
565: (shadow_object->copy == VM_OBJECT_NULL));
566: shadow_object->copy = VM_OBJECT_NULL;
567: vm_object_unlock(shadow_object);
568: }
569:
570: /*
571: * The pageout daemon might be playing with our pages.
572: * Now that the object is dead, it won't touch any more
573: * pages, but some pages might already be on their way out.
574: * Hence, we wait until the active paging activities have ceased.
575: */
576:
577: vm_object_paging_wait(object, FALSE);
578:
579: /*
580: * Clean or free the pages, as appropriate.
581: * It is possible for us to find busy/absent pages,
582: * if some faults on this object were aborted.
583: */
584:
585: if ((object->temporary) || (object->pager == IP_NULL)) {
586: while (!queue_empty(&object->memq)) {
587: p = (vm_page_t) queue_first(&object->memq);
588:
589: VM_PAGE_CHECK(p);
590:
591: if (p->busy && !p->absent)
592: panic("vm_object_terminate.2 0x%x 0x%x",
593: object, p);
594:
595: VM_PAGE_FREE(p);
596: }
597: } else while (!queue_empty(&object->memq)) {
598: p = (vm_page_t) queue_first(&object->memq);
599:
600: VM_PAGE_CHECK(p);
601:
602: if (p->busy && !p->absent)
603: panic("vm_object_terminate.3 0x%x 0x%x", object, p);
604:
605: vm_page_lock_queues();
606: VM_PAGE_QUEUES_REMOVE(p);
607: vm_page_unlock_queues();
608:
609: if (p->absent || p->private) {
610:
611: /*
612: * For private pages, VM_PAGE_FREE just
613: * leaves the page structure around for
614: * its owner to clean up. For absent
615: * pages, the structure is returned to
616: * the appropriate pool.
617: */
618:
619: goto free_page;
620: }
621:
622: if (p->fictitious)
623: panic("vm_object_terminate.4 0x%x 0x%x", object, p);
624:
625: if (!p->dirty)
626: p->dirty = pmap_is_modified(p->phys_addr);
627:
628: if (p->dirty || p->precious) {
629: p->busy = TRUE;
630: vm_pageout_page(p, FALSE, TRUE); /* flush page */
631: } else {
632: free_page:
633: VM_PAGE_FREE(p);
634: }
635: }
636:
637: assert(object->ref_count == 0);
638: assert(object->paging_in_progress == 0);
639:
640: /*
641: * Throw away port rights... note that they may
642: * already have been thrown away (by vm_object_destroy
643: * or memory_object_destroy).
644: *
645: * Instead of destroying the control and name ports,
646: * we send all rights off to the memory manager instead,
647: * using memory_object_terminate.
648: */
649:
650: vm_object_unlock(object);
651:
652: if (object->pager != IP_NULL) {
653: /* consumes our rights for pager, pager_request, pager_name */
654: memory_object_release(object->pager,
655: object->pager_request,
656: object->pager_name);
657: } else if (object->pager_name != IP_NULL) {
658: /* consumes our right for pager_name */
659: #if NORMA_VM
660: ipc_port_release_send(object->pager_name);
661: #else /* NORMA_VM */
662: ipc_port_dealloc_kernel(object->pager_name);
663: #endif /* NORMA_VM */
664: }
665:
666: #if MACH_PAGEMAP
667: vm_external_destroy(object->existence_info);
668: #endif /* MACH_PAGEMAP */
669:
670: /*
671: * Free the space for the object.
672: */
673:
674: zfree(vm_object_zone, (vm_offset_t) object);
675: }
676:
677: /*
678: * Routine: vm_object_pager_wakeup
679: * Purpose: Wake up anyone waiting for IKOT_PAGER_TERMINATING
680: */
681:
682: void
683: vm_object_pager_wakeup(
684: ipc_port_t pager)
685: {
686: boolean_t someone_waiting;
687:
688: /*
689: * If anyone was waiting for the memory_object_terminate
690: * to be queued, wake them up now.
691: */
692: vm_object_cache_lock();
693: assert(ip_kotype(pager) == IKOT_PAGER_TERMINATING);
694: someone_waiting = (pager->ip_kobject != IKO_NULL);
695: if (ip_active(pager))
696: ipc_kobject_set(pager, IKO_NULL, IKOT_NONE);
697: vm_object_cache_unlock();
698: if (someone_waiting) {
699: thread_wakeup((event_t) pager);
700: }
701: }
702:
703: /*
704: * Routine: memory_object_release
705: * Purpose: Terminate the pager and release port rights,
706: * just like memory_object_terminate, except
707: * that we wake up anyone blocked in vm_object_enter
708: * waiting for termination message to be queued
709: * before calling memory_object_init.
710: */
711: void memory_object_release(
712: ipc_port_t pager,
713: pager_request_t pager_request,
714: ipc_port_t pager_name)
715: {
716:
717: /*
718: * Keep a reference to pager port;
719: * the terminate might otherwise release all references.
720: */
721: ip_reference(pager);
722:
723: /*
724: * Terminate the pager.
725: */
726: (void) memory_object_terminate(pager, pager_request, pager_name);
727:
728: /*
729: * Wakeup anyone waiting for this terminate
730: */
731: vm_object_pager_wakeup(pager);
732:
733: /*
734: * Release reference to pager port.
735: */
736: ip_release(pager);
737: }
738:
739: /*
740: * Routine: vm_object_abort_activity [internal use only]
741: * Purpose:
742: * Abort paging requests pending on this object.
743: * In/out conditions:
744: * The object is locked on entry and exit.
745: */
746: void vm_object_abort_activity(
747: vm_object_t object)
748: {
749: register
750: vm_page_t p;
751: vm_page_t next;
752:
753: /*
754: * Abort all activity that would be waiting
755: * for a result on this memory object.
756: *
757: * We could also choose to destroy all pages
758: * that we have in memory for this object, but
759: * we don't.
760: */
761:
762: p = (vm_page_t) queue_first(&object->memq);
763: while (!queue_end(&object->memq, (queue_entry_t) p)) {
764: next = (vm_page_t) queue_next(&p->listq);
765:
766: /*
767: * If it's being paged in, destroy it.
768: * If an unlock has been requested, start it again.
769: */
770:
771: if (p->busy && p->absent) {
772: VM_PAGE_FREE(p);
773: }
774: else {
775: if (p->unlock_request != VM_PROT_NONE)
776: p->unlock_request = VM_PROT_NONE;
777: PAGE_WAKEUP(p);
778: }
779:
780: p = next;
781: }
782:
783: /*
784: * Wake up threads waiting for the memory object to
785: * become ready.
786: */
787:
788: object->pager_ready = TRUE;
789: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
790: }
791:
792: /*
793: * Routine: memory_object_destroy [user interface]
794: * Purpose:
795: * Shut down a memory object, despite the
796: * presence of address map (or other) references
797: * to the vm_object.
798: * Note:
799: * This routine may be called either from the user interface,
800: * or from port destruction handling (via vm_object_destroy).
801: */
802: kern_return_t memory_object_destroy(
803: register
804: vm_object_t object,
805: kern_return_t reason)
806: {
807: ipc_port_t old_object, old_name;
808: pager_request_t old_control;
809:
810: #ifdef lint
811: reason++;
812: #endif /* lint */
813:
814: if (object == VM_OBJECT_NULL)
815: return KERN_SUCCESS;
816:
817: /*
818: * Remove the port associations immediately.
819: *
820: * This will prevent the memory manager from further
821: * meddling. [If it wanted to flush data or make
822: * other changes, it should have done so before performing
823: * the destroy call.]
824: */
825:
826: vm_object_cache_lock();
827: vm_object_lock(object);
828: vm_object_remove(object);
829: object->can_persist = FALSE;
830: vm_object_cache_unlock();
831:
832: /*
833: * Rip out the ports from the vm_object now... this
834: * will prevent new memory_object calls from succeeding.
835: */
836:
837: old_object = object->pager;
838: object->pager = IP_NULL;
839:
840: old_control = object->pager_request;
841: object->pager_request = PAGER_REQUEST_NULL;
842:
843: old_name = object->pager_name;
844: object->pager_name = IP_NULL;
845:
846:
847: /*
848: * Wait for existing paging activity (that might
849: * have the old ports) to subside.
850: */
851:
852: vm_object_paging_wait(object, FALSE);
853: vm_object_unlock(object);
854:
855: /*
856: * Shut down the ports now.
857: *
858: * [Paging operations may be proceeding concurrently --
859: * they'll get the null values established above.]
860: */
861:
862: if (old_object != IP_NULL) {
863: /* consumes our rights for object, control, name */
864: memory_object_release(old_object, old_control,
865: old_name);
866: } else if (old_name != IP_NULL) {
867: /* consumes our right for name */
868: #if NORMA_VM
869: ipc_port_release_send(object->pager_name);
870: #else /* NORMA_VM */
871: ipc_port_dealloc_kernel(object->pager_name);
872: #endif /* NORMA_VM */
873: }
874:
875: /*
876: * Lose the reference that was donated for this routine
877: */
878:
879: vm_object_deallocate(object);
880:
881: return KERN_SUCCESS;
882: }
883:
884: /*
885: * vm_object_deactivate_pages
886: *
887: * Deactivate all pages in the specified object. (Keep its pages
888: * in memory even though it is no longer referenced.)
889: *
890: * The object must be locked.
891: */
892: void vm_object_deactivate_pages(
893: register vm_object_t object)
894: {
895: register vm_page_t p;
896:
897: queue_iterate(&object->memq, p, vm_page_t, listq) {
898: vm_page_lock_queues();
899: if (!p->busy)
900: vm_page_deactivate(p);
901: vm_page_unlock_queues();
902: }
903: }
904:
905:
906: /*
907: * Routine: vm_object_pmap_protect
908: *
909: * Purpose:
910: * Reduces the permission for all physical
911: * pages in the specified object range.
912: *
913: * If removing write permission only, it is
914: * sufficient to protect only the pages in
915: * the top-level object; only those pages may
916: * have write permission.
917: *
918: * If removing all access, we must follow the
919: * shadow chain from the top-level object to
920: * remove access to all pages in shadowed objects.
921: *
922: * The object must *not* be locked. The object must
923: * be temporary/internal.
924: *
925: * If pmap is not NULL, this routine assumes that
926: * the only mappings for the pages are in that
927: * pmap.
928: */
929: boolean_t vm_object_pmap_protect_by_page = FALSE;
930:
931: void vm_object_pmap_protect(
932: register vm_object_t object,
933: register vm_offset_t offset,
934: vm_offset_t size,
935: pmap_t pmap,
936: vm_offset_t pmap_start,
937: vm_prot_t prot)
938: {
939: if (object == VM_OBJECT_NULL)
940: return;
941:
942: vm_object_lock(object);
943:
944: assert(object->temporary && object->internal);
945:
946: while (TRUE) {
947: if (object->resident_page_count > atop(size) / 2 &&
948: pmap != PMAP_NULL) {
949: vm_object_unlock(object);
950: pmap_protect(pmap, pmap_start, pmap_start + size, prot);
951: return;
952: }
953:
954: {
955: register vm_page_t p;
956: register vm_offset_t end;
957:
958: end = offset + size;
959:
960: queue_iterate(&object->memq, p, vm_page_t, listq) {
961: if (!p->fictitious &&
962: (offset <= p->offset) &&
963: (p->offset < end)) {
964: if ((pmap == PMAP_NULL) ||
965: vm_object_pmap_protect_by_page) {
966: pmap_page_protect(p->phys_addr,
967: prot & ~p->page_lock);
968: } else {
969: vm_offset_t start =
970: pmap_start +
971: (p->offset - offset);
972:
973: pmap_protect(pmap,
974: start,
975: start + PAGE_SIZE,
976: prot);
977: }
978: }
979: }
980: }
981:
982: if (prot == VM_PROT_NONE) {
983: /*
984: * Must follow shadow chain to remove access
985: * to pages in shadowed objects.
986: */
987: register vm_object_t next_object;
988:
989: next_object = object->shadow;
990: if (next_object != VM_OBJECT_NULL) {
991: offset += object->shadow_offset;
992: vm_object_lock(next_object);
993: vm_object_unlock(object);
994: object = next_object;
995: }
996: else {
997: /*
998: * End of chain - we are done.
999: */
1000: break;
1001: }
1002: }
1003: else {
1004: /*
1005: * Pages in shadowed objects may never have
1006: * write permission - we may stop here.
1007: */
1008: break;
1009: }
1010: }
1011:
1012: vm_object_unlock(object);
1013: }
1014:
1015: /*
1016: * vm_object_pmap_remove:
1017: *
1018: * Removes all physical pages in the specified
1019: * object range from all physical maps.
1020: *
1021: * The object must *not* be locked.
1022: */
1023: void vm_object_pmap_remove(
1024: register vm_object_t object,
1025: register vm_offset_t start,
1026: register vm_offset_t end)
1027: {
1028: register vm_page_t p;
1029:
1030: if (object == VM_OBJECT_NULL)
1031: return;
1032:
1033: vm_object_lock(object);
1034: queue_iterate(&object->memq, p, vm_page_t, listq) {
1035: if (!p->fictitious &&
1036: (start <= p->offset) &&
1037: (p->offset < end))
1038: pmap_page_protect(p->phys_addr, VM_PROT_NONE);
1039: }
1040: vm_object_unlock(object);
1041: }
1042:
1043: /*
1044: * Routine: vm_object_copy_slowly
1045: *
1046: * Description:
1047: * Copy the specified range of the source
1048: * virtual memory object without using
1049: * protection-based optimizations (such
1050: * as copy-on-write). The pages in the
1051: * region are actually copied.
1052: *
1053: * In/out conditions:
1054: * The caller must hold a reference and a lock
1055: * for the source virtual memory object. The source
1056: * object will be returned *unlocked*.
1057: *
1058: * Results:
1059: * If the copy is completed successfully, KERN_SUCCESS is
1060: * returned. If the caller asserted the interruptible
1061: * argument, and an interruption occurred while waiting
1062: * for a user-generated event, MACH_SEND_INTERRUPTED is
1063: * returned. Other values may be returned to indicate
1064: * hard errors during the copy operation.
1065: *
1066: * A new virtual memory object is returned in a
1067: * parameter (_result_object). The contents of this
1068: * new object, starting at a zero offset, are a copy
1069: * of the source memory region. In the event of
1070: * an error, this parameter will contain the value
1071: * VM_OBJECT_NULL.
1072: */
1073: kern_return_t vm_object_copy_slowly(
1074: register
1075: vm_object_t src_object,
1076: vm_offset_t src_offset,
1077: vm_size_t size,
1078: boolean_t interruptible,
1079: vm_object_t *_result_object) /* OUT */
1080: {
1081: vm_object_t new_object;
1082: vm_offset_t new_offset;
1083:
1084: if (size == 0) {
1085: vm_object_unlock(src_object);
1086: *_result_object = VM_OBJECT_NULL;
1087: return KERN_INVALID_ARGUMENT;
1088: }
1089:
1090: /*
1091: * Prevent destruction of the source object while we copy.
1092: */
1093:
1094: assert(src_object->ref_count > 0);
1095: src_object->ref_count++;
1096: vm_object_unlock(src_object);
1097:
1098: /*
1099: * Create a new object to hold the copied pages.
1100: * A few notes:
1101: * We fill the new object starting at offset 0,
1102: * regardless of the input offset.
1103: * We don't bother to lock the new object within
1104: * this routine, since we have the only reference.
1105: */
1106:
1107: new_object = vm_object_allocate(size);
1108: new_offset = 0;
1109:
1110: assert(size == trunc_page(size)); /* Will the loop terminate? */
1111:
1112: for ( ;
1113: size != 0 ;
1114: src_offset += PAGE_SIZE, new_offset += PAGE_SIZE, size -= PAGE_SIZE
1115: ) {
1116: vm_page_t new_page;
1117: vm_fault_return_t result;
1118:
1119: while ((new_page = vm_page_alloc(new_object, new_offset))
1120: == VM_PAGE_NULL) {
1121: VM_PAGE_WAIT((void (*)()) 0);
1122: }
1123:
1124: do {
1125: vm_prot_t prot = VM_PROT_READ;
1126: vm_page_t _result_page;
1127: vm_page_t top_page;
1128: register
1129: vm_page_t result_page;
1130:
1131: vm_object_lock(src_object);
1132: src_object->paging_in_progress++;
1133:
1134: result = vm_fault_page(src_object, src_offset,
1135: VM_PROT_READ, FALSE, interruptible,
1136: &prot, &_result_page, &top_page,
1137: FALSE, (void (*)()) 0);
1138:
1139: switch(result) {
1140: case VM_FAULT_SUCCESS:
1141: result_page = _result_page;
1142:
1143: /*
1144: * We don't need to hold the object
1145: * lock -- the busy page will be enough.
1146: * [We don't care about picking up any
1147: * new modifications.]
1148: *
1149: * Copy the page to the new object.
1150: *
1151: * POLICY DECISION:
1152: * If result_page is clean,
1153: * we could steal it instead
1154: * of copying.
1155: */
1156:
1157: vm_object_unlock(result_page->object);
1158: vm_page_copy(result_page, new_page);
1159:
1160: /*
1161: * Let go of both pages (make them
1162: * not busy, perform wakeup, activate).
1163: */
1164:
1165: new_page->busy = FALSE;
1166: new_page->dirty = TRUE;
1167: vm_object_lock(result_page->object);
1168: PAGE_WAKEUP_DONE(result_page);
1169:
1170: vm_page_lock_queues();
1171: if (!result_page->active &&
1172: !result_page->inactive)
1173: vm_page_activate(result_page);
1174: vm_page_activate(new_page);
1175: vm_page_unlock_queues();
1176:
1177: /*
1178: * Release paging references and
1179: * top-level placeholder page, if any.
1180: */
1181:
1182: vm_fault_cleanup(result_page->object,
1183: top_page);
1184:
1185: break;
1186:
1187: case VM_FAULT_RETRY:
1188: break;
1189:
1190: case VM_FAULT_MEMORY_SHORTAGE:
1191: VM_PAGE_WAIT((void (*)()) 0);
1192: break;
1193:
1194: case VM_FAULT_FICTITIOUS_SHORTAGE:
1195: vm_page_more_fictitious();
1196: break;
1197:
1198: case VM_FAULT_INTERRUPTED:
1199: vm_page_free(new_page);
1200: vm_object_deallocate(new_object);
1201: vm_object_deallocate(src_object);
1202: *_result_object = VM_OBJECT_NULL;
1203: return MACH_SEND_INTERRUPTED;
1204:
1205: case VM_FAULT_MEMORY_ERROR:
1206: /*
1207: * A policy choice:
1208: * (a) ignore pages that we can't
1209: * copy
1210: * (b) return the null object if
1211: * any page fails [chosen]
1212: */
1213:
1214: vm_page_free(new_page);
1215: vm_object_deallocate(new_object);
1216: vm_object_deallocate(src_object);
1217: *_result_object = VM_OBJECT_NULL;
1218: return KERN_MEMORY_ERROR;
1219: }
1220: } while (result != VM_FAULT_SUCCESS);
1221: }
1222:
1223: /*
1224: * Lose the extra reference, and return our object.
1225: */
1226:
1227: vm_object_deallocate(src_object);
1228: *_result_object = new_object;
1229: return KERN_SUCCESS;
1230: }
1231:
1232: /*
1233: * Routine: vm_object_copy_temporary
1234: *
1235: * Purpose:
1236: * Copy the specified range of the source virtual
1237: * memory object, if it can be done without blocking.
1238: *
1239: * Results:
1240: * If the copy is successful, the copy is returned in
1241: * the arguments; otherwise, the arguments are not
1242: * affected.
1243: *
1244: * In/out conditions:
1245: * The object should be unlocked on entry and exit.
1246: */
1247:
1248: vm_object_t vm_object_copy_delayed(); /* forward declaration */
1249:
1250: boolean_t vm_object_copy_temporary(
1251: vm_object_t *_object, /* INOUT */
1252: vm_offset_t *_offset, /* INOUT */
1253: boolean_t *_src_needs_copy, /* OUT */
1254: boolean_t *_dst_needs_copy) /* OUT */
1255: {
1256: vm_object_t object = *_object;
1257:
1258: #ifdef lint
1259: ++*_offset;
1260: #endif /* lint */
1261:
1262: if (object == VM_OBJECT_NULL) {
1263: *_src_needs_copy = FALSE;
1264: *_dst_needs_copy = FALSE;
1265: return TRUE;
1266: }
1267:
1268: /*
1269: * If the object is temporary, we can perform
1270: * a symmetric copy-on-write without asking.
1271: */
1272:
1273: vm_object_lock(object);
1274: if (object->temporary) {
1275:
1276: /*
1277: * Shared objects use delayed copy
1278: */
1279: if (object->use_shared_copy) {
1280:
1281: /*
1282: * Asymmetric copy strategy. Destination
1283: * must be copied (to allow copy object reuse).
1284: * Source is unaffected.
1285: */
1286: vm_object_unlock(object);
1287: object = vm_object_copy_delayed(object);
1288: *_object = object;
1289: *_src_needs_copy = FALSE;
1290: *_dst_needs_copy = TRUE;
1291: return TRUE;
1292: }
1293:
1294: /*
1295: * Make another reference to the object.
1296: *
1297: * Leave object/offset unchanged.
1298: */
1299:
1300: assert(object->ref_count > 0);
1301: object->ref_count++;
1302: object->shadowed = TRUE;
1303: vm_object_unlock(object);
1304:
1305: /*
1306: * Both source and destination must make
1307: * shadows, and the source must be made
1308: * read-only if not already.
1309: */
1310:
1311: *_src_needs_copy = TRUE;
1312: *_dst_needs_copy = TRUE;
1313: return TRUE;
1314: }
1315:
1316: if (object->pager_ready &&
1317: (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY)) {
1318: /* XXX Do something intelligent (see temporary code above) */
1319: }
1320: vm_object_unlock(object);
1321:
1322: return FALSE;
1323: }
1324:
1325: /*
1326: * Routine: vm_object_copy_call [internal]
1327: *
1328: * Description:
1329: * Copy the specified (src_offset, size) portion
1330: * of the source object (src_object), using the
1331: * user-managed copy algorithm.
1332: *
1333: * In/out conditions:
1334: * The source object must be locked on entry. It
1335: * will be *unlocked* on exit.
1336: *
1337: * Results:
1338: * If the copy is successful, KERN_SUCCESS is returned.
1339: * This routine is interruptible; if a wait for
1340: * a user-generated event is interrupted, MACH_SEND_INTERRUPTED
1341: * is returned. Other return values indicate hard errors
1342: * in creating the user-managed memory object for the copy.
1343: *
1344: * A new object that represents the copied virtual
1345: * memory is returned in a parameter (*_result_object).
1346: * If the return value indicates an error, this parameter
1347: * is not valid.
1348: */
1349: kern_return_t vm_object_copy_call(
1350: vm_object_t src_object,
1351: vm_offset_t src_offset,
1352: vm_size_t size,
1353: vm_object_t *_result_object) /* OUT */
1354: {
1355: vm_offset_t src_end = src_offset + size;
1356: ipc_port_t new_memory_object;
1357: vm_object_t new_object;
1358: vm_page_t p;
1359:
1360: /*
1361: * Set the backing object for the new
1362: * temporary object.
1363: */
1364:
1365: assert(src_object->ref_count > 0);
1366: src_object->ref_count++;
1367: vm_object_paging_begin(src_object);
1368: vm_object_unlock(src_object);
1369:
1370: /*
1371: * Create a memory object port to be associated
1372: * with this new vm_object.
1373: *
1374: * Since the kernel has the only rights to this
1375: * port, we need not hold the cache lock.
1376: *
1377: * Since we have the only object reference, we
1378: * need not be worried about collapse operations.
1379: *
1380: */
1381:
1382: new_memory_object = ipc_port_alloc_kernel();
1383: if (new_memory_object == IP_NULL) {
1384: panic("vm_object_copy_call: allocate memory object port");
1385: /* XXX Shouldn't panic here. */
1386: }
1387:
1388: /* we hold a naked receive right for new_memory_object */
1389: (void) ipc_port_make_send(new_memory_object);
1390: /* now we also hold a naked send right for new_memory_object */
1391:
1392: /*
1393: * Let the memory manager know that a copy operation
1394: * is in progress. Note that we're using the old
1395: * memory object's ports (for which we're holding
1396: * a paging reference)... the memory manager cannot
1397: * yet affect the new memory object.
1398: */
1399:
1400: (void) memory_object_copy(src_object->pager,
1401: src_object->pager_request,
1402: src_offset, size,
1403: new_memory_object);
1404: /* no longer hold the naked receive right for new_memory_object */
1405:
1406: vm_object_lock(src_object);
1407: vm_object_paging_end(src_object);
1408:
1409: /*
1410: * Remove write access from all of the pages of
1411: * the old memory object that we can.
1412: */
1413:
1414: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1415: if (!p->fictitious &&
1416: (src_offset <= p->offset) &&
1417: (p->offset < src_end) &&
1418: !(p->page_lock & VM_PROT_WRITE)) {
1419: p->page_lock |= VM_PROT_WRITE;
1420: pmap_page_protect(p->phys_addr, VM_PROT_ALL & ~p->page_lock);
1421: }
1422: }
1423:
1424: vm_object_unlock(src_object);
1425:
1426: /*
1427: * Initialize the rest of the paging stuff
1428: */
1429:
1430: new_object = vm_object_enter(new_memory_object, size, FALSE);
1431: new_object->shadow = src_object;
1432: new_object->shadow_offset = src_offset;
1433:
1434: /*
1435: * Drop the reference for new_memory_object taken above.
1436: */
1437:
1438: ipc_port_release_send(new_memory_object);
1439: /* no longer hold the naked send right for new_memory_object */
1440:
1441: *_result_object = new_object;
1442: return KERN_SUCCESS;
1443: }
1444:
1445: /*
1446: * Routine: vm_object_copy_delayed [internal]
1447: *
1448: * Description:
1449: * Copy the specified virtual memory object, using
1450: * the asymmetric copy-on-write algorithm.
1451: *
1452: * In/out conditions:
1453: * The object must be unlocked on entry.
1454: *
1455: * This routine will not block waiting for user-generated
1456: * events. It is not interruptible.
1457: */
1458: vm_object_t vm_object_copy_delayed(
1459: vm_object_t src_object)
1460: {
1461: vm_object_t new_copy;
1462: vm_object_t old_copy;
1463: vm_page_t p;
1464:
1465: /*
1466: * The user-level memory manager wants to see
1467: * all of the changes to this object, but it
1468: * has promised not to make any changes on its own.
1469: *
1470: * Perform an asymmetric copy-on-write, as follows:
1471: * Create a new object, called a "copy object"
1472: * to hold pages modified by the new mapping
1473: * (i.e., the copy, not the original mapping).
1474: * Record the original object as the backing
1475: * object for the copy object. If the
1476: * original mapping does not change a page,
1477: * it may be used read-only by the copy.
1478: * Record the copy object in the original
1479: * object. When the original mapping causes
1480: * a page to be modified, it must be copied
1481: * to a new page that is "pushed" to the
1482: * copy object.
1483: * Mark the new mapping (the copy object)
1484: * copy-on-write. This makes the copy
1485: * object itself read-only, allowing it
1486: * to be reused if the original mapping
1487: * makes no changes, and simplifying the
1488: * synchronization required in the "push"
1489: * operation described above.
1490: *
1491: * The copy-on-write is said to be assymetric because
1492: * the original object is *not* marked copy-on-write.
1493: * A copied page is pushed to the copy object, regardless
1494: * which party attempted to modify the page.
1495: *
1496: * Repeated asymmetric copy operations may be done.
1497: * If the original object has not been changed since
1498: * the last copy, its copy object can be reused.
1499: * Otherwise, a new copy object can be inserted
1500: * between the original object and its previous
1501: * copy object. Since any copy object is read-only,
1502: * this cannot affect the contents of the previous copy
1503: * object.
1504: *
1505: * Note that a copy object is higher in the object
1506: * tree than the original object; therefore, use of
1507: * the copy object recorded in the original object
1508: * must be done carefully, to avoid deadlock.
1509: */
1510:
1511: /*
1512: * Allocate a new copy object before locking, even
1513: * though we may not need it later.
1514: */
1515:
1516: new_copy = vm_object_allocate(src_object->size);
1517:
1518: vm_object_lock(src_object);
1519:
1520: /*
1521: * See whether we can reuse the result of a previous
1522: * copy operation.
1523: */
1524: Retry:
1525: old_copy = src_object->copy;
1526: if (old_copy != VM_OBJECT_NULL) {
1527: /*
1528: * Try to get the locks (out of order)
1529: */
1530: if (!vm_object_lock_try(old_copy)) {
1531: vm_object_unlock(src_object);
1532:
1533: simple_lock_pause(); /* wait a bit */
1534:
1535: vm_object_lock(src_object);
1536: goto Retry;
1537: }
1538:
1539: /*
1540: * Determine whether the old copy object has
1541: * been modified.
1542: */
1543:
1544: if (old_copy->resident_page_count == 0 &&
1545: !old_copy->pager_created) {
1546: /*
1547: * It has not been modified.
1548: *
1549: * Return another reference to
1550: * the existing copy-object.
1551: */
1552: assert(old_copy->ref_count > 0);
1553: old_copy->ref_count++;
1554: vm_object_unlock(old_copy);
1555: vm_object_unlock(src_object);
1556:
1557: vm_object_deallocate(new_copy);
1558:
1559: return old_copy;
1560: }
1561:
1562: /*
1563: * The copy-object is always made large enough to
1564: * completely shadow the original object, since
1565: * it may have several users who want to shadow
1566: * the original object at different points.
1567: */
1568:
1569: assert((old_copy->shadow == src_object) &&
1570: (old_copy->shadow_offset == (vm_offset_t) 0));
1571:
1572: /*
1573: * Make the old copy-object shadow the new one.
1574: * It will receive no more pages from the original
1575: * object.
1576: */
1577:
1578: src_object->ref_count--; /* remove ref. from old_copy */
1579: assert(src_object->ref_count > 0);
1580: old_copy->shadow = new_copy;
1581: assert(new_copy->ref_count > 0);
1582: new_copy->ref_count++;
1583: vm_object_unlock(old_copy); /* done with old_copy */
1584: }
1585:
1586: /*
1587: * Point the new copy at the existing object.
1588: */
1589:
1590: new_copy->shadow = src_object;
1591: new_copy->shadow_offset = 0;
1592: new_copy->shadowed = TRUE; /* caller must set needs_copy */
1593: assert(src_object->ref_count > 0);
1594: src_object->ref_count++;
1595: src_object->copy = new_copy;
1596:
1597: /*
1598: * Mark all pages of the existing object copy-on-write.
1599: * This object may have a shadow chain below it, but
1600: * those pages will already be marked copy-on-write.
1601: */
1602:
1603: queue_iterate(&src_object->memq, p, vm_page_t, listq) {
1604: if (!p->fictitious)
1605: pmap_page_protect(p->phys_addr,
1606: (VM_PROT_ALL & ~VM_PROT_WRITE &
1607: ~p->page_lock));
1608: }
1609:
1610: vm_object_unlock(src_object);
1611:
1612: return new_copy;
1613: }
1614:
1615: /*
1616: * Routine: vm_object_copy_strategically
1617: *
1618: * Purpose:
1619: * Perform a copy according to the source object's
1620: * declared strategy. This operation may block,
1621: * and may be interrupted.
1622: */
1623: kern_return_t vm_object_copy_strategically(
1624: register
1625: vm_object_t src_object,
1626: vm_offset_t src_offset,
1627: vm_size_t size,
1628: vm_object_t *dst_object, /* OUT */
1629: vm_offset_t *dst_offset, /* OUT */
1630: boolean_t *dst_needs_copy) /* OUT */
1631: {
1632: kern_return_t result = KERN_SUCCESS; /* to quiet gcc warnings */
1633: boolean_t interruptible = TRUE; /* XXX */
1634:
1635: assert(src_object != VM_OBJECT_NULL);
1636:
1637: vm_object_lock(src_object);
1638:
1639: /* XXX assert(!src_object->temporary); JSB FIXME */
1640:
1641: /*
1642: * The copy strategy is only valid if the memory manager
1643: * is "ready".
1644: */
1645:
1646: while (!src_object->pager_ready) {
1647: vm_object_wait( src_object,
1648: VM_OBJECT_EVENT_PAGER_READY,
1649: interruptible);
1650: if (interruptible &&
1651: (current_thread()->wait_result != THREAD_AWAKENED)) {
1652: *dst_object = VM_OBJECT_NULL;
1653: *dst_offset = 0;
1654: *dst_needs_copy = FALSE;
1655: return MACH_SEND_INTERRUPTED;
1656: }
1657: vm_object_lock(src_object);
1658: }
1659:
1660: /*
1661: * The object may be temporary (even though it is external).
1662: * If so, do a symmetric copy.
1663: */
1664:
1665: if (src_object->temporary) {
1666: /*
1667: * XXX
1668: * This does not count as intelligent!
1669: * This buys us the object->temporary optimizations,
1670: * but we aren't using a symmetric copy,
1671: * which may confuse the vm code. The correct thing
1672: * to do here is to figure out what to call to get
1673: * a temporary shadowing set up.
1674: */
1675: src_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1676: }
1677:
1678: /*
1679: * The object is permanent. Use the appropriate copy strategy.
1680: */
1681:
1682: switch (src_object->copy_strategy) {
1683: case MEMORY_OBJECT_COPY_NONE:
1684: if ((result = vm_object_copy_slowly(
1685: src_object,
1686: src_offset,
1687: size,
1688: interruptible,
1689: dst_object))
1690: == KERN_SUCCESS) {
1691: *dst_offset = 0;
1692: *dst_needs_copy = FALSE;
1693: }
1694: break;
1695:
1696: case MEMORY_OBJECT_COPY_CALL:
1697: if ((result = vm_object_copy_call(
1698: src_object,
1699: src_offset,
1700: size,
1701: dst_object))
1702: == KERN_SUCCESS) {
1703: *dst_offset = 0;
1704: *dst_needs_copy = FALSE;
1705: }
1706: break;
1707:
1708: case MEMORY_OBJECT_COPY_DELAY:
1709: vm_object_unlock(src_object);
1710: *dst_object = vm_object_copy_delayed(src_object);
1711: *dst_offset = src_offset;
1712: *dst_needs_copy = TRUE;
1713:
1714: result = KERN_SUCCESS;
1715: break;
1716: }
1717:
1718: return result;
1719: }
1720:
1721: /*
1722: * vm_object_shadow:
1723: *
1724: * Create a new object which is backed by the
1725: * specified existing object range. The source
1726: * object reference is deallocated.
1727: *
1728: * The new object and offset into that object
1729: * are returned in the source parameters.
1730: */
1731:
1732: void vm_object_shadow(
1733: vm_object_t *object, /* IN/OUT */
1734: vm_offset_t *offset, /* IN/OUT */
1735: vm_size_t length)
1736: {
1737: register vm_object_t source;
1738: register vm_object_t result;
1739:
1740: source = *object;
1741:
1742: /*
1743: * Allocate a new object with the given length
1744: */
1745:
1746: if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
1747: panic("vm_object_shadow: no object for shadowing");
1748:
1749: /*
1750: * The new object shadows the source object, adding
1751: * a reference to it. Our caller changes his reference
1752: * to point to the new object, removing a reference to
1753: * the source object. Net result: no change of reference
1754: * count.
1755: */
1756: result->shadow = source;
1757:
1758: /*
1759: * Store the offset into the source object,
1760: * and fix up the offset into the new object.
1761: */
1762:
1763: result->shadow_offset = *offset;
1764:
1765: /*
1766: * Return the new things
1767: */
1768:
1769: *offset = 0;
1770: *object = result;
1771: }
1772:
1773: /*
1774: * The relationship between vm_object structures and
1775: * the memory_object ports requires careful synchronization.
1776: *
1777: * All associations are created by vm_object_enter. All three
1778: * port fields are filled in, as follows:
1779: * pager: the memory_object port itself, supplied by
1780: * the user requesting a mapping (or the kernel,
1781: * when initializing internal objects); the
1782: * kernel simulates holding send rights by keeping
1783: * a port reference;
1784: * pager_request:
1785: * pager_name:
1786: * the memory object control and name ports,
1787: * created by the kernel; the kernel holds
1788: * receive (and ownership) rights to these
1789: * ports, but no other references.
1790: * All of the ports are referenced by their global names.
1791: *
1792: * When initialization is complete, the "initialized" field
1793: * is asserted. Other mappings using a particular memory object,
1794: * and any references to the vm_object gained through the
1795: * port association must wait for this initialization to occur.
1796: *
1797: * In order to allow the memory manager to set attributes before
1798: * requests (notably virtual copy operations, but also data or
1799: * unlock requests) are made, a "ready" attribute is made available.
1800: * Only the memory manager may affect the value of this attribute.
1801: * Its value does not affect critical kernel functions, such as
1802: * internal object initialization or destruction. [Furthermore,
1803: * memory objects created by the kernel are assumed to be ready
1804: * immediately; the default memory manager need not explicitly
1805: * set the "ready" attribute.]
1806: *
1807: * [Both the "initialized" and "ready" attribute wait conditions
1808: * use the "pager" field as the wait event.]
1809: *
1810: * The port associations can be broken down by any of the
1811: * following routines:
1812: * vm_object_terminate:
1813: * No references to the vm_object remain, and
1814: * the object cannot (or will not) be cached.
1815: * This is the normal case, and is done even
1816: * though one of the other cases has already been
1817: * done.
1818: * vm_object_destroy:
1819: * The memory_object port has been destroyed,
1820: * meaning that the kernel cannot flush dirty
1821: * pages or request new data or unlock existing
1822: * data.
1823: * memory_object_destroy:
1824: * The memory manager has requested that the
1825: * kernel relinquish rights to the memory object
1826: * port. [The memory manager may not want to
1827: * destroy the port, but may wish to refuse or
1828: * tear down existing memory mappings.]
1829: * Each routine that breaks an association must break all of
1830: * them at once. At some later time, that routine must clear
1831: * the vm_object port fields and release the port rights.
1832: * [Furthermore, each routine must cope with the simultaneous
1833: * or previous operations of the others.]
1834: *
1835: * In addition to the lock on the object, the vm_object_cache_lock
1836: * governs the port associations. References gained through the
1837: * port association require use of the cache lock.
1838: *
1839: * Because the port fields may be cleared spontaneously, they
1840: * cannot be used to determine whether a memory object has
1841: * ever been associated with a particular vm_object. [This
1842: * knowledge is important to the shadow object mechanism.]
1843: * For this reason, an additional "created" attribute is
1844: * provided.
1845: *
1846: * During various paging operations, the port values found in the
1847: * vm_object must be valid. To prevent these port rights from being
1848: * released, and to prevent the port associations from changing
1849: * (other than being removed, i.e., made null), routines may use
1850: * the vm_object_paging_begin/end routines [actually, macros].
1851: * The implementation uses the "paging_in_progress" and "wanted" fields.
1852: * [Operations that alter the validity of the port values include the
1853: * termination routines and vm_object_collapse.]
1854: */
1855:
1856: vm_object_t vm_object_lookup(
1857: ipc_port_t port)
1858: {
1859: vm_object_t object = VM_OBJECT_NULL;
1860:
1861: if (IP_VALID(port)) {
1862: ip_lock(port);
1863: if (ip_active(port) &&
1864: #if NORMA_VM
1865: (ip_kotype(port) == IKOT_PAGER)) {
1866: #else /* NORMA_VM */
1867: (ip_kotype(port) == IKOT_PAGING_REQUEST)) {
1868: #endif /* NORMA_VM */
1869: vm_object_cache_lock();
1870: object = (vm_object_t) port->ip_kobject;
1871: vm_object_lock(object);
1872:
1873: assert(object->alive);
1874:
1875: if (object->ref_count == 0) {
1876: queue_remove(&vm_object_cached_list, object,
1877: vm_object_t, cached_list);
1878: vm_object_cached_count--;
1879: }
1880:
1881: object->ref_count++;
1882: vm_object_unlock(object);
1883: vm_object_cache_unlock();
1884: }
1885: ip_unlock(port);
1886: }
1887:
1888: return object;
1889: }
1890:
1891: vm_object_t vm_object_lookup_name(
1892: ipc_port_t port)
1893: {
1894: vm_object_t object = VM_OBJECT_NULL;
1895:
1896: if (IP_VALID(port)) {
1897: ip_lock(port);
1898: if (ip_active(port) &&
1899: (ip_kotype(port) == IKOT_PAGING_NAME)) {
1900: vm_object_cache_lock();
1901: object = (vm_object_t) port->ip_kobject;
1902: vm_object_lock(object);
1903:
1904: assert(object->alive);
1905:
1906: if (object->ref_count == 0) {
1907: queue_remove(&vm_object_cached_list, object,
1908: vm_object_t, cached_list);
1909: vm_object_cached_count--;
1910: }
1911:
1912: object->ref_count++;
1913: vm_object_unlock(object);
1914: vm_object_cache_unlock();
1915: }
1916: ip_unlock(port);
1917: }
1918:
1919: return object;
1920: }
1921:
1922: void vm_object_destroy(
1923: ipc_port_t pager)
1924: {
1925: vm_object_t object;
1926: pager_request_t old_request;
1927: ipc_port_t old_name;
1928:
1929: /*
1930: * Perform essentially the same operations as in vm_object_lookup,
1931: * except that this time we look up based on the memory_object
1932: * port, not the control port.
1933: */
1934: vm_object_cache_lock();
1935: if (ip_kotype(pager) != IKOT_PAGER) {
1936: vm_object_cache_unlock();
1937: return;
1938: }
1939:
1940: object = (vm_object_t) pager->ip_kobject;
1941: vm_object_lock(object);
1942: if (object->ref_count == 0) {
1943: queue_remove(&vm_object_cached_list, object,
1944: vm_object_t, cached_list);
1945: vm_object_cached_count--;
1946: }
1947: object->ref_count++;
1948:
1949: object->can_persist = FALSE;
1950:
1951: assert(object->pager == pager);
1952:
1953: /*
1954: * Remove the port associations.
1955: *
1956: * Note that the memory_object itself is dead, so
1957: * we don't bother with it.
1958: */
1959:
1960: object->pager = IP_NULL;
1961: vm_object_remove(object);
1962:
1963: old_request = object->pager_request;
1964: object->pager_request = PAGER_REQUEST_NULL;
1965:
1966: old_name = object->pager_name;
1967: object->pager_name = IP_NULL;
1968:
1969: vm_object_unlock(object);
1970: vm_object_cache_unlock();
1971:
1972: /*
1973: * Clean up the port references. Note that there's no
1974: * point in trying the memory_object_terminate call
1975: * because the memory_object itself is dead.
1976: */
1977:
1978: ipc_port_release_send(pager);
1979: #if !NORMA_VM
1980: if (old_request != IP_NULL)
1981: ipc_port_dealloc_kernel(old_request);
1982: #endif /* !NORMA_VM */
1983: if (old_name != IP_NULL)
1984: #if NORMA_VM
1985: ipc_port_release_send(old_name);
1986: #else /* NORMA_VM */
1987: ipc_port_dealloc_kernel(old_name);
1988: #endif /* NORMA_VM */
1989:
1990: /*
1991: * Restart pending page requests
1992: */
1993:
1994: vm_object_abort_activity(object);
1995:
1996: /*
1997: * Lose the object reference.
1998: */
1999:
2000: vm_object_deallocate(object);
2001: }
2002:
2003: boolean_t vm_object_accept_old_init_protocol = FALSE;
2004:
2005: /*
2006: * Routine: vm_object_enter
2007: * Purpose:
2008: * Find a VM object corresponding to the given
2009: * pager; if no such object exists, create one,
2010: * and initialize the pager.
2011: */
2012: vm_object_t vm_object_enter(
2013: ipc_port_t pager,
2014: vm_size_t size,
2015: boolean_t internal)
2016: {
2017: register
2018: vm_object_t object;
2019: vm_object_t new_object;
2020: boolean_t must_init;
2021: ipc_kobject_type_t po;
2022:
2023: restart:
2024: if (!IP_VALID(pager))
2025: return vm_object_allocate(size);
2026:
2027: new_object = VM_OBJECT_NULL;
2028: must_init = FALSE;
2029:
2030: /*
2031: * Look for an object associated with this port.
2032: */
2033:
2034: vm_object_cache_lock();
2035: for (;;) {
2036: po = ip_kotype(pager);
2037:
2038: /*
2039: * If a previous object is being terminated,
2040: * we must wait for the termination message
2041: * to be queued.
2042: *
2043: * We set kobject to a non-null value to let the
2044: * terminator know that someone is waiting.
2045: * Among the possibilities is that the port
2046: * could die while we're waiting. Must restart
2047: * instead of continuing the loop.
2048: */
2049:
2050: if (po == IKOT_PAGER_TERMINATING) {
2051: pager->ip_kobject = (ipc_kobject_t) pager;
2052: assert_wait((event_t) pager, FALSE);
2053: vm_object_cache_unlock();
2054: thread_block((void (*)()) 0);
2055: goto restart;
2056: }
2057:
2058: /*
2059: * Bail if there is already a kobject associated
2060: * with the pager port.
2061: */
2062: if (po != IKOT_NONE) {
2063: break;
2064: }
2065:
2066: /*
2067: * We must unlock to create a new object;
2068: * if we do so, we must try the lookup again.
2069: */
2070:
2071: if (new_object == VM_OBJECT_NULL) {
2072: vm_object_cache_unlock();
2073: new_object = vm_object_allocate(size);
2074: vm_object_cache_lock();
2075: } else {
2076: /*
2077: * Lookup failed twice, and we have something
2078: * to insert; set the object.
2079: */
2080:
2081: ipc_kobject_set(pager,
2082: (ipc_kobject_t) new_object,
2083: IKOT_PAGER);
2084: new_object = VM_OBJECT_NULL;
2085: must_init = TRUE;
2086: }
2087: }
2088:
2089: if (internal)
2090: must_init = TRUE;
2091:
2092: /*
2093: * It's only good if it's a VM object!
2094: */
2095:
2096: object = (po == IKOT_PAGER) ? (vm_object_t) pager->ip_kobject
2097: : VM_OBJECT_NULL;
2098:
2099: if ((object != VM_OBJECT_NULL) && !must_init) {
2100: vm_object_lock(object);
2101: if (object->ref_count == 0) {
2102: queue_remove(&vm_object_cached_list, object,
2103: vm_object_t, cached_list);
2104: vm_object_cached_count--;
2105: }
2106: object->ref_count++;
2107: vm_object_unlock(object);
2108:
2109: vm_stat.hits++;
2110: }
2111: assert((object == VM_OBJECT_NULL) || (object->ref_count > 0) ||
2112: ((object->paging_in_progress != 0) && internal));
2113:
2114: vm_stat.lookups++;
2115:
2116: vm_object_cache_unlock();
2117:
2118: /*
2119: * If we raced to create a vm_object but lost, let's
2120: * throw away ours.
2121: */
2122:
2123: if (new_object != VM_OBJECT_NULL)
2124: vm_object_deallocate(new_object);
2125:
2126: if (object == VM_OBJECT_NULL)
2127: return(object);
2128:
2129: if (must_init) {
2130: /*
2131: * Copy the naked send right we were given.
2132: */
2133:
2134: pager = ipc_port_copy_send(pager);
2135: if (!IP_VALID(pager))
2136: panic("vm_object_enter: port died"); /* XXX */
2137:
2138: object->pager_created = TRUE;
2139: object->pager = pager;
2140:
2141: #if NORMA_VM
2142:
2143: /*
2144: * Let the xmm system know that we want to use the pager.
2145: *
2146: * Name port will be provided by the xmm system
2147: * when set_attributes_common is called.
2148: */
2149:
2150: object->internal = internal;
2151: object->pager_ready = internal;
2152: if (internal) {
2153: assert(object->temporary);
2154: } else {
2155: object->temporary = FALSE;
2156: }
2157: object->pager_name = IP_NULL;
2158:
2159: (void) xmm_memory_object_init(object);
2160: #else /* NORMA_VM */
2161:
2162: /*
2163: * Allocate request port.
2164: */
2165:
2166: object->pager_request = ipc_port_alloc_kernel();
2167: if (object->pager_request == IP_NULL)
2168: panic("vm_object_enter: pager request alloc");
2169:
2170: ipc_kobject_set(object->pager_request,
2171: (ipc_kobject_t) object,
2172: IKOT_PAGING_REQUEST);
2173:
2174: /*
2175: * Let the pager know we're using it.
2176: */
2177:
2178: if (internal) {
2179: /* acquire a naked send right for the DMM */
2180: ipc_port_t DMM = memory_manager_default_reference();
2181:
2182: /* mark the object internal */
2183: object->internal = TRUE;
2184: assert(object->temporary);
2185:
2186: /* default-pager objects are ready immediately */
2187: object->pager_ready = TRUE;
2188:
2189: /* consumes the naked send right for DMM */
2190: (void) memory_object_create(DMM,
2191: pager,
2192: object->size,
2193: object->pager_request,
2194: object->pager_name,
2195: PAGE_SIZE);
2196: } else {
2197: /* the object is external and not temporary */
2198: object->internal = FALSE;
2199: object->temporary = FALSE;
2200:
2201: /* user pager objects are not ready until marked so */
2202: object->pager_ready = FALSE;
2203:
2204: (void) memory_object_init(pager,
2205: object->pager_request,
2206: object->pager_name,
2207: PAGE_SIZE);
2208:
2209: }
2210: #endif /* NORMA_VM */
2211:
2212: vm_object_lock(object);
2213: object->pager_initialized = TRUE;
2214:
2215: if (vm_object_accept_old_init_protocol)
2216: object->pager_ready = TRUE;
2217:
2218: vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
2219: } else {
2220: vm_object_lock(object);
2221: }
2222: /*
2223: * [At this point, the object must be locked]
2224: */
2225:
2226: /*
2227: * Wait for the work above to be done by the first
2228: * thread to map this object.
2229: */
2230:
2231: while (!object->pager_initialized) {
2232: vm_object_wait( object,
2233: VM_OBJECT_EVENT_INITIALIZED,
2234: FALSE);
2235: vm_object_lock(object);
2236: }
2237: vm_object_unlock(object);
2238:
2239: return object;
2240: }
2241:
2242: /*
2243: * Routine: vm_object_pager_create
2244: * Purpose:
2245: * Create a memory object for an internal object.
2246: * In/out conditions:
2247: * The object is locked on entry and exit;
2248: * it may be unlocked within this call.
2249: * Limitations:
2250: * Only one thread may be performing a
2251: * vm_object_pager_create on an object at
2252: * a time. Presumably, only the pageout
2253: * daemon will be using this routine.
2254: */
2255: void vm_object_pager_create(
2256: register
2257: vm_object_t object)
2258: {
2259: ipc_port_t pager;
2260:
2261: if (object->pager_created) {
2262: /*
2263: * Someone else got to it first...
2264: * wait for them to finish initializing
2265: */
2266:
2267: while (!object->pager_initialized) {
2268: vm_object_wait( object,
2269: VM_OBJECT_EVENT_PAGER_READY,
2270: FALSE);
2271: vm_object_lock(object);
2272: }
2273: return;
2274: }
2275:
2276: /*
2277: * Indicate that a memory object has been assigned
2278: * before dropping the lock, to prevent a race.
2279: */
2280:
2281: object->pager_created = TRUE;
2282:
2283: /*
2284: * Prevent collapse or termination by
2285: * holding a paging reference
2286: */
2287:
2288: vm_object_paging_begin(object);
2289: vm_object_unlock(object);
2290:
2291: #if MACH_PAGEMAP
2292: object->existence_info = vm_external_create(
2293: object->size +
2294: object->paging_offset);
2295: assert((object->size + object->paging_offset) >=
2296: object->size);
2297: #endif /* MACH_PAGEMAP */
2298:
2299: /*
2300: * Create the pager, and associate with it
2301: * this object.
2302: *
2303: * Note that we only make the port association
2304: * so that vm_object_enter can properly look up
2305: * the object to complete the initialization...
2306: * we do not expect any user to ever map this
2307: * object.
2308: *
2309: * Since the kernel has the only rights to the
2310: * port, it's safe to install the association
2311: * without holding the cache lock.
2312: */
2313:
2314: pager = ipc_port_alloc_kernel();
2315: if (pager == IP_NULL)
2316: panic("vm_object_pager_create: allocate pager port");
2317:
2318: (void) ipc_port_make_send(pager);
2319: ipc_kobject_set(pager, (ipc_kobject_t) object, IKOT_PAGER);
2320:
2321: /*
2322: * Initialize the rest of the paging stuff
2323: */
2324:
2325: if (vm_object_enter(pager, object->size, TRUE) != object)
2326: panic("vm_object_pager_create: mismatch");
2327:
2328: /*
2329: * Drop the naked send right taken above.
2330: */
2331:
2332: ipc_port_release_send(pager);
2333:
2334: /*
2335: * Release the paging reference
2336: */
2337:
2338: vm_object_lock(object);
2339: vm_object_paging_end(object);
2340: }
2341:
2342: /*
2343: * Routine: vm_object_remove
2344: * Purpose:
2345: * Eliminate the pager/object association
2346: * for this pager.
2347: * Conditions:
2348: * The object cache must be locked.
2349: */
2350: void vm_object_remove(
2351: vm_object_t object)
2352: {
2353: ipc_port_t port;
2354:
2355: if ((port = object->pager) != IP_NULL) {
2356: if (ip_kotype(port) == IKOT_PAGER)
2357: ipc_kobject_set(port, IKO_NULL,
2358: IKOT_PAGER_TERMINATING);
2359: else if (ip_kotype(port) != IKOT_NONE)
2360: panic("vm_object_remove: bad object port");
2361: }
2362: #if !NORMA_VM
2363: if ((port = object->pager_request) != IP_NULL) {
2364: if (ip_kotype(port) == IKOT_PAGING_REQUEST)
2365: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2366: else if (ip_kotype(port) != IKOT_NONE)
2367: panic("vm_object_remove: bad request port");
2368: }
2369: if ((port = object->pager_name) != IP_NULL) {
2370: if (ip_kotype(port) == IKOT_PAGING_NAME)
2371: ipc_kobject_set(port, IKO_NULL, IKOT_NONE);
2372: else if (ip_kotype(port) != IKOT_NONE)
2373: panic("vm_object_remove: bad name port");
2374: }
2375: #endif /* !NORMA_VM */
2376: }
2377:
2378: /*
2379: * Global variables for vm_object_collapse():
2380: *
2381: * Counts for normal collapses and bypasses.
2382: * Debugging variables, to watch or disable collapse.
2383: */
2384: long object_collapses = 0;
2385: long object_bypasses = 0;
2386:
2387: int vm_object_collapse_debug = 0;
2388: boolean_t vm_object_collapse_allowed = TRUE;
2389: boolean_t vm_object_collapse_bypass_allowed = TRUE;
2390:
2391: /*
2392: * vm_object_collapse:
2393: *
2394: * Collapse an object with the object backing it.
2395: * Pages in the backing object are moved into the
2396: * parent, and the backing object is deallocated.
2397: *
2398: * Requires that the object be locked and the page
2399: * queues be unlocked. May unlock/relock the object,
2400: * so the caller should hold a reference for the object.
2401: */
2402: void vm_object_collapse(
2403: register vm_object_t object)
2404: {
2405: register vm_object_t backing_object;
2406: register vm_offset_t backing_offset;
2407: register vm_size_t size;
2408: register vm_offset_t new_offset;
2409: register vm_page_t p, pp;
2410: ipc_port_t old_name_port;
2411:
2412: if (!vm_object_collapse_allowed)
2413: return;
2414:
2415: while (TRUE) {
2416: /*
2417: * Verify that the conditions are right for collapse:
2418: *
2419: * The object exists and no pages in it are currently
2420: * being paged out (or have ever been paged out).
2421: *
2422: * This check is probably overkill -- if a memory
2423: * object has not been created, the fault handler
2424: * shouldn't release the object lock while paging
2425: * is in progress or absent pages exist.
2426: */
2427: if (object == VM_OBJECT_NULL ||
2428: object->pager_created ||
2429: object->paging_in_progress != 0 ||
2430: object->absent_count != 0)
2431: return;
2432:
2433: /*
2434: * There is a backing object, and
2435: */
2436:
2437: if ((backing_object = object->shadow) == VM_OBJECT_NULL)
2438: return;
2439:
2440: vm_object_lock(backing_object);
2441: /*
2442: * ...
2443: * The backing object is not read_only,
2444: * and no pages in the backing object are
2445: * currently being paged out.
2446: * The backing object is internal.
2447: *
2448: * XXX It may be sufficient for the backing
2449: * XXX object to be temporary.
2450: */
2451:
2452: if (!backing_object->internal ||
2453: backing_object->paging_in_progress != 0) {
2454: vm_object_unlock(backing_object);
2455: return;
2456: }
2457:
2458: /*
2459: * The backing object can't be a copy-object:
2460: * the shadow_offset for the copy-object must stay
2461: * as 0. Furthermore (for the 'we have all the
2462: * pages' case), if we bypass backing_object and
2463: * just shadow the next object in the chain, old
2464: * pages from that object would then have to be copied
2465: * BOTH into the (former) backing_object and into the
2466: * parent object.
2467: */
2468: if (backing_object->shadow != VM_OBJECT_NULL &&
2469: backing_object->shadow->copy != VM_OBJECT_NULL) {
2470: vm_object_unlock(backing_object);
2471: return;
2472: }
2473:
2474: /*
2475: * We know that we can either collapse the backing
2476: * object (if the parent is the only reference to
2477: * it) or (perhaps) remove the parent's reference
2478: * to it.
2479: */
2480:
2481: backing_offset = object->shadow_offset;
2482: size = object->size;
2483:
2484: /*
2485: * If there is exactly one reference to the backing
2486: * object, we can collapse it into the parent.
2487: */
2488:
2489: if (backing_object->ref_count == 1) {
2490: if (!vm_object_cache_lock_try()) {
2491: vm_object_unlock(backing_object);
2492: return;
2493: }
2494:
2495: /*
2496: * We can collapse the backing object.
2497: *
2498: * Move all in-memory pages from backing_object
2499: * to the parent. Pages that have been paged out
2500: * will be overwritten by any of the parent's
2501: * pages that shadow them.
2502: */
2503:
2504: while (!queue_empty(&backing_object->memq)) {
2505:
2506: p = (vm_page_t)
2507: queue_first(&backing_object->memq);
2508:
2509: new_offset = (p->offset - backing_offset);
2510:
2511: assert(!p->busy || p->absent);
2512:
2513: /*
2514: * If the parent has a page here, or if
2515: * this page falls outside the parent,
2516: * dispose of it.
2517: *
2518: * Otherwise, move it as planned.
2519: */
2520:
2521: if (p->offset < backing_offset ||
2522: new_offset >= size) {
2523: vm_page_lock_queues();
2524: vm_page_free(p);
2525: vm_page_unlock_queues();
2526: } else {
2527: pp = vm_page_lookup(object, new_offset);
2528: if (pp != VM_PAGE_NULL && !pp->absent) {
2529: /*
2530: * Parent object has a real page.
2531: * Throw away the backing object's
2532: * page.
2533: */
2534: vm_page_lock_queues();
2535: vm_page_free(p);
2536: vm_page_unlock_queues();
2537: }
2538: else {
2539: if (pp != VM_PAGE_NULL) {
2540: /*
2541: * Parent has an absent page...
2542: * it's not being paged in, so
2543: * it must really be missing from
2544: * the parent.
2545: *
2546: * Throw out the absent page...
2547: * any faults looking for that
2548: * page will restart with the new
2549: * one.
2550: */
2551:
2552: /*
2553: * This should never happen -- the
2554: * parent cannot have ever had an
2555: * external memory object, and thus
2556: * cannot have absent pages.
2557: */
2558: panic("vm_object_collapse: bad case");
2559:
2560: vm_page_lock_queues();
2561: vm_page_free(pp);
2562: vm_page_unlock_queues();
2563:
2564: /*
2565: * Fall through to move the backing
2566: * object's page up.
2567: */
2568: }
2569: /*
2570: * Parent now has no page.
2571: * Move the backing object's page up.
2572: */
2573: vm_page_rename(p, object, new_offset);
2574: }
2575: }
2576: }
2577:
2578: /*
2579: * Move the pager from backing_object to object.
2580: *
2581: * XXX We're only using part of the paging space
2582: * for keeps now... we ought to discard the
2583: * unused portion.
2584: */
2585:
2586: switch (vm_object_collapse_debug) {
2587: case 0:
2588: break;
2589: case 1:
2590: if ((backing_object->pager == IP_NULL) &&
2591: (backing_object->pager_request ==
2592: PAGER_REQUEST_NULL))
2593: break;
2594: /* Fall through to... */
2595:
2596: default:
2597: printf("vm_object_collapse: %#x (pager %#x, request %#x) up to %#x\n",
2598: backing_object, backing_object->pager, backing_object->pager_request,
2599: object);
2600: if (vm_object_collapse_debug > 2)
2601: Debugger("vm_object_collapse");
2602: }
2603:
2604: object->pager = backing_object->pager;
2605: if (object->pager != IP_NULL)
2606: ipc_kobject_set(object->pager,
2607: (ipc_kobject_t) object,
2608: IKOT_PAGER);
2609: object->pager_initialized = backing_object->pager_initialized;
2610: object->pager_ready = backing_object->pager_ready;
2611: object->pager_created = backing_object->pager_created;
2612:
2613: object->pager_request = backing_object->pager_request;
2614: #if NORMA_VM
2615: old_name_port = object->pager_name;
2616: object->pager_name = backing_object->pager_name;
2617: #else /* NORMA_VM */
2618: if (object->pager_request != IP_NULL)
2619: ipc_kobject_set(object->pager_request,
2620: (ipc_kobject_t) object,
2621: IKOT_PAGING_REQUEST);
2622: old_name_port = object->pager_name;
2623: if (old_name_port != IP_NULL)
2624: ipc_kobject_set(old_name_port,
2625: IKO_NULL, IKOT_NONE);
2626: object->pager_name = backing_object->pager_name;
2627: if (object->pager_name != IP_NULL)
2628: ipc_kobject_set(object->pager_name,
2629: (ipc_kobject_t) object,
2630: IKOT_PAGING_NAME);
2631: #endif /* NORMA_VM */
2632:
2633: vm_object_cache_unlock();
2634:
2635: /*
2636: * If there is no pager, leave paging-offset alone.
2637: */
2638: if (object->pager != IP_NULL)
2639: object->paging_offset =
2640: backing_object->paging_offset +
2641: backing_offset;
2642:
2643: #if MACH_PAGEMAP
2644: assert(object->existence_info == VM_EXTERNAL_NULL);
2645: object->existence_info = backing_object->existence_info;
2646: #endif /* MACH_PAGEMAP */
2647:
2648: /*
2649: * Object now shadows whatever backing_object did.
2650: * Note that the reference to backing_object->shadow
2651: * moves from within backing_object to within object.
2652: */
2653:
2654: object->shadow = backing_object->shadow;
2655: object->shadow_offset += backing_object->shadow_offset;
2656: if (object->shadow != VM_OBJECT_NULL &&
2657: object->shadow->copy != VM_OBJECT_NULL) {
2658: panic("vm_object_collapse: we collapsed a copy-object!");
2659: }
2660: /*
2661: * Discard backing_object.
2662: *
2663: * Since the backing object has no pages, no
2664: * pager left, and no object references within it,
2665: * all that is necessary is to dispose of it.
2666: */
2667:
2668: assert(
2669: (backing_object->ref_count == 1) &&
2670: (backing_object->resident_page_count == 0) &&
2671: (backing_object->paging_in_progress == 0)
2672: );
2673:
2674: assert(backing_object->alive);
2675: backing_object->alive = FALSE;
2676: vm_object_unlock(backing_object);
2677:
2678: vm_object_unlock(object);
2679: if (old_name_port != IP_NULL)
2680: #if NORMA_VM
2681: ipc_port_release_send(old_name_port);
2682: #else /* NORMA_VM */
2683: ipc_port_dealloc_kernel(old_name_port);
2684: #endif /* NORMA_VM */
2685: zfree(vm_object_zone, (vm_offset_t) backing_object);
2686: vm_object_lock(object);
2687:
2688: object_collapses++;
2689: }
2690: else {
2691: if (!vm_object_collapse_bypass_allowed) {
2692: vm_object_unlock(backing_object);
2693: return;
2694: }
2695:
2696: /*
2697: * If all of the pages in the backing object are
2698: * shadowed by the parent object, the parent
2699: * object no longer has to shadow the backing
2700: * object; it can shadow the next one in the
2701: * chain.
2702: *
2703: * The backing object must not be paged out - we'd
2704: * have to check all of the paged-out pages, as
2705: * well.
2706: */
2707:
2708: if (backing_object->pager_created) {
2709: vm_object_unlock(backing_object);
2710: return;
2711: }
2712:
2713: /*
2714: * Should have a check for a 'small' number
2715: * of pages here.
2716: */
2717:
2718: queue_iterate(&backing_object->memq, p,
2719: vm_page_t, listq)
2720: {
2721: new_offset = (p->offset - backing_offset);
2722:
2723: /*
2724: * If the parent has a page here, or if
2725: * this page falls outside the parent,
2726: * keep going.
2727: *
2728: * Otherwise, the backing_object must be
2729: * left in the chain.
2730: */
2731:
2732: if (p->offset >= backing_offset &&
2733: new_offset <= size &&
2734: (pp = vm_page_lookup(object, new_offset))
2735: == VM_PAGE_NULL) {
2736: /*
2737: * Page still needed.
2738: * Can't go any further.
2739: */
2740: vm_object_unlock(backing_object);
2741: return;
2742: }
2743: }
2744:
2745: /*
2746: * Make the parent shadow the next object
2747: * in the chain. Deallocating backing_object
2748: * will not remove it, since its reference
2749: * count is at least 2.
2750: */
2751:
2752: vm_object_reference(object->shadow = backing_object->shadow);
2753: object->shadow_offset += backing_object->shadow_offset;
2754:
2755: /*
2756: * Backing object might have had a copy pointer
2757: * to us. If it did, clear it.
2758: */
2759: if (backing_object->copy == object)
2760: backing_object->copy = VM_OBJECT_NULL;
2761:
2762: /*
2763: * Drop the reference count on backing_object.
2764: * Since its ref_count was at least 2, it
2765: * will not vanish; so we don't need to call
2766: * vm_object_deallocate.
2767: */
2768: backing_object->ref_count--;
2769: assert(backing_object->ref_count > 0);
2770: vm_object_unlock(backing_object);
2771:
2772: object_bypasses ++;
2773:
2774: }
2775:
2776: /*
2777: * Try again with this object's new backing object.
2778: */
2779: }
2780: }
2781:
2782: /*
2783: * Routine: vm_object_page_remove: [internal]
2784: * Purpose:
2785: * Removes all physical pages in the specified
2786: * object range from the object's list of pages.
2787: *
2788: * In/out conditions:
2789: * The object must be locked.
2790: */
2791: unsigned int vm_object_page_remove_lookup = 0;
2792: unsigned int vm_object_page_remove_iterate = 0;
2793:
2794: void vm_object_page_remove(
2795: register vm_object_t object,
2796: register vm_offset_t start,
2797: register vm_offset_t end)
2798: {
2799: register vm_page_t p, next;
2800:
2801: /*
2802: * One and two page removals are most popular.
2803: * The factor of 16 here is somewhat arbitrary.
2804: * It balances vm_object_lookup vs iteration.
2805: */
2806:
2807: if (atop(end - start) < (unsigned)object->resident_page_count/16) {
2808: vm_object_page_remove_lookup++;
2809:
2810: for (; start < end; start += PAGE_SIZE) {
2811: p = vm_page_lookup(object, start);
2812: if (p != VM_PAGE_NULL) {
2813: if (!p->fictitious)
2814: pmap_page_protect(p->phys_addr,
2815: VM_PROT_NONE);
2816: vm_page_lock_queues();
2817: vm_page_free(p);
2818: vm_page_unlock_queues();
2819: }
2820: }
2821: } else {
2822: vm_object_page_remove_iterate++;
2823:
2824: p = (vm_page_t) queue_first(&object->memq);
2825: while (!queue_end(&object->memq, (queue_entry_t) p)) {
2826: next = (vm_page_t) queue_next(&p->listq);
2827: if ((start <= p->offset) && (p->offset < end)) {
2828: if (!p->fictitious)
2829: pmap_page_protect(p->phys_addr,
2830: VM_PROT_NONE);
2831: vm_page_lock_queues();
2832: vm_page_free(p);
2833: vm_page_unlock_queues();
2834: }
2835: p = next;
2836: }
2837: }
2838: }
2839:
2840: /*
2841: * Routine: vm_object_coalesce
2842: * Function: Coalesces two objects backing up adjoining
2843: * regions of memory into a single object.
2844: *
2845: * returns TRUE if objects were combined.
2846: *
2847: * NOTE: Only works at the moment if the second object is NULL -
2848: * if it's not, which object do we lock first?
2849: *
2850: * Parameters:
2851: * prev_object First object to coalesce
2852: * prev_offset Offset into prev_object
2853: * next_object Second object into coalesce
2854: * next_offset Offset into next_object
2855: *
2856: * prev_size Size of reference to prev_object
2857: * next_size Size of reference to next_object
2858: *
2859: * Conditions:
2860: * The object must *not* be locked.
2861: */
2862:
2863: boolean_t vm_object_coalesce(
2864: register vm_object_t prev_object,
2865: vm_object_t next_object,
2866: vm_offset_t prev_offset,
2867: vm_offset_t next_offset,
2868: vm_size_t prev_size,
2869: vm_size_t next_size)
2870: {
2871: vm_size_t newsize;
2872:
2873: #ifdef lint
2874: next_offset++;
2875: #endif /* lint */
2876:
2877: if (next_object != VM_OBJECT_NULL) {
2878: return FALSE;
2879: }
2880:
2881: if (prev_object == VM_OBJECT_NULL) {
2882: return TRUE;
2883: }
2884:
2885: vm_object_lock(prev_object);
2886:
2887: /*
2888: * Try to collapse the object first
2889: */
2890: vm_object_collapse(prev_object);
2891:
2892: /*
2893: * Can't coalesce if pages not mapped to
2894: * prev_entry may be in use anyway:
2895: * . more than one reference
2896: * . paged out
2897: * . shadows another object
2898: * . has a copy elsewhere
2899: * . paging references (pages might be in page-list)
2900: */
2901:
2902: if ((prev_object->ref_count > 1) ||
2903: prev_object->pager_created ||
2904: (prev_object->shadow != VM_OBJECT_NULL) ||
2905: (prev_object->copy != VM_OBJECT_NULL) ||
2906: (prev_object->paging_in_progress != 0)) {
2907: vm_object_unlock(prev_object);
2908: return FALSE;
2909: }
2910:
2911: /*
2912: * Remove any pages that may still be in the object from
2913: * a previous deallocation.
2914: */
2915:
2916: vm_object_page_remove(prev_object,
2917: prev_offset + prev_size,
2918: prev_offset + prev_size + next_size);
2919:
2920: /*
2921: * Extend the object if necessary.
2922: */
2923: newsize = prev_offset + prev_size + next_size;
2924: if (newsize > prev_object->size)
2925: prev_object->size = newsize;
2926:
2927: vm_object_unlock(prev_object);
2928: return TRUE;
2929: }
2930:
2931: vm_object_t vm_object_request_object(
2932: ipc_port_t p)
2933: {
2934: return vm_object_lookup(p);
2935: }
2936:
2937: /*
2938: * Routine: vm_object_name
2939: * Purpose:
2940: * Returns a naked send right to the "name" port associated
2941: * with this object.
2942: */
2943: ipc_port_t vm_object_name(
2944: vm_object_t object)
2945: {
2946: ipc_port_t p;
2947:
2948: if (object == VM_OBJECT_NULL)
2949: return IP_NULL;
2950:
2951: vm_object_lock(object);
2952:
2953: while (object->shadow != VM_OBJECT_NULL) {
2954: vm_object_t new_object = object->shadow;
2955: vm_object_lock(new_object);
2956: vm_object_unlock(object);
2957: object = new_object;
2958: }
2959:
2960: p = object->pager_name;
2961: if (p != IP_NULL)
2962: #if NORMA_VM
2963: p = ipc_port_copy_send(p);
2964: #else /* NORMA_VM */
2965: p = ipc_port_make_send(p);
2966: #endif /* NORMA_VM */
2967: vm_object_unlock(object);
2968:
2969: return p;
2970: }
2971:
2972: /*
2973: * Attach a set of physical pages to an object, so that they can
2974: * be mapped by mapping the object. Typically used to map IO memory.
2975: *
2976: * The mapping function and its private data are used to obtain the
2977: * physical addresses for each page to be mapped.
2978: */
2979: void
2980: vm_object_page_map(
2981: vm_object_t object,
2982: vm_offset_t offset,
2983: vm_size_t size,
2984: vm_offset_t (*map_fn)(void *, vm_offset_t),
2985: void * map_fn_data) /* private to map_fn */
2986: {
2987: int num_pages;
2988: int i;
2989: vm_page_t m;
2990: vm_page_t old_page;
2991: vm_offset_t addr;
2992:
2993: num_pages = atop(size);
2994:
2995: for (i = 0; i < num_pages; i++, offset += PAGE_SIZE) {
2996:
2997: addr = (*map_fn)(map_fn_data, offset);
2998:
2999: while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
3000: vm_page_more_fictitious();
3001:
3002: vm_object_lock(object);
3003: if ((old_page = vm_page_lookup(object, offset))
3004: != VM_PAGE_NULL)
3005: {
3006: vm_page_lock_queues();
3007: vm_page_free(old_page);
3008: vm_page_unlock_queues();
3009: }
3010:
3011: vm_page_init(m, addr);
3012: m->private = TRUE; /* don`t free page */
3013: m->wire_count = 1;
3014: vm_page_lock_queues();
3015: vm_page_insert(m, object, offset);
3016: vm_page_unlock_queues();
3017:
3018: PAGE_WAKEUP_DONE(m);
3019: vm_object_unlock(object);
3020: }
3021: }
3022:
3023: #include <mach_kdb.h>
3024:
3025:
3026: #if MACH_KDB
3027: #define printf kdbprintf
3028:
3029: boolean_t vm_object_print_pages = FALSE;
3030:
3031: /*
3032: * vm_object_print: [ debug ]
3033: */
3034: void vm_object_print(
3035: vm_object_t object)
3036: {
3037: register vm_page_t p;
3038: extern indent;
3039:
3040: register int count;
3041:
3042: if (object == VM_OBJECT_NULL)
3043: return;
3044:
3045: iprintf("Object 0x%X: size=0x%X",
3046: (vm_offset_t) object, (vm_offset_t) object->size);
3047: printf(", %d references, %d resident pages,", object->ref_count,
3048: object->resident_page_count);
3049: printf(" %d absent pages,", object->absent_count);
3050: printf(" %d paging ops\n", object->paging_in_progress);
3051: indent += 2;
3052: iprintf("memory object=0x%X (offset=0x%X),",
3053: (vm_offset_t) object->pager, (vm_offset_t) object->paging_offset);
3054: printf("control=0x%X, name=0x%X\n",
3055: (vm_offset_t) object->pager_request, (vm_offset_t) object->pager_name);
3056: iprintf("%s%s",
3057: object->pager_ready ? " ready" : "",
3058: object->pager_created ? " created" : "");
3059: printf("%s,%s ",
3060: object->pager_initialized ? "" : "uninitialized",
3061: object->temporary ? "temporary" : "permanent");
3062: printf("%s%s,",
3063: object->internal ? "internal" : "external",
3064: object->can_persist ? " cacheable" : "");
3065: printf("copy_strategy=%d\n", (vm_offset_t)object->copy_strategy);
3066: iprintf("shadow=0x%X (offset=0x%X),",
3067: (vm_offset_t) object->shadow, (vm_offset_t) object->shadow_offset);
3068: printf("copy=0x%X\n", (vm_offset_t) object->copy);
3069:
3070: indent += 2;
3071:
3072: if (vm_object_print_pages) {
3073: count = 0;
3074: p = (vm_page_t) queue_first(&object->memq);
3075: while (!queue_end(&object->memq, (queue_entry_t) p)) {
3076: if (count == 0) iprintf("memory:=");
3077: else if (count == 4) {printf("\n"); iprintf(" ..."); count = 0;}
3078: else printf(",");
3079: count++;
3080:
3081: printf("(off=0x%X,page=0x%X)", p->offset, (vm_offset_t) p);
3082: p = (vm_page_t) queue_next(&p->listq);
3083: }
3084: if (count != 0)
3085: printf("\n");
3086: }
3087: indent -= 4;
3088: }
3089:
3090: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.