![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_pageout.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52: /*
53: * File: vm/vm_pageout.c
54: * Author: Avadis Tevanian, Jr., Michael Wayne Young
55: * Date: 1985
56: *
57: * The proverbial page-out daemon.
58: */
59: #ifdef MACH_BSD
60: /* remove after component merge */
61: extern int vnode_pager_workaround;
62: #endif
63:
64: #include <mach_pagemap.h>
65: #include <mach_cluster_stats.h>
66: #include <mach_kdb.h>
67: #include <dipc.h>
68: #include <advisory_pageout.h>
69:
70: #include <mach/mach_types.h>
71: #include <mach/memory_object.h>
72: #include <mach/memory_object_default.h>
73: #include <mach/mach_host_server.h>
74: #include <mach/vm_param.h>
75: #include <mach/vm_statistics.h>
76: #include <kern/host_statistics.h>
77: #include <kern/counters.h>
78: #include <kern/thread.h>
79: #include <kern/thread_swap.h>
80: #include <kern/xpr.h>
81: #include <vm/pmap.h>
82: #include <vm/vm_map.h>
83: #include <vm/vm_object.h>
84: #include <vm/vm_page.h>
85: #include <vm/vm_pageout.h>
86: #include <machine/vm_tuning.h>
87: #include <kern/misc_protos.h>
88:
89: extern ipc_port_t memory_manager_default;
90:
91: #ifndef VM_PAGE_LAUNDRY_MAX
92: #define VM_PAGE_LAUNDRY_MAX 10 /* outstanding DMM page cleans */
93: #endif /* VM_PAGEOUT_LAUNDRY_MAX */
94:
95: #ifndef VM_PAGEOUT_BURST_MAX
96: #define VM_PAGEOUT_BURST_MAX 10 /* simultaneous EMM page cleans */
97: #endif /* VM_PAGEOUT_BURST_MAX */
98:
99: #ifndef VM_PAGEOUT_DISCARD_MAX
100: #define VM_PAGEOUT_DISCARD_MAX 68 /* simultaneous EMM page cleans */
101: #endif /* VM_PAGEOUT_DISCARD_MAX */
102:
103: #ifndef VM_PAGEOUT_BURST_WAIT
104: #define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */
105: #endif /* VM_PAGEOUT_BURST_WAIT */
106:
107: #ifndef VM_PAGEOUT_EMPTY_WAIT
108: #define VM_PAGEOUT_EMPTY_WAIT 200 /* milliseconds */
109: #endif /* VM_PAGEOUT_EMPTY_WAIT */
110:
111: /*
112: * To obtain a reasonable LRU approximation, the inactive queue
113: * needs to be large enough to give pages on it a chance to be
114: * referenced a second time. This macro defines the fraction
115: * of active+inactive pages that should be inactive.
116: * The pageout daemon uses it to update vm_page_inactive_target.
117: *
118: * If vm_page_free_count falls below vm_page_free_target and
119: * vm_page_inactive_count is below vm_page_inactive_target,
120: * then the pageout daemon starts running.
121: */
122:
123: #ifndef VM_PAGE_INACTIVE_TARGET
124: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3)
125: #endif /* VM_PAGE_INACTIVE_TARGET */
126:
127: /*
128: * Once the pageout daemon starts running, it keeps going
129: * until vm_page_free_count meets or exceeds vm_page_free_target.
130: */
131:
132: #ifndef VM_PAGE_FREE_TARGET
133: #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
134: #endif /* VM_PAGE_FREE_TARGET */
135:
136: /*
137: * The pageout daemon always starts running once vm_page_free_count
138: * falls below vm_page_free_min.
139: */
140:
141: #ifndef VM_PAGE_FREE_MIN
142: #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
143: #endif /* VM_PAGE_FREE_MIN */
144:
145: /*
146: * When vm_page_free_count falls below vm_page_free_reserved,
147: * only vm-privileged threads can allocate pages. vm-privilege
148: * allows the pageout daemon and default pager (and any other
149: * associated threads needed for default pageout) to continue
150: * operation by dipping into the reserved pool of pages.
151: */
152:
153: #ifndef VM_PAGE_FREE_RESERVED
154: #define VM_PAGE_FREE_RESERVED \
155: ((8 * VM_PAGE_LAUNDRY_MAX) + NCPUS)
156: #endif /* VM_PAGE_FREE_RESERVED */
157:
158:
159: /*
160: * Forward declarations for internal routines.
161: */
162: extern void vm_pageout_continue(void);
163: extern void vm_pageout_scan(void);
164: extern void vm_pageout_throttle(vm_page_t m);
165: extern vm_page_t vm_pageout_cluster_page(
166: vm_object_t object,
167: vm_offset_t offset,
168: boolean_t precious_clean);
169:
170: unsigned int vm_pageout_reserved_internal = 0;
171: unsigned int vm_pageout_reserved_really = 0;
172:
173: unsigned int vm_page_laundry_max = 0; /* # of clusters outstanding */
174: unsigned int vm_page_laundry_min = 0;
175: unsigned int vm_pageout_burst_max = 0;
176: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */
177: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
178: unsigned int vm_pageout_burst_min = 0;
179: unsigned int vm_pageout_pause_count = 0;
180: unsigned int vm_pageout_pause_max = 0;
181: unsigned int vm_free_page_pause = 100; /* milliseconds */
182:
183: /*
184: * These variables record the pageout daemon's actions:
185: * how many pages it looks at and what happens to those pages.
186: * No locking needed because only one thread modifies the variables.
187: */
188:
189: unsigned int vm_pageout_active = 0; /* debugging */
190: unsigned int vm_pageout_inactive = 0; /* debugging */
191: unsigned int vm_pageout_inactive_throttled = 0; /* debugging */
192: unsigned int vm_pageout_inactive_forced = 0; /* debugging */
193: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
194: unsigned int vm_pageout_inactive_avoid = 0; /* debugging */
195: unsigned int vm_pageout_inactive_busy = 0; /* debugging */
196: unsigned int vm_pageout_inactive_absent = 0; /* debugging */
197: unsigned int vm_pageout_inactive_used = 0; /* debugging */
198: unsigned int vm_pageout_inactive_clean = 0; /* debugging */
199: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
200: unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */
201: unsigned int vm_pageout_inactive_pinned = 0; /* debugging */
202: unsigned int vm_pageout_inactive_limbo = 0; /* debugging */
203: unsigned int vm_pageout_setup_limbo = 0; /* debugging */
204: unsigned int vm_pageout_setup_unprepped = 0; /* debugging */
205: unsigned int vm_stat_discard = 0; /* debugging */
206: unsigned int vm_stat_discard_sent = 0; /* debugging */
207: unsigned int vm_stat_discard_failure = 0; /* debugging */
208: unsigned int vm_stat_discard_throttle = 0; /* debugging */
209: unsigned int vm_pageout_scan_active_emm_throttle = 0; /* debugging */
210: unsigned int vm_pageout_scan_active_emm_throttle_success = 0; /* debugging */
211: unsigned int vm_pageout_scan_active_emm_throttle_failure = 0; /* debugging */
212: unsigned int vm_pageout_scan_inactive_emm_throttle = 0; /* debugging */
213: unsigned int vm_pageout_scan_inactive_emm_throttle_success = 0; /* debugging */
214: unsigned int vm_pageout_scan_inactive_emm_throttle_failure = 0; /* debugging */
215:
216:
217: unsigned int vm_pageout_out_of_line = 0;
218: unsigned int vm_pageout_in_place = 0;
219: /*
220: * Routine: vm_pageout_object_allocate
221: * Purpose:
222: * Allocate an object for use as out-of-line memory in a
223: * data_return/data_initialize message.
224: * The page must be in an unlocked object.
225: *
226: * If the page belongs to a trusted pager, cleaning in place
227: * will be used, which utilizes a special "pageout object"
228: * containing private alias pages for the real page frames.
229: * Untrusted pagers use normal out-of-line memory.
230: */
231: vm_object_t
232: vm_pageout_object_allocate(
233: vm_page_t m,
234: vm_size_t size,
235: vm_offset_t offset)
236: {
237: vm_object_t object = m->object;
238: vm_object_t new_object;
239:
240: assert(object->pager_ready);
241:
242: if (object->pager_trusted || object->internal)
243: vm_pageout_throttle(m);
244:
245: new_object = vm_object_allocate(size);
246:
247: if (object->pager_trusted) {
248: assert (offset < object->size);
249:
250: vm_object_lock(new_object);
251: new_object->pageout = TRUE;
252: new_object->shadow = object;
253: new_object->can_persist = FALSE;
254: new_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
255: new_object->shadow_offset = offset;
256: vm_object_unlock(new_object);
257:
258: /*
259: * Take a paging reference on the object. This will be dropped
260: * in vm_pageout_object_terminate()
261: */
262: vm_object_lock(object);
263: vm_object_paging_begin(object);
264: vm_object_unlock(object);
265:
266: vm_pageout_in_place++;
267: } else
268: vm_pageout_out_of_line++;
269: return(new_object);
270: }
271:
272: #if MACH_CLUSTER_STATS
273: unsigned long vm_pageout_cluster_dirtied = 0;
274: unsigned long vm_pageout_cluster_cleaned = 0;
275: unsigned long vm_pageout_cluster_collisions = 0;
276: unsigned long vm_pageout_cluster_clusters = 0;
277: unsigned long vm_pageout_cluster_conversions = 0;
278: unsigned long vm_pageout_target_collisions = 0;
279: unsigned long vm_pageout_target_page_dirtied = 0;
280: unsigned long vm_pageout_target_page_freed = 0;
281: unsigned long vm_pageout_target_page_pinned = 0;
282: unsigned long vm_pageout_target_page_limbo = 0;
283: #define CLUSTER_STAT(clause) clause
284: #else /* MACH_CLUSTER_STATS */
285: #define CLUSTER_STAT(clause)
286: #endif /* MACH_CLUSTER_STATS */
287:
288: /*
289: * Routine: vm_pageout_object_terminate
290: * Purpose:
291: * Destroy the pageout_object allocated by
292: * vm_pageout_object_allocate(), and perform all of the
293: * required cleanup actions.
294: *
295: * In/Out conditions:
296: * The object must be locked, and will be returned locked.
297: */
298: void
299: vm_pageout_object_terminate(
300: vm_object_t object)
301: {
302: vm_object_t shadow_object;
303:
304: /*
305: * Deal with the deallocation (last reference) of a pageout object
306: * (used for cleaning-in-place) by dropping the paging references/
307: * freeing pages in the original object.
308: */
309:
310: assert(object->pageout);
311: shadow_object = object->shadow;
312: vm_object_lock(shadow_object);
313:
314: while (!queue_empty(&object->memq)) {
315: vm_page_t p, m;
316: vm_offset_t offset;
317:
318: p = (vm_page_t) queue_first(&object->memq);
319:
320: assert(p->private);
321: assert(p->pageout);
322: p->pageout = FALSE;
323: assert(!p->cleaning);
324:
325: offset = p->offset;
326: VM_PAGE_FREE(p);
327: p = VM_PAGE_NULL;
328:
329: m = vm_page_lookup(shadow_object,
330: offset + object->shadow_offset);
331:
332: if(m == VM_PAGE_NULL)
333: continue;
334: assert(m->cleaning);
335:
336: /*
337: * Account for the paging reference taken when
338: * m->cleaning was set on this page.
339: */
340: vm_object_paging_end(shadow_object);
341: assert((m->dirty) || (m->precious) ||
342: (m->busy && m->cleaning));
343:
344: /*
345: * Handle the trusted pager throttle.
346: */
347: vm_page_lock_queues();
348: if (m->laundry) {
349: vm_page_laundry_count--;
350: m->laundry = FALSE;
351: if (vm_page_laundry_count < vm_page_laundry_min) {
352: vm_page_laundry_min = 0;
353: thread_wakeup((event_t) &vm_page_laundry_count);
354: }
355: }
356:
357: /*
358: * Handle the "target" page(s). These pages are to be freed if
359: * successfully cleaned. Target pages are always busy, and are
360: * wired exactly once. The initial target pages are not mapped,
361: * (so cannot be referenced or modified) but converted target
362: * pages may have been modified between the selection as an
363: * adjacent page and conversion to a target.
364: */
365: if (m->pageout) {
366: assert(m->busy);
367: assert(m->wire_count == 1);
368: m->cleaning = FALSE;
369: m->pageout = FALSE;
370: #if MACH_CLUSTER_STATS
371: if (m->wanted) vm_pageout_target_collisions++;
372: #endif
373: /*
374: * Revoke all access to the page. Since the object is
375: * locked, and the page is busy, this prevents the page
376: * from being dirtied after the pmap_is_modified() call
377: * returns.
378: */
379: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
380:
381: /*
382: * Since the page is left "dirty" but "not modifed", we
383: * can detect whether the page was redirtied during
384: * pageout by checking the modify state.
385: */
386: m->dirty = pmap_is_modified(m->phys_addr);
387:
388: if (m->dirty) {
389: CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
390: vm_page_unwire(m);/* reactivates */
391: VM_STAT(reactivations++);
392: PAGE_WAKEUP_DONE(m);
393: } else if (m->prep_pin_count != 0) {
394: vm_page_pin_lock();
395: if (m->pin_count != 0) {
396: /* page is pinned; reactivate */
397: CLUSTER_STAT(
398: vm_pageout_target_page_pinned++;)
399: vm_page_unwire(m);/* reactivates */
400: VM_STAT(reactivations++);
401: PAGE_WAKEUP_DONE(m);
402: } else {
403: /*
404: * page is prepped but not pinned; send
405: * it into limbo. Note that
406: * vm_page_free (which will be called
407: * after releasing the pin lock) knows
408: * how to handle a page with limbo set.
409: */
410: m->limbo = TRUE;
411: CLUSTER_STAT(
412: vm_pageout_target_page_limbo++;)
413: }
414: vm_page_pin_unlock();
415: if (m->limbo)
416: vm_page_free(m);
417: } else {
418: CLUSTER_STAT(vm_pageout_target_page_freed++;)
419: vm_page_free(m);/* clears busy, etc. */
420: }
421: vm_page_unlock_queues();
422: continue;
423: }
424: /*
425: * Handle the "adjacent" pages. These pages were cleaned in
426: * place, and should be left alone.
427: * If prep_pin_count is nonzero, then someone is using the
428: * page, so make it active.
429: */
430: if (!m->active && !m->inactive) {
431: if (m->reference || m->prep_pin_count != 0)
432: vm_page_activate(m);
433: else
434: vm_page_deactivate(m);
435: }
436: if((m->busy) && (m->cleaning)) {
437:
438: /* the request_page_list case, (COPY_OUT_FROM FALSE) */
439: m->busy = FALSE;
440:
441: /* We do not re-set m->dirty ! */
442: /* The page was busy so no extraneous activity */
443: /* could have occured. COPY_INTO is a read into the */
444: /* new pages. CLEAN_IN_PLACE does actually write */
445: /* out the pages but handling outside of this code */
446: /* will take care of resetting dirty. We clear the */
447: /* modify however for the Programmed I/O case. */
448: pmap_clear_modify(m->phys_addr);
449: if(m->absent) {
450: m->absent = FALSE;
451: if(shadow_object->absent_count == 1)
452: vm_object_absent_release(shadow_object);
453: else
454: shadow_object->absent_count--;
455: }
456: m->overwriting = FALSE;
457: } else if (m->overwriting) {
458: /* alternate request page list, write to page_list */
459: /* case. Occurs when the original page was wired */
460: /* at the time of the list request */
461: assert(m->wire_count != 0);
462: vm_page_unwire(m);/* reactivates */
463: m->overwriting = FALSE;
464: } else {
465: /*
466: * Set the dirty state according to whether or not the page was
467: * modified during the pageout. Note that we purposefully do
468: * NOT call pmap_clear_modify since the page is still mapped.
469: * If the page were to be dirtied between the 2 calls, this
470: * this fact would be lost. This code is only necessary to
471: * maintain statistics, since the pmap module is always
472: * consulted if m->dirty is false.
473: */
474: #if MACH_CLUSTER_STATS
475: m->dirty = pmap_is_modified(m->phys_addr);
476:
477: if (m->dirty) vm_pageout_cluster_dirtied++;
478: else vm_pageout_cluster_cleaned++;
479: if (m->wanted) vm_pageout_cluster_collisions++;
480: #else
481: m->dirty = 0;
482: #endif
483: }
484: m->cleaning = FALSE;
485:
486:
487: /*
488: * Wakeup any thread waiting for the page to be un-cleaning.
489: */
490: PAGE_WAKEUP(m);
491: vm_page_unlock_queues();
492: }
493: /*
494: * Account for the paging reference taken in vm_paging_object_allocate.
495: */
496: vm_object_paging_end(shadow_object);
497: vm_object_unlock(shadow_object);
498:
499: assert(object->ref_count == 0);
500: assert(object->paging_in_progress == 0);
501: assert(object->resident_page_count == 0);
502: return;
503: }
504:
505: /*
506: * Routine: vm_pageout_setup
507: * Purpose:
508: * Set up a page for pageout (clean & flush).
509: *
510: * Move the page to a new object, as part of which it will be
511: * sent to its memory manager in a memory_object_data_write or
512: * memory_object_initialize message.
513: *
514: * The "new_object" and "new_offset" arguments
515: * indicate where the page should be moved.
516: *
517: * In/Out conditions:
518: * The page in question must not be on any pageout queues,
519: * and must be busy. The object to which it belongs
520: * must be unlocked, and the caller must hold a paging
521: * reference to it. The new_object must not be locked.
522: *
523: * This routine returns a pointer to a place-holder page,
524: * inserted at the same offset, to block out-of-order
525: * requests for the page. The place-holder page must
526: * be freed after the data_write or initialize message
527: * has been sent.
528: *
529: * The original page is put on a paging queue and marked
530: * not busy on exit.
531: */
532: vm_page_t
533: vm_pageout_setup(
534: register vm_page_t m,
535: register vm_object_t new_object,
536: vm_offset_t new_offset)
537: {
538: register vm_object_t old_object = m->object;
539: vm_offset_t paging_offset;
540: vm_offset_t offset;
541: register vm_page_t holding_page;
542: register vm_page_t new_m;
543: register vm_page_t new_page;
544: boolean_t need_to_wire = FALSE;
545:
546:
547: XPR(XPR_VM_PAGEOUT,
548: "vm_pageout_setup, obj 0x%X off 0x%X page 0x%X new obj 0x%X offset 0x%X\n",
549: (integer_t)m->object, (integer_t)m->offset,
550: (integer_t)m, (integer_t)new_object,
551: (integer_t)new_offset);
552: assert(m && m->busy && !m->absent && !m->fictitious && !m->error &&
553: !m->restart);
554:
555: assert(m->dirty || m->precious);
556:
557: /*
558: * Create a place-holder page where the old one was, to prevent
559: * attempted pageins of this page while we're unlocked.
560: * If the pageout daemon put this page in limbo and we're not
561: * going to clean in place, get another fictitious page to
562: * exchange for it now.
563: */
564: VM_PAGE_GRAB_FICTITIOUS(holding_page);
565:
566: if (m->limbo)
567: VM_PAGE_GRAB_FICTITIOUS(new_page);
568:
569: vm_object_lock(old_object);
570:
571: offset = m->offset;
572: paging_offset = offset + old_object->paging_offset;
573:
574: if (old_object->pager_trusted) {
575: /*
576: * This pager is trusted, so we can clean this page
577: * in place. Leave it in the old object, and mark it
578: * cleaning & pageout.
579: */
580: new_m = holding_page;
581: holding_page = VM_PAGE_NULL;
582:
583: /*
584: * If the pageout daemon put this page in limbo, exchange the
585: * identities of the limbo page and the new fictitious page,
586: * and continue with the new page, unless the prep count has
587: * gone to zero in the meantime (which means no one is
588: * interested in the page any more). In that case, just clear
589: * the limbo bit and free the extra fictitious page.
590: */
591: if (m->limbo) {
592: if (m->prep_pin_count == 0) {
593: /* page doesn't have to be in limbo any more */
594: m->limbo = FALSE;
595: vm_page_free(new_page);
596: vm_pageout_setup_unprepped++;
597: } else {
598: vm_page_lock_queues();
599: VM_PAGE_QUEUES_REMOVE(m);
600: vm_page_remove(m);
601: vm_page_limbo_exchange(m, new_page);
602: vm_pageout_setup_limbo++;
603: vm_page_release_limbo(m);
604: m = new_page;
605: vm_page_insert(m, old_object, offset);
606: vm_page_unlock_queues();
607: }
608: }
609:
610: /*
611: * Set up new page to be private shadow of real page.
612: */
613: new_m->phys_addr = m->phys_addr;
614: new_m->fictitious = FALSE;
615: new_m->private = TRUE;
616: new_m->pageout = TRUE;
617:
618: /*
619: * Mark real page as cleaning (indicating that we hold a
620: * paging reference to be released via m_o_d_r_c) and
621: * pageout (indicating that the page should be freed
622: * when the pageout completes).
623: */
624: pmap_clear_modify(m->phys_addr);
625: vm_page_lock_queues();
626: vm_page_wire(new_m);
627: m->cleaning = TRUE;
628: m->pageout = TRUE;
629:
630: vm_page_wire(m);
631: assert(m->wire_count == 1);
632: vm_page_unlock_queues();
633:
634: m->dirty = TRUE;
635: m->precious = FALSE;
636: m->page_lock = VM_PROT_NONE;
637: m->unusual = FALSE;
638: m->unlock_request = VM_PROT_NONE;
639: } else {
640: /*
641: * Cannot clean in place, so rip the old page out of the
642: * object, and stick the holding page in. Set new_m to the
643: * page in the new object.
644: */
645: vm_page_lock_queues();
646: VM_PAGE_QUEUES_REMOVE(m);
647: vm_page_remove(m);
648:
649: /*
650: * If the pageout daemon put this page in limbo, exchange the
651: * identities of the limbo page and the new fictitious page,
652: * and continue with the new page, unless the prep count has
653: * gone to zero in the meantime (which means no one is
654: * interested in the page any more). In that case, just clear
655: * the limbo bit and free the extra fictitious page.
656: */
657: if (m->limbo) {
658: if (m->prep_pin_count == 0) {
659: /* page doesn't have to be in limbo any more */
660: m->limbo = FALSE;
661: vm_page_free(new_page);
662: vm_pageout_setup_unprepped++;
663: } else {
664: vm_page_limbo_exchange(m, new_page);
665: vm_pageout_setup_limbo++;
666: vm_page_release_limbo(m);
667: m = new_page;
668: }
669: }
670:
671: vm_page_insert(holding_page, old_object, offset);
672: vm_page_unlock_queues();
673:
674: m->dirty = TRUE;
675: m->precious = FALSE;
676: new_m = m;
677: new_m->page_lock = VM_PROT_NONE;
678: new_m->unlock_request = VM_PROT_NONE;
679:
680: if (old_object->internal)
681: need_to_wire = TRUE;
682: }
683: /*
684: * Record that this page has been written out
685: */
686: #if MACH_PAGEMAP
687: vm_external_state_set(old_object->existence_map, offset);
688: #endif /* MACH_PAGEMAP */
689:
690: vm_object_unlock(old_object);
691:
692: vm_object_lock(new_object);
693:
694: /*
695: * Put the page into the new object. If it is a not wired
696: * (if it's the real page) it will be activated.
697: */
698:
699: vm_page_lock_queues();
700: vm_page_insert(new_m, new_object, new_offset);
701: if (need_to_wire)
702: vm_page_wire(new_m);
703: else
704: vm_page_activate(new_m);
705: PAGE_WAKEUP_DONE(new_m);
706: vm_page_unlock_queues();
707:
708: vm_object_unlock(new_object);
709:
710: /*
711: * Return the placeholder page to simplify cleanup.
712: */
713: return (holding_page);
714: }
715:
716: /*
717: * Routine: vm_pageclean_setup
718: *
719: * Purpose: setup a page to be cleaned (made non-dirty), but not
720: * necessarily flushed from the VM page cache.
721: * This is accomplished by cleaning in place.
722: *
723: * The page must not be busy, and the object and page
724: * queues must be locked.
725: *
726: */
727: void
728: vm_pageclean_setup(
729: vm_page_t m,
730: vm_page_t new_m,
731: vm_object_t new_object,
732: vm_offset_t new_offset)
733: {
734: vm_object_t old_object = m->object;
735: assert(!m->busy);
736: assert(!m->cleaning);
737:
738: XPR(XPR_VM_PAGEOUT,
739: "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
740: (integer_t)old_object, m->offset, (integer_t)m,
741: (integer_t)new_m, new_offset);
742:
743: pmap_clear_modify(m->phys_addr);
744: vm_object_paging_begin(old_object);
745:
746: /*
747: * Record that this page has been written out
748: */
749: #if MACH_PAGEMAP
750: vm_external_state_set(old_object->existence_map, m->offset);
751: #endif /*MACH_PAGEMAP*/
752:
753: /*
754: * Mark original page as cleaning in place.
755: */
756: m->cleaning = TRUE;
757: m->dirty = TRUE;
758: m->precious = FALSE;
759:
760: /*
761: * Convert the fictitious page to a private shadow of
762: * the real page.
763: */
764: assert(new_m->fictitious);
765: new_m->fictitious = FALSE;
766: new_m->private = TRUE;
767: new_m->pageout = TRUE;
768: new_m->phys_addr = m->phys_addr;
769: vm_page_wire(new_m);
770:
771: vm_page_insert(new_m, new_object, new_offset);
772: assert(!new_m->wanted);
773: new_m->busy = FALSE;
774: }
775:
776: void
777: vm_pageclean_copy(
778: vm_page_t m,
779: vm_page_t new_m,
780: vm_object_t new_object,
781: vm_offset_t new_offset)
782: {
783: XPR(XPR_VM_PAGEOUT,
784: "vm_pageclean_copy, page 0x%X new_m 0x%X new_obj 0x%X offset 0x%X\n",
785: m, new_m, new_object, new_offset, 0);
786:
787: assert((!m->busy) && (!m->cleaning));
788:
789: assert(!new_m->private && !new_m->fictitious);
790:
791: pmap_clear_modify(m->phys_addr);
792:
793: m->busy = TRUE;
794: vm_object_paging_begin(m->object);
795: vm_page_unlock_queues();
796: vm_object_unlock(m->object);
797:
798: /*
799: * Copy the original page to the new page.
800: */
801: vm_page_copy(m, new_m);
802:
803: /*
804: * Mark the old page as clean. A request to pmap_is_modified
805: * will get the right answer.
806: */
807: vm_object_lock(m->object);
808: m->dirty = FALSE;
809:
810: vm_object_paging_end(m->object);
811:
812: vm_page_lock_queues();
813: if (!m->active && !m->inactive)
814: vm_page_activate(m);
815: PAGE_WAKEUP_DONE(m);
816:
817: vm_page_insert(new_m, new_object, new_offset);
818: vm_page_activate(new_m);
819: new_m->busy = FALSE; /* No other thread can be waiting */
820: }
821:
822:
823: /*
824: * Routine: vm_pageout_initialize_page
825: * Purpose:
826: * Causes the specified page to be initialized in
827: * the appropriate memory object. This routine is used to push
828: * pages into a copy-object when they are modified in the
829: * permanent object.
830: *
831: * The page is moved to a temporary object and paged out.
832: *
833: * In/out conditions:
834: * The page in question must not be on any pageout queues.
835: * The object to which it belongs must be locked.
836: * The page must be busy, but not hold a paging reference.
837: *
838: * Implementation:
839: * Move this page to a completely new object.
840: */
841: void
842: vm_pageout_initialize_page(
843: vm_page_t m)
844: {
845: vm_map_copy_t copy;
846: vm_object_t new_object;
847: vm_object_t object;
848: vm_offset_t paging_offset;
849: vm_page_t holding_page;
850:
851:
852: XPR(XPR_VM_PAGEOUT,
853: "vm_pageout_initialize_page, page 0x%X\n",
854: (integer_t)m, 0, 0, 0, 0);
855: assert(m->busy);
856:
857: /*
858: * Verify that we really want to clean this page
859: */
860: assert(!m->absent);
861: assert(!m->error);
862: assert(m->dirty);
863:
864: /*
865: * Create a paging reference to let us play with the object.
866: */
867: object = m->object;
868: paging_offset = m->offset + object->paging_offset;
869: vm_object_paging_begin(object);
870: vm_object_unlock(object);
871: if (m->absent || m->error || m->restart ||
872: (!m->dirty && !m->precious)) {
873: VM_PAGE_FREE(m);
874: panic("reservation without pageout?"); /* alan */
875: return;
876: }
877:
878: /* set the page for future call to vm_fault_list_request */
879: holding_page = NULL;
880: vm_object_lock(m->object);
881: vm_page_lock_queues();
882: pmap_clear_modify(m->phys_addr);
883: m->dirty = TRUE;
884: m->busy = TRUE;
885: m->list_req_pending = TRUE;
886: m->cleaning = TRUE;
887: m->pageout = TRUE;
888: vm_page_wire(m);
889: vm_page_unlock_queues();
890: vm_object_unlock(m->object);
891: vm_pageout_throttle(m);
892: copy = NULL;
893:
894: VM_STAT(pageouts++);
895: /* VM_STAT(pages_pagedout++); */
896:
897: /*
898: * Write the data to its pager.
899: * Note that the data is passed by naming the new object,
900: * not a virtual address; the pager interface has been
901: * manipulated to use the "internal memory" data type.
902: * [The object reference from its allocation is donated
903: * to the eventual recipient.]
904: */
905: memory_object_data_initialize(object->pager,
906: object->pager_request,
907: paging_offset,
908: POINTER_T(copy),
909: PAGE_SIZE);
910:
911: vm_object_lock(object);
912: }
913:
914: #if MACH_CLUSTER_STATS
915: #define MAXCLUSTERPAGES 16
916: struct {
917: unsigned long pages_in_cluster;
918: unsigned long pages_at_higher_offsets;
919: unsigned long pages_at_lower_offsets;
920: } cluster_stats[MAXCLUSTERPAGES];
921: #endif /* MACH_CLUSTER_STATS */
922:
923: boolean_t allow_clustered_pageouts = TRUE;
924:
925: /*
926: * vm_pageout_cluster:
927: *
928: * Given a page, page it out, and attempt to clean adjacent pages
929: * in the same operation.
930: *
931: * The page must be busy, and the object unlocked w/ paging reference
932: * to prevent deallocation or collapse. The page must not be on any
933: * pageout queue.
934: */
935: void
936: vm_pageout_cluster(
937: vm_page_t m)
938: {
939: vm_object_t object = m->object;
940: vm_offset_t offset = m->offset; /* from vm_object start */
941: vm_offset_t paging_offset = m->offset + object->paging_offset;
942: vm_object_t new_object;
943: vm_offset_t new_offset;
944: vm_size_t cluster_size;
945: vm_offset_t cluster_offset; /* from memory_object start */
946: vm_offset_t cluster_lower_bound; /* from vm_object_start */
947: vm_offset_t cluster_upper_bound; /* from vm_object_start */
948: vm_offset_t cluster_start, cluster_end; /* from vm_object start */
949: vm_offset_t offset_within_cluster;
950: vm_size_t length_of_data;
951: vm_page_t friend, holding_page;
952: vm_map_copy_t copy;
953: kern_return_t rc;
954: boolean_t precious_clean = FALSE;
955: int pages_in_cluster;
956:
957: CLUSTER_STAT(int pages_at_higher_offsets = 0;)
958: CLUSTER_STAT(int pages_at_lower_offsets = 0;)
959:
960: XPR(XPR_VM_PAGEOUT,
961: "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
962: (integer_t)object, offset, (integer_t)m, 0, 0);
963:
964: CLUSTER_STAT(vm_pageout_cluster_clusters++;)
965: /*
966: * Only a certain kind of page is appreciated here.
967: */
968: assert(m->busy && (m->dirty || m->precious) && (m->wire_count == 0));
969: assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
970:
971: vm_object_lock(object);
972: cluster_size = object->cluster_size;
973:
974: assert(cluster_size >= PAGE_SIZE);
975: if (cluster_size < PAGE_SIZE) cluster_size = PAGE_SIZE;
976: assert(object->pager_created && object->pager_initialized);
977: assert(object->internal || object->pager_ready);
978:
979: if (m->precious && !m->dirty)
980: precious_clean = TRUE;
981:
982: if (!object->pager_trusted || !allow_clustered_pageouts)
983: cluster_size = PAGE_SIZE;
984: vm_object_unlock(object);
985:
986: cluster_offset = paging_offset & (cluster_size - 1);
987: /* bytes from beginning of cluster */
988: /*
989: * Due to unaligned mappings, we have to be careful
990: * of negative offsets into the VM object. Clip the cluster
991: * boundary to the VM object, not the memory object.
992: */
993: if (offset > cluster_offset) {
994: cluster_lower_bound = offset - cluster_offset;
995: /* from vm_object */
996: } else {
997: cluster_lower_bound = 0;
998: }
999: cluster_upper_bound = (offset - cluster_offset) + cluster_size;
1000:
1001: /* set the page for future call to vm_fault_list_request */
1002: holding_page = NULL;
1003: vm_object_lock(m->object);
1004: vm_page_lock_queues();
1005: pmap_clear_modify(m->phys_addr);
1006: m->dirty = TRUE;
1007: m->busy = TRUE;
1008: m->list_req_pending = TRUE;
1009: m->cleaning = TRUE;
1010: m->pageout = TRUE;
1011: vm_page_wire(m);
1012: vm_page_unlock_queues();
1013: vm_object_unlock(m->object);
1014: vm_pageout_throttle(m);
1015:
1016: /*
1017: * Search backward for adjacent eligible pages to clean in
1018: * this operation.
1019: */
1020:
1021: cluster_start = offset;
1022: if (offset) { /* avoid wrap-around at zero */
1023: for (cluster_start = offset - PAGE_SIZE;
1024: cluster_start >= cluster_lower_bound;
1025: cluster_start -= PAGE_SIZE) {
1026: assert(cluster_size > PAGE_SIZE);
1027:
1028: vm_object_lock(object);
1029: vm_page_lock_queues();
1030:
1031: if ((friend = vm_pageout_cluster_page(object, cluster_start,
1032: precious_clean)) == VM_PAGE_NULL) {
1033: vm_page_unlock_queues();
1034: vm_object_unlock(object);
1035: break;
1036: }
1037: new_offset = (cluster_start + object->paging_offset)
1038: & (cluster_size - 1);
1039:
1040: assert(new_offset < cluster_offset);
1041: pmap_clear_modify(m->phys_addr);
1042: m->dirty = TRUE;
1043: m->list_req_pending = TRUE;
1044: m->cleaning = TRUE;
1045: /* do nothing except advance the write request, all we really need to */
1046: /* do is push the target page and let the code at the other end decide */
1047: /* what is really the right size */
1048: if (vm_page_free_count <= vm_page_free_reserved) {
1049: m->busy = TRUE;
1050: m->pageout = TRUE;
1051: vm_page_wire(m);
1052: }
1053:
1054: vm_page_unlock_queues();
1055: vm_object_unlock(object);
1056: CLUSTER_STAT(pages_at_lower_offsets++;)
1057:
1058: }
1059: cluster_start += PAGE_SIZE;
1060: }
1061: assert(cluster_start >= cluster_lower_bound);
1062: assert(cluster_start <= offset);
1063: /*
1064: * Search forward for adjacent eligible pages to clean in
1065: * this operation.
1066: */
1067: for (cluster_end = offset + PAGE_SIZE;
1068: cluster_end < cluster_upper_bound;
1069: cluster_end += PAGE_SIZE) {
1070: assert(cluster_size > PAGE_SIZE);
1071:
1072: vm_object_lock(object);
1073: vm_page_lock_queues();
1074:
1075: if ((friend = vm_pageout_cluster_page(object, cluster_end,
1076: precious_clean)) == VM_PAGE_NULL) {
1077: vm_page_unlock_queues();
1078: vm_object_unlock(object);
1079: break;
1080: }
1081: new_offset = (cluster_end + object->paging_offset)
1082: & (cluster_size - 1);
1083:
1084: assert(new_offset < cluster_size);
1085:
1086: pmap_clear_modify(m->phys_addr);
1087: m->dirty = TRUE;
1088: m->list_req_pending = TRUE;
1089: m->cleaning = TRUE;
1090: /* do nothing except advance the write request, all we really need to */
1091: /* do is push the target page and let the code at the other end decide */
1092: /* what is really the right size */
1093: if (vm_page_free_count <= vm_page_free_reserved) {
1094: m->busy = TRUE;
1095: m->pageout = TRUE;
1096: vm_page_wire(m);
1097: }
1098:
1099: vm_page_unlock_queues();
1100: vm_object_unlock(object);
1101: CLUSTER_STAT(pages_at_higher_offsets++;)
1102: }
1103: assert(cluster_end <= cluster_upper_bound);
1104: assert(cluster_end >= offset + PAGE_SIZE);
1105:
1106: /*
1107: * (offset - cluster_offset) is beginning of cluster_object
1108: * relative to vm_object start.
1109: */
1110: offset_within_cluster = cluster_start - (offset - cluster_offset);
1111: length_of_data = cluster_end - cluster_start;
1112:
1113: assert(offset_within_cluster < cluster_size);
1114: assert((offset_within_cluster + length_of_data) <= cluster_size);
1115:
1116: rc = KERN_SUCCESS;
1117: assert(rc == KERN_SUCCESS);
1118:
1119: pages_in_cluster = length_of_data/PAGE_SIZE;
1120: VM_STAT(pageouts++);
1121: /* VM_STAT(pages_pagedout += pages_in_cluster); */
1122:
1123: #if MACH_CLUSTER_STATS
1124: (cluster_stats[pages_at_lower_offsets].pages_at_lower_offsets)++;
1125: (cluster_stats[pages_at_higher_offsets].pages_at_higher_offsets)++;
1126: (cluster_stats[pages_in_cluster].pages_in_cluster)++;
1127: #endif /* MACH_CLUSTER_STATS */
1128:
1129: /*
1130: * Send the data to the pager.
1131: */
1132: paging_offset = cluster_start + object->paging_offset;
1133: #ifdef MACH_BSD
1134: if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) ==
1135: ((rpc_subsystem_t) &vnode_pager_workaround)) {
1136: rc = vnode_pager_data_return(object->pager,
1137: object->pager_request,
1138: paging_offset,
1139: POINTER_T(copy),
1140: length_of_data,
1141: !precious_clean,
1142: FALSE);
1143: } else {
1144: rc = memory_object_data_return(object->pager,
1145: object->pager_request,
1146: paging_offset,
1147: POINTER_T(copy),
1148: length_of_data,
1149: !precious_clean,
1150: FALSE);
1151: }
1152: #else
1153: rc = memory_object_data_return(object->pager,
1154: object->pager_request,
1155: paging_offset,
1156: POINTER_T(copy),
1157: length_of_data,
1158: !precious_clean,
1159: FALSE);
1160: #endif
1161: vm_object_paging_end(object);
1162:
1163: if (rc != KERN_SUCCESS)
1164: vm_map_copy_discard(copy);
1165:
1166: if (holding_page) {
1167: assert(!object->pager_trusted);
1168: vm_object_lock(object);
1169: VM_PAGE_FREE(holding_page);
1170: vm_object_paging_end(object);
1171: vm_object_unlock(object);
1172: }
1173: }
1174:
1175: /*
1176: * vm_pageout_return_write_pages
1177: * Recover pages from an aborted write attempt
1178: *
1179: */
1180:
1181: vm_pageout_return_write_pages(
1182: ipc_port_t control_port,
1183: vm_offset_t object_offset,
1184: vm_map_copy_t copy)
1185: {
1186: vm_object_t object;
1187: int offset;
1188: int size;
1189: int shadow_offset;
1190: int copy_offset;
1191: int j;
1192: vm_page_t m;
1193:
1194:
1195: object = copy->cpy_object;
1196: copy_offset = copy->offset;
1197: size = copy->size;
1198:
1199: if((copy->type != VM_MAP_COPY_OBJECT) || (object->shadow == 0)) {
1200: object = (vm_object_t)control_port->ip_kobject;
1201: shadow_offset = (object_offset - object->paging_offset)
1202: - copy->offset;
1203: } else {
1204: /* get the offset from the copy object */
1205: shadow_offset = object->shadow_offset;
1206: /* find the backing object */
1207: object = object->shadow;
1208: }
1209: vm_object_lock(object);
1210:
1211: for(offset = 0, j=0; offset < size; offset+=page_size, j++) {
1212: m = vm_page_lookup(object,
1213: offset + shadow_offset + copy_offset);
1214: if((m == VM_PAGE_NULL) || m->fictitious) {
1215:
1216: vm_page_t p;
1217: int i;
1218: vm_object_t copy_object;
1219:
1220: /* m might be fictitious if the original page */
1221: /* was found to be in limbo at the time of */
1222: /* vm_pageout_setup */
1223:
1224: if((m != VM_PAGE_NULL) && m->fictitious) {
1225: m->cleaning = FALSE;
1226: vm_page_remove(m);
1227: /* if object is not pager trusted then */
1228: /* this fictitious page will be removed */
1229: /* as the holding page in vm_pageout_cluster */
1230: if (object->pager_trusted)
1231: vm_page_free(m);
1232: if(vm_page_laundry_count)
1233: vm_page_laundry_count--;
1234: if (vm_page_laundry_count
1235: < vm_page_laundry_min) {
1236: vm_page_laundry_min = 0;
1237: thread_wakeup((event_t)
1238: &vm_page_laundry_count);
1239: }
1240: }
1241: else if ((object->pager_trusted) &&
1242: (copy->type == VM_MAP_COPY_OBJECT)) {
1243: vm_object_paging_end(object);
1244: }
1245:
1246: copy_object = copy->cpy_object;
1247:
1248: if(copy->type == VM_MAP_COPY_OBJECT) {
1249: p = (vm_page_t) queue_first(©_object->memq);
1250:
1251: for(i = 0;
1252: i < copy_object->resident_page_count;
1253: i++) {
1254: if(p->offset == (offset + copy_offset))
1255: break;
1256: p = (vm_page_t) queue_next(&p->listq);
1257: }
1258:
1259: vm_page_remove(p);
1260: } else {
1261: p = copy->cpy_page_list[j];
1262: copy->cpy_page_list[j] = 0;
1263: p->gobbled = FALSE;
1264: }
1265:
1266: vm_page_insert(p, object,
1267: offset + shadow_offset + copy_offset);
1268: p->busy = TRUE;
1269: p->dirty = TRUE;
1270: p->laundry = FALSE;
1271: if (p->pageout) {
1272: p->pageout = FALSE; /*dont throw away target*/
1273: vm_page_unwire(p);/* reactivates */
1274: }
1275: } else if(m->pageout) {
1276: m->pageout = FALSE; /* dont throw away target pages */
1277: vm_page_unwire(m);/* reactivates */
1278: }
1279: }
1280:
1281: vm_object_unlock(object);
1282: vm_map_copy_discard(copy);
1283: vm_object_lock(object);
1284:
1285: for(offset = 0; offset < size; offset+=page_size) {
1286: m = vm_page_lookup(object,
1287: offset + shadow_offset + copy_offset);
1288: m->dirty = TRUE; /* we'll send the pages home later */
1289: m->busy = FALSE; /* allow system access again */
1290: }
1291:
1292: vm_object_unlock(object);
1293: }
1294:
1295: /*
1296: * Trusted pager throttle.
1297: * Object must be unlocked, page queues must be unlocked.
1298: */
1299: void
1300: vm_pageout_throttle(
1301: register vm_page_t m)
1302: {
1303: vm_page_lock_queues();
1304: assert(!m->laundry);
1305: m->laundry = TRUE;
1306: while (vm_page_laundry_count >= vm_page_laundry_max) {
1307: /*
1308: * Set the threshold for when vm_page_free()
1309: * should wake us up.
1310: */
1311: vm_page_laundry_min = vm_page_laundry_max/2;
1312: assert_wait((event_t) &vm_page_laundry_count, THREAD_UNINT);
1313: vm_page_unlock_queues();
1314:
1315: /*
1316: * Pause to let the default pager catch up.
1317: */
1318: thread_block((void (*)(void)) 0);
1319: vm_page_lock_queues();
1320: }
1321: vm_page_laundry_count++;
1322: vm_page_unlock_queues();
1323: }
1324:
1325: /*
1326: * The global variable vm_pageout_clean_active_pages controls whether
1327: * active pages are considered valid to be cleaned in place during a
1328: * clustered pageout. Performance measurements are necessary to determine
1329: * the best policy.
1330: */
1331: int vm_pageout_clean_active_pages = 1;
1332: /*
1333: * vm_pageout_cluster_page: [Internal]
1334: *
1335: * return a vm_page_t to the page at (object,offset) if it is appropriate
1336: * to clean in place. Pages that are non-existent, busy, absent, already
1337: * cleaning, or not dirty are not eligible to be cleaned as an adjacent
1338: * page in a cluster.
1339: *
1340: * The object must be locked on entry, and remains locked throughout
1341: * this call.
1342: */
1343:
1344: vm_page_t
1345: vm_pageout_cluster_page(
1346: vm_object_t object,
1347: vm_offset_t offset,
1348: boolean_t precious_clean)
1349: {
1350: vm_page_t m;
1351:
1352: XPR(XPR_VM_PAGEOUT,
1353: "vm_pageout_cluster_page, object 0x%X offset 0x%X\n",
1354: (integer_t)object, offset, 0, 0, 0);
1355:
1356: if ((m = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
1357: return(VM_PAGE_NULL);
1358:
1359: if (m->busy || m->absent || m->cleaning ||
1360: m->prep_pin_count != 0 ||
1361: (m->wire_count != 0) || m->error)
1362: return(VM_PAGE_NULL);
1363:
1364: if (vm_pageout_clean_active_pages) {
1365: if (!m->active && !m->inactive) return(VM_PAGE_NULL);
1366: } else {
1367: if (!m->inactive) return(VM_PAGE_NULL);
1368: }
1369:
1370: assert(!m->private);
1371: assert(!m->fictitious);
1372:
1373: if (!m->dirty) m->dirty = pmap_is_modified(m->phys_addr);
1374:
1375: if (precious_clean) {
1376: if (!m->precious || m->dirty)
1377: return(VM_PAGE_NULL);
1378: } else {
1379: if (!m->dirty)
1380: return(VM_PAGE_NULL);
1381: }
1382: return(m);
1383: }
1384:
1385: /*
1386: * vm_pageout_scan does the dirty work for the pageout daemon.
1387: * It returns with vm_page_queue_free_lock held and
1388: * vm_page_free_wanted == 0.
1389: */
1390: extern void vm_pageout_scan_continue(void); /* forward; */
1391:
1392: void
1393: vm_pageout_scan(void)
1394: {
1395: unsigned int burst_count;
1396: boolean_t now = FALSE;
1397: unsigned int laundry_pages;
1398: boolean_t need_more_inactive_pages;
1399: unsigned int loop_detect;
1400:
1401: XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
1402:
1403: /*???*/ /*
1404: * We want to gradually dribble pages from the active queue
1405: * to the inactive queue. If we let the inactive queue get
1406: * very small, and then suddenly dump many pages into it,
1407: * those pages won't get a sufficient chance to be referenced
1408: * before we start taking them from the inactive queue.
1409: *
1410: * We must limit the rate at which we send pages to the pagers.
1411: * data_write messages consume memory, for message buffers and
1412: * for map-copy objects. If we get too far ahead of the pagers,
1413: * we can potentially run out of memory.
1414: *
1415: * We can use the laundry count to limit directly the number
1416: * of pages outstanding to the default pager. A similar
1417: * strategy for external pagers doesn't work, because
1418: * external pagers don't have to deallocate the pages sent them,
1419: * and because we might have to send pages to external pagers
1420: * even if they aren't processing writes. So we also
1421: * use a burst count to limit writes to external pagers.
1422: *
1423: * When memory is very tight, we can't rely on external pagers to
1424: * clean pages. They probably aren't running, because they
1425: * aren't vm-privileged. If we kept sending dirty pages to them,
1426: * we could exhaust the free list. However, we can't just ignore
1427: * pages belonging to external objects, because there might be no
1428: * pages belonging to internal objects. Hence, we get the page
1429: * into an internal object and then immediately double-page it,
1430: * sending it to the default pager.
1431: *
1432: * consider_zone_gc should be last, because the other operations
1433: * might return memory to zones.
1434: */
1435:
1436: Restart:
1437:
1438: mutex_lock(&vm_page_queue_free_lock);
1439: now = (vm_page_free_count < vm_page_free_min);
1440: mutex_unlock(&vm_page_queue_free_lock);
1441: #if THREAD_SWAPPER
1442: swapout_threads(now);
1443: #endif /* THREAD_SWAPPER */
1444:
1445: stack_collect();
1446: consider_task_collect();
1447: consider_thread_collect();
1448: cleanup_limbo_queue();
1449: consider_zone_gc();
1450: consider_machine_collect();
1451:
1452: loop_detect = vm_page_active_count + vm_page_inactive_count;
1453:
1454: if (vm_page_free_count <= vm_page_free_reserved) {
1455: need_more_inactive_pages = TRUE;
1456: } else {
1457: need_more_inactive_pages = FALSE;
1458: }
1459: for (burst_count = 0;;) {
1460: register vm_page_t m;
1461: register vm_object_t object;
1462: unsigned int free_count;
1463:
1464: /*
1465: * Recalculate vm_page_inactivate_target.
1466: */
1467:
1468: vm_page_lock_queues();
1469: vm_page_inactive_target =
1470: VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
1471: vm_page_inactive_count);
1472:
1473: /*
1474: * Move pages from active to inactive.
1475: */
1476:
1477: while ((vm_page_inactive_count < vm_page_inactive_target ||
1478: need_more_inactive_pages) &&
1479: !queue_empty(&vm_page_queue_active)) {
1480: register vm_object_t object;
1481:
1482: vm_pageout_active++;
1483: m = (vm_page_t) queue_first(&vm_page_queue_active);
1484:
1485: /*
1486: * If we're getting really low on memory,
1487: * try selecting a page that will go
1488: * directly to the default_pager.
1489: * If there are no such pages, we have to
1490: * page out a page backed by an EMM,
1491: * so that the default_pager can recover
1492: * it eventually.
1493: */
1494: if (need_more_inactive_pages &&
1495: (IP_VALID(memory_manager_default))) {
1496: vm_pageout_scan_active_emm_throttle++;
1497: do {
1498: assert(m->active && !m->inactive);
1499: object = m->object;
1500:
1501: if (vm_object_lock_try(object)) {
1502: if (object->pager_trusted ||
1503: object->internal) {
1504: /* found one ! */
1505: vm_pageout_scan_active_emm_throttle_success++;
1506: goto object_locked_active;
1507: }
1508: vm_object_unlock(object);
1509: }
1510: m = (vm_page_t) queue_next(&m->pageq);
1511: } while (!queue_end(&vm_page_queue_active,
1512: (queue_entry_t) m));
1513: if (queue_end(&vm_page_queue_active,
1514: (queue_entry_t) m)) {
1515: vm_pageout_scan_active_emm_throttle_failure++;
1516: m = (vm_page_t)
1517: queue_first(&vm_page_queue_active);
1518: }
1519: }
1520:
1521: assert(m->active && !m->inactive);
1522:
1523: object = m->object;
1524: if (!vm_object_lock_try(object)) {
1525: /*
1526: * Move page to end and continue.
1527: */
1528:
1529: queue_remove(&vm_page_queue_active, m,
1530: vm_page_t, pageq);
1531: queue_enter(&vm_page_queue_active, m,
1532: vm_page_t, pageq);
1533: vm_page_unlock_queues();
1534: mutex_pause();
1535: vm_page_lock_queues();
1536: continue;
1537: }
1538:
1539: object_locked_active:
1540: /*
1541: * If the page is busy, then we pull it
1542: * off the active queue and leave it alone.
1543: */
1544:
1545: if (m->busy) {
1546: vm_object_unlock(object);
1547: queue_remove(&vm_page_queue_active, m,
1548: vm_page_t, pageq);
1549: m->active = FALSE;
1550: if (!m->fictitious)
1551: vm_page_active_count--;
1552: continue;
1553: }
1554:
1555: /*
1556: * Deactivate the page while holding the object
1557: * locked, so we know the page is still not busy.
1558: * This should prevent races between pmap_enter
1559: * and pmap_clear_reference. The page might be
1560: * absent or fictitious, but vm_page_deactivate
1561: * can handle that.
1562: */
1563:
1564: vm_page_deactivate(m);
1565: vm_object_unlock(object);
1566: }
1567:
1568: /*
1569: * We are done if we have met our target *and*
1570: * nobody is still waiting for a page.
1571: */
1572:
1573: mutex_lock(&vm_page_queue_free_lock);
1574: free_count = vm_page_free_count;
1575: if ((free_count >= vm_page_free_target) &&
1576: (vm_page_free_wanted == 0)) {
1577: vm_page_unlock_queues();
1578: break;
1579: }
1580: mutex_unlock(&vm_page_queue_free_lock);
1581:
1582: /*
1583: * Sometimes we have to pause:
1584: * 1) No inactive pages - nothing to do.
1585: * 2) Flow control - wait for untrusted pagers to catch up.
1586: */
1587:
1588: if (queue_empty(&vm_page_queue_inactive) ||
1589: ((--loop_detect) == 0) ||
1590: (burst_count >= vm_pageout_burst_max)) {
1591: unsigned int pages, msecs;
1592:
1593: /*
1594: * vm_pageout_burst_wait is msecs/page.
1595: * If there is nothing for us to do, we wait
1596: * at least vm_pageout_empty_wait msecs.
1597: */
1598: pages = burst_count;
1599:
1600: if (loop_detect == 0) {
1601: printf("Warning: No physical memory suitable for pageout or reclaim, pageout thread temporarily going to sleep\n");
1602: msecs = vm_free_page_pause;
1603: }
1604: else {
1605: msecs = burst_count * vm_pageout_burst_wait;
1606: }
1607:
1608: if (queue_empty(&vm_page_queue_inactive) &&
1609: (msecs < vm_pageout_empty_wait))
1610: msecs = vm_pageout_empty_wait;
1611: vm_page_unlock_queues();
1612: assert_wait_timeout(msecs, THREAD_INTERRUPTIBLE);
1613: counter(c_vm_pageout_scan_block++);
1614:
1615: /*
1616: * Unfortunately, we don't have call_continuation
1617: * so we can't rely on tail-recursion.
1618: */
1619: thread_block((void (*)(void)) 0);
1620: thread_cancel_timer();
1621: vm_pageout_scan_continue();
1622: goto Restart;
1623: /*NOTREACHED*/
1624: }
1625:
1626: vm_pageout_inactive++;
1627: m = (vm_page_t) queue_first(&vm_page_queue_inactive);
1628:
1629: if ((vm_page_free_count <= vm_page_free_reserved) &&
1630: (IP_VALID(memory_manager_default))) {
1631: /*
1632: * We're really low on memory. Try to select a page that
1633: * would go directly to the default_pager.
1634: * If there are no such pages, we have to page out a
1635: * page backed by an EMM, so that the default_pager
1636: * can recover it eventually.
1637: */
1638: vm_pageout_scan_inactive_emm_throttle++;
1639: do {
1640: assert(!m->active && m->inactive);
1641: object = m->object;
1642:
1643: if (vm_object_lock_try(object)) {
1644: if (object->pager_trusted ||
1645: object->internal) {
1646: /* found one ! */
1647: vm_pageout_scan_inactive_emm_throttle_success++;
1648: goto object_locked_inactive;
1649: }
1650: vm_object_unlock(object);
1651: }
1652: m = (vm_page_t) queue_next(&m->pageq);
1653: } while (!queue_end(&vm_page_queue_inactive,
1654: (queue_entry_t) m));
1655: if (queue_end(&vm_page_queue_inactive,
1656: (queue_entry_t) m)) {
1657: vm_pageout_scan_inactive_emm_throttle_failure++;
1658: /*
1659: * We should check the "active" queue
1660: * for good candidates to page out.
1661: */
1662: need_more_inactive_pages = TRUE;
1663:
1664: m = (vm_page_t)
1665: queue_first(&vm_page_queue_inactive);
1666: }
1667: }
1668:
1669: assert(!m->active && m->inactive);
1670: object = m->object;
1671:
1672: /*
1673: * Try to lock object; since we've got the
1674: * page queues lock, we can only try for this one.
1675: */
1676:
1677: if (!vm_object_lock_try(object)) {
1678: /*
1679: * Move page to end and continue.
1680: */
1681: queue_remove(&vm_page_queue_inactive, m,
1682: vm_page_t, pageq);
1683: queue_enter(&vm_page_queue_inactive, m,
1684: vm_page_t, pageq);
1685: vm_page_unlock_queues();
1686: mutex_pause();
1687: vm_pageout_inactive_nolock++;
1688: continue;
1689: }
1690:
1691: object_locked_inactive:
1692: /*
1693: * Paging out pages of objects which pager is being
1694: * created by another thread must be avoided, because
1695: * this thread may claim for memory, thus leading to a
1696: * possible dead lock between it and the pageout thread
1697: * which will wait for pager creation, if such pages are
1698: * finally chosen. The remaining assumption is that there
1699: * will finally be enough available pages in the inactive
1700: * pool to page out in order to satisfy all memory claimed
1701: * by the thread which concurrently creates the pager.
1702: */
1703:
1704: if (!object->pager_initialized && object->pager_created) {
1705: /*
1706: * Move page to end and continue, hoping that
1707: * there will be enough other inactive pages to
1708: * page out so that the thread which currently
1709: * initializes the pager will succeed.
1710: */
1711: queue_remove(&vm_page_queue_inactive, m,
1712: vm_page_t, pageq);
1713: queue_enter(&vm_page_queue_inactive, m,
1714: vm_page_t, pageq);
1715: vm_page_unlock_queues();
1716: vm_object_unlock(object);
1717: vm_pageout_inactive_avoid++;
1718: continue;
1719: }
1720:
1721: /*
1722: * Remove the page from the inactive list.
1723: */
1724:
1725: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
1726: m->inactive = FALSE;
1727: if (!m->fictitious)
1728: vm_page_inactive_count--;
1729:
1730: if (m->busy || !object->alive) {
1731: /*
1732: * Somebody is already playing with this page.
1733: * Leave it off the pageout queues.
1734: */
1735:
1736: vm_page_unlock_queues();
1737: vm_object_unlock(object);
1738: vm_pageout_inactive_busy++;
1739: continue;
1740: }
1741:
1742: /*
1743: * If it's absent or in error, we can reclaim the page.
1744: */
1745:
1746: if (m->absent || m->error) {
1747: vm_pageout_inactive_absent++;
1748: reclaim_page:
1749: vm_page_free(m);
1750: vm_page_unlock_queues();
1751: vm_object_unlock(object);
1752: continue;
1753: }
1754:
1755: assert(!m->private);
1756: assert(!m->fictitious);
1757:
1758: /*
1759: * If already cleaning this page in place, convert from
1760: * "adjacent" to "target". We can leave the page mapped,
1761: * and vm_pageout_object_terminate will determine whether
1762: * to free or reactivate.
1763: */
1764:
1765: if (m->cleaning) {
1766: #if MACH_CLUSTER_STATS
1767: vm_pageout_cluster_conversions++;
1768: #endif
1769: if (m->prep_pin_count == 0) {
1770: m->busy = TRUE;
1771: m->pageout = TRUE;
1772: vm_page_wire(m);
1773: }
1774: vm_object_unlock(object);
1775: vm_page_unlock_queues();
1776: continue;
1777: }
1778:
1779: /*
1780: * If it's being used, reactivate.
1781: * (Fictitious pages are either busy or absent.)
1782: */
1783:
1784: if (m->reference || pmap_is_referenced(m->phys_addr)) {
1785: vm_pageout_inactive_used++;
1786: reactivate_page:
1787: #if ADVISORY_PAGEOUT
1788: if (m->discard_request) {
1789: m->discard_request = FALSE;
1790: }
1791: #endif /* ADVISORY_PAGEOUT */
1792: vm_object_unlock(object);
1793: vm_page_activate(m);
1794: VM_STAT(reactivations++);
1795: vm_page_unlock_queues();
1796: continue;
1797: }
1798:
1799: if (m->prep_pin_count != 0) {
1800: boolean_t pinned = FALSE;
1801:
1802: vm_page_pin_lock();
1803: if (m->pin_count != 0) {
1804: /* skip and reactivate pinned page */
1805: pinned = TRUE;
1806: vm_pageout_inactive_pinned++;
1807: } else {
1808: /* page is prepped; send it into limbo */
1809: m->limbo = TRUE;
1810: vm_pageout_inactive_limbo++;
1811: }
1812: vm_page_pin_unlock();
1813: if (pinned)
1814: goto reactivate_page;
1815: }
1816:
1817: #if ADVISORY_PAGEOUT
1818: if (object->advisory_pageout) {
1819: boolean_t do_throttle;
1820: ipc_port_t port;
1821: vm_offset_t discard_offset;
1822:
1823: if (m->discard_request) {
1824: vm_stat_discard_failure++;
1825: goto mandatory_pageout;
1826: }
1827:
1828: assert(object->pager_initialized);
1829: m->discard_request = TRUE;
1830: port = object->pager;
1831:
1832: /* system-wide throttle */
1833: do_throttle = (vm_page_free_count <=
1834: vm_page_free_reserved);
1835: if (!do_throttle) {
1836: /* throttle on this pager */
1837: /* XXX lock ordering ? */
1838: ip_lock(port);
1839: do_throttle= imq_full(&port->ip_messages);
1840: ip_unlock(port);
1841: }
1842: if (do_throttle) {
1843: vm_stat_discard_throttle++;
1844: #if 0
1845: /* ignore this page and skip to next */
1846: vm_page_unlock_queues();
1847: vm_object_unlock(object);
1848: continue;
1849: #else
1850: /* force mandatory pageout */
1851: goto mandatory_pageout;
1852: #endif
1853: }
1854:
1855: /* proceed with discard_request */
1856: vm_page_activate(m);
1857: vm_stat_discard++;
1858: VM_STAT(reactivations++);
1859: discard_offset = m->offset + object->paging_offset;
1860: vm_stat_discard_sent++;
1861: vm_page_unlock_queues();
1862: vm_object_unlock(object);
1863: memory_object_discard_request(object->pager,
1864: object->pager_request,
1865: discard_offset,
1866: PAGE_SIZE);
1867: continue;
1868: }
1869: mandatory_pageout:
1870: #endif /* ADVISORY_PAGEOUT */
1871:
1872: XPR(XPR_VM_PAGEOUT,
1873: "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
1874: (integer_t)object, (integer_t)m->offset, (integer_t)m, 0,0);
1875:
1876: /*
1877: * Eliminate all mappings.
1878: */
1879:
1880: m->busy = TRUE;
1881: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
1882: if (!m->dirty)
1883: m->dirty = pmap_is_modified(m->phys_addr);
1884:
1885: /*
1886: * If it's clean and not precious, we can free the page.
1887: */
1888:
1889: if (!m->dirty && !m->precious) {
1890: vm_pageout_inactive_clean++;
1891: goto reclaim_page;
1892: }
1893: vm_page_unlock_queues();
1894:
1895: /*
1896: * If there is no memory object for the page, create
1897: * one and hand it to the default pager.
1898: * [First try to collapse, so we don't create
1899: * one unnecessarily.]
1900: */
1901:
1902: if (!object->pager_initialized)
1903: vm_object_collapse(object);
1904: if (!object->pager_initialized)
1905: vm_object_pager_create(object);
1906: if (!object->pager_initialized) {
1907: /*
1908: * Still no pager for the object.
1909: * Reactivate the page.
1910: *
1911: * Should only happen if there is no
1912: * default pager.
1913: */
1914: vm_page_lock_queues();
1915: vm_page_activate(m);
1916: vm_page_unlock_queues();
1917:
1918: /*
1919: * And we are done with it.
1920: */
1921: PAGE_WAKEUP_DONE(m);
1922: vm_object_unlock(object);
1923:
1924: /*
1925: * break here to get back to the preemption
1926: * point in the outer loop so that we don't
1927: * spin forever if there is no default pager.
1928: */
1929: vm_pageout_dirty_no_pager++;
1930: /*
1931: * Well there's no pager, but we can still reclaim
1932: * free pages out of the inactive list. Go back
1933: * to top of loop and look for suitable pages.
1934: */
1935: continue;
1936: }
1937:
1938: if (object->pager_initialized && object->pager == IP_NULL) {
1939: /*
1940: * This pager has been destroyed by either
1941: * memory_object_destroy or vm_object_destroy, and
1942: * so there is nowhere for the page to go.
1943: * Just free the page.
1944: */
1945: VM_PAGE_FREE(m);
1946: vm_object_unlock(object);
1947: continue;
1948: }
1949:
1950: vm_pageout_inactive_dirty++;
1951: if (!object->internal)
1952: burst_count++;
1953: vm_object_paging_begin(object);
1954: vm_object_unlock(object);
1955: vm_pageout_cluster(m); /* flush it */
1956: }
1957: }
1958:
1959: counter(unsigned int c_vm_pageout_scan_continue = 0;)
1960:
1961: void
1962: vm_pageout_scan_continue(void)
1963: {
1964: /*
1965: * We just paused to let the pagers catch up.
1966: * If vm_page_laundry_count is still high,
1967: * then we aren't waiting long enough.
1968: * If we have paused some vm_pageout_pause_max times without
1969: * adjusting vm_pageout_burst_wait, it might be too big,
1970: * so we decrease it.
1971: */
1972:
1973: vm_page_lock_queues();
1974: counter(++c_vm_pageout_scan_continue);
1975: if (vm_page_laundry_count > vm_pageout_burst_min) {
1976: vm_pageout_burst_wait++;
1977: vm_pageout_pause_count = 0;
1978: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) {
1979: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4;
1980: if (vm_pageout_burst_wait < 1)
1981: vm_pageout_burst_wait = 1;
1982: vm_pageout_pause_count = 0;
1983: }
1984: vm_page_unlock_queues();
1985: }
1986:
1987: void vm_page_free_reserve(int pages);
1988: int vm_page_free_count_init;
1989:
1990: void
1991: vm_page_free_reserve(
1992: int pages)
1993: {
1994: int free_after_reserve;
1995:
1996: vm_page_free_reserved += pages;
1997:
1998: free_after_reserve = vm_page_free_count_init - vm_page_free_reserved;
1999:
2000: vm_page_free_min = vm_page_free_reserved +
2001: VM_PAGE_FREE_MIN(free_after_reserve);
2002:
2003: vm_page_free_target = vm_page_free_reserved +
2004: VM_PAGE_FREE_TARGET(free_after_reserve);
2005:
2006: if (vm_page_free_target < vm_page_free_min + 5)
2007: vm_page_free_target = vm_page_free_min + 5;
2008: }
2009:
2010: /*
2011: * vm_pageout is the high level pageout daemon.
2012: */
2013:
2014:
2015: void
2016: vm_pageout(void)
2017: {
2018: thread_t thread;
2019: processor_set_t pset;
2020: kern_return_t ret;
2021: policy_base_t base;
2022: policy_limit_t limit;
2023: policy_fifo_base_data_t fifo_base;
2024: policy_fifo_limit_data_t fifo_limit;
2025:
2026: /*
2027: * Set thread privileges.
2028: */
2029: thread = current_thread();
2030: thread->vm_privilege = TRUE;
2031: stack_privilege(thread);
2032: thread_swappable(current_act(), FALSE);
2033:
2034: /*
2035: * Set thread scheduling priority and policy.
2036: */
2037: pset = thread->processor_set;
2038: base = (policy_base_t) &fifo_base;
2039: limit = (policy_limit_t) &fifo_limit;
2040: fifo_base.base_priority = BASEPRI_SYSTEM;
2041: fifo_limit.max_priority = BASEPRI_SYSTEM;
2042: ret = thread_set_policy(thread->top_act, pset, POLICY_FIFO,
2043: base, POLICY_TIMESHARE_BASE_COUNT,
2044: limit, POLICY_TIMESHARE_LIMIT_COUNT);
2045: if (ret != KERN_SUCCESS)
2046: printf("WARNING: vm_pageout_thread is being TIMESHARED!\n");
2047:
2048: /*
2049: * Initialize some paging parameters.
2050: */
2051:
2052: if (vm_page_laundry_max == 0)
2053: vm_page_laundry_max = VM_PAGE_LAUNDRY_MAX;
2054:
2055: if (vm_pageout_burst_max == 0)
2056: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX;
2057:
2058: if (vm_pageout_burst_wait == 0)
2059: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
2060:
2061: if (vm_pageout_empty_wait == 0)
2062: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
2063:
2064: vm_page_free_count_init = vm_page_free_count;
2065: /*
2066: * even if we've already called vm_page_free_reserve
2067: * call it again here to insure that the targets are
2068: * accurately calculated (it uses vm_page_free_count_init)
2069: * calling it with an arg of 0 will not change the reserve
2070: * but will re-calculate free_min and free_target
2071: */
2072: if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED)
2073: vm_page_free_reserve(VM_PAGE_FREE_RESERVED - vm_page_free_reserved);
2074: else
2075: vm_page_free_reserve(0);
2076:
2077: /*
2078: * vm_pageout_scan will set vm_page_inactive_target.
2079: *
2080: * The pageout daemon is never done, so loop forever.
2081: * We should call vm_pageout_scan at least once each
2082: * time we are woken, even if vm_page_free_wanted is
2083: * zero, to check vm_page_free_target and
2084: * vm_page_inactive_target.
2085: */
2086: for (;;) {
2087: vm_pageout_scan();
2088: /* we hold vm_page_queue_free_lock now */
2089: assert(vm_page_free_wanted == 0);
2090: assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
2091: mutex_unlock(&vm_page_queue_free_lock);
2092: counter(c_vm_pageout_block++);
2093: thread_block((void (*)(void)) 0);
2094: }
2095: /*NOTREACHED*/
2096: }
2097:
2098:
2099:
2100: /*
2101: * Routine: vm_fault_list_request
2102: * Purpose:
2103: * Cause the population of a portion of a vm_object.
2104: * Depending on the nature of the request, the pages
2105: * returned may be contain valid data or be uninitialized.
2106: * A page list structure, listing the physical pages
2107: * will be returned upon request.
2108: * This function is called by the file system or any other
2109: * supplier of backing store to a pager.
2110: * IMPORTANT NOTE: The caller must still respect the relationship
2111: * between the vm_object and its backing memory object. The
2112: * caller MUST NOT substitute changes in the backing file
2113: * without first doing a memory_object_lock_request on the
2114: * target range unless it is know that the pages are not
2115: * shared with another entity at the pager level.
2116: * Copy_in_to:
2117: * if a page list structure is present
2118: * return the mapped physical pages, where a
2119: * page is not present, return a non-initialized
2120: * one. If the no_sync bit is turned on, don't
2121: * call the pager unlock to synchronize with other
2122: * possible copies of the page. Leave pages busy
2123: * in the original object, if a page list structure
2124: * was specified. When a commit of the page list
2125: * pages is done, the dirty bit will be set for each one.
2126: * Copy_out_from:
2127: * If a page list structure is present, return
2128: * all mapped pages. Where a page does not exist
2129: * map a zero filled one. Leave pages busy in
2130: * the original object. If a page list structure
2131: * is not specified, this call is a no-op.
2132: *
2133: * Note: access of default pager objects has a rather interesting
2134: * twist. The caller of this routine, presumably the file system
2135: * page cache handling code, will never actually make a request
2136: * against a default pager backed object. Only the default
2137: * pager will make requests on backing store related vm_objects
2138: * In this way the default pager can maintain the relationship
2139: * between backing store files (abstract memory objects) and
2140: * the vm_objects (cache objects), they support.
2141: *
2142: */
2143: kern_return_t
2144: vm_fault_list_request(
2145: vm_object_t object,
2146: vm_object_offset_t offset,
2147: vm_size_t size,
2148: upl_t *upl_ptr,
2149: upl_page_info_t **user_page_list_ptr,
2150: int page_list_count,
2151: int cntrl_flags)
2152: {
2153: vm_page_t dst_page;
2154: vm_object_offset_t dst_offset = offset;
2155: upl_page_info_t *user_page_list;
2156: vm_size_t xfer_size = size;
2157: boolean_t do_m_lock = FALSE;
2158: boolean_t dirty;
2159: upl_t upl = NULL;
2160: int entry;
2161:
2162: if(cntrl_flags & UPL_SET_INTERNAL)
2163: page_list_count = 20;
2164: if(user_page_list_ptr && (page_list_count < (size/page_size)))
2165: return KERN_INVALID_ARGUMENT;
2166:
2167: if((!object->internal) && (object->paging_offset != 0))
2168: panic("vm_fault_list_request: vnode object with non-zero paging offset\n");
2169:
2170: if((cntrl_flags & UPL_COPYOUT_FROM) && (upl_ptr == NULL)) {
2171: return KERN_SUCCESS;
2172: }
2173: if(upl_ptr) {
2174: if(cntrl_flags & UPL_SET_INTERNAL) {
2175: upl = upl_create(TRUE);
2176: user_page_list = (upl_page_info_t *)
2177: (((vm_offset_t)upl) + sizeof(struct upl));
2178: if(user_page_list_ptr)
2179: *user_page_list_ptr = user_page_list;
2180: else
2181: user_page_list = NULL;
2182: upl->flags |= UPL_INTERNAL;
2183: } else {
2184: upl = upl_create(FALSE);
2185: if(user_page_list_ptr)
2186: user_page_list = *user_page_list_ptr;
2187: else
2188: user_page_list = NULL;
2189: }
2190: upl->map_object = vm_object_allocate(size);
2191: vm_object_lock(upl->map_object);
2192: upl->map_object->shadow = object;
2193: upl->size = size;
2194: upl->offset = offset + object->paging_offset;
2195: upl->map_object->pageout = TRUE;
2196: upl->map_object->can_persist = FALSE;
2197: upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2198: upl->map_object->shadow_offset = offset;
2199: vm_object_unlock(upl->map_object);
2200: *upl_ptr = upl;
2201: }
2202: vm_object_lock(object);
2203: vm_object_paging_begin(object);
2204: entry = 0;
2205: if(cntrl_flags & UPL_COPYOUT_FROM) {
2206: while (xfer_size) {
2207: vm_page_t alias_page;
2208:
2209: if(((dst_page = vm_page_lookup(object,
2210: dst_offset)) == VM_PAGE_NULL) ||
2211: dst_page->fictitious ||
2212: dst_page->absent ||
2213: dst_page->error ||
2214: (dst_page->wire_count != 0 &&
2215: !dst_page->pageout) ||
2216: ((!(dst_page->dirty || dst_page->precious)) &&
2217: (cntrl_flags & UPL_RET_ONLY_DIRTY))) {
2218: if(user_page_list)
2219: user_page_list[entry].phys_addr = 0;
2220: } else {
2221:
2222: if(dst_page->busy &&
2223: (!(dst_page->list_req_pending &&
2224: dst_page->pageout))) {
2225: if(cntrl_flags & UPL_NOBLOCK) {
2226: if(user_page_list)
2227: user_page_list[entry]
2228: .phys_addr = 0;
2229: entry++;
2230: dst_offset += PAGE_SIZE;
2231: xfer_size -= PAGE_SIZE;
2232: continue;
2233: }
2234: /*someone else is playing with the */
2235: /* page. We will have to wait. */
2236: PAGE_ASSERT_WAIT(
2237: dst_page, THREAD_UNINT);
2238: vm_object_unlock(object);
2239: thread_block((void(*)(void))0);
2240: vm_object_lock(object);
2241: continue;
2242: }
2243: /* Someone else already cleaning the page? */
2244: if((dst_page->cleaning || dst_page->absent ||
2245: dst_page->prep_pin_count != 0 ||
2246: dst_page->wire_count != 0) &&
2247: !dst_page->list_req_pending) {
2248: if(user_page_list)
2249: user_page_list[entry].phys_addr = 0;
2250: entry++;
2251: dst_offset += PAGE_SIZE;
2252: xfer_size -= PAGE_SIZE;
2253: continue;
2254: }
2255: VM_PAGE_GRAB_FICTITIOUS(alias_page);
2256:
2257: /* eliminate all mappings from the */
2258: /* original object and its prodigy */
2259:
2260: vm_page_lock_queues();
2261: pmap_page_protect(dst_page->phys_addr,
2262: VM_PROT_NONE);
2263: /* Turn off busy indication on pending */
2264: /* pageout. Note: we can only get here */
2265: /* in the request pending case. */
2266: dst_page->list_req_pending = FALSE;
2267: dst_page->busy = FALSE;
2268: dst_page->cleaning = FALSE;
2269:
2270: dirty = pmap_is_modified(dst_page->phys_addr);
2271: dirty = dirty ? TRUE : dst_page->dirty;
2272:
2273: /* use pageclean setup, it is more convenient */
2274: /* even for the pageout cases here */
2275: vm_pageclean_setup(dst_page, alias_page,
2276: upl->map_object, size - xfer_size);
2277:
2278: if(!dirty) {
2279: dst_page->dirty = FALSE;
2280: dst_page->precious = TRUE;
2281: }
2282:
2283: if(dst_page->pageout)
2284: dst_page->busy = TRUE;
2285:
2286: alias_page->absent = FALSE;
2287: if(!(cntrl_flags & UPL_CLEAN_IN_PLACE)) {
2288: /* deny access to the target page */
2289: /* while it is being worked on */
2290: if((!dst_page->pageout) &&
2291: (dst_page->wire_count == 0)) {
2292: dst_page->busy = TRUE;
2293: dst_page->pageout = TRUE;
2294: vm_page_wire(dst_page);
2295: }
2296: }
2297: if(user_page_list) {
2298: user_page_list[entry].phys_addr
2299: = dst_page->phys_addr;
2300: user_page_list[entry].dirty =
2301: dst_page->dirty;
2302: user_page_list[entry].pageout =
2303: dst_page->pageout;
2304: user_page_list[entry].absent =
2305: dst_page->absent;
2306: }
2307:
2308: vm_page_unlock_queues();
2309: }
2310: entry++;
2311: dst_offset += PAGE_SIZE;
2312: xfer_size -= PAGE_SIZE;
2313: }
2314: } else {
2315: while (xfer_size) {
2316: dst_page = vm_page_lookup(object, dst_offset);
2317: if(dst_page != VM_PAGE_NULL) {
2318: if((dst_page->cleaning) &&
2319: !(dst_page->list_req_pending)) {
2320: /*someone else is writing to the */
2321: /* page. We will have to wait. */
2322: PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT);
2323: vm_object_unlock(object);
2324: thread_block((void(*)(void))0);
2325: vm_object_lock(object);
2326: continue;
2327: }
2328: if ((dst_page->fictitious &&
2329: dst_page->list_req_pending)) {
2330: /* dump the fictitious page */
2331: dst_page->list_req_pending = FALSE;
2332: dst_page->clustered = FALSE;
2333: vm_page_lock_queues();
2334: vm_page_free(dst_page);
2335: vm_page_unlock_queues();
2336: } else if ((dst_page->absent &&
2337: dst_page->list_req_pending)) {
2338: /* the default_pager case */
2339: dst_page->list_req_pending = FALSE;
2340: dst_page->busy = FALSE;
2341: dst_page->clustered = FALSE;
2342: }
2343: }
2344: if((dst_page = vm_page_lookup(
2345: object, dst_offset)) == VM_PAGE_NULL) {
2346: /* need to allocate a page */
2347: dst_page = vm_page_alloc(object, dst_offset);
2348: if (dst_page == VM_PAGE_NULL) {
2349: vm_object_unlock(object);
2350: VM_PAGE_WAIT();
2351: vm_object_lock(object);
2352: continue;
2353: }
2354: dst_page->busy = FALSE;
2355: if(cntrl_flags & UPL_NO_SYNC) {
2356: dst_page->page_lock = 0;
2357: dst_page->unlock_request = 0;
2358: }
2359: dst_page->absent = TRUE;
2360: object->absent_count++;
2361: }
2362: dst_page->overwriting = TRUE;
2363: if(dst_page->fictitious) {
2364: panic("need corner case for fictitious page");
2365: }
2366: if(dst_page->page_lock) {
2367: do_m_lock = TRUE;
2368: }
2369: if(upl_ptr) {
2370: vm_page_t alias_page;
2371:
2372: VM_PAGE_GRAB_FICTITIOUS(alias_page);
2373: /* eliminate all mappings from the */
2374: /* original object and its prodigy */
2375:
2376: if(dst_page->busy) {
2377: /*someone else is playing with the */
2378: /* page. We will have to wait. */
2379: PAGE_ASSERT_WAIT(
2380: dst_page, THREAD_UNINT);
2381: vm_object_unlock(object);
2382: thread_block((void(*)(void))0);
2383: vm_object_lock(object);
2384: continue;
2385: }
2386:
2387: vm_page_lock_queues();
2388: pmap_page_protect(dst_page->phys_addr,
2389: VM_PROT_NONE);
2390: dirty = pmap_is_modified(dst_page->phys_addr);
2391: dirty = dirty ? TRUE : dst_page->dirty;
2392:
2393: vm_pageclean_setup(dst_page, alias_page,
2394: upl->map_object, size - xfer_size);
2395:
2396: if(cntrl_flags & UPL_CLEAN_IN_PLACE) {
2397: /* clean in place for read implies */
2398: /* that a write will be done on all */
2399: /* the pages that are dirty before */
2400: /* a upl commit is done. The caller */
2401: /* is obligated to preserve the */
2402: /* contents of all pages marked */
2403: /* dirty. */
2404: upl->flags |= UPL_CLEAR_DIRTY;
2405: }
2406:
2407: if(!dirty) {
2408: dst_page->dirty = FALSE;
2409: dst_page->precious = TRUE;
2410: }
2411:
2412: if (dst_page->wire_count == 0) {
2413: /* deny access to the target page while */
2414: /* it is being worked on */
2415: dst_page->busy = TRUE;
2416: } else {
2417: vm_page_wire(dst_page);
2418: }
2419: /* expect the page to be used */
2420: dst_page->reference = TRUE;
2421: dst_page->precious =
2422: (cntrl_flags & UPL_PRECIOUS)
2423: ? TRUE : FALSE;
2424: alias_page->absent = FALSE;
2425: if(user_page_list) {
2426: user_page_list[entry].phys_addr
2427: = dst_page->phys_addr;
2428: user_page_list[entry].dirty =
2429: dst_page->dirty;
2430: user_page_list[entry].pageout =
2431: dst_page->pageout;
2432: user_page_list[entry].absent =
2433: dst_page->absent;
2434: }
2435: vm_page_unlock_queues();
2436: }
2437: entry++;
2438: dst_offset += PAGE_SIZE;
2439: xfer_size -= PAGE_SIZE;
2440: }
2441: }
2442: if(do_m_lock) {
2443: vm_prot_t access_required;
2444: /* call back all associated pages from other users of the pager */
2445: /* all future updates will be on data which is based on the */
2446: /* changes we are going to make here. Note: it is assumed that */
2447: /* we already hold copies of the data so we will not be seeing */
2448: /* an avalanche of incoming data from the pager */
2449: access_required = (cntrl_flags & UPL_COPYOUT_FROM)
2450: ? VM_PROT_READ : VM_PROT_WRITE;
2451: while (TRUE) {
2452: kern_return_t rc;
2453: thread_t thread;
2454:
2455: if(!object->pager_ready) {
2456: thread = current_thread();
2457: vm_object_assert_wait(object,
2458: VM_OBJECT_EVENT_PAGER_READY, THREAD_UNINT);
2459: vm_object_unlock(object);
2460: thread_block((void (*)(void))0);
2461: if (thread->wait_result != THREAD_AWAKENED) {
2462: return(KERN_FAILURE);
2463: }
2464: vm_object_lock(object);
2465: continue;
2466: }
2467:
2468: vm_object_unlock(object);
2469:
2470: if (rc = memory_object_data_unlock(
2471: object->pager,
2472: object->pager_request,
2473: dst_offset + object->paging_offset,
2474: size,
2475: access_required)) {
2476: if (rc == MACH_SEND_INTERRUPTED)
2477: continue;
2478: else
2479: return KERN_FAILURE;
2480: }
2481: break;
2482:
2483: }
2484: /* lets wait on the last page requested */
2485: /* NOTE: we will have to update lock completed routine to signal */
2486: if(dst_page != VM_PAGE_NULL &&
2487: (access_required & dst_page->page_lock) != access_required) {
2488: PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT);
2489: thread_block((void (*)(void))0);
2490: vm_object_lock(object);
2491: }
2492: }
2493: vm_object_unlock(object);
2494: return KERN_SUCCESS;
2495: }
2496:
2497: kern_return_t
2498: upl_system_list_request(
2499: vm_object_t object,
2500: vm_object_offset_t offset,
2501: vm_size_t size,
2502: vm_size_t super_cluster,
2503: upl_t *upl,
2504: upl_page_info_t **user_page_list_ptr,
2505: int page_list_count,
2506: int cntrl_flags)
2507: {
2508: if(object->paging_offset > offset)
2509: return KERN_FAILURE;
2510: offset = offset - object->paging_offset;
2511:
2512: /* turns off super cluster exercised by the default_pager */
2513: /*
2514: super_cluster = size;
2515: */
2516: if ((super_cluster > size) &&
2517: (vm_page_free_count > vm_page_free_reserved)) {
2518:
2519: vm_offset_t base_offset;
2520: vm_size_t super_size;
2521:
2522: base_offset = (vm_offset_t)(offset &
2523: ~((vm_offset_t) super_cluster - 1));
2524: super_size = (offset+size) > (base_offset + super_cluster) ?
2525: super_cluster<<1 : super_cluster;
2526: super_size = ((base_offset + super_size) > object->size) ?
2527: (object->size - base_offset) : super_size;
2528: if(offset > (base_offset + super_size))
2529: panic("upl_system_list_request: Missed target pageout 0x%x,0x%x, 0x%x, 0x%x, 0x%x, 0x%x\n", offset, base_offset, super_size, super_cluster, size, object->paging_offset);
2530: /* apparently there is a case where the vm requests a */
2531: /* page to be written out who's offset is beyond the */
2532: /* object size */
2533: if((offset + size) > (base_offset + super_size))
2534: super_size = (offset + size) - base_offset;
2535:
2536: offset = base_offset;
2537: size = super_size;
2538: }
2539: vm_fault_list_request(object, offset, size, upl, user_page_list_ptr,
2540: page_list_count, cntrl_flags);
2541: }
2542:
2543:
2544: kern_return_t
2545: upl_map(
2546: vm_map_t map,
2547: upl_t upl,
2548: vm_offset_t *dst_addr)
2549: {
2550: vm_size_t size;
2551: vm_offset_t offset;
2552: vm_offset_t addr;
2553: vm_page_t m;
2554: kern_return_t kr;
2555:
2556: /* check to see if already mapped */
2557: if(UPL_PAGE_LIST_MAPPED & upl->flags)
2558: return KERN_FAILURE;
2559:
2560: offset = 0; /* Always map the entire object */
2561: size = upl->size;
2562:
2563: vm_object_lock(upl->map_object);
2564: upl->map_object->ref_count++;
2565: vm_object_res_reference(upl->map_object);
2566: vm_object_unlock(upl->map_object);
2567:
2568: *dst_addr = 0;
2569:
2570:
2571: /* NEED A UPL_MAP ALIAS */
2572: kr = vm_map_enter(map, dst_addr, size, (vm_offset_t) 0, TRUE,
2573: upl->map_object, offset, FALSE,
2574: VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
2575:
2576: if (kr != KERN_SUCCESS)
2577: return(kr);
2578:
2579: for(addr=*dst_addr; size > 0; size-=PAGE_SIZE,addr+=PAGE_SIZE) {
2580: m = vm_page_lookup(upl->map_object, offset);
2581: if(m) {
2582: PMAP_ENTER(map->pmap, addr, m, VM_PROT_ALL, TRUE);
2583: }
2584: offset+=PAGE_SIZE;
2585: }
2586:
2587: upl->flags |= UPL_PAGE_LIST_MAPPED;
2588: upl->kaddr = *dst_addr;
2589: return KERN_SUCCESS;
2590: }
2591:
2592:
2593: kern_return_t
2594: upl_un_map(
2595: vm_map_t map,
2596: upl_t upl)
2597: {
2598: vm_size_t size;
2599:
2600: if(upl->flags & UPL_PAGE_LIST_MAPPED) {
2601: size = upl->size;
2602: vm_deallocate(map, upl->kaddr, size);
2603: upl->flags &= ~UPL_PAGE_LIST_MAPPED;
2604: upl->kaddr = (vm_offset_t) 0;
2605: return KERN_SUCCESS;
2606: } else {
2607: return KERN_FAILURE;
2608: }
2609: }
2610:
2611: kern_return_t
2612: upl_commit_range(
2613: upl_t upl,
2614: vm_offset_t offset,
2615: vm_size_t size,
2616: boolean_t free_on_empty,
2617: upl_page_info_t *page_list)
2618: {
2619: vm_offset_t xfer_size = size;
2620: vm_object_t shadow_object = upl->map_object->shadow;
2621: vm_object_t object = upl->map_object;
2622: vm_offset_t target_offset;
2623: vm_offset_t page_offset;
2624: int entry;
2625:
2626: if ((offset + size) > upl->size)
2627: return KERN_FAILURE;
2628:
2629: vm_object_lock(shadow_object);
2630:
2631: entry = offset/PAGE_SIZE;
2632: target_offset = offset;
2633: while(xfer_size) {
2634: vm_page_t t,m;
2635: upl_page_info_t *p;
2636:
2637: if((t = vm_page_lookup(object,
2638: (vm_offset_t)target_offset)) != NULL) {
2639:
2640: t->pageout = FALSE;
2641: page_offset = (vm_offset_t)t->offset;
2642: VM_PAGE_FREE(t);
2643: t = VM_PAGE_NULL;
2644: m = vm_page_lookup(shadow_object,
2645: page_offset + object->shadow_offset);
2646: if(m != VM_PAGE_NULL) {
2647: vm_object_paging_end(shadow_object);
2648: vm_page_lock_queues();
2649: if (upl->flags & UPL_CLEAR_DIRTY) {
2650: pmap_clear_modify(m->phys_addr);
2651: m->dirty = FALSE;
2652: }
2653: if(page_list) {
2654: p = &(page_list[entry]);
2655: if(p->phys_addr && p->pageout && !m->pageout) {
2656: m->busy = TRUE;
2657: m->pageout = TRUE;
2658: vm_page_wire(m);
2659: } else if (page_list[entry].phys_addr &&
2660: !p->pageout && m->pageout) {
2661: m->pageout = FALSE;
2662: vm_page_unwire(m);
2663: PAGE_WAKEUP_DONE(m);
2664: }
2665: page_list[entry].phys_addr = 0;
2666: }
2667: if(m->laundry) {
2668: vm_page_laundry_count--;
2669: m->laundry = FALSE;
2670: if (vm_page_laundry_count < vm_page_laundry_min) {
2671: vm_page_laundry_min = 0;
2672: thread_wakeup((event_t)
2673: &vm_page_laundry_count);
2674: }
2675: }
2676: if(m->pageout) {
2677: m->cleaning = FALSE;
2678: m->pageout = FALSE;
2679: #if MACH_CLUSTER_STATS
2680: if (m->wanted) vm_pageout_target_collisions++;
2681: #endif
2682: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
2683: m->dirty = pmap_is_modified(m->phys_addr);
2684: if(m->dirty) {
2685: CLUSTER_STAT(
2686: vm_pageout_target_page_dirtied++;)
2687: vm_page_unwire(m);/* reactivates */
2688: VM_STAT(reactivations++);
2689: PAGE_WAKEUP_DONE(m);
2690: } else if (m->prep_pin_count != 0) {
2691: vm_page_pin_lock();
2692: if (m->pin_count != 0) {
2693: /* page is pinned; reactivate */
2694: CLUSTER_STAT(
2695: vm_pageout_target_page_pinned++;)
2696: vm_page_unwire(m);/* reactivates */
2697: VM_STAT(reactivations++);
2698: PAGE_WAKEUP_DONE(m);
2699: } else {
2700: /*
2701: * page is prepped but not pinned;
2702: * send it into limbo. Note that
2703: * vm_page_free (which will be
2704: * called after releasing the pin
2705: * lock) knows how to handle a page
2706: * with limbo set.
2707: */
2708: m->limbo = TRUE;
2709: CLUSTER_STAT(
2710: vm_pageout_target_page_limbo++;)
2711: }
2712: vm_page_pin_unlock();
2713: if (m->limbo)
2714: vm_page_free(m);
2715: } else {
2716: CLUSTER_STAT(
2717: vm_pageout_target_page_freed++;)
2718: vm_page_free(m);/* clears busy, etc. */
2719: }
2720: vm_page_unlock_queues();
2721: target_offset += PAGE_SIZE;
2722: xfer_size -= PAGE_SIZE;
2723: entry++;
2724: continue;
2725: }
2726: if (!m->active && !m->inactive) {
2727: if (m->reference || m->prep_pin_count != 0)
2728: vm_page_activate(m);
2729: else
2730: vm_page_deactivate(m);
2731: }
2732: #if MACH_CLUSTER_STATS
2733: m->dirty = pmap_is_modified(m->phys_addr);
2734:
2735: if (m->dirty) vm_pageout_cluster_dirtied++;
2736: else vm_pageout_cluster_cleaned++;
2737: if (m->wanted) vm_pageout_cluster_collisions++;
2738: #else
2739: m->dirty = 0;
2740: #endif
2741:
2742: if((m->busy) && (m->cleaning)) {
2743: /* the request_page_list case */
2744: if(m->absent) {
2745: m->absent = FALSE;
2746: if(shadow_object->absent_count == 1)
2747: vm_object_absent_release(shadow_object);
2748: else
2749: shadow_object->absent_count--;
2750: }
2751: m->overwriting = FALSE;
2752: m->busy = FALSE;
2753: m->dirty = FALSE;
2754: }
2755: else if (m->overwriting) {
2756: /* alternate request page list, write to
2757: /* page_list case. Occurs when the original
2758: /* page was wired at the time of the list
2759: /* request */
2760: assert(m->wire_count != 0);
2761: vm_page_unwire(m);/* reactivates */
2762: m->overwriting = FALSE;
2763: }
2764: m->cleaning = FALSE;
2765: /*
2766: * Wakeup any thread waiting for the page to be un-cleaning.
2767: */
2768: PAGE_WAKEUP(m);
2769: vm_page_unlock_queues();
2770:
2771: }
2772: }
2773: target_offset += PAGE_SIZE;
2774: xfer_size -= PAGE_SIZE;
2775: entry++;
2776: }
2777:
2778: vm_object_unlock(shadow_object);
2779: if(free_on_empty) {
2780: if(queue_empty(&upl->map_object->memq)) {
2781: upl_destroy(upl);
2782: }
2783: }
2784: return KERN_SUCCESS;
2785: }
2786:
2787: upl_abort_range(
2788: upl_t upl,
2789: vm_offset_t offset,
2790: vm_size_t size,
2791: int error)
2792: {
2793: vm_offset_t xfer_size = size;
2794: vm_object_t shadow_object = upl->map_object->shadow;
2795: vm_object_t object = upl->map_object;
2796: vm_offset_t target_offset;
2797: vm_offset_t page_offset;
2798: int entry;
2799:
2800: if ((offset + size) > upl->size)
2801: return KERN_FAILURE;
2802:
2803: vm_object_lock(shadow_object);
2804:
2805: entry = offset/PAGE_SIZE;
2806: target_offset = offset;
2807: while(xfer_size) {
2808: vm_page_t t,m;
2809: upl_page_info_t *p;
2810:
2811: if((t = vm_page_lookup(object,
2812: (vm_offset_t)target_offset)) != NULL) {
2813:
2814: t->pageout = FALSE;
2815: page_offset = (vm_offset_t)t->offset;
2816: VM_PAGE_FREE(t);
2817: t = VM_PAGE_NULL;
2818: m = vm_page_lookup(shadow_object,
2819: page_offset + object->shadow_offset);
2820: if(m != VM_PAGE_NULL) {
2821: vm_object_paging_end(m->object);
2822: vm_page_lock_queues();
2823: if(m->absent) {
2824: /* COPYOUT = FALSE case */
2825: /* check for error conditions which must */
2826: /* be passed back to the pages customer */
2827: if(error & UPL_ABORT_RESTART) {
2828: m->restart = TRUE;
2829: m->absent = FALSE;
2830: vm_object_absent_release(m->object);
2831: m->page_error = KERN_MEMORY_ERROR;
2832: m->error = TRUE;
2833: } else if(error & UPL_ABORT_UNAVAILABLE) {
2834: m->restart = FALSE;
2835: m->unusual = TRUE;
2836: m->clustered = FALSE;
2837: } else if(error & UPL_ABORT_ERROR) {
2838: m->restart = FALSE;
2839: m->absent = FALSE;
2840: vm_object_absent_release(m->object);
2841: m->page_error = KERN_MEMORY_ERROR;
2842: m->error = TRUE;
2843: } else {
2844: m->clustered = TRUE;
2845: }
2846:
2847:
2848: m->cleaning = FALSE;
2849: m->overwriting = FALSE;
2850: PAGE_WAKEUP_DONE(m);
2851: if(m->clustered) {
2852: vm_page_free(m);
2853: } else {
2854: vm_page_activate(m);
2855: }
2856:
2857: vm_page_unlock_queues();
2858: target_offset += PAGE_SIZE;
2859: xfer_size -= PAGE_SIZE;
2860: entry++;
2861: continue;
2862: }
2863: /*
2864: * Handle the trusted pager throttle.
2865: */
2866: if (m->laundry) {
2867: vm_page_laundry_count--;
2868: m->laundry = FALSE;
2869: if (vm_page_laundry_count
2870: < vm_page_laundry_min) {
2871: vm_page_laundry_min = 0;
2872: thread_wakeup((event_t)
2873: &vm_page_laundry_count);
2874: }
2875: }
2876: if(m->pageout) {
2877: assert(m->busy);
2878: assert(m->wire_count == 1);
2879: m->pageout = FALSE;
2880: vm_page_unwire(m);
2881: }
2882: m->dirty = TRUE;
2883: m->cleaning = FALSE;
2884: m->busy = FALSE;
2885: m->overwriting = FALSE;
2886: PAGE_WAKEUP(m);
2887: vm_page_unlock_queues();
2888: }
2889: }
2890: target_offset += PAGE_SIZE;
2891: xfer_size -= PAGE_SIZE;
2892: entry++;
2893: }
2894: vm_object_unlock(shadow_object);
2895: if(error & UPL_ABORT_FREE_ON_EMPTY) {
2896: if(queue_empty(&upl->map_object->memq)) {
2897: upl_destroy(upl);
2898: }
2899: }
2900: return KERN_SUCCESS;
2901: }
2902:
2903: kern_return_t
2904: upl_abort(
2905: upl_t upl,
2906: int error)
2907: {
2908: vm_object_t object = NULL;
2909: vm_object_t shadow_object = NULL;
2910: vm_offset_t offset;
2911: vm_object_offset_t shadow_offset;
2912: vm_object_offset_t target_offset;
2913: int i;
2914: vm_page_t t,m;
2915:
2916: object = upl->map_object;
2917:
2918: if(object == NULL) {
2919: panic("upl_abort: upl object is not backed by an object");
2920: return KERN_INVALID_ARGUMENT;
2921: }
2922:
2923: shadow_object = upl->map_object->shadow;
2924: shadow_offset = upl->map_object->shadow_offset;
2925: offset = 0;
2926: vm_object_lock(shadow_object);
2927: for(i = 0; i<(upl->size); i+=PAGE_SIZE) {
2928: if((t = vm_page_lookup(object,
2929: (vm_offset_t)offset)) != NULL) {
2930: target_offset = t->offset + shadow_offset;
2931: if((m = vm_page_lookup(shadow_object, target_offset)) != NULL) {
2932: vm_object_paging_end(m->object);
2933: vm_page_lock_queues();
2934: if(m->absent) {
2935: /* COPYOUT = FALSE case */
2936: /* check for error conditions which must */
2937: /* be passed back to the pages customer */
2938: if(error & UPL_ABORT_RESTART) {
2939: m->restart = TRUE;
2940: m->absent = FALSE;
2941: vm_object_absent_release(m->object);
2942: m->page_error = KERN_MEMORY_ERROR;
2943: m->error = TRUE;
2944: } else if(error & UPL_ABORT_UNAVAILABLE) {
2945: m->restart = FALSE;
2946: m->unusual = TRUE;
2947: m->clustered = FALSE;
2948: } else if(error & UPL_ABORT_ERROR) {
2949: m->restart = FALSE;
2950: m->absent = FALSE;
2951: vm_object_absent_release(m->object);
2952: m->page_error = KERN_MEMORY_ERROR;
2953: m->error = TRUE;
2954: } else {
2955: m->clustered = TRUE;
2956: }
2957:
2958:
2959: m->cleaning = FALSE;
2960: m->overwriting = FALSE;
2961: PAGE_WAKEUP_DONE(m);
2962: if(m->clustered) {
2963: vm_page_free(m);
2964: } else {
2965: vm_page_activate(m);
2966: }
2967:
2968: vm_page_unlock_queues();
2969: continue;
2970: }
2971: /*
2972: * Handle the trusted pager throttle.
2973: */
2974: if (m->laundry) {
2975: vm_page_laundry_count--;
2976: m->laundry = FALSE;
2977: if (vm_page_laundry_count
2978: < vm_page_laundry_min) {
2979: vm_page_laundry_min = 0;
2980: thread_wakeup((event_t)
2981: &vm_page_laundry_count);
2982: }
2983: }
2984: if(m->pageout) {
2985: assert(m->busy);
2986: assert(m->wire_count == 1);
2987: m->pageout = FALSE;
2988: vm_page_unwire(m);
2989: }
2990: m->dirty = TRUE;
2991: m->cleaning = FALSE;
2992: m->busy = FALSE;
2993: m->overwriting = FALSE;
2994: PAGE_WAKEUP(m);
2995: vm_page_unlock_queues();
2996: }
2997: }
2998: offset += PAGE_SIZE;
2999: }
3000: vm_object_unlock(shadow_object);
3001: /* Remove all the pages from the map object so */
3002: /* vm_pageout_object_terminate will work properly. */
3003: while (!queue_empty(&upl->map_object->memq)) {
3004: vm_page_t p;
3005:
3006: p = (vm_page_t) queue_first(&upl->map_object->memq);
3007:
3008: assert(p->private);
3009: assert(p->pageout);
3010: p->pageout = FALSE;
3011: assert(!p->cleaning);
3012:
3013: VM_PAGE_FREE(p);
3014: }
3015: upl_destroy(upl);
3016: return KERN_SUCCESS;
3017: }
3018:
3019: /* an option on commit should be wire */
3020: kern_return_t
3021: upl_commit(
3022: upl_t upl,
3023: upl_page_info_t *page_list)
3024: {
3025: if (upl->flags & UPL_CLEAR_DIRTY) {
3026: vm_object_t shadow_object = upl->map_object->shadow;
3027: vm_object_t object = upl->map_object;
3028: vm_object_offset_t target_offset;
3029: vm_size_t xfer_end;
3030:
3031: vm_page_t t,m;
3032:
3033: vm_object_lock(shadow_object);
3034:
3035: target_offset = object->shadow_offset;
3036: xfer_end = upl->size + object->shadow_offset;
3037:
3038: while(target_offset <= xfer_end) {
3039: if ((t = vm_page_lookup(object,
3040: target_offset - object->shadow_offset))
3041: != NULL) {
3042: m = vm_page_lookup(
3043: shadow_object, target_offset);
3044: if(m != VM_PAGE_NULL) {
3045: pmap_clear_modify(m->phys_addr);
3046: m->dirty = FALSE;
3047: }
3048: }
3049: target_offset += PAGE_SIZE;
3050: }
3051: vm_object_unlock(shadow_object);
3052: }
3053: if (page_list) {
3054: vm_object_t shadow_object = upl->map_object->shadow;
3055: vm_object_t object = upl->map_object;
3056: vm_object_offset_t target_offset;
3057: vm_size_t xfer_end;
3058: int entry;
3059:
3060: vm_page_t m;
3061: upl_page_info_t *p;
3062:
3063: vm_object_lock(shadow_object);
3064:
3065: entry = 0;
3066: target_offset = object->shadow_offset;
3067: xfer_end = upl->size + object->shadow_offset;
3068:
3069: while(target_offset <= xfer_end) {
3070:
3071: m = vm_page_lookup(shadow_object, target_offset);
3072: if(m != VM_PAGE_NULL) {
3073: p = &(page_list[entry]);
3074: if(page_list[entry].phys_addr &&
3075: p->pageout && !m->pageout) {
3076: vm_page_lock_queues();
3077: m->busy = TRUE;
3078: m->pageout = TRUE;
3079: vm_page_wire(m);
3080: vm_page_unlock_queues();
3081: } else if (page_list[entry].phys_addr &&
3082: !p->pageout && m->pageout) {
3083: vm_page_lock_queues();
3084: m->pageout = FALSE;
3085: vm_page_unwire(m);
3086: PAGE_WAKEUP_DONE(m);
3087: vm_page_unlock_queues();
3088: }
3089: page_list[entry].phys_addr = 0;
3090: }
3091: target_offset += PAGE_SIZE;
3092: entry++;
3093: }
3094:
3095: vm_object_unlock(shadow_object);
3096: }
3097: upl_destroy(upl);
3098: return KERN_SUCCESS;
3099: }
3100:
3101: upl_t
3102: upl_create(
3103: boolean_t internal)
3104: {
3105: upl_t upl;
3106:
3107: if(internal) {
3108: upl = (upl_t)kalloc(sizeof(struct upl)
3109: + (sizeof(struct upl_page_info)*20));
3110: } else {
3111: upl = (upl_t)kalloc(sizeof(struct upl));
3112: }
3113: upl->flags = 0;
3114: upl->src_object = NULL;
3115: upl->kaddr = (vm_offset_t)0;
3116: upl->size = 0;
3117: upl->map_object = NULL;
3118: upl->ref_count = 1;
3119: upl_lock_init(upl);
3120: return(upl);
3121: }
3122:
3123: void
3124: upl_destroy(
3125: upl_t upl)
3126: {
3127: vm_object_deallocate(upl->map_object);
3128: if(upl->flags & UPL_INTERNAL) {
3129: kfree((vm_offset_t)upl,
3130: sizeof(struct upl) + (sizeof(struct upl_page_info)*20));
3131: } else {
3132: kfree((vm_offset_t)upl, sizeof(struct upl));
3133: }
3134: }
3135:
3136: vm_size_t
3137: upl_get_internal_pagelist_offset()
3138: {
3139: return sizeof(struct upl);
3140: }
3141:
3142: void
3143: upl_set_dirty(
3144: upl_t upl)
3145: {
3146: upl->flags |= UPL_CLEAR_DIRTY;
3147: }
3148:
3149: void
3150: upl_clear_dirty(
3151: upl_t upl)
3152: {
3153: upl->flags &= ~UPL_CLEAR_DIRTY;
3154: }
3155:
3156:
3157:
3158:
3159: #if MACH_KDB
3160: #include <ddb/db_output.h>
3161: #include <ddb/db_print.h>
3162: #include <vm/vm_print.h>
3163:
3164: #define printf kdbprintf
3165: extern int db_indent;
3166: void db_pageout(void);
3167:
3168: void
3169: db_vm(void)
3170: {
3171: extern int vm_page_gobble_count;
3172: extern int vm_page_limbo_count, vm_page_limbo_real_count;
3173: extern int vm_page_pin_count;
3174:
3175: iprintf("VM Statistics:\n");
3176: db_indent += 2;
3177: iprintf("pages:\n");
3178: db_indent += 2;
3179: iprintf("activ %5d inact %5d free %5d",
3180: vm_page_active_count, vm_page_inactive_count,
3181: vm_page_free_count);
3182: printf(" wire %5d gobbl %5d\n",
3183: vm_page_wire_count, vm_page_gobble_count);
3184: iprintf("laund %5d limbo %5d lim_r %5d pin %5d\n",
3185: vm_page_laundry_count, vm_page_limbo_count,
3186: vm_page_limbo_real_count, vm_page_pin_count);
3187: db_indent -= 2;
3188: iprintf("target:\n");
3189: db_indent += 2;
3190: iprintf("min %5d inact %5d free %5d",
3191: vm_page_free_min, vm_page_inactive_target,
3192: vm_page_free_target);
3193: printf(" resrv %5d\n", vm_page_free_reserved);
3194: db_indent -= 2;
3195:
3196: iprintf("burst:\n");
3197: db_indent += 2;
3198: iprintf("max %5d min %5d wait %5d empty %5d\n",
3199: vm_pageout_burst_max, vm_pageout_burst_min,
3200: vm_pageout_burst_wait, vm_pageout_empty_wait);
3201: db_indent -= 2;
3202: iprintf("pause:\n");
3203: db_indent += 2;
3204: iprintf("count %5d max %5d\n",
3205: vm_pageout_pause_count, vm_pageout_pause_max);
3206: #if MACH_COUNTERS
3207: iprintf("scan_continue called %8d\n", c_vm_pageout_scan_continue);
3208: #endif /* MACH_COUNTERS */
3209: db_indent -= 2;
3210: db_pageout();
3211: db_indent -= 2;
3212: }
3213:
3214: void
3215: db_pageout(void)
3216: {
3217: extern int c_limbo_page_free;
3218: extern int c_limbo_convert;
3219: #if MACH_COUNTERS
3220: extern int c_laundry_pages_freed;
3221: #endif /* MACH_COUNTERS */
3222:
3223: iprintf("Pageout Statistics:\n");
3224: db_indent += 2;
3225: iprintf("active %5d inactv %5d\n",
3226: vm_pageout_active, vm_pageout_inactive);
3227: iprintf("nolock %5d avoid %5d busy %5d absent %5d\n",
3228: vm_pageout_inactive_nolock, vm_pageout_inactive_avoid,
3229: vm_pageout_inactive_busy, vm_pageout_inactive_absent);
3230: iprintf("used %5d clean %5d dirty %5d\n",
3231: vm_pageout_inactive_used, vm_pageout_inactive_clean,
3232: vm_pageout_inactive_dirty);
3233: iprintf("pinned %5d limbo %5d setup_limbo %5d setup_unprep %5d\n",
3234: vm_pageout_inactive_pinned, vm_pageout_inactive_limbo,
3235: vm_pageout_setup_limbo, vm_pageout_setup_unprepped);
3236: iprintf("limbo_page_free %5d limbo_convert %5d\n",
3237: c_limbo_page_free, c_limbo_convert);
3238: #if MACH_COUNTERS
3239: iprintf("laundry_pages_freed %d\n", c_laundry_pages_freed);
3240: #endif /* MACH_COUNTERS */
3241: #if MACH_CLUSTER_STATS
3242: iprintf("Cluster Statistics:\n");
3243: db_indent += 2;
3244: iprintf("dirtied %5d cleaned %5d collisions %5d\n",
3245: vm_pageout_cluster_dirtied, vm_pageout_cluster_cleaned,
3246: vm_pageout_cluster_collisions);
3247: iprintf("clusters %5d conversions %5d\n",
3248: vm_pageout_cluster_clusters, vm_pageout_cluster_conversions);
3249: db_indent -= 2;
3250: iprintf("Target Statistics:\n");
3251: db_indent += 2;
3252: iprintf("collisions %5d page_dirtied %5d page_freed %5d\n",
3253: vm_pageout_target_collisions, vm_pageout_target_page_dirtied,
3254: vm_pageout_target_page_freed);
3255: iprintf("page_pinned %5d page_limbo %5d\n",
3256: vm_pageout_target_page_pinned, vm_pageout_target_page_limbo);
3257: db_indent -= 2;
3258: #endif /* MACH_CLUSTER_STATS */
3259: db_indent -= 2;
3260: }
3261:
3262: #if MACH_CLUSTER_STATS
3263: unsigned long vm_pageout_cluster_dirtied = 0;
3264: unsigned long vm_pageout_cluster_cleaned = 0;
3265: unsigned long vm_pageout_cluster_collisions = 0;
3266: unsigned long vm_pageout_cluster_clusters = 0;
3267: unsigned long vm_pageout_cluster_conversions = 0;
3268: unsigned long vm_pageout_target_collisions = 0;
3269: unsigned long vm_pageout_target_page_dirtied = 0;
3270: unsigned long vm_pageout_target_page_freed = 0;
3271: unsigned long vm_pageout_target_page_pinned = 0;
3272: unsigned long vm_pageout_target_page_limbo = 0;
3273: #define CLUSTER_STAT(clause) clause
3274: #else /* MACH_CLUSTER_STATS */
3275: #define CLUSTER_STAT(clause)
3276: #endif /* MACH_CLUSTER_STATS */
3277:
3278: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.