![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_pageout.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_pageout.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * The proverbial page-out daemon.
35: */
36:
1.1.1.3 root 37: #include <device/net_io.h>
1.1 root 38: #include <mach/mach_types.h>
39: #include <mach/memory_object.h>
1.1.1.3 root 40: #include <vm/memory_object_default.user.h>
41: #include <vm/memory_object_user.user.h>
1.1 root 42: #include <mach/vm_param.h>
43: #include <mach/vm_statistics.h>
44: #include <kern/counters.h>
1.1.1.3 root 45: #include <kern/debug.h>
46: #include <kern/slab.h>
47: #include <kern/task.h>
1.1 root 48: #include <kern/thread.h>
49: #include <vm/pmap.h>
50: #include <vm/vm_map.h>
51: #include <vm/vm_object.h>
52: #include <vm/vm_page.h>
53: #include <vm/vm_pageout.h>
1.1.1.3 root 54: #include <machine/locore.h>
1.1 root 55:
56:
57:
58: #ifndef VM_PAGEOUT_BURST_MAX
59: #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */
1.1.1.2 root 60: #endif /* VM_PAGEOUT_BURST_MAX */
1.1 root 61:
62: #ifndef VM_PAGEOUT_BURST_MIN
63: #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */
1.1.1.2 root 64: #endif /* VM_PAGEOUT_BURST_MIN */
1.1 root 65:
66: #ifndef VM_PAGEOUT_BURST_WAIT
1.1.1.2 root 67: #define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds per page */
68: #endif /* VM_PAGEOUT_BURST_WAIT */
1.1 root 69:
70: #ifndef VM_PAGEOUT_EMPTY_WAIT
1.1.1.2 root 71: #define VM_PAGEOUT_EMPTY_WAIT 75 /* milliseconds */
72: #endif /* VM_PAGEOUT_EMPTY_WAIT */
1.1 root 73:
74: #ifndef VM_PAGEOUT_PAUSE_MAX
75: #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */
1.1.1.2 root 76: #endif /* VM_PAGEOUT_PAUSE_MAX */
1.1 root 77:
78: /*
79: * To obtain a reasonable LRU approximation, the inactive queue
80: * needs to be large enough to give pages on it a chance to be
81: * referenced a second time. This macro defines the fraction
82: * of active+inactive pages that should be inactive.
83: * The pageout daemon uses it to update vm_page_inactive_target.
84: *
1.1.1.6 ! root 85: * If the number of free pages falls below vm_page_free_target and
1.1 root 86: * vm_page_inactive_count is below vm_page_inactive_target,
87: * then the pageout daemon starts running.
88: */
89:
90: #ifndef VM_PAGE_INACTIVE_TARGET
91: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3)
1.1.1.2 root 92: #endif /* VM_PAGE_INACTIVE_TARGET */
1.1 root 93:
94: /*
95: * Once the pageout daemon starts running, it keeps going
1.1.1.6 ! root 96: * until the number of free pages meets or exceeds vm_page_free_target.
1.1 root 97: */
98:
99: #ifndef VM_PAGE_FREE_TARGET
1.1.1.4 root 100: #define VM_PAGE_FREE_TARGET(free) (150 + (free) * 10 / 100)
1.1.1.2 root 101: #endif /* VM_PAGE_FREE_TARGET */
1.1 root 102:
103: /*
1.1.1.6 ! root 104: * The pageout daemon always starts running once the number of free pages
1.1 root 105: * falls below vm_page_free_min.
106: */
107:
108: #ifndef VM_PAGE_FREE_MIN
1.1.1.4 root 109: #define VM_PAGE_FREE_MIN(free) (100 + (free) * 8 / 100)
1.1.1.2 root 110: #endif /* VM_PAGE_FREE_MIN */
111:
112: /* When vm_page_external_count exceeds vm_page_external_limit,
113: * allocations of externally paged pages stops.
114: */
115:
116: #ifndef VM_PAGE_EXTERNAL_LIMIT
117: #define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2)
118: #endif /* VM_PAGE_EXTERNAL_LIMIT */
119:
120: /* Attempt to keep the number of externally paged pages less
121: * than vm_pages_external_target.
122: */
123: #ifndef VM_PAGE_EXTERNAL_TARGET
124: #define VM_PAGE_EXTERNAL_TARGET(free) ((free) / 4)
125: #endif /* VM_PAGE_EXTERNAL_TARGET */
1.1 root 126:
127: /*
1.1.1.6 ! root 128: * When the number of free pages falls below vm_page_free_reserved,
1.1 root 129: * only vm-privileged threads can allocate pages. vm-privilege
130: * allows the pageout daemon and default pager (and any other
131: * associated threads needed for default pageout) to continue
1.1.1.2 root 132: * operation by dipping into the reserved pool of pages. */
1.1 root 133:
134: #ifndef VM_PAGE_FREE_RESERVED
1.1.1.4 root 135: #define VM_PAGE_FREE_RESERVED 500
1.1.1.2 root 136: #endif /* VM_PAGE_FREE_RESERVED */
1.1 root 137:
138: /*
1.1.1.6 ! root 139: * When the number of free pages falls below vm_pageout_reserved_internal,
1.1 root 140: * the pageout daemon no longer trusts external pagers to clean pages.
141: * External pagers are probably all wedged waiting for a free page.
142: * It forcibly double-pages dirty pages belonging to external objects,
143: * getting the pages to the default pager to clean.
144: */
145:
146: #ifndef VM_PAGEOUT_RESERVED_INTERNAL
1.1.1.4 root 147: #define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 250)
1.1.1.2 root 148: #endif /* VM_PAGEOUT_RESERVED_INTERNAL */
1.1 root 149:
150: /*
1.1.1.6 ! root 151: * When the number of free pages falls below vm_pageout_reserved_really,
1.1 root 152: * the pageout daemon stops work entirely to let the default pager
153: * catch up (assuming the default pager has pages to clean).
154: * Beyond this point, it is too dangerous to consume memory
155: * even for memory_object_data_write messages to the default pager.
156: */
157:
158: #ifndef VM_PAGEOUT_RESERVED_REALLY
1.1.1.4 root 159: #define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 400)
1.1.1.2 root 160: #endif /* VM_PAGEOUT_RESERVED_REALLY */
1.1 root 161:
162: unsigned int vm_pageout_reserved_internal = 0;
163: unsigned int vm_pageout_reserved_really = 0;
164:
1.1.1.2 root 165: unsigned int vm_page_external_target = 0;
166:
1.1 root 167: unsigned int vm_pageout_burst_max = 0;
168: unsigned int vm_pageout_burst_min = 0;
169: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */
170: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
171: unsigned int vm_pageout_pause_count = 0;
172: unsigned int vm_pageout_pause_max = 0;
173:
174: /*
175: * These variables record the pageout daemon's actions:
176: * how many pages it looks at and what happens to those pages.
177: * No locking needed because only one thread modifies the variables.
178: */
179:
180: unsigned int vm_pageout_active = 0; /* debugging */
181: unsigned int vm_pageout_inactive = 0; /* debugging */
182: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
183: unsigned int vm_pageout_inactive_busy = 0; /* debugging */
184: unsigned int vm_pageout_inactive_absent = 0; /* debugging */
185: unsigned int vm_pageout_inactive_used = 0; /* debugging */
186: unsigned int vm_pageout_inactive_clean = 0; /* debugging */
187: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
188: unsigned int vm_pageout_inactive_double = 0; /* debugging */
1.1.1.2 root 189: unsigned int vm_pageout_inactive_cleaned_external = 0;
1.1 root 190:
191: /*
192: * Routine: vm_pageout_setup
193: * Purpose:
194: * Set up a page for pageout.
195: *
196: * Move or copy the page to a new object, as part
197: * of which it will be sent to its memory manager
198: * in a memory_object_data_write or memory_object_initialize
199: * message.
200: *
201: * The "paging_offset" argument specifies the offset
202: * of the page within its external memory object.
203: *
204: * The "new_object" and "new_offset" arguments
205: * indicate where the page should be moved.
206: *
207: * The "flush" argument specifies whether the page
208: * should be flushed from its object. If not, a
209: * copy of the page is moved to the new object.
210: *
211: * In/Out conditions:
212: * The page in question must not be on any pageout queues,
213: * and must be busy. The object to which it belongs
214: * must be unlocked, and the caller must hold a paging
215: * reference to it. The new_object must not be locked.
216: *
217: * If the page is flushed from its original object,
218: * this routine returns a pointer to a place-holder page,
219: * inserted at the same offset, to block out-of-order
220: * requests for the page. The place-holder page must
221: * be freed after the data_write or initialize message
222: * has been sent. If the page is copied,
223: * the holding page is VM_PAGE_NULL.
224: *
225: * The original page is put on a paging queue and marked
226: * not busy on exit.
227: */
228: vm_page_t
1.1.1.4 root 229: vm_pageout_setup(
230: vm_page_t m,
231: vm_offset_t paging_offset,
232: vm_object_t new_object,
233: vm_offset_t new_offset,
234: boolean_t flush)
1.1 root 235: {
1.1.1.4 root 236: vm_object_t old_object = m->object;
237: vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/
238: vm_page_t new_m;
1.1 root 239:
240: assert(m->busy && !m->absent && !m->fictitious);
241:
242: /*
243: * If we are not flushing the page, allocate a
244: * page in the object. If we cannot get the
245: * page, flush instead.
246: */
247: if (!flush) {
248: vm_object_lock(new_object);
249: new_m = vm_page_alloc(new_object, new_offset);
250: if (new_m == VM_PAGE_NULL)
251: flush = TRUE;
252: vm_object_unlock(new_object);
253: }
254:
255: if (flush) {
256: /*
257: * Create a place-holder page where the old one was,
258: * to prevent anyone from attempting to page in this
259: * page while we`re unlocked.
260: */
261: while ((holding_page = vm_page_grab_fictitious())
262: == VM_PAGE_NULL)
263: vm_page_more_fictitious();
264:
265: vm_object_lock(old_object);
266: vm_page_lock_queues();
267: vm_page_remove(m);
268: vm_page_unlock_queues();
269: PAGE_WAKEUP_DONE(m);
270:
271: vm_page_lock_queues();
272: vm_page_insert(holding_page, old_object, m->offset);
273: vm_page_unlock_queues();
274:
275: /*
276: * Record that this page has been written out
277: */
278: #if MACH_PAGEMAP
279: vm_external_state_set(old_object->existence_info,
280: paging_offset,
281: VM_EXTERNAL_STATE_EXISTS);
1.1.1.2 root 282: #endif /* MACH_PAGEMAP */
1.1 root 283:
284: vm_object_unlock(old_object);
285:
286: vm_object_lock(new_object);
287:
288: /*
289: * Move this page into the new object
290: */
291:
292: vm_page_lock_queues();
293: vm_page_insert(m, new_object, new_offset);
294: vm_page_unlock_queues();
295:
296: m->dirty = TRUE;
297: m->precious = FALSE;
298: m->page_lock = VM_PROT_NONE;
299: m->unlock_request = VM_PROT_NONE;
300: }
301: else {
302: /*
303: * Copy the data into the new page,
304: * and mark the new page as clean.
305: */
306: vm_page_copy(m, new_m);
307:
308: vm_object_lock(old_object);
309: m->dirty = FALSE;
310: pmap_clear_modify(m->phys_addr);
311:
312: /*
313: * Deactivate old page.
314: */
315: vm_page_lock_queues();
316: vm_page_deactivate(m);
317: vm_page_unlock_queues();
318:
319: PAGE_WAKEUP_DONE(m);
320:
321: /*
322: * Record that this page has been written out
323: */
324:
325: #if MACH_PAGEMAP
326: vm_external_state_set(old_object->existence_info,
327: paging_offset,
328: VM_EXTERNAL_STATE_EXISTS);
1.1.1.2 root 329: #endif /* MACH_PAGEMAP */
1.1 root 330:
331: vm_object_unlock(old_object);
332:
333: vm_object_lock(new_object);
334:
335: /*
336: * Use the new page below.
337: */
338: m = new_m;
339: m->dirty = TRUE;
340: assert(!m->precious);
341: PAGE_WAKEUP_DONE(m);
342: }
343:
344: /*
345: * Make the old page eligible for replacement again; if a
346: * user-supplied memory manager fails to release the page,
347: * it will be paged out again to the default memory manager.
348: *
349: * Note that pages written to the default memory manager
350: * must be wired down -- in return, it guarantees to free
351: * this page, rather than reusing it.
352: */
353:
354: vm_page_lock_queues();
355: vm_stat.pageouts++;
356: if (m->laundry) {
357: /*
358: * vm_pageout_scan is telling us to put this page
359: * at the front of the inactive queue, so it will
360: * be immediately paged out to the default pager.
361: */
362:
363: assert(!old_object->internal);
364: m->laundry = FALSE;
365:
366: queue_enter_first(&vm_page_queue_inactive, m,
367: vm_page_t, pageq);
368: m->inactive = TRUE;
369: vm_page_inactive_count++;
370: } else if (old_object->internal) {
371: m->laundry = TRUE;
372: vm_page_laundry_count++;
373:
374: vm_page_wire(m);
375: } else
376: vm_page_activate(m);
377: vm_page_unlock_queues();
378:
379: /*
380: * Since IPC operations may block, we drop locks now.
381: * [The placeholder page is busy, and we still have
382: * paging_in_progress incremented.]
383: */
384:
385: vm_object_unlock(new_object);
386:
387: /*
388: * Return the placeholder page to simplify cleanup.
389: */
390: return (flush ? holding_page : VM_PAGE_NULL);
391: }
392:
393: /*
394: * Routine: vm_pageout_page
395: * Purpose:
396: * Causes the specified page to be written back to
397: * the appropriate memory object.
398: *
399: * The "initial" argument specifies whether this
400: * data is an initialization only, and should use
401: * memory_object_data_initialize instead of
402: * memory_object_data_write.
403: *
404: * The "flush" argument specifies whether the page
405: * should be flushed from the object. If not, a
406: * copy of the data is sent to the memory object.
407: *
408: * In/out conditions:
409: * The page in question must not be on any pageout queues.
410: * The object to which it belongs must be locked.
411: * Implementation:
412: * Move this page to a completely new object, if flushing;
413: * copy to a new page in a new object, if not.
414: */
415: void
1.1.1.4 root 416: vm_pageout_page(
417: vm_page_t m,
418: boolean_t initial,
419: boolean_t flush)
1.1 root 420: {
421: vm_map_copy_t copy;
1.1.1.4 root 422: vm_object_t old_object;
423: vm_object_t new_object;
424: vm_page_t holding_page;
1.1 root 425: vm_offset_t paging_offset;
426: kern_return_t rc;
427: boolean_t precious_clean;
428:
429: assert(m->busy);
430:
431: /*
432: * Cleaning but not flushing a clean precious page is a
433: * no-op. Remember whether page is clean and precious now
434: * because vm_pageout_setup will mark it dirty and not precious.
435: *
436: * XXX Check if precious_clean && !flush can really happen.
437: */
438: precious_clean = (!m->dirty) && m->precious;
439: if (precious_clean && !flush) {
440: PAGE_WAKEUP_DONE(m);
441: return;
442: }
443:
444: /*
445: * Verify that we really want to clean this page.
446: */
447: if (m->absent || m->error || (!m->dirty && !m->precious)) {
448: VM_PAGE_FREE(m);
449: return;
450: }
451:
452: /*
453: * Create a paging reference to let us play with the object.
454: */
455: old_object = m->object;
456: paging_offset = m->offset + old_object->paging_offset;
457: vm_object_paging_begin(old_object);
458: vm_object_unlock(old_object);
459:
460: /*
461: * Allocate a new object into which we can put the page.
462: */
463: new_object = vm_object_allocate(PAGE_SIZE);
1.1.1.6 ! root 464: new_object->used_for_pageout = TRUE;
1.1 root 465:
466: /*
467: * Move the page into the new object.
468: */
469: holding_page = vm_pageout_setup(m,
470: paging_offset,
471: new_object,
472: 0, /* new offset */
473: flush); /* flush */
474:
475: rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©);
476: assert(rc == KERN_SUCCESS);
477:
478: if (initial || old_object->use_old_pageout) {
479: rc = (*(initial ? memory_object_data_initialize
480: : memory_object_data_write))
481: (old_object->pager,
482: old_object->pager_request,
483: paging_offset, (pointer_t) copy, PAGE_SIZE);
484: }
485: else {
486: rc = memory_object_data_return(
487: old_object->pager,
488: old_object->pager_request,
489: paging_offset, (pointer_t) copy, PAGE_SIZE,
490: !precious_clean, !flush);
491: }
492:
493: if (rc != KERN_SUCCESS)
494: vm_map_copy_discard(copy);
495:
496: /*
497: * Clean up.
498: */
499: vm_object_lock(old_object);
500: if (holding_page != VM_PAGE_NULL)
501: VM_PAGE_FREE(holding_page);
502: vm_object_paging_end(old_object);
503: }
504:
505: /*
506: * vm_pageout_scan does the dirty work for the pageout daemon.
507: * It returns with vm_page_queue_free_lock held and
508: * vm_page_free_wanted == 0.
509: */
510:
1.1.1.4 root 511: void vm_pageout_scan(void)
1.1 root 512: {
513: unsigned int burst_count;
1.1.1.2 root 514: unsigned int want_pages;
1.1 root 515:
516: /*
517: * We want to gradually dribble pages from the active queue
518: * to the inactive queue. If we let the inactive queue get
519: * very small, and then suddenly dump many pages into it,
520: * those pages won't get a sufficient chance to be referenced
521: * before we start taking them from the inactive queue.
522: *
523: * We must limit the rate at which we send pages to the pagers.
524: * data_write messages consume memory, for message buffers and
525: * for map-copy objects. If we get too far ahead of the pagers,
526: * we can potentially run out of memory.
527: *
528: * We can use the laundry count to limit directly the number
529: * of pages outstanding to the default pager. A similar
530: * strategy for external pagers doesn't work, because
531: * external pagers don't have to deallocate the pages sent them,
532: * and because we might have to send pages to external pagers
533: * even if they aren't processing writes. So we also
534: * use a burst count to limit writes to external pagers.
535: *
536: * When memory is very tight, we can't rely on external pagers to
537: * clean pages. They probably aren't running, because they
538: * aren't vm-privileged. If we kept sending dirty pages to them,
539: * we could exhaust the free list. However, we can't just ignore
540: * pages belonging to external objects, because there might be no
541: * pages belonging to internal objects. Hence, we get the page
542: * into an internal object and then immediately double-page it,
543: * sending it to the default pager.
544: *
1.1.1.3 root 545: * slab_collect should be last, because the other operations
546: * might return memory to caches. When we pause we use
1.1 root 547: * vm_pageout_scan_continue as our continuation, so we will
548: * reenter vm_pageout_scan periodically and attempt to reclaim
549: * internal memory even if we never reach vm_page_free_target.
550: */
551:
552: stack_collect();
553: net_kmsg_collect();
554: consider_task_collect();
1.1.1.4 root 555: if (0) /* XXX: pcb_collect doesn't do anything yet, so it is
556: pointless to call consider_thread_collect. */
1.1 root 557: consider_thread_collect();
1.1.1.3 root 558: slab_collect();
1.1 root 559:
560: for (burst_count = 0;;) {
1.1.1.4 root 561: vm_page_t m;
562: vm_object_t object;
1.1.1.6 ! root 563: unsigned long free_count;
1.1 root 564:
565: /*
566: * Recalculate vm_page_inactivate_target.
567: */
568:
569: vm_page_lock_queues();
570: vm_page_inactive_target =
571: VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
572: vm_page_inactive_count);
573:
574: /*
575: * Move pages from active to inactive.
576: */
577:
578: while ((vm_page_inactive_count < vm_page_inactive_target) &&
579: !queue_empty(&vm_page_queue_active)) {
1.1.1.4 root 580: vm_object_t obj;
1.1 root 581:
582: vm_pageout_active++;
583: m = (vm_page_t) queue_first(&vm_page_queue_active);
584: assert(m->active && !m->inactive);
585:
586: obj = m->object;
587: if (!vm_object_lock_try(obj)) {
588: /*
589: * Move page to end and continue.
590: */
591:
592: queue_remove(&vm_page_queue_active, m,
593: vm_page_t, pageq);
594: queue_enter(&vm_page_queue_active, m,
595: vm_page_t, pageq);
596: vm_page_unlock_queues();
597: vm_page_lock_queues();
598: continue;
599: }
600:
601: /*
602: * If the page is busy, then we pull it
603: * off the active queue and leave it alone.
604: */
605:
606: if (m->busy) {
607: vm_object_unlock(obj);
608: queue_remove(&vm_page_queue_active, m,
609: vm_page_t, pageq);
610: m->active = FALSE;
611: vm_page_active_count--;
612: continue;
613: }
614:
615: /*
616: * Deactivate the page while holding the object
617: * locked, so we know the page is still not busy.
618: * This should prevent races between pmap_enter
619: * and pmap_clear_reference. The page might be
620: * absent or fictitious, but vm_page_deactivate
621: * can handle that.
622: */
623:
624: vm_page_deactivate(m);
625: vm_object_unlock(obj);
626: }
627:
628: /*
1.1.1.2 root 629: * We are done if we have met our targets *and*
1.1 root 630: * nobody is still waiting for a page.
631: */
632:
633: simple_lock(&vm_page_queue_free_lock);
1.1.1.6 ! root 634: free_count = vm_page_mem_free();
1.1.1.2 root 635: if ((free_count >= vm_page_free_target) &&
636: (vm_page_external_count <= vm_page_external_target) &&
1.1 root 637: (vm_page_free_wanted == 0)) {
638: vm_page_unlock_queues();
639: break;
640: }
1.1.1.2 root 641: want_pages = ((free_count < vm_page_free_target) ||
642: vm_page_free_wanted);
1.1 root 643: simple_unlock(&vm_page_queue_free_lock);
644:
645: /*
646: * Sometimes we have to pause:
647: * 1) No inactive pages - nothing to do.
648: * 2) Flow control - wait for pagers to catch up.
649: * 3) Extremely low memory - sending out dirty pages
650: * consumes memory. We don't take the risk of doing
651: * this if the default pager already has work to do.
652: */
1.1.1.2 root 653: pause:
1.1 root 654: if (queue_empty(&vm_page_queue_inactive) ||
655: (burst_count >= vm_pageout_burst_max) ||
656: (vm_page_laundry_count >= vm_pageout_burst_max) ||
657: ((free_count < vm_pageout_reserved_really) &&
658: (vm_page_laundry_count > 0))) {
659: unsigned int pages, msecs;
660:
661: /*
662: * vm_pageout_burst_wait is msecs/page.
663: * If there is nothing for us to do, we wait
664: * at least vm_pageout_empty_wait msecs.
665: */
666:
667: if (vm_page_laundry_count > burst_count)
668: pages = vm_page_laundry_count;
669: else
670: pages = burst_count;
671: msecs = pages * vm_pageout_burst_wait;
672:
673: if (queue_empty(&vm_page_queue_inactive) &&
674: (msecs < vm_pageout_empty_wait))
675: msecs = vm_pageout_empty_wait;
676: vm_page_unlock_queues();
677:
678: thread_will_wait_with_timeout(current_thread(), msecs);
679: counter(c_vm_pageout_scan_block++);
680: thread_block(vm_pageout_scan_continue);
681: call_continuation(vm_pageout_scan_continue);
682: /*NOTREACHED*/
683: }
684:
685: vm_pageout_inactive++;
1.1.1.2 root 686:
687: /* Find a page we are interested in paging out. If we
688: need pages, then we'll page anything out; otherwise
689: we only page out external pages. */
690: m = (vm_page_t) queue_first (&vm_page_queue_inactive);
691: while (1)
692: {
693: assert (!m->active && m->inactive);
694: if (want_pages || m->external)
695: break;
696:
1.1.1.5 root 697: m = (vm_page_t) queue_next (&m->pageq);
1.1.1.2 root 698: if (!m)
699: goto pause;
700: }
701:
1.1 root 702: object = m->object;
703:
704: /*
705: * Try to lock object; since we've got the
706: * page queues lock, we can only try for this one.
707: */
708:
709: if (!vm_object_lock_try(object)) {
710: /*
711: * Move page to end and continue.
712: */
713:
714: queue_remove(&vm_page_queue_inactive, m,
715: vm_page_t, pageq);
716: queue_enter(&vm_page_queue_inactive, m,
717: vm_page_t, pageq);
718: vm_page_unlock_queues();
719: vm_pageout_inactive_nolock++;
720: continue;
721: }
722:
723: /*
724: * Remove the page from the inactive list.
725: */
726:
727: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
728: vm_page_inactive_count--;
729: m->inactive = FALSE;
730:
731: if (m->busy || !object->alive) {
732: /*
733: * Somebody is already playing with this page.
734: * Leave it off the pageout queues.
735: */
736:
737: vm_page_unlock_queues();
738: vm_object_unlock(object);
739: vm_pageout_inactive_busy++;
740: continue;
741: }
742:
743: /*
744: * If it's absent, we can reclaim the page.
745: */
746:
1.1.1.2 root 747: if (want_pages && m->absent) {
1.1 root 748: vm_pageout_inactive_absent++;
749: reclaim_page:
750: vm_page_free(m);
751: vm_page_unlock_queues();
1.1.1.6 ! root 752:
! 753: if (vm_object_collectable(object))
! 754: vm_object_collect(object);
! 755: else
! 756: vm_object_unlock(object);
! 757:
1.1 root 758: continue;
759: }
760:
761: /*
762: * If it's being used, reactivate.
763: * (Fictitious pages are either busy or absent.)
764: */
765:
766: assert(!m->fictitious);
767: if (m->reference || pmap_is_referenced(m->phys_addr)) {
768: vm_object_unlock(object);
769: vm_page_activate(m);
770: vm_stat.reactivations++;
1.1.1.3 root 771: current_task()->reactivations++;
1.1 root 772: vm_page_unlock_queues();
773: vm_pageout_inactive_used++;
774: continue;
775: }
776:
777: /*
778: * Eliminate all mappings.
779: */
780:
781: m->busy = TRUE;
782: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
783: if (!m->dirty)
784: m->dirty = pmap_is_modified(m->phys_addr);
785:
1.1.1.2 root 786: if (m->external) {
787: /* Figure out if we still care about this
788: page in the limit of externally managed pages.
789: Clean pages don't actually cause system hosage,
790: so it's ok to stop considering them as
791: "consumers" of memory. */
792: if (m->dirty && !m->extcounted) {
793: m->extcounted = TRUE;
794: vm_page_external_count++;
795: } else if (!m->dirty && m->extcounted) {
796: m->extcounted = FALSE;
797: vm_page_external_count--;
798: }
799: }
800:
801: /* If we don't actually need more memory, and the page
802: is not dirty, put it on the tail of the inactive queue
803: and move on to the next page. */
804: if (!want_pages && !m->dirty) {
805: queue_remove (&vm_page_queue_inactive, m,
806: vm_page_t, pageq);
807: queue_enter (&vm_page_queue_inactive, m,
808: vm_page_t, pageq);
809: vm_page_unlock_queues();
810: vm_pageout_inactive_cleaned_external++;
811: continue;
812: }
813:
1.1 root 814: /*
815: * If it's clean and not precious, we can free the page.
816: */
817:
818: if (!m->dirty && !m->precious) {
819: vm_pageout_inactive_clean++;
820: goto reclaim_page;
821: }
822:
823: /*
824: * If we are very low on memory, then we can't
825: * rely on an external pager to clean a dirty page,
826: * because external pagers are not vm-privileged.
827: *
828: * The laundry bit tells vm_pageout_setup to
829: * put the page back at the front of the inactive
830: * queue instead of activating the page. Hence,
831: * we will pick the page up again immediately and
832: * resend it to the default pager.
833: */
834:
835: assert(!m->laundry);
836: if ((free_count < vm_pageout_reserved_internal) &&
837: !object->internal) {
838: m->laundry = TRUE;
839: vm_pageout_inactive_double++;
840: }
841: vm_page_unlock_queues();
842:
843: /*
844: * If there is no memory object for the page, create
845: * one and hand it to the default pager.
846: * [First try to collapse, so we don't create
847: * one unnecessarily.]
848: */
849:
850: if (!object->pager_initialized)
851: vm_object_collapse(object);
852: if (!object->pager_initialized)
853: vm_object_pager_create(object);
854: if (!object->pager_initialized)
855: panic("vm_pageout_scan");
856:
857: vm_pageout_inactive_dirty++;
858: vm_pageout_page(m, FALSE, TRUE); /* flush it */
859: vm_object_unlock(object);
860: burst_count++;
861: }
862: }
863:
1.1.1.4 root 864: void vm_pageout_scan_continue(void)
1.1 root 865: {
866: /*
867: * We just paused to let the pagers catch up.
868: * If vm_page_laundry_count is still high,
869: * then we aren't waiting long enough.
870: * If we have paused some vm_pageout_pause_max times without
871: * adjusting vm_pageout_burst_wait, it might be too big,
872: * so we decrease it.
873: */
874:
875: vm_page_lock_queues();
876: if (vm_page_laundry_count > vm_pageout_burst_min) {
877: vm_pageout_burst_wait++;
878: vm_pageout_pause_count = 0;
879: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) {
880: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4;
881: if (vm_pageout_burst_wait < 1)
882: vm_pageout_burst_wait = 1;
883: vm_pageout_pause_count = 0;
884: }
885: vm_page_unlock_queues();
886:
887: vm_pageout_continue();
888: /*NOTREACHED*/
889: }
890:
891: /*
892: * vm_pageout is the high level pageout daemon.
893: */
894:
1.1.1.4 root 895: void vm_pageout_continue(void)
1.1 root 896: {
897: /*
898: * The pageout daemon is never done, so loop forever.
899: * We should call vm_pageout_scan at least once each
900: * time we are woken, even if vm_page_free_wanted is
901: * zero, to check vm_page_free_target and
902: * vm_page_inactive_target.
903: */
904:
905: for (;;) {
906: vm_pageout_scan();
907: /* we hold vm_page_queue_free_lock now */
908: assert(vm_page_free_wanted == 0);
909:
910: assert_wait(&vm_page_free_wanted, FALSE);
911: simple_unlock(&vm_page_queue_free_lock);
912: counter(c_vm_pageout_block++);
913: thread_block(vm_pageout_continue);
914: }
915: }
916:
1.1.1.4 root 917: void vm_pageout(void)
1.1 root 918: {
1.1.1.6 ! root 919: unsigned long free_after_reserve;
1.1 root 920:
921: current_thread()->vm_privilege = TRUE;
922: stack_privilege(current_thread());
1.1.1.4 root 923: thread_set_own_priority(0);
1.1 root 924:
925: /*
926: * Initialize some paging parameters.
927: */
928:
929: if (vm_pageout_burst_max == 0)
930: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX;
931:
932: if (vm_pageout_burst_min == 0)
933: vm_pageout_burst_min = VM_PAGEOUT_BURST_MIN;
934:
935: if (vm_pageout_burst_wait == 0)
936: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
937:
938: if (vm_pageout_empty_wait == 0)
939: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
940:
941: if (vm_page_free_reserved == 0)
942: vm_page_free_reserved = VM_PAGE_FREE_RESERVED;
943:
944: if (vm_pageout_pause_max == 0)
945: vm_pageout_pause_max = VM_PAGEOUT_PAUSE_MAX;
946:
947: if (vm_pageout_reserved_internal == 0)
948: vm_pageout_reserved_internal =
949: VM_PAGEOUT_RESERVED_INTERNAL(vm_page_free_reserved);
950:
951: if (vm_pageout_reserved_really == 0)
952: vm_pageout_reserved_really =
953: VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved);
954:
1.1.1.6 ! root 955: free_after_reserve = vm_page_mem_free() - vm_page_free_reserved;
1.1 root 956:
1.1.1.2 root 957: if (vm_page_external_limit == 0)
958: vm_page_external_limit =
959: VM_PAGE_EXTERNAL_LIMIT (free_after_reserve);
960:
961: if (vm_page_external_target == 0)
962: vm_page_external_target =
963: VM_PAGE_EXTERNAL_TARGET (free_after_reserve);
964:
1.1 root 965: if (vm_page_free_min == 0)
966: vm_page_free_min = vm_page_free_reserved +
967: VM_PAGE_FREE_MIN(free_after_reserve);
968:
969: if (vm_page_free_target == 0)
970: vm_page_free_target = vm_page_free_reserved +
971: VM_PAGE_FREE_TARGET(free_after_reserve);
972:
973: if (vm_page_free_target < vm_page_free_min + 5)
974: vm_page_free_target = vm_page_free_min + 5;
975:
976: /*
977: * vm_pageout_scan will set vm_page_inactive_target.
978: */
979:
980: vm_pageout_continue();
981: /*NOTREACHED*/
982: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.