![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_pageout.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_pageout.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * The proverbial page-out daemon.
35: */
36:
37: #include <mach_pagemap.h>
38: #include <norma_vm.h>
39:
40: #include <mach/mach_types.h>
41: #include <mach/memory_object.h>
42: #include "memory_object_default.h"
43: #include "memory_object_user.h"
44: #include <mach/vm_param.h>
45: #include <mach/vm_statistics.h>
46: #include <kern/counters.h>
47: #include <kern/thread.h>
48: #include <vm/pmap.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
52: #include <vm/vm_pageout.h>
53: #include <machine/vm_tuning.h>
54:
55:
56:
57: #ifndef VM_PAGEOUT_BURST_MAX
58: #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */
1.1.1.2 ! root 59: #endif /* VM_PAGEOUT_BURST_MAX */
1.1 root 60:
61: #ifndef VM_PAGEOUT_BURST_MIN
62: #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */
1.1.1.2 ! root 63: #endif /* VM_PAGEOUT_BURST_MIN */
1.1 root 64:
65: #ifndef VM_PAGEOUT_BURST_WAIT
1.1.1.2 ! root 66: #define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds per page */
! 67: #endif /* VM_PAGEOUT_BURST_WAIT */
1.1 root 68:
69: #ifndef VM_PAGEOUT_EMPTY_WAIT
1.1.1.2 ! root 70: #define VM_PAGEOUT_EMPTY_WAIT 75 /* milliseconds */
! 71: #endif /* VM_PAGEOUT_EMPTY_WAIT */
1.1 root 72:
73: #ifndef VM_PAGEOUT_PAUSE_MAX
74: #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */
1.1.1.2 ! root 75: #endif /* VM_PAGEOUT_PAUSE_MAX */
1.1 root 76:
77: /*
78: * To obtain a reasonable LRU approximation, the inactive queue
79: * needs to be large enough to give pages on it a chance to be
80: * referenced a second time. This macro defines the fraction
81: * of active+inactive pages that should be inactive.
82: * The pageout daemon uses it to update vm_page_inactive_target.
83: *
84: * If vm_page_free_count falls below vm_page_free_target and
85: * vm_page_inactive_count is below vm_page_inactive_target,
86: * then the pageout daemon starts running.
87: */
88:
89: #ifndef VM_PAGE_INACTIVE_TARGET
90: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3)
1.1.1.2 ! root 91: #endif /* VM_PAGE_INACTIVE_TARGET */
1.1 root 92:
93: /*
94: * Once the pageout daemon starts running, it keeps going
95: * until vm_page_free_count meets or exceeds vm_page_free_target.
96: */
97:
98: #ifndef VM_PAGE_FREE_TARGET
99: #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
1.1.1.2 ! root 100: #endif /* VM_PAGE_FREE_TARGET */
1.1 root 101:
102: /*
103: * The pageout daemon always starts running once vm_page_free_count
104: * falls below vm_page_free_min.
105: */
106:
107: #ifndef VM_PAGE_FREE_MIN
108: #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
1.1.1.2 ! root 109: #endif /* VM_PAGE_FREE_MIN */
! 110:
! 111: /* When vm_page_external_count exceeds vm_page_external_limit,
! 112: * allocations of externally paged pages stops.
! 113: */
! 114:
! 115: #ifndef VM_PAGE_EXTERNAL_LIMIT
! 116: #define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2)
! 117: #endif /* VM_PAGE_EXTERNAL_LIMIT */
! 118:
! 119: /* Attempt to keep the number of externally paged pages less
! 120: * than vm_pages_external_target.
! 121: */
! 122: #ifndef VM_PAGE_EXTERNAL_TARGET
! 123: #define VM_PAGE_EXTERNAL_TARGET(free) ((free) / 4)
! 124: #endif /* VM_PAGE_EXTERNAL_TARGET */
1.1 root 125:
126: /*
127: * When vm_page_free_count falls below vm_page_free_reserved,
128: * only vm-privileged threads can allocate pages. vm-privilege
129: * allows the pageout daemon and default pager (and any other
130: * associated threads needed for default pageout) to continue
1.1.1.2 ! root 131: * operation by dipping into the reserved pool of pages. */
1.1 root 132:
133: #ifndef VM_PAGE_FREE_RESERVED
1.1.1.2 ! root 134: #define VM_PAGE_FREE_RESERVED 50
! 135: #endif /* VM_PAGE_FREE_RESERVED */
1.1 root 136:
137: /*
138: * When vm_page_free_count falls below vm_pageout_reserved_internal,
139: * the pageout daemon no longer trusts external pagers to clean pages.
140: * External pagers are probably all wedged waiting for a free page.
141: * It forcibly double-pages dirty pages belonging to external objects,
142: * getting the pages to the default pager to clean.
143: */
144:
145: #ifndef VM_PAGEOUT_RESERVED_INTERNAL
1.1.1.2 ! root 146: #define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 25)
! 147: #endif /* VM_PAGEOUT_RESERVED_INTERNAL */
1.1 root 148:
149: /*
150: * When vm_page_free_count falls below vm_pageout_reserved_really,
151: * the pageout daemon stops work entirely to let the default pager
152: * catch up (assuming the default pager has pages to clean).
153: * Beyond this point, it is too dangerous to consume memory
154: * even for memory_object_data_write messages to the default pager.
155: */
156:
157: #ifndef VM_PAGEOUT_RESERVED_REALLY
1.1.1.2 ! root 158: #define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 40)
! 159: #endif /* VM_PAGEOUT_RESERVED_REALLY */
1.1 root 160:
161: extern void vm_pageout_continue();
162: extern void vm_pageout_scan_continue();
163:
164: unsigned int vm_pageout_reserved_internal = 0;
165: unsigned int vm_pageout_reserved_really = 0;
166:
1.1.1.2 ! root 167: unsigned int vm_page_external_target = 0;
! 168:
1.1 root 169: unsigned int vm_pageout_burst_max = 0;
170: unsigned int vm_pageout_burst_min = 0;
171: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */
172: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
173: unsigned int vm_pageout_pause_count = 0;
174: unsigned int vm_pageout_pause_max = 0;
175:
176: /*
177: * These variables record the pageout daemon's actions:
178: * how many pages it looks at and what happens to those pages.
179: * No locking needed because only one thread modifies the variables.
180: */
181:
182: unsigned int vm_pageout_active = 0; /* debugging */
183: unsigned int vm_pageout_inactive = 0; /* debugging */
184: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
185: unsigned int vm_pageout_inactive_busy = 0; /* debugging */
186: unsigned int vm_pageout_inactive_absent = 0; /* debugging */
187: unsigned int vm_pageout_inactive_used = 0; /* debugging */
188: unsigned int vm_pageout_inactive_clean = 0; /* debugging */
189: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
190: unsigned int vm_pageout_inactive_double = 0; /* debugging */
1.1.1.2 ! root 191: unsigned int vm_pageout_inactive_cleaned_external = 0;
1.1 root 192:
193: #if NORMA_VM
194: /*
195: * Define them here, since they won't be defined by memory_object_user.h.
196: */
197: extern kern_return_t memory_object_data_initialize();
198: extern kern_return_t memory_object_data_write();
1.1.1.2 ! root 199: #endif /* NORMA_VM */
1.1 root 200:
201: /*
202: * Routine: vm_pageout_setup
203: * Purpose:
204: * Set up a page for pageout.
205: *
206: * Move or copy the page to a new object, as part
207: * of which it will be sent to its memory manager
208: * in a memory_object_data_write or memory_object_initialize
209: * message.
210: *
211: * The "paging_offset" argument specifies the offset
212: * of the page within its external memory object.
213: *
214: * The "new_object" and "new_offset" arguments
215: * indicate where the page should be moved.
216: *
217: * The "flush" argument specifies whether the page
218: * should be flushed from its object. If not, a
219: * copy of the page is moved to the new object.
220: *
221: * In/Out conditions:
222: * The page in question must not be on any pageout queues,
223: * and must be busy. The object to which it belongs
224: * must be unlocked, and the caller must hold a paging
225: * reference to it. The new_object must not be locked.
226: *
227: * If the page is flushed from its original object,
228: * this routine returns a pointer to a place-holder page,
229: * inserted at the same offset, to block out-of-order
230: * requests for the page. The place-holder page must
231: * be freed after the data_write or initialize message
232: * has been sent. If the page is copied,
233: * the holding page is VM_PAGE_NULL.
234: *
235: * The original page is put on a paging queue and marked
236: * not busy on exit.
237: */
238: vm_page_t
239: vm_pageout_setup(m, paging_offset, new_object, new_offset, flush)
240: register vm_page_t m;
241: vm_offset_t paging_offset;
242: register vm_object_t new_object;
243: vm_offset_t new_offset;
244: boolean_t flush;
245: {
246: register vm_object_t old_object = m->object;
247: register vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/
248: register vm_page_t new_m;
249:
250: assert(m->busy && !m->absent && !m->fictitious);
251:
252: /*
253: * If we are not flushing the page, allocate a
254: * page in the object. If we cannot get the
255: * page, flush instead.
256: */
257: if (!flush) {
258: vm_object_lock(new_object);
259: new_m = vm_page_alloc(new_object, new_offset);
260: if (new_m == VM_PAGE_NULL)
261: flush = TRUE;
262: vm_object_unlock(new_object);
263: }
264:
265: if (flush) {
266: /*
267: * Create a place-holder page where the old one was,
268: * to prevent anyone from attempting to page in this
269: * page while we`re unlocked.
270: */
271: while ((holding_page = vm_page_grab_fictitious())
272: == VM_PAGE_NULL)
273: vm_page_more_fictitious();
274:
275: vm_object_lock(old_object);
276: vm_page_lock_queues();
277: vm_page_remove(m);
278: vm_page_unlock_queues();
279: PAGE_WAKEUP_DONE(m);
280:
281: vm_page_lock_queues();
282: vm_page_insert(holding_page, old_object, m->offset);
283: vm_page_unlock_queues();
284:
285: /*
286: * Record that this page has been written out
287: */
288: #if MACH_PAGEMAP
289: vm_external_state_set(old_object->existence_info,
290: paging_offset,
291: VM_EXTERNAL_STATE_EXISTS);
1.1.1.2 ! root 292: #endif /* MACH_PAGEMAP */
1.1 root 293:
294: vm_object_unlock(old_object);
295:
296: vm_object_lock(new_object);
297:
298: /*
299: * Move this page into the new object
300: */
301:
302: vm_page_lock_queues();
303: vm_page_insert(m, new_object, new_offset);
304: vm_page_unlock_queues();
305:
306: m->dirty = TRUE;
307: m->precious = FALSE;
308: m->page_lock = VM_PROT_NONE;
309: m->unlock_request = VM_PROT_NONE;
310: }
311: else {
312: /*
313: * Copy the data into the new page,
314: * and mark the new page as clean.
315: */
316: vm_page_copy(m, new_m);
317:
318: vm_object_lock(old_object);
319: m->dirty = FALSE;
320: pmap_clear_modify(m->phys_addr);
321:
322: /*
323: * Deactivate old page.
324: */
325: vm_page_lock_queues();
326: vm_page_deactivate(m);
327: vm_page_unlock_queues();
328:
329: PAGE_WAKEUP_DONE(m);
330:
331: /*
332: * Record that this page has been written out
333: */
334:
335: #if MACH_PAGEMAP
336: vm_external_state_set(old_object->existence_info,
337: paging_offset,
338: VM_EXTERNAL_STATE_EXISTS);
1.1.1.2 ! root 339: #endif /* MACH_PAGEMAP */
1.1 root 340:
341: vm_object_unlock(old_object);
342:
343: vm_object_lock(new_object);
344:
345: /*
346: * Use the new page below.
347: */
348: m = new_m;
349: m->dirty = TRUE;
350: assert(!m->precious);
351: PAGE_WAKEUP_DONE(m);
352: }
353:
354: /*
355: * Make the old page eligible for replacement again; if a
356: * user-supplied memory manager fails to release the page,
357: * it will be paged out again to the default memory manager.
358: *
359: * Note that pages written to the default memory manager
360: * must be wired down -- in return, it guarantees to free
361: * this page, rather than reusing it.
362: */
363:
364: vm_page_lock_queues();
365: vm_stat.pageouts++;
366: if (m->laundry) {
367: /*
368: * vm_pageout_scan is telling us to put this page
369: * at the front of the inactive queue, so it will
370: * be immediately paged out to the default pager.
371: */
372:
373: assert(!old_object->internal);
374: m->laundry = FALSE;
375:
376: queue_enter_first(&vm_page_queue_inactive, m,
377: vm_page_t, pageq);
378: m->inactive = TRUE;
379: vm_page_inactive_count++;
380: } else if (old_object->internal) {
381: m->laundry = TRUE;
382: vm_page_laundry_count++;
383:
384: vm_page_wire(m);
385: } else
386: vm_page_activate(m);
387: vm_page_unlock_queues();
388:
389: /*
390: * Since IPC operations may block, we drop locks now.
391: * [The placeholder page is busy, and we still have
392: * paging_in_progress incremented.]
393: */
394:
395: vm_object_unlock(new_object);
396:
397: /*
398: * Return the placeholder page to simplify cleanup.
399: */
400: return (flush ? holding_page : VM_PAGE_NULL);
401: }
402:
403: /*
404: * Routine: vm_pageout_page
405: * Purpose:
406: * Causes the specified page to be written back to
407: * the appropriate memory object.
408: *
409: * The "initial" argument specifies whether this
410: * data is an initialization only, and should use
411: * memory_object_data_initialize instead of
412: * memory_object_data_write.
413: *
414: * The "flush" argument specifies whether the page
415: * should be flushed from the object. If not, a
416: * copy of the data is sent to the memory object.
417: *
418: * In/out conditions:
419: * The page in question must not be on any pageout queues.
420: * The object to which it belongs must be locked.
421: * Implementation:
422: * Move this page to a completely new object, if flushing;
423: * copy to a new page in a new object, if not.
424: */
425: void
426: vm_pageout_page(m, initial, flush)
427: register vm_page_t m;
428: boolean_t initial;
429: boolean_t flush;
430: {
431: vm_map_copy_t copy;
432: register vm_object_t old_object;
433: register vm_object_t new_object;
434: register vm_page_t holding_page;
435: vm_offset_t paging_offset;
436: kern_return_t rc;
437: boolean_t precious_clean;
438:
439: assert(m->busy);
440:
441: /*
442: * Cleaning but not flushing a clean precious page is a
443: * no-op. Remember whether page is clean and precious now
444: * because vm_pageout_setup will mark it dirty and not precious.
445: *
446: * XXX Check if precious_clean && !flush can really happen.
447: */
448: precious_clean = (!m->dirty) && m->precious;
449: if (precious_clean && !flush) {
450: PAGE_WAKEUP_DONE(m);
451: return;
452: }
453:
454: /*
455: * Verify that we really want to clean this page.
456: */
457: if (m->absent || m->error || (!m->dirty && !m->precious)) {
458: VM_PAGE_FREE(m);
459: return;
460: }
461:
462: /*
463: * Create a paging reference to let us play with the object.
464: */
465: old_object = m->object;
466: paging_offset = m->offset + old_object->paging_offset;
467: vm_object_paging_begin(old_object);
468: vm_object_unlock(old_object);
469:
470: /*
471: * Allocate a new object into which we can put the page.
472: */
473: new_object = vm_object_allocate(PAGE_SIZE);
474:
475: /*
476: * Move the page into the new object.
477: */
478: holding_page = vm_pageout_setup(m,
479: paging_offset,
480: new_object,
481: 0, /* new offset */
482: flush); /* flush */
483:
484: rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©);
485: assert(rc == KERN_SUCCESS);
486:
487: if (initial || old_object->use_old_pageout) {
488: rc = (*(initial ? memory_object_data_initialize
489: : memory_object_data_write))
490: (old_object->pager,
491: old_object->pager_request,
492: paging_offset, (pointer_t) copy, PAGE_SIZE);
493: }
494: else {
495: rc = memory_object_data_return(
496: old_object->pager,
497: old_object->pager_request,
498: paging_offset, (pointer_t) copy, PAGE_SIZE,
499: !precious_clean, !flush);
500: }
501:
502: if (rc != KERN_SUCCESS)
503: vm_map_copy_discard(copy);
504:
505: /*
506: * Clean up.
507: */
508: vm_object_lock(old_object);
509: if (holding_page != VM_PAGE_NULL)
510: VM_PAGE_FREE(holding_page);
511: vm_object_paging_end(old_object);
512: }
513:
514: /*
515: * vm_pageout_scan does the dirty work for the pageout daemon.
516: * It returns with vm_page_queue_free_lock held and
517: * vm_page_free_wanted == 0.
518: */
519:
520: void vm_pageout_scan()
521: {
522: unsigned int burst_count;
1.1.1.2 ! root 523: unsigned int want_pages;
1.1 root 524:
525: /*
526: * We want to gradually dribble pages from the active queue
527: * to the inactive queue. If we let the inactive queue get
528: * very small, and then suddenly dump many pages into it,
529: * those pages won't get a sufficient chance to be referenced
530: * before we start taking them from the inactive queue.
531: *
532: * We must limit the rate at which we send pages to the pagers.
533: * data_write messages consume memory, for message buffers and
534: * for map-copy objects. If we get too far ahead of the pagers,
535: * we can potentially run out of memory.
536: *
537: * We can use the laundry count to limit directly the number
538: * of pages outstanding to the default pager. A similar
539: * strategy for external pagers doesn't work, because
540: * external pagers don't have to deallocate the pages sent them,
541: * and because we might have to send pages to external pagers
542: * even if they aren't processing writes. So we also
543: * use a burst count to limit writes to external pagers.
544: *
545: * When memory is very tight, we can't rely on external pagers to
546: * clean pages. They probably aren't running, because they
547: * aren't vm-privileged. If we kept sending dirty pages to them,
548: * we could exhaust the free list. However, we can't just ignore
549: * pages belonging to external objects, because there might be no
550: * pages belonging to internal objects. Hence, we get the page
551: * into an internal object and then immediately double-page it,
552: * sending it to the default pager.
553: *
554: * consider_zone_gc should be last, because the other operations
555: * might return memory to zones. When we pause we use
556: * vm_pageout_scan_continue as our continuation, so we will
557: * reenter vm_pageout_scan periodically and attempt to reclaim
558: * internal memory even if we never reach vm_page_free_target.
559: */
560:
561: Restart:
562: stack_collect();
563: net_kmsg_collect();
564: consider_task_collect();
565: consider_thread_collect();
566: consider_zone_gc();
567:
568: for (burst_count = 0;;) {
569: register vm_page_t m;
570: register vm_object_t object;
571: unsigned int free_count;
572:
573: /*
574: * Recalculate vm_page_inactivate_target.
575: */
576:
577: vm_page_lock_queues();
578: vm_page_inactive_target =
579: VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
580: vm_page_inactive_count);
581:
582: /*
583: * Move pages from active to inactive.
584: */
585:
586: while ((vm_page_inactive_count < vm_page_inactive_target) &&
587: !queue_empty(&vm_page_queue_active)) {
588: register vm_object_t obj;
589:
590: vm_pageout_active++;
591: m = (vm_page_t) queue_first(&vm_page_queue_active);
592: assert(m->active && !m->inactive);
593:
594: obj = m->object;
595: if (!vm_object_lock_try(obj)) {
596: /*
597: * Move page to end and continue.
598: */
599:
600: queue_remove(&vm_page_queue_active, m,
601: vm_page_t, pageq);
602: queue_enter(&vm_page_queue_active, m,
603: vm_page_t, pageq);
604: vm_page_unlock_queues();
605: vm_page_lock_queues();
606: continue;
607: }
608:
609: /*
610: * If the page is busy, then we pull it
611: * off the active queue and leave it alone.
612: */
613:
614: if (m->busy) {
615: vm_object_unlock(obj);
616: queue_remove(&vm_page_queue_active, m,
617: vm_page_t, pageq);
618: m->active = FALSE;
619: vm_page_active_count--;
620: continue;
621: }
622:
623: /*
624: * Deactivate the page while holding the object
625: * locked, so we know the page is still not busy.
626: * This should prevent races between pmap_enter
627: * and pmap_clear_reference. The page might be
628: * absent or fictitious, but vm_page_deactivate
629: * can handle that.
630: */
631:
632: vm_page_deactivate(m);
633: vm_object_unlock(obj);
634: }
635:
636: /*
1.1.1.2 ! root 637: * We are done if we have met our targets *and*
1.1 root 638: * nobody is still waiting for a page.
639: */
640:
641: simple_lock(&vm_page_queue_free_lock);
642: free_count = vm_page_free_count;
1.1.1.2 ! root 643: if ((free_count >= vm_page_free_target) &&
! 644: (vm_page_external_count <= vm_page_external_target) &&
1.1 root 645: (vm_page_free_wanted == 0)) {
646: vm_page_unlock_queues();
647: break;
648: }
1.1.1.2 ! root 649: want_pages = ((free_count < vm_page_free_target) ||
! 650: vm_page_free_wanted);
1.1 root 651: simple_unlock(&vm_page_queue_free_lock);
652:
653: /*
654: * Sometimes we have to pause:
655: * 1) No inactive pages - nothing to do.
656: * 2) Flow control - wait for pagers to catch up.
657: * 3) Extremely low memory - sending out dirty pages
658: * consumes memory. We don't take the risk of doing
659: * this if the default pager already has work to do.
660: */
1.1.1.2 ! root 661: pause:
1.1 root 662: if (queue_empty(&vm_page_queue_inactive) ||
663: (burst_count >= vm_pageout_burst_max) ||
664: (vm_page_laundry_count >= vm_pageout_burst_max) ||
665: ((free_count < vm_pageout_reserved_really) &&
666: (vm_page_laundry_count > 0))) {
667: unsigned int pages, msecs;
668:
669: /*
670: * vm_pageout_burst_wait is msecs/page.
671: * If there is nothing for us to do, we wait
672: * at least vm_pageout_empty_wait msecs.
673: */
674:
675: if (vm_page_laundry_count > burst_count)
676: pages = vm_page_laundry_count;
677: else
678: pages = burst_count;
679: msecs = pages * vm_pageout_burst_wait;
680:
681: if (queue_empty(&vm_page_queue_inactive) &&
682: (msecs < vm_pageout_empty_wait))
683: msecs = vm_pageout_empty_wait;
684: vm_page_unlock_queues();
685:
686: thread_will_wait_with_timeout(current_thread(), msecs);
687: counter(c_vm_pageout_scan_block++);
688: thread_block(vm_pageout_scan_continue);
689: #ifndef CONTINUATIONS
690: /*
691: * Unfortunately, we don't have call_continuation
692: * so we can't rely on tail-recursion.
693: */
694:
695: vm_pageout_scan_continue();
696: goto Restart;
697: #else /* CONTINUATIONS */
698: call_continuation(vm_pageout_scan_continue);
699: /*NOTREACHED*/
700: #endif /* CONTINUATIONS */
701: }
702:
703: vm_pageout_inactive++;
1.1.1.2 ! root 704:
! 705: /* Find a page we are interested in paging out. If we
! 706: need pages, then we'll page anything out; otherwise
! 707: we only page out external pages. */
! 708: m = (vm_page_t) queue_first (&vm_page_queue_inactive);
! 709: while (1)
! 710: {
! 711: assert (!m->active && m->inactive);
! 712: if (want_pages || m->external)
! 713: break;
! 714:
! 715: m = (vm_page_t) queue_next (m);
! 716: if (!m)
! 717: goto pause;
! 718: }
! 719:
1.1 root 720: object = m->object;
721:
722: /*
723: * Try to lock object; since we've got the
724: * page queues lock, we can only try for this one.
725: */
726:
727: if (!vm_object_lock_try(object)) {
728: /*
729: * Move page to end and continue.
730: */
731:
732: queue_remove(&vm_page_queue_inactive, m,
733: vm_page_t, pageq);
734: queue_enter(&vm_page_queue_inactive, m,
735: vm_page_t, pageq);
736: vm_page_unlock_queues();
737: vm_pageout_inactive_nolock++;
738: continue;
739: }
740:
741: /*
742: * Remove the page from the inactive list.
743: */
744:
745: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
746: vm_page_inactive_count--;
747: m->inactive = FALSE;
748:
749: if (m->busy || !object->alive) {
750: /*
751: * Somebody is already playing with this page.
752: * Leave it off the pageout queues.
753: */
754:
755: vm_page_unlock_queues();
756: vm_object_unlock(object);
757: vm_pageout_inactive_busy++;
758: continue;
759: }
760:
761: /*
762: * If it's absent, we can reclaim the page.
763: */
764:
1.1.1.2 ! root 765: if (want_pages && m->absent) {
1.1 root 766: vm_pageout_inactive_absent++;
767: reclaim_page:
768: vm_page_free(m);
769: vm_page_unlock_queues();
770: vm_object_unlock(object);
771: continue;
772: }
773:
774: /*
775: * If it's being used, reactivate.
776: * (Fictitious pages are either busy or absent.)
777: */
778:
779: assert(!m->fictitious);
780: if (m->reference || pmap_is_referenced(m->phys_addr)) {
781: vm_object_unlock(object);
782: vm_page_activate(m);
783: vm_stat.reactivations++;
784: vm_page_unlock_queues();
785: vm_pageout_inactive_used++;
786: continue;
787: }
788:
789: /*
790: * Eliminate all mappings.
791: */
792:
793: m->busy = TRUE;
794: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
795: if (!m->dirty)
796: m->dirty = pmap_is_modified(m->phys_addr);
797:
1.1.1.2 ! root 798: if (m->external) {
! 799: /* Figure out if we still care about this
! 800: page in the limit of externally managed pages.
! 801: Clean pages don't actually cause system hosage,
! 802: so it's ok to stop considering them as
! 803: "consumers" of memory. */
! 804: if (m->dirty && !m->extcounted) {
! 805: m->extcounted = TRUE;
! 806: vm_page_external_count++;
! 807: } else if (!m->dirty && m->extcounted) {
! 808: m->extcounted = FALSE;
! 809: vm_page_external_count--;
! 810: }
! 811: }
! 812:
! 813: /* If we don't actually need more memory, and the page
! 814: is not dirty, put it on the tail of the inactive queue
! 815: and move on to the next page. */
! 816: if (!want_pages && !m->dirty) {
! 817: queue_remove (&vm_page_queue_inactive, m,
! 818: vm_page_t, pageq);
! 819: queue_enter (&vm_page_queue_inactive, m,
! 820: vm_page_t, pageq);
! 821: vm_page_unlock_queues();
! 822: vm_pageout_inactive_cleaned_external++;
! 823: continue;
! 824: }
! 825:
1.1 root 826: /*
827: * If it's clean and not precious, we can free the page.
828: */
829:
830: if (!m->dirty && !m->precious) {
831: vm_pageout_inactive_clean++;
832: goto reclaim_page;
833: }
834:
835: /*
836: * If we are very low on memory, then we can't
837: * rely on an external pager to clean a dirty page,
838: * because external pagers are not vm-privileged.
839: *
840: * The laundry bit tells vm_pageout_setup to
841: * put the page back at the front of the inactive
842: * queue instead of activating the page. Hence,
843: * we will pick the page up again immediately and
844: * resend it to the default pager.
845: */
846:
847: assert(!m->laundry);
848: if ((free_count < vm_pageout_reserved_internal) &&
849: !object->internal) {
850: m->laundry = TRUE;
851: vm_pageout_inactive_double++;
852: }
853: vm_page_unlock_queues();
854:
855: /*
856: * If there is no memory object for the page, create
857: * one and hand it to the default pager.
858: * [First try to collapse, so we don't create
859: * one unnecessarily.]
860: */
861:
862: if (!object->pager_initialized)
863: vm_object_collapse(object);
864: if (!object->pager_initialized)
865: vm_object_pager_create(object);
866: if (!object->pager_initialized)
867: panic("vm_pageout_scan");
868:
869: vm_pageout_inactive_dirty++;
870: vm_pageout_page(m, FALSE, TRUE); /* flush it */
871: vm_object_unlock(object);
872: burst_count++;
873: }
874: }
875:
876: void vm_pageout_scan_continue()
877: {
878: /*
879: * We just paused to let the pagers catch up.
880: * If vm_page_laundry_count is still high,
881: * then we aren't waiting long enough.
882: * If we have paused some vm_pageout_pause_max times without
883: * adjusting vm_pageout_burst_wait, it might be too big,
884: * so we decrease it.
885: */
886:
887: vm_page_lock_queues();
888: if (vm_page_laundry_count > vm_pageout_burst_min) {
889: vm_pageout_burst_wait++;
890: vm_pageout_pause_count = 0;
891: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) {
892: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4;
893: if (vm_pageout_burst_wait < 1)
894: vm_pageout_burst_wait = 1;
895: vm_pageout_pause_count = 0;
896: }
897: vm_page_unlock_queues();
898:
899: #ifdef CONTINUATIONS
900: vm_pageout_continue();
901: /*NOTREACHED*/
902: #endif /* CONTINUATIONS */
903: }
904:
905: /*
906: * vm_pageout is the high level pageout daemon.
907: */
908:
909: void vm_pageout_continue()
910: {
911: /*
912: * The pageout daemon is never done, so loop forever.
913: * We should call vm_pageout_scan at least once each
914: * time we are woken, even if vm_page_free_wanted is
915: * zero, to check vm_page_free_target and
916: * vm_page_inactive_target.
917: */
918:
919: for (;;) {
920: vm_pageout_scan();
921: /* we hold vm_page_queue_free_lock now */
922: assert(vm_page_free_wanted == 0);
923:
924: assert_wait(&vm_page_free_wanted, FALSE);
925: simple_unlock(&vm_page_queue_free_lock);
926: counter(c_vm_pageout_block++);
927: thread_block(vm_pageout_continue);
928: }
929: }
930:
931: void vm_pageout()
932: {
933: int free_after_reserve;
934:
935: current_thread()->vm_privilege = TRUE;
936: stack_privilege(current_thread());
937:
938: /*
939: * Initialize some paging parameters.
940: */
941:
942: if (vm_pageout_burst_max == 0)
943: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX;
944:
945: if (vm_pageout_burst_min == 0)
946: vm_pageout_burst_min = VM_PAGEOUT_BURST_MIN;
947:
948: if (vm_pageout_burst_wait == 0)
949: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
950:
951: if (vm_pageout_empty_wait == 0)
952: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
953:
954: if (vm_page_free_reserved == 0)
955: vm_page_free_reserved = VM_PAGE_FREE_RESERVED;
956:
957: if (vm_pageout_pause_max == 0)
958: vm_pageout_pause_max = VM_PAGEOUT_PAUSE_MAX;
959:
960: if (vm_pageout_reserved_internal == 0)
961: vm_pageout_reserved_internal =
962: VM_PAGEOUT_RESERVED_INTERNAL(vm_page_free_reserved);
963:
964: if (vm_pageout_reserved_really == 0)
965: vm_pageout_reserved_really =
966: VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved);
967:
968: free_after_reserve = vm_page_free_count - vm_page_free_reserved;
969:
1.1.1.2 ! root 970: if (vm_page_external_limit == 0)
! 971: vm_page_external_limit =
! 972: VM_PAGE_EXTERNAL_LIMIT (free_after_reserve);
! 973:
! 974: if (vm_page_external_target == 0)
! 975: vm_page_external_target =
! 976: VM_PAGE_EXTERNAL_TARGET (free_after_reserve);
! 977:
1.1 root 978: if (vm_page_free_min == 0)
979: vm_page_free_min = vm_page_free_reserved +
980: VM_PAGE_FREE_MIN(free_after_reserve);
981:
982: if (vm_page_free_target == 0)
983: vm_page_free_target = vm_page_free_reserved +
984: VM_PAGE_FREE_TARGET(free_after_reserve);
985:
986: if (vm_page_free_target < vm_page_free_min + 5)
987: vm_page_free_target = vm_page_free_min + 5;
988:
989: /*
990: * vm_pageout_scan will set vm_page_inactive_target.
991: */
992:
993: vm_pageout_continue();
994: /*NOTREACHED*/
995: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.