![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_pageout.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_pageout.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * The proverbial page-out daemon.
35: */
36:
37: #include <mach_pagemap.h>
38: #include <norma_vm.h>
39:
40: #include <mach/mach_types.h>
41: #include <mach/memory_object.h>
42: #include "memory_object_default.h"
43: #include "memory_object_user.h"
44: #include <mach/vm_param.h>
45: #include <mach/vm_statistics.h>
46: #include <kern/counters.h>
47: #include <kern/thread.h>
48: #include <vm/pmap.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
52: #include <vm/vm_pageout.h>
53: #include <machine/vm_tuning.h>
54:
55:
56:
57: #ifndef VM_PAGEOUT_BURST_MAX
58: #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */
59: #endif /* VM_PAGEOUT_BURST_MAX */
60:
61: #ifndef VM_PAGEOUT_BURST_MIN
62: #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */
63: #endif /* VM_PAGEOUT_BURST_MIN */
64:
65: #ifndef VM_PAGEOUT_BURST_WAIT
66: #define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds per page */
67: #endif /* VM_PAGEOUT_BURST_WAIT */
68:
69: #ifndef VM_PAGEOUT_EMPTY_WAIT
70: #define VM_PAGEOUT_EMPTY_WAIT 75 /* milliseconds */
71: #endif /* VM_PAGEOUT_EMPTY_WAIT */
72:
73: #ifndef VM_PAGEOUT_PAUSE_MAX
74: #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */
75: #endif /* VM_PAGEOUT_PAUSE_MAX */
76:
77: /*
78: * To obtain a reasonable LRU approximation, the inactive queue
79: * needs to be large enough to give pages on it a chance to be
80: * referenced a second time. This macro defines the fraction
81: * of active+inactive pages that should be inactive.
82: * The pageout daemon uses it to update vm_page_inactive_target.
83: *
84: * If vm_page_free_count falls below vm_page_free_target and
85: * vm_page_inactive_count is below vm_page_inactive_target,
86: * then the pageout daemon starts running.
87: */
88:
89: #ifndef VM_PAGE_INACTIVE_TARGET
90: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3)
91: #endif /* VM_PAGE_INACTIVE_TARGET */
92:
93: /*
94: * Once the pageout daemon starts running, it keeps going
95: * until vm_page_free_count meets or exceeds vm_page_free_target.
96: */
97:
98: #ifndef VM_PAGE_FREE_TARGET
99: #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
100: #endif /* VM_PAGE_FREE_TARGET */
101:
102: /*
103: * The pageout daemon always starts running once vm_page_free_count
104: * falls below vm_page_free_min.
105: */
106:
107: #ifndef VM_PAGE_FREE_MIN
108: #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
109: #endif /* VM_PAGE_FREE_MIN */
110:
111: /* When vm_page_external_count exceeds vm_page_external_limit,
112: * allocations of externally paged pages stops.
113: */
114:
115: #ifndef VM_PAGE_EXTERNAL_LIMIT
116: #define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2)
117: #endif /* VM_PAGE_EXTERNAL_LIMIT */
118:
119: /* Attempt to keep the number of externally paged pages less
120: * than vm_pages_external_target.
121: */
122: #ifndef VM_PAGE_EXTERNAL_TARGET
123: #define VM_PAGE_EXTERNAL_TARGET(free) ((free) / 4)
124: #endif /* VM_PAGE_EXTERNAL_TARGET */
125:
126: /*
127: * When vm_page_free_count falls below vm_page_free_reserved,
128: * only vm-privileged threads can allocate pages. vm-privilege
129: * allows the pageout daemon and default pager (and any other
130: * associated threads needed for default pageout) to continue
131: * operation by dipping into the reserved pool of pages. */
132:
133: #ifndef VM_PAGE_FREE_RESERVED
134: #define VM_PAGE_FREE_RESERVED 50
135: #endif /* VM_PAGE_FREE_RESERVED */
136:
137: /*
138: * When vm_page_free_count falls below vm_pageout_reserved_internal,
139: * the pageout daemon no longer trusts external pagers to clean pages.
140: * External pagers are probably all wedged waiting for a free page.
141: * It forcibly double-pages dirty pages belonging to external objects,
142: * getting the pages to the default pager to clean.
143: */
144:
145: #ifndef VM_PAGEOUT_RESERVED_INTERNAL
146: #define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 25)
147: #endif /* VM_PAGEOUT_RESERVED_INTERNAL */
148:
149: /*
150: * When vm_page_free_count falls below vm_pageout_reserved_really,
151: * the pageout daemon stops work entirely to let the default pager
152: * catch up (assuming the default pager has pages to clean).
153: * Beyond this point, it is too dangerous to consume memory
154: * even for memory_object_data_write messages to the default pager.
155: */
156:
157: #ifndef VM_PAGEOUT_RESERVED_REALLY
158: #define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 40)
159: #endif /* VM_PAGEOUT_RESERVED_REALLY */
160:
161: extern void vm_pageout_continue();
162: extern void vm_pageout_scan_continue();
163:
164: unsigned int vm_pageout_reserved_internal = 0;
165: unsigned int vm_pageout_reserved_really = 0;
166:
167: unsigned int vm_page_external_target = 0;
168:
169: unsigned int vm_pageout_burst_max = 0;
170: unsigned int vm_pageout_burst_min = 0;
171: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */
172: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
173: unsigned int vm_pageout_pause_count = 0;
174: unsigned int vm_pageout_pause_max = 0;
175:
176: /*
177: * These variables record the pageout daemon's actions:
178: * how many pages it looks at and what happens to those pages.
179: * No locking needed because only one thread modifies the variables.
180: */
181:
182: unsigned int vm_pageout_active = 0; /* debugging */
183: unsigned int vm_pageout_inactive = 0; /* debugging */
184: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
185: unsigned int vm_pageout_inactive_busy = 0; /* debugging */
186: unsigned int vm_pageout_inactive_absent = 0; /* debugging */
187: unsigned int vm_pageout_inactive_used = 0; /* debugging */
188: unsigned int vm_pageout_inactive_clean = 0; /* debugging */
189: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
190: unsigned int vm_pageout_inactive_double = 0; /* debugging */
191: unsigned int vm_pageout_inactive_cleaned_external = 0;
192:
193: #if NORMA_VM
194: /*
195: * Define them here, since they won't be defined by memory_object_user.h.
196: */
197: extern kern_return_t memory_object_data_initialize();
198: extern kern_return_t memory_object_data_write();
199: #endif /* NORMA_VM */
200:
201: /*
202: * Routine: vm_pageout_setup
203: * Purpose:
204: * Set up a page for pageout.
205: *
206: * Move or copy the page to a new object, as part
207: * of which it will be sent to its memory manager
208: * in a memory_object_data_write or memory_object_initialize
209: * message.
210: *
211: * The "paging_offset" argument specifies the offset
212: * of the page within its external memory object.
213: *
214: * The "new_object" and "new_offset" arguments
215: * indicate where the page should be moved.
216: *
217: * The "flush" argument specifies whether the page
218: * should be flushed from its object. If not, a
219: * copy of the page is moved to the new object.
220: *
221: * In/Out conditions:
222: * The page in question must not be on any pageout queues,
223: * and must be busy. The object to which it belongs
224: * must be unlocked, and the caller must hold a paging
225: * reference to it. The new_object must not be locked.
226: *
227: * If the page is flushed from its original object,
228: * this routine returns a pointer to a place-holder page,
229: * inserted at the same offset, to block out-of-order
230: * requests for the page. The place-holder page must
231: * be freed after the data_write or initialize message
232: * has been sent. If the page is copied,
233: * the holding page is VM_PAGE_NULL.
234: *
235: * The original page is put on a paging queue and marked
236: * not busy on exit.
237: */
238: vm_page_t
239: vm_pageout_setup(m, paging_offset, new_object, new_offset, flush)
240: register vm_page_t m;
241: vm_offset_t paging_offset;
242: register vm_object_t new_object;
243: vm_offset_t new_offset;
244: boolean_t flush;
245: {
246: register vm_object_t old_object = m->object;
247: register vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/
248: register vm_page_t new_m;
249:
250: assert(m->busy && !m->absent && !m->fictitious);
251:
252: /*
253: * If we are not flushing the page, allocate a
254: * page in the object. If we cannot get the
255: * page, flush instead.
256: */
257: if (!flush) {
258: vm_object_lock(new_object);
259: new_m = vm_page_alloc(new_object, new_offset);
260: if (new_m == VM_PAGE_NULL)
261: flush = TRUE;
262: vm_object_unlock(new_object);
263: }
264:
265: if (flush) {
266: /*
267: * Create a place-holder page where the old one was,
268: * to prevent anyone from attempting to page in this
269: * page while we`re unlocked.
270: */
271: while ((holding_page = vm_page_grab_fictitious())
272: == VM_PAGE_NULL)
273: vm_page_more_fictitious();
274:
275: vm_object_lock(old_object);
276: vm_page_lock_queues();
277: vm_page_remove(m);
278: vm_page_unlock_queues();
279: PAGE_WAKEUP_DONE(m);
280:
281: vm_page_lock_queues();
282: vm_page_insert(holding_page, old_object, m->offset);
283: vm_page_unlock_queues();
284:
285: /*
286: * Record that this page has been written out
287: */
288: #if MACH_PAGEMAP
289: vm_external_state_set(old_object->existence_info,
290: paging_offset,
291: VM_EXTERNAL_STATE_EXISTS);
292: #endif /* MACH_PAGEMAP */
293:
294: vm_object_unlock(old_object);
295:
296: vm_object_lock(new_object);
297:
298: /*
299: * Move this page into the new object
300: */
301:
302: vm_page_lock_queues();
303: vm_page_insert(m, new_object, new_offset);
304: vm_page_unlock_queues();
305:
306: m->dirty = TRUE;
307: m->precious = FALSE;
308: m->page_lock = VM_PROT_NONE;
309: m->unlock_request = VM_PROT_NONE;
310: }
311: else {
312: /*
313: * Copy the data into the new page,
314: * and mark the new page as clean.
315: */
316: vm_page_copy(m, new_m);
317:
318: vm_object_lock(old_object);
319: m->dirty = FALSE;
320: pmap_clear_modify(m->phys_addr);
321:
322: /*
323: * Deactivate old page.
324: */
325: vm_page_lock_queues();
326: vm_page_deactivate(m);
327: vm_page_unlock_queues();
328:
329: PAGE_WAKEUP_DONE(m);
330:
331: /*
332: * Record that this page has been written out
333: */
334:
335: #if MACH_PAGEMAP
336: vm_external_state_set(old_object->existence_info,
337: paging_offset,
338: VM_EXTERNAL_STATE_EXISTS);
339: #endif /* MACH_PAGEMAP */
340:
341: vm_object_unlock(old_object);
342:
343: vm_object_lock(new_object);
344:
345: /*
346: * Use the new page below.
347: */
348: m = new_m;
349: m->dirty = TRUE;
350: assert(!m->precious);
351: PAGE_WAKEUP_DONE(m);
352: }
353:
354: /*
355: * Make the old page eligible for replacement again; if a
356: * user-supplied memory manager fails to release the page,
357: * it will be paged out again to the default memory manager.
358: *
359: * Note that pages written to the default memory manager
360: * must be wired down -- in return, it guarantees to free
361: * this page, rather than reusing it.
362: */
363:
364: vm_page_lock_queues();
365: vm_stat.pageouts++;
366: if (m->laundry) {
367: /*
368: * vm_pageout_scan is telling us to put this page
369: * at the front of the inactive queue, so it will
370: * be immediately paged out to the default pager.
371: */
372:
373: assert(!old_object->internal);
374: m->laundry = FALSE;
375:
376: queue_enter_first(&vm_page_queue_inactive, m,
377: vm_page_t, pageq);
378: m->inactive = TRUE;
379: vm_page_inactive_count++;
380: } else if (old_object->internal) {
381: m->laundry = TRUE;
382: vm_page_laundry_count++;
383:
384: vm_page_wire(m);
385: } else
386: vm_page_activate(m);
387: vm_page_unlock_queues();
388:
389: /*
390: * Since IPC operations may block, we drop locks now.
391: * [The placeholder page is busy, and we still have
392: * paging_in_progress incremented.]
393: */
394:
395: vm_object_unlock(new_object);
396:
397: /*
398: * Return the placeholder page to simplify cleanup.
399: */
400: return (flush ? holding_page : VM_PAGE_NULL);
401: }
402:
403: /*
404: * Routine: vm_pageout_page
405: * Purpose:
406: * Causes the specified page to be written back to
407: * the appropriate memory object.
408: *
409: * The "initial" argument specifies whether this
410: * data is an initialization only, and should use
411: * memory_object_data_initialize instead of
412: * memory_object_data_write.
413: *
414: * The "flush" argument specifies whether the page
415: * should be flushed from the object. If not, a
416: * copy of the data is sent to the memory object.
417: *
418: * In/out conditions:
419: * The page in question must not be on any pageout queues.
420: * The object to which it belongs must be locked.
421: * Implementation:
422: * Move this page to a completely new object, if flushing;
423: * copy to a new page in a new object, if not.
424: */
425: void
426: vm_pageout_page(m, initial, flush)
427: register vm_page_t m;
428: boolean_t initial;
429: boolean_t flush;
430: {
431: vm_map_copy_t copy;
432: register vm_object_t old_object;
433: register vm_object_t new_object;
434: register vm_page_t holding_page;
435: vm_offset_t paging_offset;
436: kern_return_t rc;
437: boolean_t precious_clean;
438:
439: assert(m->busy);
440:
441: /*
442: * Cleaning but not flushing a clean precious page is a
443: * no-op. Remember whether page is clean and precious now
444: * because vm_pageout_setup will mark it dirty and not precious.
445: *
446: * XXX Check if precious_clean && !flush can really happen.
447: */
448: precious_clean = (!m->dirty) && m->precious;
449: if (precious_clean && !flush) {
450: PAGE_WAKEUP_DONE(m);
451: return;
452: }
453:
454: /*
455: * Verify that we really want to clean this page.
456: */
457: if (m->absent || m->error || (!m->dirty && !m->precious)) {
458: VM_PAGE_FREE(m);
459: return;
460: }
461:
462: /*
463: * Create a paging reference to let us play with the object.
464: */
465: old_object = m->object;
466: paging_offset = m->offset + old_object->paging_offset;
467: vm_object_paging_begin(old_object);
468: vm_object_unlock(old_object);
469:
470: /*
471: * Allocate a new object into which we can put the page.
472: */
473: new_object = vm_object_allocate(PAGE_SIZE);
474:
475: /*
476: * Move the page into the new object.
477: */
478: holding_page = vm_pageout_setup(m,
479: paging_offset,
480: new_object,
481: 0, /* new offset */
482: flush); /* flush */
483:
484: rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©);
485: assert(rc == KERN_SUCCESS);
486:
487: if (initial || old_object->use_old_pageout) {
488: rc = (*(initial ? memory_object_data_initialize
489: : memory_object_data_write))
490: (old_object->pager,
491: old_object->pager_request,
492: paging_offset, (pointer_t) copy, PAGE_SIZE);
493: }
494: else {
495: rc = memory_object_data_return(
496: old_object->pager,
497: old_object->pager_request,
498: paging_offset, (pointer_t) copy, PAGE_SIZE,
499: !precious_clean, !flush);
500: }
501:
502: if (rc != KERN_SUCCESS)
503: vm_map_copy_discard(copy);
504:
505: /*
506: * Clean up.
507: */
508: vm_object_lock(old_object);
509: if (holding_page != VM_PAGE_NULL)
510: VM_PAGE_FREE(holding_page);
511: vm_object_paging_end(old_object);
512: }
513:
514: /*
515: * vm_pageout_scan does the dirty work for the pageout daemon.
516: * It returns with vm_page_queue_free_lock held and
517: * vm_page_free_wanted == 0.
518: */
519:
520: void vm_pageout_scan()
521: {
522: unsigned int burst_count;
523: unsigned int want_pages;
524:
525: /*
526: * We want to gradually dribble pages from the active queue
527: * to the inactive queue. If we let the inactive queue get
528: * very small, and then suddenly dump many pages into it,
529: * those pages won't get a sufficient chance to be referenced
530: * before we start taking them from the inactive queue.
531: *
532: * We must limit the rate at which we send pages to the pagers.
533: * data_write messages consume memory, for message buffers and
534: * for map-copy objects. If we get too far ahead of the pagers,
535: * we can potentially run out of memory.
536: *
537: * We can use the laundry count to limit directly the number
538: * of pages outstanding to the default pager. A similar
539: * strategy for external pagers doesn't work, because
540: * external pagers don't have to deallocate the pages sent them,
541: * and because we might have to send pages to external pagers
542: * even if they aren't processing writes. So we also
543: * use a burst count to limit writes to external pagers.
544: *
545: * When memory is very tight, we can't rely on external pagers to
546: * clean pages. They probably aren't running, because they
547: * aren't vm-privileged. If we kept sending dirty pages to them,
548: * we could exhaust the free list. However, we can't just ignore
549: * pages belonging to external objects, because there might be no
550: * pages belonging to internal objects. Hence, we get the page
551: * into an internal object and then immediately double-page it,
552: * sending it to the default pager.
553: *
554: * consider_zone_gc should be last, because the other operations
555: * might return memory to zones. When we pause we use
556: * vm_pageout_scan_continue as our continuation, so we will
557: * reenter vm_pageout_scan periodically and attempt to reclaim
558: * internal memory even if we never reach vm_page_free_target.
559: */
560:
561: Restart:
562: stack_collect();
563: net_kmsg_collect();
564: consider_lmm_collect();
565: consider_task_collect();
566: consider_thread_collect();
567: consider_zone_gc();
568:
569: for (burst_count = 0;;) {
570: register vm_page_t m;
571: register vm_object_t object;
572: unsigned int free_count;
573:
574: /*
575: * Recalculate vm_page_inactivate_target.
576: */
577:
578: vm_page_lock_queues();
579: vm_page_inactive_target =
580: VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
581: vm_page_inactive_count);
582:
583: /*
584: * Move pages from active to inactive.
585: */
586:
587: while ((vm_page_inactive_count < vm_page_inactive_target) &&
588: !queue_empty(&vm_page_queue_active)) {
589: register vm_object_t obj;
590:
591: vm_pageout_active++;
592: m = (vm_page_t) queue_first(&vm_page_queue_active);
593: assert(m->active && !m->inactive);
594:
595: obj = m->object;
596: if (!vm_object_lock_try(obj)) {
597: /*
598: * Move page to end and continue.
599: */
600:
601: queue_remove(&vm_page_queue_active, m,
602: vm_page_t, pageq);
603: queue_enter(&vm_page_queue_active, m,
604: vm_page_t, pageq);
605: vm_page_unlock_queues();
606: vm_page_lock_queues();
607: continue;
608: }
609:
610: /*
611: * If the page is busy, then we pull it
612: * off the active queue and leave it alone.
613: */
614:
615: if (m->busy) {
616: vm_object_unlock(obj);
617: queue_remove(&vm_page_queue_active, m,
618: vm_page_t, pageq);
619: m->active = FALSE;
620: vm_page_active_count--;
621: continue;
622: }
623:
624: /*
625: * Deactivate the page while holding the object
626: * locked, so we know the page is still not busy.
627: * This should prevent races between pmap_enter
628: * and pmap_clear_reference. The page might be
629: * absent or fictitious, but vm_page_deactivate
630: * can handle that.
631: */
632:
633: vm_page_deactivate(m);
634: vm_object_unlock(obj);
635: }
636:
637: /*
638: * We are done if we have met our targets *and*
639: * nobody is still waiting for a page.
640: */
641:
642: simple_lock(&vm_page_queue_free_lock);
643: free_count = vm_page_free_count;
644: if ((free_count >= vm_page_free_target) &&
645: (vm_page_external_count <= vm_page_external_target) &&
646: (vm_page_free_wanted == 0)) {
647: vm_page_unlock_queues();
648: break;
649: }
650: want_pages = ((free_count < vm_page_free_target) ||
651: vm_page_free_wanted);
652: simple_unlock(&vm_page_queue_free_lock);
653:
654: /*
655: * Sometimes we have to pause:
656: * 1) No inactive pages - nothing to do.
657: * 2) Flow control - wait for pagers to catch up.
658: * 3) Extremely low memory - sending out dirty pages
659: * consumes memory. We don't take the risk of doing
660: * this if the default pager already has work to do.
661: */
662: pause:
663: if (queue_empty(&vm_page_queue_inactive) ||
664: (burst_count >= vm_pageout_burst_max) ||
665: (vm_page_laundry_count >= vm_pageout_burst_max) ||
666: ((free_count < vm_pageout_reserved_really) &&
667: (vm_page_laundry_count > 0))) {
668: unsigned int pages, msecs;
669:
670: /*
671: * vm_pageout_burst_wait is msecs/page.
672: * If there is nothing for us to do, we wait
673: * at least vm_pageout_empty_wait msecs.
674: */
675:
676: if (vm_page_laundry_count > burst_count)
677: pages = vm_page_laundry_count;
678: else
679: pages = burst_count;
680: msecs = pages * vm_pageout_burst_wait;
681:
682: if (queue_empty(&vm_page_queue_inactive) &&
683: (msecs < vm_pageout_empty_wait))
684: msecs = vm_pageout_empty_wait;
685: vm_page_unlock_queues();
686:
687: thread_will_wait_with_timeout(current_thread(), msecs);
688: counter(c_vm_pageout_scan_block++);
689: thread_block(vm_pageout_scan_continue);
690: #ifndef CONTINUATIONS
691: /*
692: * Unfortunately, we don't have call_continuation
693: * so we can't rely on tail-recursion.
694: */
695:
696: vm_pageout_scan_continue();
697: goto Restart;
698: #else /* CONTINUATIONS */
699: call_continuation(vm_pageout_scan_continue);
700: /*NOTREACHED*/
701: #endif /* CONTINUATIONS */
702: }
703:
704: vm_pageout_inactive++;
705:
706: /* Find a page we are interested in paging out. If we
707: need pages, then we'll page anything out; otherwise
708: we only page out external pages. */
709: m = (vm_page_t) queue_first (&vm_page_queue_inactive);
710: while (1)
711: {
712: assert (!m->active && m->inactive);
713: if (want_pages || m->external)
714: break;
715:
716: m = (vm_page_t) queue_next (m);
717: if (!m)
718: goto pause;
719: }
720:
721: object = m->object;
722:
723: /*
724: * Try to lock object; since we've got the
725: * page queues lock, we can only try for this one.
726: */
727:
728: if (!vm_object_lock_try(object)) {
729: /*
730: * Move page to end and continue.
731: */
732:
733: queue_remove(&vm_page_queue_inactive, m,
734: vm_page_t, pageq);
735: queue_enter(&vm_page_queue_inactive, m,
736: vm_page_t, pageq);
737: vm_page_unlock_queues();
738: vm_pageout_inactive_nolock++;
739: continue;
740: }
741:
742: /*
743: * Remove the page from the inactive list.
744: */
745:
746: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
747: vm_page_inactive_count--;
748: m->inactive = FALSE;
749:
750: if (m->busy || !object->alive) {
751: /*
752: * Somebody is already playing with this page.
753: * Leave it off the pageout queues.
754: */
755:
756: vm_page_unlock_queues();
757: vm_object_unlock(object);
758: vm_pageout_inactive_busy++;
759: continue;
760: }
761:
762: /*
763: * If it's absent, we can reclaim the page.
764: */
765:
766: if (want_pages && m->absent) {
767: vm_pageout_inactive_absent++;
768: reclaim_page:
769: vm_page_free(m);
770: vm_page_unlock_queues();
771: vm_object_unlock(object);
772: continue;
773: }
774:
775: /*
776: * If it's being used, reactivate.
777: * (Fictitious pages are either busy or absent.)
778: */
779:
780: assert(!m->fictitious);
781: if (m->reference || pmap_is_referenced(m->phys_addr)) {
782: vm_object_unlock(object);
783: vm_page_activate(m);
784: vm_stat.reactivations++;
785: vm_page_unlock_queues();
786: vm_pageout_inactive_used++;
787: continue;
788: }
789:
790: /*
791: * Eliminate all mappings.
792: */
793:
794: m->busy = TRUE;
795: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
796: if (!m->dirty)
797: m->dirty = pmap_is_modified(m->phys_addr);
798:
799: if (m->external) {
800: /* Figure out if we still care about this
801: page in the limit of externally managed pages.
802: Clean pages don't actually cause system hosage,
803: so it's ok to stop considering them as
804: "consumers" of memory. */
805: if (m->dirty && !m->extcounted) {
806: m->extcounted = TRUE;
807: vm_page_external_count++;
808: } else if (!m->dirty && m->extcounted) {
809: m->extcounted = FALSE;
810: vm_page_external_count--;
811: }
812: }
813:
814: /* If we don't actually need more memory, and the page
815: is not dirty, put it on the tail of the inactive queue
816: and move on to the next page. */
817: if (!want_pages && !m->dirty) {
818: queue_remove (&vm_page_queue_inactive, m,
819: vm_page_t, pageq);
820: queue_enter (&vm_page_queue_inactive, m,
821: vm_page_t, pageq);
822: vm_page_unlock_queues();
823: vm_pageout_inactive_cleaned_external++;
824: continue;
825: }
826:
827: /*
828: * If it's clean and not precious, we can free the page.
829: */
830:
831: if (!m->dirty && !m->precious) {
832: vm_pageout_inactive_clean++;
833: goto reclaim_page;
834: }
835:
836: /*
837: * If we are very low on memory, then we can't
838: * rely on an external pager to clean a dirty page,
839: * because external pagers are not vm-privileged.
840: *
841: * The laundry bit tells vm_pageout_setup to
842: * put the page back at the front of the inactive
843: * queue instead of activating the page. Hence,
844: * we will pick the page up again immediately and
845: * resend it to the default pager.
846: */
847:
848: assert(!m->laundry);
849: if ((free_count < vm_pageout_reserved_internal) &&
850: !object->internal) {
851: m->laundry = TRUE;
852: vm_pageout_inactive_double++;
853: }
854: vm_page_unlock_queues();
855:
856: /*
857: * If there is no memory object for the page, create
858: * one and hand it to the default pager.
859: * [First try to collapse, so we don't create
860: * one unnecessarily.]
861: */
862:
863: if (!object->pager_initialized)
864: vm_object_collapse(object);
865: if (!object->pager_initialized)
866: vm_object_pager_create(object);
867: if (!object->pager_initialized)
868: panic("vm_pageout_scan");
869:
870: vm_pageout_inactive_dirty++;
871: vm_pageout_page(m, FALSE, TRUE); /* flush it */
872: vm_object_unlock(object);
873: burst_count++;
874: }
875: }
876:
877: void vm_pageout_scan_continue()
878: {
879: /*
880: * We just paused to let the pagers catch up.
881: * If vm_page_laundry_count is still high,
882: * then we aren't waiting long enough.
883: * If we have paused some vm_pageout_pause_max times without
884: * adjusting vm_pageout_burst_wait, it might be too big,
885: * so we decrease it.
886: */
887:
888: vm_page_lock_queues();
889: if (vm_page_laundry_count > vm_pageout_burst_min) {
890: vm_pageout_burst_wait++;
891: vm_pageout_pause_count = 0;
892: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) {
893: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4;
894: if (vm_pageout_burst_wait < 1)
895: vm_pageout_burst_wait = 1;
896: vm_pageout_pause_count = 0;
897: }
898: vm_page_unlock_queues();
899:
900: #ifdef CONTINUATIONS
901: vm_pageout_continue();
902: /*NOTREACHED*/
903: #endif /* CONTINUATIONS */
904: }
905:
906: /*
907: * vm_pageout is the high level pageout daemon.
908: */
909:
910: void vm_pageout_continue()
911: {
912: /*
913: * The pageout daemon is never done, so loop forever.
914: * We should call vm_pageout_scan at least once each
915: * time we are woken, even if vm_page_free_wanted is
916: * zero, to check vm_page_free_target and
917: * vm_page_inactive_target.
918: */
919:
920: for (;;) {
921: vm_pageout_scan();
922: /* we hold vm_page_queue_free_lock now */
923: assert(vm_page_free_wanted == 0);
924:
925: assert_wait(&vm_page_free_wanted, FALSE);
926: simple_unlock(&vm_page_queue_free_lock);
927: counter(c_vm_pageout_block++);
928: thread_block(vm_pageout_continue);
929: }
930: }
931:
932: void vm_pageout()
933: {
934: int free_after_reserve;
935:
936: current_thread()->vm_privilege = TRUE;
937: stack_privilege(current_thread());
938:
939: /*
940: * Initialize some paging parameters.
941: */
942:
943: if (vm_pageout_burst_max == 0)
944: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX;
945:
946: if (vm_pageout_burst_min == 0)
947: vm_pageout_burst_min = VM_PAGEOUT_BURST_MIN;
948:
949: if (vm_pageout_burst_wait == 0)
950: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
951:
952: if (vm_pageout_empty_wait == 0)
953: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
954:
955: if (vm_page_free_reserved == 0)
956: vm_page_free_reserved = VM_PAGE_FREE_RESERVED;
957:
958: if (vm_pageout_pause_max == 0)
959: vm_pageout_pause_max = VM_PAGEOUT_PAUSE_MAX;
960:
961: if (vm_pageout_reserved_internal == 0)
962: vm_pageout_reserved_internal =
963: VM_PAGEOUT_RESERVED_INTERNAL(vm_page_free_reserved);
964:
965: if (vm_pageout_reserved_really == 0)
966: vm_pageout_reserved_really =
967: VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved);
968:
969: free_after_reserve = vm_page_free_count - vm_page_free_reserved;
970:
971: if (vm_page_external_limit == 0)
972: vm_page_external_limit =
973: VM_PAGE_EXTERNAL_LIMIT (free_after_reserve);
974:
975: if (vm_page_external_target == 0)
976: vm_page_external_target =
977: VM_PAGE_EXTERNAL_TARGET (free_after_reserve);
978:
979: if (vm_page_free_min == 0)
980: vm_page_free_min = vm_page_free_reserved +
981: VM_PAGE_FREE_MIN(free_after_reserve);
982:
983: if (vm_page_free_target == 0)
984: vm_page_free_target = vm_page_free_reserved +
985: VM_PAGE_FREE_TARGET(free_after_reserve);
986:
987: if (vm_page_free_target < vm_page_free_min + 5)
988: vm_page_free_target = vm_page_free_min + 5;
989:
990: /*
991: * vm_pageout_scan will set vm_page_inactive_target.
992: */
993:
994: vm_pageout_continue();
995: /*NOTREACHED*/
996: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.