![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_pageout.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 ! root 1: /* ! 2: * Mach Operating System ! 3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University. ! 4: * Copyright (c) 1993,1994 The University of Utah and ! 5: * the Computer Systems Laboratory (CSL). ! 6: * All rights reserved. ! 7: * ! 8: * Permission to use, copy, modify and distribute this software and its ! 9: * documentation is hereby granted, provided that both the copyright ! 10: * notice and this permission notice appear in all copies of the ! 11: * software, derivative works or modified versions, and any portions ! 12: * thereof, and that both notices appear in supporting documentation. ! 13: * ! 14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF ! 15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY ! 16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF ! 17: * THIS SOFTWARE. ! 18: * ! 19: * Carnegie Mellon requests users of this software to return to ! 20: * ! 21: * Software Distribution Coordinator or [email protected] ! 22: * School of Computer Science ! 23: * Carnegie Mellon University ! 24: * Pittsburgh PA 15213-3890 ! 25: * ! 26: * any improvements or extensions that they make and grant Carnegie Mellon ! 27: * the rights to redistribute these changes. ! 28: */ ! 29: /* ! 30: * File: vm/vm_pageout.c ! 31: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 32: * Date: 1985 ! 33: * ! 34: * The proverbial page-out daemon. ! 35: */ ! 36: ! 37: #include <mach_pagemap.h> ! 38: #include <norma_vm.h> ! 39: ! 40: #include <mach/mach_types.h> ! 41: #include <mach/memory_object.h> ! 42: #include "memory_object_default.h" ! 43: #include "memory_object_user.h" ! 44: #include <mach/vm_param.h> ! 45: #include <mach/vm_statistics.h> ! 46: #include <kern/counters.h> ! 47: #include <kern/thread.h> ! 48: #include <vm/pmap.h> ! 49: #include <vm/vm_map.h> ! 50: #include <vm/vm_object.h> ! 51: #include <vm/vm_page.h> ! 52: #include <vm/vm_pageout.h> ! 53: #include <machine/vm_tuning.h> ! 54: ! 55: ! 56: ! 57: #ifndef VM_PAGEOUT_BURST_MAX ! 58: #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */ ! 59: #endif VM_PAGEOUT_BURST_MAX ! 60: ! 61: #ifndef VM_PAGEOUT_BURST_MIN ! 62: #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */ ! 63: #endif VM_PAGEOUT_BURST_MIN ! 64: ! 65: #ifndef VM_PAGEOUT_BURST_WAIT ! 66: #define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */ ! 67: #endif VM_PAGEOUT_BURST_WAIT ! 68: ! 69: #ifndef VM_PAGEOUT_EMPTY_WAIT ! 70: #define VM_PAGEOUT_EMPTY_WAIT 200 /* milliseconds */ ! 71: #endif VM_PAGEOUT_EMPTY_WAIT ! 72: ! 73: #ifndef VM_PAGEOUT_PAUSE_MAX ! 74: #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */ ! 75: #endif VM_PAGEOUT_PAUSE_MAX ! 76: ! 77: /* ! 78: * To obtain a reasonable LRU approximation, the inactive queue ! 79: * needs to be large enough to give pages on it a chance to be ! 80: * referenced a second time. This macro defines the fraction ! 81: * of active+inactive pages that should be inactive. ! 82: * The pageout daemon uses it to update vm_page_inactive_target. ! 83: * ! 84: * If vm_page_free_count falls below vm_page_free_target and ! 85: * vm_page_inactive_count is below vm_page_inactive_target, ! 86: * then the pageout daemon starts running. ! 87: */ ! 88: ! 89: #ifndef VM_PAGE_INACTIVE_TARGET ! 90: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3) ! 91: #endif VM_PAGE_INACTIVE_TARGET ! 92: ! 93: /* ! 94: * Once the pageout daemon starts running, it keeps going ! 95: * until vm_page_free_count meets or exceeds vm_page_free_target. ! 96: */ ! 97: ! 98: #ifndef VM_PAGE_FREE_TARGET ! 99: #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) ! 100: #endif VM_PAGE_FREE_TARGET ! 101: ! 102: /* ! 103: * The pageout daemon always starts running once vm_page_free_count ! 104: * falls below vm_page_free_min. ! 105: */ ! 106: ! 107: #ifndef VM_PAGE_FREE_MIN ! 108: #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) ! 109: #endif VM_PAGE_FREE_MIN ! 110: ! 111: /* ! 112: * When vm_page_free_count falls below vm_page_free_reserved, ! 113: * only vm-privileged threads can allocate pages. vm-privilege ! 114: * allows the pageout daemon and default pager (and any other ! 115: * associated threads needed for default pageout) to continue ! 116: * operation by dipping into the reserved pool of pages. ! 117: */ ! 118: ! 119: #ifndef VM_PAGE_FREE_RESERVED ! 120: #define VM_PAGE_FREE_RESERVED 15 ! 121: #endif VM_PAGE_FREE_RESERVED ! 122: ! 123: /* ! 124: * When vm_page_free_count falls below vm_pageout_reserved_internal, ! 125: * the pageout daemon no longer trusts external pagers to clean pages. ! 126: * External pagers are probably all wedged waiting for a free page. ! 127: * It forcibly double-pages dirty pages belonging to external objects, ! 128: * getting the pages to the default pager to clean. ! 129: */ ! 130: ! 131: #ifndef VM_PAGEOUT_RESERVED_INTERNAL ! 132: #define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 5) ! 133: #endif VM_PAGEOUT_RESERVED_INTERNAL ! 134: ! 135: /* ! 136: * When vm_page_free_count falls below vm_pageout_reserved_really, ! 137: * the pageout daemon stops work entirely to let the default pager ! 138: * catch up (assuming the default pager has pages to clean). ! 139: * Beyond this point, it is too dangerous to consume memory ! 140: * even for memory_object_data_write messages to the default pager. ! 141: */ ! 142: ! 143: #ifndef VM_PAGEOUT_RESERVED_REALLY ! 144: #define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 10) ! 145: #endif VM_PAGEOUT_RESERVED_REALLY ! 146: ! 147: extern void vm_pageout_continue(); ! 148: extern void vm_pageout_scan_continue(); ! 149: ! 150: unsigned int vm_pageout_reserved_internal = 0; ! 151: unsigned int vm_pageout_reserved_really = 0; ! 152: ! 153: unsigned int vm_pageout_burst_max = 0; ! 154: unsigned int vm_pageout_burst_min = 0; ! 155: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */ ! 156: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */ ! 157: unsigned int vm_pageout_pause_count = 0; ! 158: unsigned int vm_pageout_pause_max = 0; ! 159: ! 160: /* ! 161: * These variables record the pageout daemon's actions: ! 162: * how many pages it looks at and what happens to those pages. ! 163: * No locking needed because only one thread modifies the variables. ! 164: */ ! 165: ! 166: unsigned int vm_pageout_active = 0; /* debugging */ ! 167: unsigned int vm_pageout_inactive = 0; /* debugging */ ! 168: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */ ! 169: unsigned int vm_pageout_inactive_busy = 0; /* debugging */ ! 170: unsigned int vm_pageout_inactive_absent = 0; /* debugging */ ! 171: unsigned int vm_pageout_inactive_used = 0; /* debugging */ ! 172: unsigned int vm_pageout_inactive_clean = 0; /* debugging */ ! 173: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */ ! 174: unsigned int vm_pageout_inactive_double = 0; /* debugging */ ! 175: ! 176: #if NORMA_VM ! 177: /* ! 178: * Define them here, since they won't be defined by memory_object_user.h. ! 179: */ ! 180: extern kern_return_t memory_object_data_initialize(); ! 181: extern kern_return_t memory_object_data_write(); ! 182: #endif NORMA_VM ! 183: ! 184: /* ! 185: * Routine: vm_pageout_setup ! 186: * Purpose: ! 187: * Set up a page for pageout. ! 188: * ! 189: * Move or copy the page to a new object, as part ! 190: * of which it will be sent to its memory manager ! 191: * in a memory_object_data_write or memory_object_initialize ! 192: * message. ! 193: * ! 194: * The "paging_offset" argument specifies the offset ! 195: * of the page within its external memory object. ! 196: * ! 197: * The "new_object" and "new_offset" arguments ! 198: * indicate where the page should be moved. ! 199: * ! 200: * The "flush" argument specifies whether the page ! 201: * should be flushed from its object. If not, a ! 202: * copy of the page is moved to the new object. ! 203: * ! 204: * In/Out conditions: ! 205: * The page in question must not be on any pageout queues, ! 206: * and must be busy. The object to which it belongs ! 207: * must be unlocked, and the caller must hold a paging ! 208: * reference to it. The new_object must not be locked. ! 209: * ! 210: * If the page is flushed from its original object, ! 211: * this routine returns a pointer to a place-holder page, ! 212: * inserted at the same offset, to block out-of-order ! 213: * requests for the page. The place-holder page must ! 214: * be freed after the data_write or initialize message ! 215: * has been sent. If the page is copied, ! 216: * the holding page is VM_PAGE_NULL. ! 217: * ! 218: * The original page is put on a paging queue and marked ! 219: * not busy on exit. ! 220: */ ! 221: vm_page_t ! 222: vm_pageout_setup(m, paging_offset, new_object, new_offset, flush) ! 223: register vm_page_t m; ! 224: vm_offset_t paging_offset; ! 225: register vm_object_t new_object; ! 226: vm_offset_t new_offset; ! 227: boolean_t flush; ! 228: { ! 229: register vm_object_t old_object = m->object; ! 230: register vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/ ! 231: register vm_page_t new_m; ! 232: ! 233: assert(m->busy && !m->absent && !m->fictitious); ! 234: ! 235: /* ! 236: * If we are not flushing the page, allocate a ! 237: * page in the object. If we cannot get the ! 238: * page, flush instead. ! 239: */ ! 240: if (!flush) { ! 241: vm_object_lock(new_object); ! 242: new_m = vm_page_alloc(new_object, new_offset); ! 243: if (new_m == VM_PAGE_NULL) ! 244: flush = TRUE; ! 245: vm_object_unlock(new_object); ! 246: } ! 247: ! 248: if (flush) { ! 249: /* ! 250: * Create a place-holder page where the old one was, ! 251: * to prevent anyone from attempting to page in this ! 252: * page while we`re unlocked. ! 253: */ ! 254: while ((holding_page = vm_page_grab_fictitious()) ! 255: == VM_PAGE_NULL) ! 256: vm_page_more_fictitious(); ! 257: ! 258: vm_object_lock(old_object); ! 259: vm_page_lock_queues(); ! 260: vm_page_remove(m); ! 261: vm_page_unlock_queues(); ! 262: PAGE_WAKEUP_DONE(m); ! 263: ! 264: vm_page_lock_queues(); ! 265: vm_page_insert(holding_page, old_object, m->offset); ! 266: vm_page_unlock_queues(); ! 267: ! 268: /* ! 269: * Record that this page has been written out ! 270: */ ! 271: #if MACH_PAGEMAP ! 272: vm_external_state_set(old_object->existence_info, ! 273: paging_offset, ! 274: VM_EXTERNAL_STATE_EXISTS); ! 275: #endif MACH_PAGEMAP ! 276: ! 277: vm_object_unlock(old_object); ! 278: ! 279: vm_object_lock(new_object); ! 280: ! 281: /* ! 282: * Move this page into the new object ! 283: */ ! 284: ! 285: vm_page_lock_queues(); ! 286: vm_page_insert(m, new_object, new_offset); ! 287: vm_page_unlock_queues(); ! 288: ! 289: m->dirty = TRUE; ! 290: m->precious = FALSE; ! 291: m->page_lock = VM_PROT_NONE; ! 292: m->unlock_request = VM_PROT_NONE; ! 293: } ! 294: else { ! 295: /* ! 296: * Copy the data into the new page, ! 297: * and mark the new page as clean. ! 298: */ ! 299: vm_page_copy(m, new_m); ! 300: ! 301: vm_object_lock(old_object); ! 302: m->dirty = FALSE; ! 303: pmap_clear_modify(m->phys_addr); ! 304: ! 305: /* ! 306: * Deactivate old page. ! 307: */ ! 308: vm_page_lock_queues(); ! 309: vm_page_deactivate(m); ! 310: vm_page_unlock_queues(); ! 311: ! 312: PAGE_WAKEUP_DONE(m); ! 313: ! 314: /* ! 315: * Record that this page has been written out ! 316: */ ! 317: ! 318: #if MACH_PAGEMAP ! 319: vm_external_state_set(old_object->existence_info, ! 320: paging_offset, ! 321: VM_EXTERNAL_STATE_EXISTS); ! 322: #endif MACH_PAGEMAP ! 323: ! 324: vm_object_unlock(old_object); ! 325: ! 326: vm_object_lock(new_object); ! 327: ! 328: /* ! 329: * Use the new page below. ! 330: */ ! 331: m = new_m; ! 332: m->dirty = TRUE; ! 333: assert(!m->precious); ! 334: PAGE_WAKEUP_DONE(m); ! 335: } ! 336: ! 337: /* ! 338: * Make the old page eligible for replacement again; if a ! 339: * user-supplied memory manager fails to release the page, ! 340: * it will be paged out again to the default memory manager. ! 341: * ! 342: * Note that pages written to the default memory manager ! 343: * must be wired down -- in return, it guarantees to free ! 344: * this page, rather than reusing it. ! 345: */ ! 346: ! 347: vm_page_lock_queues(); ! 348: vm_stat.pageouts++; ! 349: if (m->laundry) { ! 350: /* ! 351: * vm_pageout_scan is telling us to put this page ! 352: * at the front of the inactive queue, so it will ! 353: * be immediately paged out to the default pager. ! 354: */ ! 355: ! 356: assert(!old_object->internal); ! 357: m->laundry = FALSE; ! 358: ! 359: queue_enter_first(&vm_page_queue_inactive, m, ! 360: vm_page_t, pageq); ! 361: m->inactive = TRUE; ! 362: vm_page_inactive_count++; ! 363: } else if (old_object->internal) { ! 364: m->laundry = TRUE; ! 365: vm_page_laundry_count++; ! 366: ! 367: vm_page_wire(m); ! 368: } else ! 369: vm_page_activate(m); ! 370: vm_page_unlock_queues(); ! 371: ! 372: /* ! 373: * Since IPC operations may block, we drop locks now. ! 374: * [The placeholder page is busy, and we still have ! 375: * paging_in_progress incremented.] ! 376: */ ! 377: ! 378: vm_object_unlock(new_object); ! 379: ! 380: /* ! 381: * Return the placeholder page to simplify cleanup. ! 382: */ ! 383: return (flush ? holding_page : VM_PAGE_NULL); ! 384: } ! 385: ! 386: /* ! 387: * Routine: vm_pageout_page ! 388: * Purpose: ! 389: * Causes the specified page to be written back to ! 390: * the appropriate memory object. ! 391: * ! 392: * The "initial" argument specifies whether this ! 393: * data is an initialization only, and should use ! 394: * memory_object_data_initialize instead of ! 395: * memory_object_data_write. ! 396: * ! 397: * The "flush" argument specifies whether the page ! 398: * should be flushed from the object. If not, a ! 399: * copy of the data is sent to the memory object. ! 400: * ! 401: * In/out conditions: ! 402: * The page in question must not be on any pageout queues. ! 403: * The object to which it belongs must be locked. ! 404: * Implementation: ! 405: * Move this page to a completely new object, if flushing; ! 406: * copy to a new page in a new object, if not. ! 407: */ ! 408: void ! 409: vm_pageout_page(m, initial, flush) ! 410: register vm_page_t m; ! 411: boolean_t initial; ! 412: boolean_t flush; ! 413: { ! 414: vm_map_copy_t copy; ! 415: register vm_object_t old_object; ! 416: register vm_object_t new_object; ! 417: register vm_page_t holding_page; ! 418: vm_offset_t paging_offset; ! 419: kern_return_t rc; ! 420: boolean_t precious_clean; ! 421: ! 422: assert(m->busy); ! 423: ! 424: /* ! 425: * Cleaning but not flushing a clean precious page is a ! 426: * no-op. Remember whether page is clean and precious now ! 427: * because vm_pageout_setup will mark it dirty and not precious. ! 428: * ! 429: * XXX Check if precious_clean && !flush can really happen. ! 430: */ ! 431: precious_clean = (!m->dirty) && m->precious; ! 432: if (precious_clean && !flush) { ! 433: PAGE_WAKEUP_DONE(m); ! 434: return; ! 435: } ! 436: ! 437: /* ! 438: * Verify that we really want to clean this page. ! 439: */ ! 440: if (m->absent || m->error || (!m->dirty && !m->precious)) { ! 441: VM_PAGE_FREE(m); ! 442: return; ! 443: } ! 444: ! 445: /* ! 446: * Create a paging reference to let us play with the object. ! 447: */ ! 448: old_object = m->object; ! 449: paging_offset = m->offset + old_object->paging_offset; ! 450: vm_object_paging_begin(old_object); ! 451: vm_object_unlock(old_object); ! 452: ! 453: /* ! 454: * Allocate a new object into which we can put the page. ! 455: */ ! 456: new_object = vm_object_allocate(PAGE_SIZE); ! 457: ! 458: /* ! 459: * Move the page into the new object. ! 460: */ ! 461: holding_page = vm_pageout_setup(m, ! 462: paging_offset, ! 463: new_object, ! 464: 0, /* new offset */ ! 465: flush); /* flush */ ! 466: ! 467: rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©); ! 468: assert(rc == KERN_SUCCESS); ! 469: ! 470: if (initial || old_object->use_old_pageout) { ! 471: rc = (*(initial ? memory_object_data_initialize ! 472: : memory_object_data_write)) ! 473: (old_object->pager, ! 474: old_object->pager_request, ! 475: paging_offset, (pointer_t) copy, PAGE_SIZE); ! 476: } ! 477: else { ! 478: rc = memory_object_data_return( ! 479: old_object->pager, ! 480: old_object->pager_request, ! 481: paging_offset, (pointer_t) copy, PAGE_SIZE, ! 482: !precious_clean, !flush); ! 483: } ! 484: ! 485: if (rc != KERN_SUCCESS) ! 486: vm_map_copy_discard(copy); ! 487: ! 488: /* ! 489: * Clean up. ! 490: */ ! 491: vm_object_lock(old_object); ! 492: if (holding_page != VM_PAGE_NULL) ! 493: VM_PAGE_FREE(holding_page); ! 494: vm_object_paging_end(old_object); ! 495: } ! 496: ! 497: /* ! 498: * vm_pageout_scan does the dirty work for the pageout daemon. ! 499: * It returns with vm_page_queue_free_lock held and ! 500: * vm_page_free_wanted == 0. ! 501: */ ! 502: ! 503: void vm_pageout_scan() ! 504: { ! 505: unsigned int burst_count; ! 506: ! 507: /* ! 508: * We want to gradually dribble pages from the active queue ! 509: * to the inactive queue. If we let the inactive queue get ! 510: * very small, and then suddenly dump many pages into it, ! 511: * those pages won't get a sufficient chance to be referenced ! 512: * before we start taking them from the inactive queue. ! 513: * ! 514: * We must limit the rate at which we send pages to the pagers. ! 515: * data_write messages consume memory, for message buffers and ! 516: * for map-copy objects. If we get too far ahead of the pagers, ! 517: * we can potentially run out of memory. ! 518: * ! 519: * We can use the laundry count to limit directly the number ! 520: * of pages outstanding to the default pager. A similar ! 521: * strategy for external pagers doesn't work, because ! 522: * external pagers don't have to deallocate the pages sent them, ! 523: * and because we might have to send pages to external pagers ! 524: * even if they aren't processing writes. So we also ! 525: * use a burst count to limit writes to external pagers. ! 526: * ! 527: * When memory is very tight, we can't rely on external pagers to ! 528: * clean pages. They probably aren't running, because they ! 529: * aren't vm-privileged. If we kept sending dirty pages to them, ! 530: * we could exhaust the free list. However, we can't just ignore ! 531: * pages belonging to external objects, because there might be no ! 532: * pages belonging to internal objects. Hence, we get the page ! 533: * into an internal object and then immediately double-page it, ! 534: * sending it to the default pager. ! 535: * ! 536: * consider_zone_gc should be last, because the other operations ! 537: * might return memory to zones. When we pause we use ! 538: * vm_pageout_scan_continue as our continuation, so we will ! 539: * reenter vm_pageout_scan periodically and attempt to reclaim ! 540: * internal memory even if we never reach vm_page_free_target. ! 541: */ ! 542: ! 543: Restart: ! 544: stack_collect(); ! 545: net_kmsg_collect(); ! 546: consider_task_collect(); ! 547: consider_thread_collect(); ! 548: consider_zone_gc(); ! 549: ! 550: for (burst_count = 0;;) { ! 551: register vm_page_t m; ! 552: register vm_object_t object; ! 553: unsigned int free_count; ! 554: ! 555: /* ! 556: * Recalculate vm_page_inactivate_target. ! 557: */ ! 558: ! 559: vm_page_lock_queues(); ! 560: vm_page_inactive_target = ! 561: VM_PAGE_INACTIVE_TARGET(vm_page_active_count + ! 562: vm_page_inactive_count); ! 563: ! 564: /* ! 565: * Move pages from active to inactive. ! 566: */ ! 567: ! 568: while ((vm_page_inactive_count < vm_page_inactive_target) && ! 569: !queue_empty(&vm_page_queue_active)) { ! 570: register vm_object_t obj; ! 571: ! 572: vm_pageout_active++; ! 573: m = (vm_page_t) queue_first(&vm_page_queue_active); ! 574: assert(m->active && !m->inactive); ! 575: ! 576: obj = m->object; ! 577: if (!vm_object_lock_try(obj)) { ! 578: /* ! 579: * Move page to end and continue. ! 580: */ ! 581: ! 582: queue_remove(&vm_page_queue_active, m, ! 583: vm_page_t, pageq); ! 584: queue_enter(&vm_page_queue_active, m, ! 585: vm_page_t, pageq); ! 586: vm_page_unlock_queues(); ! 587: vm_page_lock_queues(); ! 588: continue; ! 589: } ! 590: ! 591: /* ! 592: * If the page is busy, then we pull it ! 593: * off the active queue and leave it alone. ! 594: */ ! 595: ! 596: if (m->busy) { ! 597: vm_object_unlock(obj); ! 598: queue_remove(&vm_page_queue_active, m, ! 599: vm_page_t, pageq); ! 600: m->active = FALSE; ! 601: vm_page_active_count--; ! 602: continue; ! 603: } ! 604: ! 605: /* ! 606: * Deactivate the page while holding the object ! 607: * locked, so we know the page is still not busy. ! 608: * This should prevent races between pmap_enter ! 609: * and pmap_clear_reference. The page might be ! 610: * absent or fictitious, but vm_page_deactivate ! 611: * can handle that. ! 612: */ ! 613: ! 614: vm_page_deactivate(m); ! 615: vm_object_unlock(obj); ! 616: } ! 617: ! 618: /* ! 619: * We are done if we have met our target *and* ! 620: * nobody is still waiting for a page. ! 621: */ ! 622: ! 623: simple_lock(&vm_page_queue_free_lock); ! 624: free_count = vm_page_free_count; ! 625: if ((free_count >= vm_page_free_target) & ! 626: (vm_page_free_wanted == 0)) { ! 627: vm_page_unlock_queues(); ! 628: break; ! 629: } ! 630: simple_unlock(&vm_page_queue_free_lock); ! 631: ! 632: /* ! 633: * Sometimes we have to pause: ! 634: * 1) No inactive pages - nothing to do. ! 635: * 2) Flow control - wait for pagers to catch up. ! 636: * 3) Extremely low memory - sending out dirty pages ! 637: * consumes memory. We don't take the risk of doing ! 638: * this if the default pager already has work to do. ! 639: */ ! 640: ! 641: if (queue_empty(&vm_page_queue_inactive) || ! 642: (burst_count >= vm_pageout_burst_max) || ! 643: (vm_page_laundry_count >= vm_pageout_burst_max) || ! 644: ((free_count < vm_pageout_reserved_really) && ! 645: (vm_page_laundry_count > 0))) { ! 646: unsigned int pages, msecs; ! 647: ! 648: /* ! 649: * vm_pageout_burst_wait is msecs/page. ! 650: * If there is nothing for us to do, we wait ! 651: * at least vm_pageout_empty_wait msecs. ! 652: */ ! 653: ! 654: if (vm_page_laundry_count > burst_count) ! 655: pages = vm_page_laundry_count; ! 656: else ! 657: pages = burst_count; ! 658: msecs = pages * vm_pageout_burst_wait; ! 659: ! 660: if (queue_empty(&vm_page_queue_inactive) && ! 661: (msecs < vm_pageout_empty_wait)) ! 662: msecs = vm_pageout_empty_wait; ! 663: vm_page_unlock_queues(); ! 664: ! 665: thread_will_wait_with_timeout(current_thread(), msecs); ! 666: counter(c_vm_pageout_scan_block++); ! 667: thread_block(vm_pageout_scan_continue); ! 668: #ifndef CONTINUATIONS ! 669: /* ! 670: * Unfortunately, we don't have call_continuation ! 671: * so we can't rely on tail-recursion. ! 672: */ ! 673: ! 674: vm_pageout_scan_continue(); ! 675: goto Restart; ! 676: #else /* CONTINUATIONS */ ! 677: call_continuation(vm_pageout_scan_continue); ! 678: /*NOTREACHED*/ ! 679: #endif /* CONTINUATIONS */ ! 680: } ! 681: ! 682: vm_pageout_inactive++; ! 683: m = (vm_page_t) queue_first(&vm_page_queue_inactive); ! 684: assert(!m->active && m->inactive); ! 685: object = m->object; ! 686: ! 687: /* ! 688: * Try to lock object; since we've got the ! 689: * page queues lock, we can only try for this one. ! 690: */ ! 691: ! 692: if (!vm_object_lock_try(object)) { ! 693: /* ! 694: * Move page to end and continue. ! 695: */ ! 696: ! 697: queue_remove(&vm_page_queue_inactive, m, ! 698: vm_page_t, pageq); ! 699: queue_enter(&vm_page_queue_inactive, m, ! 700: vm_page_t, pageq); ! 701: vm_page_unlock_queues(); ! 702: vm_pageout_inactive_nolock++; ! 703: continue; ! 704: } ! 705: ! 706: /* ! 707: * Remove the page from the inactive list. ! 708: */ ! 709: ! 710: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq); ! 711: vm_page_inactive_count--; ! 712: m->inactive = FALSE; ! 713: ! 714: if (m->busy || !object->alive) { ! 715: /* ! 716: * Somebody is already playing with this page. ! 717: * Leave it off the pageout queues. ! 718: */ ! 719: ! 720: vm_page_unlock_queues(); ! 721: vm_object_unlock(object); ! 722: vm_pageout_inactive_busy++; ! 723: continue; ! 724: } ! 725: ! 726: /* ! 727: * If it's absent, we can reclaim the page. ! 728: */ ! 729: ! 730: if (m->absent) { ! 731: vm_pageout_inactive_absent++; ! 732: reclaim_page: ! 733: vm_page_free(m); ! 734: vm_page_unlock_queues(); ! 735: vm_object_unlock(object); ! 736: continue; ! 737: } ! 738: ! 739: /* ! 740: * If it's being used, reactivate. ! 741: * (Fictitious pages are either busy or absent.) ! 742: */ ! 743: ! 744: assert(!m->fictitious); ! 745: if (m->reference || pmap_is_referenced(m->phys_addr)) { ! 746: vm_object_unlock(object); ! 747: vm_page_activate(m); ! 748: vm_stat.reactivations++; ! 749: vm_page_unlock_queues(); ! 750: vm_pageout_inactive_used++; ! 751: continue; ! 752: } ! 753: ! 754: /* ! 755: * Eliminate all mappings. ! 756: */ ! 757: ! 758: m->busy = TRUE; ! 759: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 760: if (!m->dirty) ! 761: m->dirty = pmap_is_modified(m->phys_addr); ! 762: ! 763: /* ! 764: * If it's clean and not precious, we can free the page. ! 765: */ ! 766: ! 767: if (!m->dirty && !m->precious) { ! 768: vm_pageout_inactive_clean++; ! 769: goto reclaim_page; ! 770: } ! 771: ! 772: /* ! 773: * If we are very low on memory, then we can't ! 774: * rely on an external pager to clean a dirty page, ! 775: * because external pagers are not vm-privileged. ! 776: * ! 777: * The laundry bit tells vm_pageout_setup to ! 778: * put the page back at the front of the inactive ! 779: * queue instead of activating the page. Hence, ! 780: * we will pick the page up again immediately and ! 781: * resend it to the default pager. ! 782: */ ! 783: ! 784: assert(!m->laundry); ! 785: if ((free_count < vm_pageout_reserved_internal) && ! 786: !object->internal) { ! 787: m->laundry = TRUE; ! 788: vm_pageout_inactive_double++; ! 789: } ! 790: vm_page_unlock_queues(); ! 791: ! 792: /* ! 793: * If there is no memory object for the page, create ! 794: * one and hand it to the default pager. ! 795: * [First try to collapse, so we don't create ! 796: * one unnecessarily.] ! 797: */ ! 798: ! 799: if (!object->pager_initialized) ! 800: vm_object_collapse(object); ! 801: if (!object->pager_initialized) ! 802: vm_object_pager_create(object); ! 803: if (!object->pager_initialized) ! 804: panic("vm_pageout_scan"); ! 805: ! 806: vm_pageout_inactive_dirty++; ! 807: vm_pageout_page(m, FALSE, TRUE); /* flush it */ ! 808: vm_object_unlock(object); ! 809: burst_count++; ! 810: } ! 811: } ! 812: ! 813: void vm_pageout_scan_continue() ! 814: { ! 815: /* ! 816: * We just paused to let the pagers catch up. ! 817: * If vm_page_laundry_count is still high, ! 818: * then we aren't waiting long enough. ! 819: * If we have paused some vm_pageout_pause_max times without ! 820: * adjusting vm_pageout_burst_wait, it might be too big, ! 821: * so we decrease it. ! 822: */ ! 823: ! 824: vm_page_lock_queues(); ! 825: if (vm_page_laundry_count > vm_pageout_burst_min) { ! 826: vm_pageout_burst_wait++; ! 827: vm_pageout_pause_count = 0; ! 828: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) { ! 829: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4; ! 830: if (vm_pageout_burst_wait < 1) ! 831: vm_pageout_burst_wait = 1; ! 832: vm_pageout_pause_count = 0; ! 833: } ! 834: vm_page_unlock_queues(); ! 835: ! 836: #ifdef CONTINUATIONS ! 837: vm_pageout_continue(); ! 838: /*NOTREACHED*/ ! 839: #endif /* CONTINUATIONS */ ! 840: } ! 841: ! 842: /* ! 843: * vm_pageout is the high level pageout daemon. ! 844: */ ! 845: ! 846: void vm_pageout_continue() ! 847: { ! 848: /* ! 849: * The pageout daemon is never done, so loop forever. ! 850: * We should call vm_pageout_scan at least once each ! 851: * time we are woken, even if vm_page_free_wanted is ! 852: * zero, to check vm_page_free_target and ! 853: * vm_page_inactive_target. ! 854: */ ! 855: ! 856: for (;;) { ! 857: vm_pageout_scan(); ! 858: /* we hold vm_page_queue_free_lock now */ ! 859: assert(vm_page_free_wanted == 0); ! 860: ! 861: assert_wait(&vm_page_free_wanted, FALSE); ! 862: simple_unlock(&vm_page_queue_free_lock); ! 863: counter(c_vm_pageout_block++); ! 864: thread_block(vm_pageout_continue); ! 865: } ! 866: } ! 867: ! 868: void vm_pageout() ! 869: { ! 870: int free_after_reserve; ! 871: ! 872: current_thread()->vm_privilege = TRUE; ! 873: stack_privilege(current_thread()); ! 874: ! 875: /* ! 876: * Initialize some paging parameters. ! 877: */ ! 878: ! 879: if (vm_pageout_burst_max == 0) ! 880: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX; ! 881: ! 882: if (vm_pageout_burst_min == 0) ! 883: vm_pageout_burst_min = VM_PAGEOUT_BURST_MIN; ! 884: ! 885: if (vm_pageout_burst_wait == 0) ! 886: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT; ! 887: ! 888: if (vm_pageout_empty_wait == 0) ! 889: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT; ! 890: ! 891: if (vm_page_free_reserved == 0) ! 892: vm_page_free_reserved = VM_PAGE_FREE_RESERVED; ! 893: ! 894: if (vm_pageout_pause_max == 0) ! 895: vm_pageout_pause_max = VM_PAGEOUT_PAUSE_MAX; ! 896: ! 897: if (vm_pageout_reserved_internal == 0) ! 898: vm_pageout_reserved_internal = ! 899: VM_PAGEOUT_RESERVED_INTERNAL(vm_page_free_reserved); ! 900: ! 901: if (vm_pageout_reserved_really == 0) ! 902: vm_pageout_reserved_really = ! 903: VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved); ! 904: ! 905: free_after_reserve = vm_page_free_count - vm_page_free_reserved; ! 906: ! 907: if (vm_page_free_min == 0) ! 908: vm_page_free_min = vm_page_free_reserved + ! 909: VM_PAGE_FREE_MIN(free_after_reserve); ! 910: ! 911: if (vm_page_free_target == 0) ! 912: vm_page_free_target = vm_page_free_reserved + ! 913: VM_PAGE_FREE_TARGET(free_after_reserve); ! 914: ! 915: if (vm_page_free_target < vm_page_free_min + 5) ! 916: vm_page_free_target = vm_page_free_min + 5; ! 917: ! 918: /* ! 919: * vm_pageout_scan will set vm_page_inactive_target. ! 920: */ ! 921: ! 922: vm_pageout_continue(); ! 923: /*NOTREACHED*/ ! 924: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.