![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_pageout.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_pageout.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 ! root 1: /* ! 2: * Mach Operating System ! 3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University. ! 4: * Copyright (c) 1993,1994 The University of Utah and ! 5: * the Computer Systems Laboratory (CSL). ! 6: * All rights reserved. ! 7: * ! 8: * Permission to use, copy, modify and distribute this software and its ! 9: * documentation is hereby granted, provided that both the copyright ! 10: * notice and this permission notice appear in all copies of the ! 11: * software, derivative works or modified versions, and any portions ! 12: * thereof, and that both notices appear in supporting documentation. ! 13: * ! 14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF ! 15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY ! 16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF ! 17: * THIS SOFTWARE. ! 18: * ! 19: * Carnegie Mellon requests users of this software to return to ! 20: * ! 21: * Software Distribution Coordinator or [email protected] ! 22: * School of Computer Science ! 23: * Carnegie Mellon University ! 24: * Pittsburgh PA 15213-3890 ! 25: * ! 26: * any improvements or extensions that they make and grant Carnegie Mellon ! 27: * the rights to redistribute these changes. ! 28: */ ! 29: /* ! 30: * File: vm/vm_pageout.c ! 31: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 32: * Date: 1985 ! 33: * ! 34: * The proverbial page-out daemon. ! 35: */ ! 36: ! 37: #include <mach_pagemap.h> ! 38: #include <norma_vm.h> ! 39: ! 40: #include <mach/mach_types.h> ! 41: #include <mach/memory_object.h> ! 42: #include "memory_object_default.h" ! 43: #include "memory_object_user.h" ! 44: #include <mach/vm_param.h> ! 45: #include <mach/vm_statistics.h> ! 46: #include <kern/counters.h> ! 47: #include <kern/thread.h> ! 48: #include <vm/pmap.h> ! 49: #include <vm/vm_map.h> ! 50: #include <vm/vm_object.h> ! 51: #include <vm/vm_page.h> ! 52: #include <vm/vm_pageout.h> ! 53: #include <machine/vm_tuning.h> ! 54: ! 55: ! 56: ! 57: #ifndef VM_PAGEOUT_BURST_MAX ! 58: #define VM_PAGEOUT_BURST_MAX 10 /* number of pages */ ! 59: #endif /* VM_PAGEOUT_BURST_MAX */ ! 60: ! 61: #ifndef VM_PAGEOUT_BURST_MIN ! 62: #define VM_PAGEOUT_BURST_MIN 5 /* number of pages */ ! 63: #endif /* VM_PAGEOUT_BURST_MIN */ ! 64: ! 65: #ifndef VM_PAGEOUT_BURST_WAIT ! 66: #define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds per page */ ! 67: #endif /* VM_PAGEOUT_BURST_WAIT */ ! 68: ! 69: #ifndef VM_PAGEOUT_EMPTY_WAIT ! 70: #define VM_PAGEOUT_EMPTY_WAIT 75 /* milliseconds */ ! 71: #endif /* VM_PAGEOUT_EMPTY_WAIT */ ! 72: ! 73: #ifndef VM_PAGEOUT_PAUSE_MAX ! 74: #define VM_PAGEOUT_PAUSE_MAX 10 /* number of pauses */ ! 75: #endif /* VM_PAGEOUT_PAUSE_MAX */ ! 76: ! 77: /* ! 78: * To obtain a reasonable LRU approximation, the inactive queue ! 79: * needs to be large enough to give pages on it a chance to be ! 80: * referenced a second time. This macro defines the fraction ! 81: * of active+inactive pages that should be inactive. ! 82: * The pageout daemon uses it to update vm_page_inactive_target. ! 83: * ! 84: * If vm_page_free_count falls below vm_page_free_target and ! 85: * vm_page_inactive_count is below vm_page_inactive_target, ! 86: * then the pageout daemon starts running. ! 87: */ ! 88: ! 89: #ifndef VM_PAGE_INACTIVE_TARGET ! 90: #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 2 / 3) ! 91: #endif /* VM_PAGE_INACTIVE_TARGET */ ! 92: ! 93: /* ! 94: * Once the pageout daemon starts running, it keeps going ! 95: * until vm_page_free_count meets or exceeds vm_page_free_target. ! 96: */ ! 97: ! 98: #ifndef VM_PAGE_FREE_TARGET ! 99: #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) ! 100: #endif /* VM_PAGE_FREE_TARGET */ ! 101: ! 102: /* ! 103: * The pageout daemon always starts running once vm_page_free_count ! 104: * falls below vm_page_free_min. ! 105: */ ! 106: ! 107: #ifndef VM_PAGE_FREE_MIN ! 108: #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) ! 109: #endif /* VM_PAGE_FREE_MIN */ ! 110: ! 111: /* When vm_page_external_count exceeds vm_page_external_limit, ! 112: * allocations of externally paged pages stops. ! 113: */ ! 114: ! 115: #ifndef VM_PAGE_EXTERNAL_LIMIT ! 116: #define VM_PAGE_EXTERNAL_LIMIT(free) ((free) / 2) ! 117: #endif /* VM_PAGE_EXTERNAL_LIMIT */ ! 118: ! 119: /* Attempt to keep the number of externally paged pages less ! 120: * than vm_pages_external_target. ! 121: */ ! 122: #ifndef VM_PAGE_EXTERNAL_TARGET ! 123: #define VM_PAGE_EXTERNAL_TARGET(free) ((free) / 4) ! 124: #endif /* VM_PAGE_EXTERNAL_TARGET */ ! 125: ! 126: /* ! 127: * When vm_page_free_count falls below vm_page_free_reserved, ! 128: * only vm-privileged threads can allocate pages. vm-privilege ! 129: * allows the pageout daemon and default pager (and any other ! 130: * associated threads needed for default pageout) to continue ! 131: * operation by dipping into the reserved pool of pages. */ ! 132: ! 133: #ifndef VM_PAGE_FREE_RESERVED ! 134: #define VM_PAGE_FREE_RESERVED 50 ! 135: #endif /* VM_PAGE_FREE_RESERVED */ ! 136: ! 137: /* ! 138: * When vm_page_free_count falls below vm_pageout_reserved_internal, ! 139: * the pageout daemon no longer trusts external pagers to clean pages. ! 140: * External pagers are probably all wedged waiting for a free page. ! 141: * It forcibly double-pages dirty pages belonging to external objects, ! 142: * getting the pages to the default pager to clean. ! 143: */ ! 144: ! 145: #ifndef VM_PAGEOUT_RESERVED_INTERNAL ! 146: #define VM_PAGEOUT_RESERVED_INTERNAL(reserve) ((reserve) - 25) ! 147: #endif /* VM_PAGEOUT_RESERVED_INTERNAL */ ! 148: ! 149: /* ! 150: * When vm_page_free_count falls below vm_pageout_reserved_really, ! 151: * the pageout daemon stops work entirely to let the default pager ! 152: * catch up (assuming the default pager has pages to clean). ! 153: * Beyond this point, it is too dangerous to consume memory ! 154: * even for memory_object_data_write messages to the default pager. ! 155: */ ! 156: ! 157: #ifndef VM_PAGEOUT_RESERVED_REALLY ! 158: #define VM_PAGEOUT_RESERVED_REALLY(reserve) ((reserve) - 40) ! 159: #endif /* VM_PAGEOUT_RESERVED_REALLY */ ! 160: ! 161: extern void vm_pageout_continue(); ! 162: extern void vm_pageout_scan_continue(); ! 163: ! 164: unsigned int vm_pageout_reserved_internal = 0; ! 165: unsigned int vm_pageout_reserved_really = 0; ! 166: ! 167: unsigned int vm_page_external_target = 0; ! 168: ! 169: unsigned int vm_pageout_burst_max = 0; ! 170: unsigned int vm_pageout_burst_min = 0; ! 171: unsigned int vm_pageout_burst_wait = 0; /* milliseconds per page */ ! 172: unsigned int vm_pageout_empty_wait = 0; /* milliseconds */ ! 173: unsigned int vm_pageout_pause_count = 0; ! 174: unsigned int vm_pageout_pause_max = 0; ! 175: ! 176: /* ! 177: * These variables record the pageout daemon's actions: ! 178: * how many pages it looks at and what happens to those pages. ! 179: * No locking needed because only one thread modifies the variables. ! 180: */ ! 181: ! 182: unsigned int vm_pageout_active = 0; /* debugging */ ! 183: unsigned int vm_pageout_inactive = 0; /* debugging */ ! 184: unsigned int vm_pageout_inactive_nolock = 0; /* debugging */ ! 185: unsigned int vm_pageout_inactive_busy = 0; /* debugging */ ! 186: unsigned int vm_pageout_inactive_absent = 0; /* debugging */ ! 187: unsigned int vm_pageout_inactive_used = 0; /* debugging */ ! 188: unsigned int vm_pageout_inactive_clean = 0; /* debugging */ ! 189: unsigned int vm_pageout_inactive_dirty = 0; /* debugging */ ! 190: unsigned int vm_pageout_inactive_double = 0; /* debugging */ ! 191: unsigned int vm_pageout_inactive_cleaned_external = 0; ! 192: ! 193: #if NORMA_VM ! 194: /* ! 195: * Define them here, since they won't be defined by memory_object_user.h. ! 196: */ ! 197: extern kern_return_t memory_object_data_initialize(); ! 198: extern kern_return_t memory_object_data_write(); ! 199: #endif /* NORMA_VM */ ! 200: ! 201: /* ! 202: * Routine: vm_pageout_setup ! 203: * Purpose: ! 204: * Set up a page for pageout. ! 205: * ! 206: * Move or copy the page to a new object, as part ! 207: * of which it will be sent to its memory manager ! 208: * in a memory_object_data_write or memory_object_initialize ! 209: * message. ! 210: * ! 211: * The "paging_offset" argument specifies the offset ! 212: * of the page within its external memory object. ! 213: * ! 214: * The "new_object" and "new_offset" arguments ! 215: * indicate where the page should be moved. ! 216: * ! 217: * The "flush" argument specifies whether the page ! 218: * should be flushed from its object. If not, a ! 219: * copy of the page is moved to the new object. ! 220: * ! 221: * In/Out conditions: ! 222: * The page in question must not be on any pageout queues, ! 223: * and must be busy. The object to which it belongs ! 224: * must be unlocked, and the caller must hold a paging ! 225: * reference to it. The new_object must not be locked. ! 226: * ! 227: * If the page is flushed from its original object, ! 228: * this routine returns a pointer to a place-holder page, ! 229: * inserted at the same offset, to block out-of-order ! 230: * requests for the page. The place-holder page must ! 231: * be freed after the data_write or initialize message ! 232: * has been sent. If the page is copied, ! 233: * the holding page is VM_PAGE_NULL. ! 234: * ! 235: * The original page is put on a paging queue and marked ! 236: * not busy on exit. ! 237: */ ! 238: vm_page_t ! 239: vm_pageout_setup(m, paging_offset, new_object, new_offset, flush) ! 240: register vm_page_t m; ! 241: vm_offset_t paging_offset; ! 242: register vm_object_t new_object; ! 243: vm_offset_t new_offset; ! 244: boolean_t flush; ! 245: { ! 246: register vm_object_t old_object = m->object; ! 247: register vm_page_t holding_page = 0; /*'=0'to quiet gcc warnings*/ ! 248: register vm_page_t new_m; ! 249: ! 250: assert(m->busy && !m->absent && !m->fictitious); ! 251: ! 252: /* ! 253: * If we are not flushing the page, allocate a ! 254: * page in the object. If we cannot get the ! 255: * page, flush instead. ! 256: */ ! 257: if (!flush) { ! 258: vm_object_lock(new_object); ! 259: new_m = vm_page_alloc(new_object, new_offset); ! 260: if (new_m == VM_PAGE_NULL) ! 261: flush = TRUE; ! 262: vm_object_unlock(new_object); ! 263: } ! 264: ! 265: if (flush) { ! 266: /* ! 267: * Create a place-holder page where the old one was, ! 268: * to prevent anyone from attempting to page in this ! 269: * page while we`re unlocked. ! 270: */ ! 271: while ((holding_page = vm_page_grab_fictitious()) ! 272: == VM_PAGE_NULL) ! 273: vm_page_more_fictitious(); ! 274: ! 275: vm_object_lock(old_object); ! 276: vm_page_lock_queues(); ! 277: vm_page_remove(m); ! 278: vm_page_unlock_queues(); ! 279: PAGE_WAKEUP_DONE(m); ! 280: ! 281: vm_page_lock_queues(); ! 282: vm_page_insert(holding_page, old_object, m->offset); ! 283: vm_page_unlock_queues(); ! 284: ! 285: /* ! 286: * Record that this page has been written out ! 287: */ ! 288: #if MACH_PAGEMAP ! 289: vm_external_state_set(old_object->existence_info, ! 290: paging_offset, ! 291: VM_EXTERNAL_STATE_EXISTS); ! 292: #endif /* MACH_PAGEMAP */ ! 293: ! 294: vm_object_unlock(old_object); ! 295: ! 296: vm_object_lock(new_object); ! 297: ! 298: /* ! 299: * Move this page into the new object ! 300: */ ! 301: ! 302: vm_page_lock_queues(); ! 303: vm_page_insert(m, new_object, new_offset); ! 304: vm_page_unlock_queues(); ! 305: ! 306: m->dirty = TRUE; ! 307: m->precious = FALSE; ! 308: m->page_lock = VM_PROT_NONE; ! 309: m->unlock_request = VM_PROT_NONE; ! 310: } ! 311: else { ! 312: /* ! 313: * Copy the data into the new page, ! 314: * and mark the new page as clean. ! 315: */ ! 316: vm_page_copy(m, new_m); ! 317: ! 318: vm_object_lock(old_object); ! 319: m->dirty = FALSE; ! 320: pmap_clear_modify(m->phys_addr); ! 321: ! 322: /* ! 323: * Deactivate old page. ! 324: */ ! 325: vm_page_lock_queues(); ! 326: vm_page_deactivate(m); ! 327: vm_page_unlock_queues(); ! 328: ! 329: PAGE_WAKEUP_DONE(m); ! 330: ! 331: /* ! 332: * Record that this page has been written out ! 333: */ ! 334: ! 335: #if MACH_PAGEMAP ! 336: vm_external_state_set(old_object->existence_info, ! 337: paging_offset, ! 338: VM_EXTERNAL_STATE_EXISTS); ! 339: #endif /* MACH_PAGEMAP */ ! 340: ! 341: vm_object_unlock(old_object); ! 342: ! 343: vm_object_lock(new_object); ! 344: ! 345: /* ! 346: * Use the new page below. ! 347: */ ! 348: m = new_m; ! 349: m->dirty = TRUE; ! 350: assert(!m->precious); ! 351: PAGE_WAKEUP_DONE(m); ! 352: } ! 353: ! 354: /* ! 355: * Make the old page eligible for replacement again; if a ! 356: * user-supplied memory manager fails to release the page, ! 357: * it will be paged out again to the default memory manager. ! 358: * ! 359: * Note that pages written to the default memory manager ! 360: * must be wired down -- in return, it guarantees to free ! 361: * this page, rather than reusing it. ! 362: */ ! 363: ! 364: vm_page_lock_queues(); ! 365: vm_stat.pageouts++; ! 366: if (m->laundry) { ! 367: /* ! 368: * vm_pageout_scan is telling us to put this page ! 369: * at the front of the inactive queue, so it will ! 370: * be immediately paged out to the default pager. ! 371: */ ! 372: ! 373: assert(!old_object->internal); ! 374: m->laundry = FALSE; ! 375: ! 376: queue_enter_first(&vm_page_queue_inactive, m, ! 377: vm_page_t, pageq); ! 378: m->inactive = TRUE; ! 379: vm_page_inactive_count++; ! 380: } else if (old_object->internal) { ! 381: m->laundry = TRUE; ! 382: vm_page_laundry_count++; ! 383: ! 384: vm_page_wire(m); ! 385: } else ! 386: vm_page_activate(m); ! 387: vm_page_unlock_queues(); ! 388: ! 389: /* ! 390: * Since IPC operations may block, we drop locks now. ! 391: * [The placeholder page is busy, and we still have ! 392: * paging_in_progress incremented.] ! 393: */ ! 394: ! 395: vm_object_unlock(new_object); ! 396: ! 397: /* ! 398: * Return the placeholder page to simplify cleanup. ! 399: */ ! 400: return (flush ? holding_page : VM_PAGE_NULL); ! 401: } ! 402: ! 403: /* ! 404: * Routine: vm_pageout_page ! 405: * Purpose: ! 406: * Causes the specified page to be written back to ! 407: * the appropriate memory object. ! 408: * ! 409: * The "initial" argument specifies whether this ! 410: * data is an initialization only, and should use ! 411: * memory_object_data_initialize instead of ! 412: * memory_object_data_write. ! 413: * ! 414: * The "flush" argument specifies whether the page ! 415: * should be flushed from the object. If not, a ! 416: * copy of the data is sent to the memory object. ! 417: * ! 418: * In/out conditions: ! 419: * The page in question must not be on any pageout queues. ! 420: * The object to which it belongs must be locked. ! 421: * Implementation: ! 422: * Move this page to a completely new object, if flushing; ! 423: * copy to a new page in a new object, if not. ! 424: */ ! 425: void ! 426: vm_pageout_page(m, initial, flush) ! 427: register vm_page_t m; ! 428: boolean_t initial; ! 429: boolean_t flush; ! 430: { ! 431: vm_map_copy_t copy; ! 432: register vm_object_t old_object; ! 433: register vm_object_t new_object; ! 434: register vm_page_t holding_page; ! 435: vm_offset_t paging_offset; ! 436: kern_return_t rc; ! 437: boolean_t precious_clean; ! 438: ! 439: assert(m->busy); ! 440: ! 441: /* ! 442: * Cleaning but not flushing a clean precious page is a ! 443: * no-op. Remember whether page is clean and precious now ! 444: * because vm_pageout_setup will mark it dirty and not precious. ! 445: * ! 446: * XXX Check if precious_clean && !flush can really happen. ! 447: */ ! 448: precious_clean = (!m->dirty) && m->precious; ! 449: if (precious_clean && !flush) { ! 450: PAGE_WAKEUP_DONE(m); ! 451: return; ! 452: } ! 453: ! 454: /* ! 455: * Verify that we really want to clean this page. ! 456: */ ! 457: if (m->absent || m->error || (!m->dirty && !m->precious)) { ! 458: VM_PAGE_FREE(m); ! 459: return; ! 460: } ! 461: ! 462: /* ! 463: * Create a paging reference to let us play with the object. ! 464: */ ! 465: old_object = m->object; ! 466: paging_offset = m->offset + old_object->paging_offset; ! 467: vm_object_paging_begin(old_object); ! 468: vm_object_unlock(old_object); ! 469: ! 470: /* ! 471: * Allocate a new object into which we can put the page. ! 472: */ ! 473: new_object = vm_object_allocate(PAGE_SIZE); ! 474: ! 475: /* ! 476: * Move the page into the new object. ! 477: */ ! 478: holding_page = vm_pageout_setup(m, ! 479: paging_offset, ! 480: new_object, ! 481: 0, /* new offset */ ! 482: flush); /* flush */ ! 483: ! 484: rc = vm_map_copyin_object(new_object, 0, PAGE_SIZE, ©); ! 485: assert(rc == KERN_SUCCESS); ! 486: ! 487: if (initial || old_object->use_old_pageout) { ! 488: rc = (*(initial ? memory_object_data_initialize ! 489: : memory_object_data_write)) ! 490: (old_object->pager, ! 491: old_object->pager_request, ! 492: paging_offset, (pointer_t) copy, PAGE_SIZE); ! 493: } ! 494: else { ! 495: rc = memory_object_data_return( ! 496: old_object->pager, ! 497: old_object->pager_request, ! 498: paging_offset, (pointer_t) copy, PAGE_SIZE, ! 499: !precious_clean, !flush); ! 500: } ! 501: ! 502: if (rc != KERN_SUCCESS) ! 503: vm_map_copy_discard(copy); ! 504: ! 505: /* ! 506: * Clean up. ! 507: */ ! 508: vm_object_lock(old_object); ! 509: if (holding_page != VM_PAGE_NULL) ! 510: VM_PAGE_FREE(holding_page); ! 511: vm_object_paging_end(old_object); ! 512: } ! 513: ! 514: /* ! 515: * vm_pageout_scan does the dirty work for the pageout daemon. ! 516: * It returns with vm_page_queue_free_lock held and ! 517: * vm_page_free_wanted == 0. ! 518: */ ! 519: ! 520: void vm_pageout_scan() ! 521: { ! 522: unsigned int burst_count; ! 523: unsigned int want_pages; ! 524: ! 525: /* ! 526: * We want to gradually dribble pages from the active queue ! 527: * to the inactive queue. If we let the inactive queue get ! 528: * very small, and then suddenly dump many pages into it, ! 529: * those pages won't get a sufficient chance to be referenced ! 530: * before we start taking them from the inactive queue. ! 531: * ! 532: * We must limit the rate at which we send pages to the pagers. ! 533: * data_write messages consume memory, for message buffers and ! 534: * for map-copy objects. If we get too far ahead of the pagers, ! 535: * we can potentially run out of memory. ! 536: * ! 537: * We can use the laundry count to limit directly the number ! 538: * of pages outstanding to the default pager. A similar ! 539: * strategy for external pagers doesn't work, because ! 540: * external pagers don't have to deallocate the pages sent them, ! 541: * and because we might have to send pages to external pagers ! 542: * even if they aren't processing writes. So we also ! 543: * use a burst count to limit writes to external pagers. ! 544: * ! 545: * When memory is very tight, we can't rely on external pagers to ! 546: * clean pages. They probably aren't running, because they ! 547: * aren't vm-privileged. If we kept sending dirty pages to them, ! 548: * we could exhaust the free list. However, we can't just ignore ! 549: * pages belonging to external objects, because there might be no ! 550: * pages belonging to internal objects. Hence, we get the page ! 551: * into an internal object and then immediately double-page it, ! 552: * sending it to the default pager. ! 553: * ! 554: * consider_zone_gc should be last, because the other operations ! 555: * might return memory to zones. When we pause we use ! 556: * vm_pageout_scan_continue as our continuation, so we will ! 557: * reenter vm_pageout_scan periodically and attempt to reclaim ! 558: * internal memory even if we never reach vm_page_free_target. ! 559: */ ! 560: ! 561: Restart: ! 562: stack_collect(); ! 563: net_kmsg_collect(); ! 564: consider_lmm_collect(); ! 565: consider_task_collect(); ! 566: consider_thread_collect(); ! 567: consider_zone_gc(); ! 568: ! 569: for (burst_count = 0;;) { ! 570: register vm_page_t m; ! 571: register vm_object_t object; ! 572: unsigned int free_count; ! 573: ! 574: /* ! 575: * Recalculate vm_page_inactivate_target. ! 576: */ ! 577: ! 578: vm_page_lock_queues(); ! 579: vm_page_inactive_target = ! 580: VM_PAGE_INACTIVE_TARGET(vm_page_active_count + ! 581: vm_page_inactive_count); ! 582: ! 583: /* ! 584: * Move pages from active to inactive. ! 585: */ ! 586: ! 587: while ((vm_page_inactive_count < vm_page_inactive_target) && ! 588: !queue_empty(&vm_page_queue_active)) { ! 589: register vm_object_t obj; ! 590: ! 591: vm_pageout_active++; ! 592: m = (vm_page_t) queue_first(&vm_page_queue_active); ! 593: assert(m->active && !m->inactive); ! 594: ! 595: obj = m->object; ! 596: if (!vm_object_lock_try(obj)) { ! 597: /* ! 598: * Move page to end and continue. ! 599: */ ! 600: ! 601: queue_remove(&vm_page_queue_active, m, ! 602: vm_page_t, pageq); ! 603: queue_enter(&vm_page_queue_active, m, ! 604: vm_page_t, pageq); ! 605: vm_page_unlock_queues(); ! 606: vm_page_lock_queues(); ! 607: continue; ! 608: } ! 609: ! 610: /* ! 611: * If the page is busy, then we pull it ! 612: * off the active queue and leave it alone. ! 613: */ ! 614: ! 615: if (m->busy) { ! 616: vm_object_unlock(obj); ! 617: queue_remove(&vm_page_queue_active, m, ! 618: vm_page_t, pageq); ! 619: m->active = FALSE; ! 620: vm_page_active_count--; ! 621: continue; ! 622: } ! 623: ! 624: /* ! 625: * Deactivate the page while holding the object ! 626: * locked, so we know the page is still not busy. ! 627: * This should prevent races between pmap_enter ! 628: * and pmap_clear_reference. The page might be ! 629: * absent or fictitious, but vm_page_deactivate ! 630: * can handle that. ! 631: */ ! 632: ! 633: vm_page_deactivate(m); ! 634: vm_object_unlock(obj); ! 635: } ! 636: ! 637: /* ! 638: * We are done if we have met our targets *and* ! 639: * nobody is still waiting for a page. ! 640: */ ! 641: ! 642: simple_lock(&vm_page_queue_free_lock); ! 643: free_count = vm_page_free_count; ! 644: if ((free_count >= vm_page_free_target) && ! 645: (vm_page_external_count <= vm_page_external_target) && ! 646: (vm_page_free_wanted == 0)) { ! 647: vm_page_unlock_queues(); ! 648: break; ! 649: } ! 650: want_pages = ((free_count < vm_page_free_target) || ! 651: vm_page_free_wanted); ! 652: simple_unlock(&vm_page_queue_free_lock); ! 653: ! 654: /* ! 655: * Sometimes we have to pause: ! 656: * 1) No inactive pages - nothing to do. ! 657: * 2) Flow control - wait for pagers to catch up. ! 658: * 3) Extremely low memory - sending out dirty pages ! 659: * consumes memory. We don't take the risk of doing ! 660: * this if the default pager already has work to do. ! 661: */ ! 662: pause: ! 663: if (queue_empty(&vm_page_queue_inactive) || ! 664: (burst_count >= vm_pageout_burst_max) || ! 665: (vm_page_laundry_count >= vm_pageout_burst_max) || ! 666: ((free_count < vm_pageout_reserved_really) && ! 667: (vm_page_laundry_count > 0))) { ! 668: unsigned int pages, msecs; ! 669: ! 670: /* ! 671: * vm_pageout_burst_wait is msecs/page. ! 672: * If there is nothing for us to do, we wait ! 673: * at least vm_pageout_empty_wait msecs. ! 674: */ ! 675: ! 676: if (vm_page_laundry_count > burst_count) ! 677: pages = vm_page_laundry_count; ! 678: else ! 679: pages = burst_count; ! 680: msecs = pages * vm_pageout_burst_wait; ! 681: ! 682: if (queue_empty(&vm_page_queue_inactive) && ! 683: (msecs < vm_pageout_empty_wait)) ! 684: msecs = vm_pageout_empty_wait; ! 685: vm_page_unlock_queues(); ! 686: ! 687: thread_will_wait_with_timeout(current_thread(), msecs); ! 688: counter(c_vm_pageout_scan_block++); ! 689: thread_block(vm_pageout_scan_continue); ! 690: #ifndef CONTINUATIONS ! 691: /* ! 692: * Unfortunately, we don't have call_continuation ! 693: * so we can't rely on tail-recursion. ! 694: */ ! 695: ! 696: vm_pageout_scan_continue(); ! 697: goto Restart; ! 698: #else /* CONTINUATIONS */ ! 699: call_continuation(vm_pageout_scan_continue); ! 700: /*NOTREACHED*/ ! 701: #endif /* CONTINUATIONS */ ! 702: } ! 703: ! 704: vm_pageout_inactive++; ! 705: ! 706: /* Find a page we are interested in paging out. If we ! 707: need pages, then we'll page anything out; otherwise ! 708: we only page out external pages. */ ! 709: m = (vm_page_t) queue_first (&vm_page_queue_inactive); ! 710: while (1) ! 711: { ! 712: assert (!m->active && m->inactive); ! 713: if (want_pages || m->external) ! 714: break; ! 715: ! 716: m = (vm_page_t) queue_next (m); ! 717: if (!m) ! 718: goto pause; ! 719: } ! 720: ! 721: object = m->object; ! 722: ! 723: /* ! 724: * Try to lock object; since we've got the ! 725: * page queues lock, we can only try for this one. ! 726: */ ! 727: ! 728: if (!vm_object_lock_try(object)) { ! 729: /* ! 730: * Move page to end and continue. ! 731: */ ! 732: ! 733: queue_remove(&vm_page_queue_inactive, m, ! 734: vm_page_t, pageq); ! 735: queue_enter(&vm_page_queue_inactive, m, ! 736: vm_page_t, pageq); ! 737: vm_page_unlock_queues(); ! 738: vm_pageout_inactive_nolock++; ! 739: continue; ! 740: } ! 741: ! 742: /* ! 743: * Remove the page from the inactive list. ! 744: */ ! 745: ! 746: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq); ! 747: vm_page_inactive_count--; ! 748: m->inactive = FALSE; ! 749: ! 750: if (m->busy || !object->alive) { ! 751: /* ! 752: * Somebody is already playing with this page. ! 753: * Leave it off the pageout queues. ! 754: */ ! 755: ! 756: vm_page_unlock_queues(); ! 757: vm_object_unlock(object); ! 758: vm_pageout_inactive_busy++; ! 759: continue; ! 760: } ! 761: ! 762: /* ! 763: * If it's absent, we can reclaim the page. ! 764: */ ! 765: ! 766: if (want_pages && m->absent) { ! 767: vm_pageout_inactive_absent++; ! 768: reclaim_page: ! 769: vm_page_free(m); ! 770: vm_page_unlock_queues(); ! 771: vm_object_unlock(object); ! 772: continue; ! 773: } ! 774: ! 775: /* ! 776: * If it's being used, reactivate. ! 777: * (Fictitious pages are either busy or absent.) ! 778: */ ! 779: ! 780: assert(!m->fictitious); ! 781: if (m->reference || pmap_is_referenced(m->phys_addr)) { ! 782: vm_object_unlock(object); ! 783: vm_page_activate(m); ! 784: vm_stat.reactivations++; ! 785: vm_page_unlock_queues(); ! 786: vm_pageout_inactive_used++; ! 787: continue; ! 788: } ! 789: ! 790: /* ! 791: * Eliminate all mappings. ! 792: */ ! 793: ! 794: m->busy = TRUE; ! 795: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 796: if (!m->dirty) ! 797: m->dirty = pmap_is_modified(m->phys_addr); ! 798: ! 799: if (m->external) { ! 800: /* Figure out if we still care about this ! 801: page in the limit of externally managed pages. ! 802: Clean pages don't actually cause system hosage, ! 803: so it's ok to stop considering them as ! 804: "consumers" of memory. */ ! 805: if (m->dirty && !m->extcounted) { ! 806: m->extcounted = TRUE; ! 807: vm_page_external_count++; ! 808: } else if (!m->dirty && m->extcounted) { ! 809: m->extcounted = FALSE; ! 810: vm_page_external_count--; ! 811: } ! 812: } ! 813: ! 814: /* If we don't actually need more memory, and the page ! 815: is not dirty, put it on the tail of the inactive queue ! 816: and move on to the next page. */ ! 817: if (!want_pages && !m->dirty) { ! 818: queue_remove (&vm_page_queue_inactive, m, ! 819: vm_page_t, pageq); ! 820: queue_enter (&vm_page_queue_inactive, m, ! 821: vm_page_t, pageq); ! 822: vm_page_unlock_queues(); ! 823: vm_pageout_inactive_cleaned_external++; ! 824: continue; ! 825: } ! 826: ! 827: /* ! 828: * If it's clean and not precious, we can free the page. ! 829: */ ! 830: ! 831: if (!m->dirty && !m->precious) { ! 832: vm_pageout_inactive_clean++; ! 833: goto reclaim_page; ! 834: } ! 835: ! 836: /* ! 837: * If we are very low on memory, then we can't ! 838: * rely on an external pager to clean a dirty page, ! 839: * because external pagers are not vm-privileged. ! 840: * ! 841: * The laundry bit tells vm_pageout_setup to ! 842: * put the page back at the front of the inactive ! 843: * queue instead of activating the page. Hence, ! 844: * we will pick the page up again immediately and ! 845: * resend it to the default pager. ! 846: */ ! 847: ! 848: assert(!m->laundry); ! 849: if ((free_count < vm_pageout_reserved_internal) && ! 850: !object->internal) { ! 851: m->laundry = TRUE; ! 852: vm_pageout_inactive_double++; ! 853: } ! 854: vm_page_unlock_queues(); ! 855: ! 856: /* ! 857: * If there is no memory object for the page, create ! 858: * one and hand it to the default pager. ! 859: * [First try to collapse, so we don't create ! 860: * one unnecessarily.] ! 861: */ ! 862: ! 863: if (!object->pager_initialized) ! 864: vm_object_collapse(object); ! 865: if (!object->pager_initialized) ! 866: vm_object_pager_create(object); ! 867: if (!object->pager_initialized) ! 868: panic("vm_pageout_scan"); ! 869: ! 870: vm_pageout_inactive_dirty++; ! 871: vm_pageout_page(m, FALSE, TRUE); /* flush it */ ! 872: vm_object_unlock(object); ! 873: burst_count++; ! 874: } ! 875: } ! 876: ! 877: void vm_pageout_scan_continue() ! 878: { ! 879: /* ! 880: * We just paused to let the pagers catch up. ! 881: * If vm_page_laundry_count is still high, ! 882: * then we aren't waiting long enough. ! 883: * If we have paused some vm_pageout_pause_max times without ! 884: * adjusting vm_pageout_burst_wait, it might be too big, ! 885: * so we decrease it. ! 886: */ ! 887: ! 888: vm_page_lock_queues(); ! 889: if (vm_page_laundry_count > vm_pageout_burst_min) { ! 890: vm_pageout_burst_wait++; ! 891: vm_pageout_pause_count = 0; ! 892: } else if (++vm_pageout_pause_count > vm_pageout_pause_max) { ! 893: vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4; ! 894: if (vm_pageout_burst_wait < 1) ! 895: vm_pageout_burst_wait = 1; ! 896: vm_pageout_pause_count = 0; ! 897: } ! 898: vm_page_unlock_queues(); ! 899: ! 900: #ifdef CONTINUATIONS ! 901: vm_pageout_continue(); ! 902: /*NOTREACHED*/ ! 903: #endif /* CONTINUATIONS */ ! 904: } ! 905: ! 906: /* ! 907: * vm_pageout is the high level pageout daemon. ! 908: */ ! 909: ! 910: void vm_pageout_continue() ! 911: { ! 912: /* ! 913: * The pageout daemon is never done, so loop forever. ! 914: * We should call vm_pageout_scan at least once each ! 915: * time we are woken, even if vm_page_free_wanted is ! 916: * zero, to check vm_page_free_target and ! 917: * vm_page_inactive_target. ! 918: */ ! 919: ! 920: for (;;) { ! 921: vm_pageout_scan(); ! 922: /* we hold vm_page_queue_free_lock now */ ! 923: assert(vm_page_free_wanted == 0); ! 924: ! 925: assert_wait(&vm_page_free_wanted, FALSE); ! 926: simple_unlock(&vm_page_queue_free_lock); ! 927: counter(c_vm_pageout_block++); ! 928: thread_block(vm_pageout_continue); ! 929: } ! 930: } ! 931: ! 932: void vm_pageout() ! 933: { ! 934: int free_after_reserve; ! 935: ! 936: current_thread()->vm_privilege = TRUE; ! 937: stack_privilege(current_thread()); ! 938: ! 939: /* ! 940: * Initialize some paging parameters. ! 941: */ ! 942: ! 943: if (vm_pageout_burst_max == 0) ! 944: vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX; ! 945: ! 946: if (vm_pageout_burst_min == 0) ! 947: vm_pageout_burst_min = VM_PAGEOUT_BURST_MIN; ! 948: ! 949: if (vm_pageout_burst_wait == 0) ! 950: vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT; ! 951: ! 952: if (vm_pageout_empty_wait == 0) ! 953: vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT; ! 954: ! 955: if (vm_page_free_reserved == 0) ! 956: vm_page_free_reserved = VM_PAGE_FREE_RESERVED; ! 957: ! 958: if (vm_pageout_pause_max == 0) ! 959: vm_pageout_pause_max = VM_PAGEOUT_PAUSE_MAX; ! 960: ! 961: if (vm_pageout_reserved_internal == 0) ! 962: vm_pageout_reserved_internal = ! 963: VM_PAGEOUT_RESERVED_INTERNAL(vm_page_free_reserved); ! 964: ! 965: if (vm_pageout_reserved_really == 0) ! 966: vm_pageout_reserved_really = ! 967: VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved); ! 968: ! 969: free_after_reserve = vm_page_free_count - vm_page_free_reserved; ! 970: ! 971: if (vm_page_external_limit == 0) ! 972: vm_page_external_limit = ! 973: VM_PAGE_EXTERNAL_LIMIT (free_after_reserve); ! 974: ! 975: if (vm_page_external_target == 0) ! 976: vm_page_external_target = ! 977: VM_PAGE_EXTERNAL_TARGET (free_after_reserve); ! 978: ! 979: if (vm_page_free_min == 0) ! 980: vm_page_free_min = vm_page_free_reserved + ! 981: VM_PAGE_FREE_MIN(free_after_reserve); ! 982: ! 983: if (vm_page_free_target == 0) ! 984: vm_page_free_target = vm_page_free_reserved + ! 985: VM_PAGE_FREE_TARGET(free_after_reserve); ! 986: ! 987: if (vm_page_free_target < vm_page_free_min + 5) ! 988: vm_page_free_target = vm_page_free_min + 5; ! 989: ! 990: /* ! 991: * vm_pageout_scan will set vm_page_inactive_target. ! 992: */ ! 993: ! 994: vm_pageout_continue(); ! 995: /*NOTREACHED*/ ! 996: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.