![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to thread_swap.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / kern|
Return to thread_swap.c CVS log | Up to [Apple XNU] / XNU / osfmk / kern |
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * @OSF_COPYRIGHT@ ! 24: */ ! 25: /* ! 26: * Mach Operating System ! 27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University ! 28: * All Rights Reserved. ! 29: * ! 30: * Permission to use, copy, modify and distribute this software and its ! 31: * documentation is hereby granted, provided that both the copyright ! 32: * notice and this permission notice appear in all copies of the ! 33: * software, derivative works or modified versions, and any portions ! 34: * thereof, and that both notices appear in supporting documentation. ! 35: * ! 36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ! 38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 39: * ! 40: * Carnegie Mellon requests users of this software to return to ! 41: * ! 42: * Software Distribution Coordinator or [email protected] ! 43: * School of Computer Science ! 44: * Carnegie Mellon University ! 45: * Pittsburgh PA 15213-3890 ! 46: * ! 47: * any improvements or extensions that they make and grant Carnegie Mellon ! 48: * the rights to redistribute these changes. ! 49: */ ! 50: /* ! 51: */ ! 52: /* ! 53: * ! 54: * File: kern/thread_swap.c ! 55: * Author: Avadis Tevanian, Jr. ! 56: * Date: 1987 ! 57: * ! 58: * Mach thread swapper: ! 59: * Find idle threads to swap, freeing up kernel stack resources ! 60: * at the expense of allowing them to execute. ! 61: * ! 62: * Swap in threads that need to be run. This is done here ! 63: * by the swapper thread since it cannot be done (in general) ! 64: * when the kernel tries to place a thread on a run queue. ! 65: * ! 66: * Note: The act of swapping a thread in Mach does not mean that ! 67: * its memory gets forcibly swapped to secondary storage. The memory ! 68: * for the task corresponding to a swapped thread is paged out ! 69: * through the normal paging mechanism. ! 70: * ! 71: */ ! 72: #if 1 ! 73: ! 74: #include <kern/thread.h> ! 75: #include <kern/lock.h> ! 76: #include <vm/vm_map.h> ! 77: #include <vm/vm_kern.h> ! 78: #include <mach/vm_param.h> ! 79: #include <kern/sched_prim.h> ! 80: #include <kern/sf.h> ! 81: #include <kern/processor.h> ! 82: #include <kern/thread_swap.h> ! 83: #include <kern/spl.h> /* for splsched */ ! 84: #include <kern/misc_protos.h> ! 85: #include <kern/counters.h> ! 86: #include <mach/policy.h> ! 87: ! 88: queue_head_t swapin_queue; ! 89: decl_simple_lock_data(, swapper_lock_data) ! 90: ! 91: #define swapper_lock() simple_lock(&swapper_lock_data) ! 92: #define swapper_unlock() simple_unlock(&swapper_lock_data) ! 93: ! 94: mach_counter_t c_swapin_thread_block; ! 95: ! 96: /* ! 97: * swapper_init: [exported] ! 98: * ! 99: * Initialize the swapper module. ! 100: */ ! 101: void swapper_init() ! 102: { ! 103: queue_init(&swapin_queue); ! 104: simple_lock_init(&swapper_lock_data, ETAP_THREAD_SWAPPER); ! 105: } ! 106: ! 107: /* ! 108: * thread_swapin: [exported] ! 109: * ! 110: * Place the specified thread in the list of threads to swapin. It ! 111: * is assumed that the thread is locked, therefore we are at splsched. ! 112: * ! 113: * We don't bother with stack_alloc_try to optimize swapin; ! 114: * our callers have already tried that route. ! 115: */ ! 116: ! 117: void thread_swapin(thread) ! 118: thread_t thread; ! 119: { ! 120: switch (thread->state & TH_STACK_STATE) { ! 121: case TH_STACK_HANDOFF: ! 122: /* ! 123: * Swapped out - queue for swapin thread. ! 124: */ ! 125: thread->state = (thread->state & ~TH_STACK_STATE) ! 126: | TH_STACK_COMING_IN; ! 127: swapper_lock(); ! 128: enqueue_tail(&swapin_queue, (queue_entry_t) thread); ! 129: swapper_unlock(); ! 130: thread_wakeup((event_t) &swapin_queue); ! 131: break; ! 132: ! 133: case TH_STACK_COMING_IN: ! 134: /* ! 135: * Already queued for swapin thread, or being ! 136: * swapped in. ! 137: */ ! 138: break; ! 139: ! 140: default: ! 141: /* ! 142: * Already swapped in. ! 143: */ ! 144: panic("thread_swapin"); ! 145: } ! 146: } ! 147: ! 148: /* ! 149: * thread_doswapin: ! 150: * ! 151: * Swapin the specified thread, if it should be runnable, then put ! 152: * it on a run queue. No locks should be held on entry, as it is ! 153: * likely that this routine will sleep (waiting for stack allocation). ! 154: */ ! 155: void thread_doswapin(thread) ! 156: register thread_t thread; ! 157: { ! 158: spl_t s; ! 159: vm_offset_t stack; ! 160: ! 161: /* ! 162: * do machdep allocation ! 163: */ ! 164: ! 165: /* ! 166: * Allocate the kernel stack. ! 167: */ ! 168: stack = stack_alloc(thread, thread_continue); ! 169: assert(stack); ! 170: ! 171: /* ! 172: * Place on run queue. ! 173: */ ! 174: ! 175: s = splsched(); ! 176: thread_lock(thread); ! 177: thread->state &= ~(TH_STACK_HANDOFF | TH_STACK_COMING_IN); ! 178: if (thread->state & TH_RUN) ! 179: thread_setrun(thread, TRUE, FALSE); ! 180: thread_unlock(thread); ! 181: (void) splx(s); ! 182: } ! 183: ! 184: /* ! 185: * swapin_thread: [exported] ! 186: * ! 187: * This procedure executes as a kernel thread. Threads that need to ! 188: * be swapped in are swapped in by this thread. ! 189: */ ! 190: void swapin_thread_continue() ! 191: { ! 192: for (;;) { ! 193: register thread_t thread; ! 194: spl_t s; ! 195: ! 196: s = splsched(); ! 197: swapper_lock(); ! 198: ! 199: while ((thread = (thread_t) dequeue_head(&swapin_queue)) ! 200: != THREAD_NULL) { ! 201: swapper_unlock(); ! 202: (void) splx(s); ! 203: ! 204: thread_doswapin(thread); /* may block */ ! 205: ! 206: s = splsched(); ! 207: swapper_lock(); ! 208: } ! 209: ! 210: assert_wait((event_t) &swapin_queue, THREAD_UNINT); ! 211: swapper_unlock(); ! 212: (void) splx(s); ! 213: counter(c_swapin_thread_block++); ! 214: #if 1 ! 215: thread_block(swapin_thread_continue); ! 216: #else ! 217: thread_block((void (*)(void)) 0); ! 218: #endif ! 219: } ! 220: } ! 221: ! 222: void swapin_thread() ! 223: { ! 224: stack_privilege(current_thread()); ! 225: current_thread()->vm_privilege = TRUE; ! 226: ! 227: swapin_thread_continue(); ! 228: /*NOTREACHED*/ ! 229: } ! 230: ! 231: #else /* UNUSED CODE FOLLOWS */ ! 232: ! 233: #include <kern/thread.h> ! 234: #include <kern/lock.h> ! 235: #include <vm/vm_map.h> ! 236: #include <vm/vm_kern.h> ! 237: #include <mach/vm_param.h> ! 238: #include <kern/sched_prim.h> ! 239: #include <kern/sf.h> ! 240: #include <kern/processor.h> ! 241: #include <kern/thread_swap.h> ! 242: #include <kern/spl.h> /* for splsched */ ! 243: #include <kern/misc_protos.h> ! 244: #include <mach/policy.h> ! 245: ! 246: queue_head_t swapin_queue; ! 247: decl_simple_lock_data(,swapper_lock) ! 248: ! 249: #define swapper_lock() simple_lock(&swapper_lock) ! 250: #define swapper_unlock() simple_unlock(&swapper_lock) ! 251: ! 252: #define THREAD_SW_DEBUG 1 ! 253: int thread_swap_debug = 0; ! 254: ! 255: /* ! 256: * MAXSLP and MAX_SWAP_RATE can be overridden in vm_param.h, or by poking ! 257: * the global variable (lower-case) at any time. ! 258: * ! 259: * MAXSLP is the amount of time a thread will be allowed to sleep ! 260: * without being a candidate for idle thread swapping. ! 261: * ! 262: * MAX_SWAP_RATE is the number of scheduler ticks (see sched_prim.c), ! 263: * between periodic scans of the idle thread list. When the pageout ! 264: * daemon detects a severe page shortage, this rate is ignored. A ! 265: * scheduler tick is usually 1 second. ! 266: */ ! 267: #ifndef MAXSLP ! 268: #define MAXSLP 10 ! 269: #endif ! 270: ! 271: int maxslp = MAXSLP; ! 272: ! 273: #ifndef MAX_SWAP_RATE ! 274: #define MAX_SWAP_RATE 60 ! 275: #endif ! 276: ! 277: int max_swap_rate = MAX_SWAP_RATE; ! 278: ! 279: /* should we unwire the kernel stack when a thread is swapped out ? */ ! 280: #ifdef HP700___ /* CHECKME! Is this true anymore?? From Cambridge branch */ ! 281: /* ! 282: * On HP/PA, kernel stacks are ghost pages that are mapped at their ! 283: * physical address (physical addr == virtual address). They're not ! 284: * on any page list or VM object, so they can't be paged out. ! 285: */ ! 286: boolean_t thread_swap_unwire_stack = FALSE; ! 287: #else /* HP700 */ ! 288: boolean_t thread_swap_unwire_stack = TRUE; ! 289: #endif /* HP700 */ ! 290: ! 291: #if THREAD_SWAP_UNWIRE_USER_STACK ! 292: boolean_t thread_swap_unwire_user_stack = TRUE; ! 293: #else /* THREAD_SWAP_UNWIRE_USER_STACK */ ! 294: #define thread_swap_unwire_user_stack FALSE ! 295: #endif /* THREAD_SWAP_UNWIRE_USER_STACK */ ! 296: ! 297: /* ! 298: * thread_swapper_init: [exported] ! 299: * ! 300: * Initialize the swapper module. ! 301: */ ! 302: void thread_swapper_init(void) ! 303: { ! 304: queue_init(&swapin_queue); ! 305: simple_lock_init(&swapper_lock, ETAP_THREAD_SWAPPER); ! 306: } ! 307: ! 308: /* ! 309: * thread_swapin: [exported] ! 310: * ! 311: * Place the specified thread in the list of threads to swapin. If ! 312: * thr_act is associated with a thread, the thread is locked; if not, ! 313: * the thr_act itself (i.e., its act_lock()) is locked. Since ! 314: * swapper_lock() must be taken at splsched(), we go to splsched() ! 315: * ourself to cover the latter case. The make_unswappable argument ! 316: * is used to ask swapin_thread to make thread unswappable. ! 317: * ! 318: * The naked references to thr_act->thread here are safe purely based ! 319: * on the assumption that only an activation itself can remove itself ! 320: * from an RPC chain. Since its thread is "swapped out", the activation ! 321: * cannot be running. Similarly, the reference to thread->top_act ! 322: * in calls to this function from scheduler code is safe purely because ! 323: * an RPC chain can change only under its own steam (i.e., if the ! 324: * associated thread is running). ! 325: * ! 326: * XXX - Shuttle cloning would break the above assumptions. ! 327: */ ! 328: ! 329: void thread_swapin( ! 330: thread_act_t thr_act, ! 331: boolean_t make_unswappable) ! 332: { ! 333: spl_t s; ! 334: ! 335: switch (thr_act->swap_state & TH_SW_STATE) { ! 336: case TH_SW_OUT: ! 337: /* ! 338: * Swapped out - queue for swapin thread ! 339: */ ! 340: thr_act->swap_state = TH_SW_COMING_IN; ! 341: s = splsched(); ! 342: swapper_lock(); ! 343: queue_enter(&swapin_queue, thr_act, thread_act_t, swap_queue); ! 344: swapper_unlock(); ! 345: thread_wakeup((event_t) &swapin_queue); ! 346: splx(s); ! 347: break; ! 348: ! 349: case TH_SW_GOING_OUT: ! 350: /* ! 351: * Being swapped out - wait until swapped out, ! 352: * then queue for swapin thread (in thread_swapout). ! 353: */ ! 354: thr_act->swap_state = TH_SW_WANT_IN; ! 355: break; ! 356: ! 357: case TH_SW_WANT_IN: ! 358: case TH_SW_COMING_IN: ! 359: /* ! 360: * Already queued for swapin thread, or being ! 361: * swapped in ! 362: */ ! 363: break; ! 364: ! 365: default: ! 366: /* ! 367: * Swapped in or unswappable ! 368: */ ! 369: panic("thread_swapin"); ! 370: } ! 371: ! 372: /* ! 373: * Set make unswappable flag if asked to. swapin thread ! 374: * will make thread unswappable. ! 375: */ ! 376: if (make_unswappable) ! 377: thr_act->swap_state |= TH_SW_MAKE_UNSWAPPABLE; ! 378: } ! 379: ! 380: /* ! 381: * thread_swapin_blocking: [exported] ! 382: * ! 383: * Mimic thread_swapin(), but block if swapin of our target thread ! 384: * has already been initiated, and call thread_doswapin() directly ! 385: * (instead of via swapin thread) if we initiate it ourself. ! 386: * ! 387: * Locking: if thr_act is on an RPC chain, the rpc_lock() for the ! 388: * attached shuttle is held; otherwise both the act_lock() for ! 389: * thr_act plus the ip_lock() for its thread_pool port are held. ! 390: * NB: if thr_act is attached to a thread, thread_lock() controls ! 391: * access to thr_act's swap_state. ! 392: */ ! 393: boolean_t thread_swapin_blocking( ! 394: thread_act_t thr_act) ! 395: { ! 396: spl_t s; ! 397: thread_t thread, othread; ! 398: ! 399: #define lock_swap_state(thr_act) \ ! 400: MACRO_BEGIN \ ! 401: if (thr_act->thread) { \ ! 402: s = splsched(); \ ! 403: thread_lock(thr_act->thread); \ ! 404: } \ ! 405: MACRO_END ! 406: ! 407: #define unlock_swap_state(thr_act) \ ! 408: MACRO_BEGIN \ ! 409: if (thr_act->thread) { \ ! 410: thread_unlock(thr_act->thread); \ ! 411: splx(s); \ ! 412: } \ ! 413: MACRO_END ! 414: ! 415: for (thread = thr_act->thread; ; ) { ! 416: lock_swap_state(thr_act); ! 417: switch (thr_act->swap_state & TH_SW_STATE) { ! 418: case TH_SW_OUT: ! 419: /* ! 420: * Swapped all the way out -- start it back in. ! 421: */ ! 422: thr_act->swap_state = TH_SW_COMING_IN; ! 423: unlock_swap_state(thr_act); ! 424: if (thread) ! 425: rpc_unlock(thread); ! 426: else { ! 427: ip_unlock(thr_act->pool_port); ! 428: act_unlock(thr_act); ! 429: } ! 430: thread_doswapin(thr_act); ! 431: relock: ! 432: othread = thread; ! 433: thread = act_lock_thread(thr_act); ! 434: /* ! 435: * We may resume with a shuttle attached ! 436: * after blocking with no shuttle attached, ! 437: * but should never move between shuttles. ! 438: */ ! 439: if (othread && thread != othread) ! 440: panic("thread_swapin_blocking: act moved"); ! 441: if (othread) { ! 442: /* ! 443: * XXX - don't care about this ip_lock (taken ! 444: * by act_lock_thread() if pool_port non-null) ! 445: */ ! 446: if (thr_act->pool_port) ! 447: ip_unlock(thr_act->pool_port); ! 448: act_unlock(thr_act); ! 449: } ! 450: else if (thread) { ! 451: /* ! 452: * XXX - don't care about this rpc_lock (taken ! 453: * by act_lock_thread() if thread non-null) ! 454: */ ! 455: rpc_unlock(thr_act->thread); ! 456: } ! 457: break; ! 458: ! 459: case TH_SW_GOING_OUT: ! 460: /* ! 461: * Not out yet -- have to wait till it is, then queue ! 462: * it for swapin. ! 463: */ ! 464: thr_act->swap_state = TH_SW_WANT_IN; ! 465: ! 466: /* fall-through */ ! 467: ! 468: case TH_SW_WANT_IN: ! 469: case TH_SW_COMING_IN: ! 470: /* ! 471: * On its way back -- wait for it to get here. ! 472: */ ! 473: assert_wait((event_t)&thr_act->swap_state, THREAD_UNINT); ! 474: unlock_swap_state(thr_act); ! 475: if (thread) ! 476: rpc_unlock(thread); ! 477: else { ! 478: ip_unlock(thr_act->pool_port); ! 479: act_unlock(thr_act); ! 480: } ! 481: thread_block((void (*)(void))0); ! 482: goto relock; ! 483: break; ! 484: ! 485: default: ! 486: /* ! 487: * Swapped in or not swappable -- finally! ! 488: */ ! 489: unlock_swap_state(thr_act); ! 490: return (TRUE); ! 491: } ! 492: if (!thr_act->active) ! 493: break; /* out of loop -- don'tcha love C? */ ! 494: } ! 495: return (FALSE); ! 496: } ! 497: ! 498: #undef lock_swap_state ! 499: #undef unlock_swap_state ! 500: ! 501: /* ! 502: * thread_doswapin: [exported] ! 503: * ! 504: * Swapin the specified thread, if it should be runnable, then put ! 505: * it on a run queue. No locks should be held on entry, as it is ! 506: * likely that this routine will sleep (waiting for page faults). ! 507: */ ! 508: void thread_doswapin( ! 509: thread_act_t thr_act) ! 510: { ! 511: register vm_offset_t addr; ! 512: register int s; ! 513: kern_return_t kr; ! 514: thread_t thread; ! 515: sched_policy_t *policy; ! 516: sf_return_t sfr; ! 517: ! 518: /* ! 519: * Wire down the kernel stack. ! 520: */ ! 521: ! 522: if (thread_swap_unwire_stack && (thread = thr_act->thread)) { ! 523: assert(KERNEL_STACK_SIZE % page_size == 0); ! 524: addr = thread->kernel_stack; ! 525: /* ! 526: * This pagein can never fail ! 527: */ ! 528: #if THREAD_SW_DEBUG ! 529: if (thread_swap_debug) { ! 530: printf("thread_doswapin(%x): wiring stack %x\n", ! 531: thr_act, trunc_page(addr)); ! 532: } ! 533: #endif /* THREAD_SW_DEBUG */ ! 534: #if MACH_ASSERT ! 535: assert(thr_act->kernel_stack_swapped_in == FALSE); ! 536: thr_act->kernel_stack_swapped_in = TRUE; ! 537: #endif /* MACH_ASSERT */ ! 538: for (;;) { ! 539: kr = vm_map_wire(kernel_map, trunc_page(addr), ! 540: round_page(addr + KERNEL_STACK_SIZE), ! 541: VM_PROT_READ|VM_PROT_WRITE, ! 542: FALSE); ! 543: if (kr == KERN_SUCCESS) { ! 544: break; ! 545: } ! 546: if (kr != KERN_MEMORY_ERROR) { ! 547: printf("vm_map_wire returned 0x%x\n", kr); ! 548: panic("thread_doswapin"); ! 549: } ! 550: thread_block((void (*)(void)) 0); ! 551: } ! 552: vm_map_simplify(kernel_map, addr); ! 553: } ! 554: ! 555: #if THREAD_SWAP_UNWIRE_USER_STACK ! 556: if (thread_swap_unwire_user_stack && thr_act->user_stack) { ! 557: vm_offset_t user_stack; ! 558: long user_stack_size; /* signed ! */ ! 559: vm_offset_t start, end; ! 560: ! 561: user_stack = thr_act->user_stack; ! 562: user_stack_size = (long) thr_act->user_stack_size; ! 563: #if 0 ! 564: assert(thr_act->user_stack_size % page_size == 0); ! 565: #endif ! 566: #if THREAD_SW_DEBUG ! 567: if (thread_swap_debug) { ! 568: printf("thread_doswapin(%x): wiring user stack %x\n", ! 569: thr_act, trunc_page(user_stack)); ! 570: } ! 571: #endif /* THREAD_SW_DEBUG */ ! 572: #if MACH_ASSERT ! 573: assert(thr_act->user_stack_swapped_in == FALSE); ! 574: thr_act->user_stack_swapped_in = TRUE; ! 575: #endif /* MACH_ASSERT */ ! 576: assert(thr_act->map != VM_MAP_NULL); ! 577: if (user_stack_size > 0) { ! 578: /* stack grows up */ ! 579: start = trunc_page(user_stack); ! 580: end = round_page(user_stack + user_stack_size - 1); ! 581: } else { ! 582: /* stack grows down */ ! 583: start = trunc_page(user_stack + user_stack_size + 1); ! 584: end = round_page(user_stack); ! 585: } ! 586: for (;;) { ! 587: kr = vm_map_wire(thr_act->map, start, end, ! 588: VM_PROT_READ|VM_PROT_WRITE, ! 589: TRUE); ! 590: if (kr == KERN_SUCCESS) { ! 591: break; ! 592: } ! 593: if (kr != KERN_MEMORY_ERROR) { ! 594: printf("vm_map_wire(user) returned 0x%x\n", kr); ! 595: panic("thread_doswapin 2"); ! 596: } ! 597: thread_block((void (*)(void)) 0); ! 598: } ! 599: vm_map_simplify(thr_act->map, start); ! 600: } ! 601: #endif /* THREAD_SWAP_UNWIRE_USER_STACK */ ! 602: ! 603: /* ! 604: * Make unswappable and wake up waiting thread(s) if needed. ! 605: * Place on run queue if appropriate. ! 606: * ! 607: * This function only operates on threads that had a state ! 608: * of TH_SW_OUT at one time, in which case, there needs ! 609: * to be a reference on the task's resident thread count. ! 610: */ ! 611: ! 612: /* ! 613: * Increment the resident thread count for the task. ! 614: * This has no direct effect here, but serves as a counter ! 615: * for the pmap_collect call in thread_swapout. ! 616: */ ! 617: if (thr_act->task != TASK_NULL) { ! 618: mutex_lock(&thr_act->task->act_list_lock); ! 619: ++thr_act->task->res_act_count; ! 620: mutex_unlock(&thr_act->task->act_list_lock); ! 621: } ! 622: ! 623: thread = act_lock_thread(thr_act); ! 624: if (thread != THREAD_NULL) { ! 625: s = splsched(); ! 626: thread_lock(thread); ! 627: } ! 628: if (thr_act->swap_state & TH_SW_MAKE_UNSWAPPABLE) { ! 629: thr_act->swap_state = TH_SW_UNSWAPPABLE; ! 630: } else { ! 631: thr_act->swap_state = TH_SW_IN; ! 632: } ! 633: ! 634: if (thread != THREAD_NULL) { ! 635: if (thread->top_act == thr_act) { ! 636: thread->state &= ~TH_SWAPPED_OUT; ! 637: if (thread->state & TH_RUN) { ! 638: policy = &sched_policy[thread->policy]; ! 639: sfr = sched_policy->sp_ops.sp_thread_unblock( ! 640: policy, thread); ! 641: assert(sfr == SF_SUCCESS); ! 642: } ! 643: } ! 644: thread_unlock(thread); ! 645: (void) splx(s); ! 646: } ! 647: thread_wakeup((event_t)&thr_act->swap_state); ! 648: act_unlock_thread(thr_act); ! 649: } ! 650: ! 651: /* ! 652: * thread_swapout: [exported] ! 653: * ! 654: * Swap out the specified thread (unwire its kernel stack). ! 655: * The thread must already be marked as 'swapping out'. ! 656: * ! 657: * Decrement the resident thread count for the task, and if ! 658: * it goes to zero, call pmap_collect on the task's pmap. ! 659: * ! 660: * The thread has an extra reference (obtained by the caller) ! 661: * to keep it from being deallocated during swapout. ! 662: */ ! 663: void thread_swapout( ! 664: thread_act_t thr_act) ! 665: { ! 666: register vm_offset_t addr; ! 667: register boolean_t make_unswappable; ! 668: boolean_t collect; ! 669: int s; ! 670: kern_return_t kr; ! 671: thread_t thread; ! 672: ! 673: /* ! 674: * Thread is marked as swapped before we swap it out; if ! 675: * it is awakened while we are swapping it out, it will ! 676: * be put on the swapin list. ! 677: */ ! 678: ! 679: /* ! 680: * Notify the pcb module that it must update any ! 681: * hardware state associated with this thread. ! 682: */ ! 683: #if 0 /* XXX */ ! 684: pcb_synch(thread); ! 685: #endif ! 686: ! 687: /* ! 688: * Unwire the kernel stack. ! 689: * No need to lock the thread to check TH_STACK_HANDOFF and ! 690: * get its kernel_stack address, since the thread is blocked ! 691: * somewhere and can't restart without being swapped back in. ! 692: */ ! 693: ! 694: if (thread_swap_unwire_stack && (thread = thr_act->thread)) { ! 695: assert(KERNEL_STACK_SIZE % page_size == 0); ! 696: addr = thread->kernel_stack; ! 697: #if THREAD_SW_DEBUG ! 698: if (thread_swap_debug) { ! 699: printf("thread_swapout(%x):unwiring stack %x\n", ! 700: thr_act, trunc_page(addr)); ! 701: } ! 702: #endif /* THREAD_SW_DEBUG */ ! 703: #if MACH_ASSERT ! 704: assert(thr_act->kernel_stack_swapped_in == TRUE); ! 705: thr_act->kernel_stack_swapped_in = FALSE; ! 706: #endif /* MACH_ASSERT */ ! 707: kr = vm_map_unwire(kernel_map, trunc_page(addr), ! 708: round_page(addr + ! 709: KERNEL_STACK_SIZE), ! 710: FALSE); ! 711: if (kr != KERN_SUCCESS) { ! 712: printf("vm_map_unwire returned %x\n", kr); ! 713: panic("thread_swapout"); ! 714: } ! 715: vm_map_simplify(kernel_map, addr); ! 716: } ! 717: ! 718: #if THREAD_SWAP_UNWIRE_USER_STACK ! 719: if (thread_swap_unwire_user_stack && thr_act->user_stack) { ! 720: vm_offset_t user_stack; ! 721: long user_stack_size; /* signed ! */ ! 722: vm_offset_t start, end; ! 723: ! 724: user_stack = thr_act->user_stack; ! 725: user_stack_size = thr_act->user_stack_size; ! 726: #if 0 ! 727: assert(user_stack_size % page_size == 0); ! 728: #endif ! 729: #if THREAD_SW_DEBUG ! 730: if (thread_swap_debug) { ! 731: printf("thread_swapout(%x):unwiring user stack %x\n", ! 732: thr_act, trunc_page(user_stack)); ! 733: } ! 734: #endif /* THREAD_SW_DEBUG */ ! 735: #if MACH_ASSERT ! 736: assert(thr_act->user_stack_swapped_in == TRUE); ! 737: thr_act->user_stack_swapped_in = FALSE; ! 738: #endif /* MACH_ASSERT */ ! 739: assert(thr_act->map != VM_MAP_NULL); ! 740: if (user_stack_size > 0) { ! 741: /* stack grows up */ ! 742: start = trunc_page(user_stack); ! 743: end = round_page(user_stack + user_stack_size - 1); ! 744: } else { ! 745: /* stack grows down */ ! 746: start = trunc_page(user_stack + user_stack_size + 1); ! 747: end = round_page(user_stack); ! 748: } ! 749: kr = vm_map_unwire(thr_act->map, start, end, TRUE); ! 750: if (kr != KERN_SUCCESS) { ! 751: printf("vm_map_unwire(user_stack) returned %x\n", kr); ! 752: panic("thread_swapout 2"); ! 753: } ! 754: vm_map_simplify(thr_act->map, start); ! 755: } ! 756: #endif /* THREAD_SWAP_UNWIRE_USER_STACK */ ! 757: ! 758: /* ! 759: * Arrange to call pmap_collect if this is the last resident ! 760: * thread for its task. The task and its map aren't going ! 761: * anywhere because of the extra reference on this thread. ! 762: */ ! 763: mutex_lock(&thr_act->task->act_list_lock); ! 764: collect = (--thr_act->task->res_act_count == 0); ! 765: mutex_unlock(&thr_act->task->act_list_lock); ! 766: ! 767: thread = act_lock_thread(thr_act); ! 768: s = splsched(); ! 769: if (thread) ! 770: thread_lock(thread); ! 771: switch (thr_act->swap_state & TH_SW_STATE) { ! 772: case TH_SW_GOING_OUT: ! 773: thr_act->swap_state = TH_SW_OUT; ! 774: break; ! 775: ! 776: case TH_SW_WANT_IN: ! 777: /* didn't get it out fast enough */ ! 778: make_unswappable = thr_act->swap_state & TH_SW_MAKE_UNSWAPPABLE; ! 779: thr_act->swap_state = TH_SW_OUT; ! 780: thread_swapin(thr_act, make_unswappable); ! 781: collect = FALSE; /* don't pmap_collect */ ! 782: break; ! 783: ! 784: default: ! 785: panic("thread_swapout"); ! 786: } ! 787: if (thread) ! 788: thread_unlock(thread); ! 789: splx(s); ! 790: act_unlock_thread(thr_act); ! 791: if (collect) { ! 792: /* task and map can't disappear yet */ ! 793: assert((thr_act->task != TASK_NULL) && ! 794: (thr_act->task->map != VM_MAP_NULL)); ! 795: pmap_collect(vm_map_pmap(thr_act->task->map)); ! 796: } ! 797: } ! 798: ! 799: /* ! 800: * swapin_thread: [exported] ! 801: * ! 802: * This procedure executes as a kernel thread. Threads that need to ! 803: * be swapped in are swapped in by this thread. ! 804: */ ! 805: void swapin_thread(void) ! 806: { ! 807: thread_swappable(current_act(), FALSE); ! 808: stack_privilege(current_thread()); ! 809: ! 810: while (TRUE) { ! 811: register thread_act_t thr_act; ! 812: register int s; ! 813: ! 814: s = splsched(); ! 815: swapper_lock(); ! 816: ! 817: while (! queue_empty(&swapin_queue)) { ! 818: queue_remove_first(&swapin_queue, thr_act, thread_act_t, ! 819: swap_queue); ! 820: swapper_unlock(); ! 821: splx(s); ! 822: ! 823: thread_doswapin(thr_act); ! 824: ! 825: s = splsched(); ! 826: swapper_lock(); ! 827: } ! 828: ! 829: assert_wait((event_t) &swapin_queue, THREAD_UNINT); ! 830: swapper_unlock(); ! 831: splx(s); ! 832: thread_block((void (*)(void)) 0); ! 833: } ! 834: } ! 835: ! 836: boolean_t thread_swapout_allowed = TRUE; ! 837: ! 838: int last_swap_tick = 0; ! 839: ! 840: /* ! 841: * swapout_threads: [exported] ! 842: * ! 843: * This procedure is called periodically by the pageout daemon. It ! 844: * determines if it should scan for threads to swap and starts that ! 845: * scan if appropriate. ! 846: * The pageout daemon sets the now flag if all the page queues are ! 847: * empty and it wants to start the swapper right away. ! 848: */ ! 849: void swapout_threads( ! 850: boolean_t now) ! 851: { ! 852: if (thread_swapout_allowed && ! 853: (now || (sched_tick > (last_swap_tick + max_swap_rate)))) { ! 854: last_swap_tick = sched_tick; ! 855: thread_wakeup((event_t) &last_swap_tick); /* poke swapper */ ! 856: thread_block((void (*)(void)) 0);/* let it run if it wants to */ ! 857: } ! 858: } ! 859: ! 860: boolean_t thread_swapout_empty_acts = FALSE; ! 861: ! 862: void swapout_scan(void); /* forward */ ! 863: ! 864: /* ! 865: * swapout_scan: ! 866: * ! 867: * Scan the list of all threads looking for threads to swap. ! 868: */ ! 869: void swapout_scan(void) ! 870: { ! 871: register int s; ! 872: register thread_t thread, prev_thread; ! 873: processor_set_t pset, prev_pset; ! 874: thread_act_t thr_act, prev_act; ! 875: task_t task, prev_task; ! 876: ! 877: zone_gc(); ! 878: ! 879: prev_thread = THREAD_NULL; ! 880: prev_act = THR_ACT_NULL; ! 881: prev_task = TASK_NULL; ! 882: prev_pset = PROCESSOR_SET_NULL; ! 883: /* ! 884: * Ugh. If an activation is terminated while being swapped ! 885: * out, we can no longer count on consistency of the lists ! 886: * we're traversing, so we have to start over. ! 887: * ! 888: * Note: we deliberately leave the "prev" variables alone if ! 889: * we restart -- we will remove a ref to them as needed in ! 890: * the normal operation of the loop. ! 891: */ ! 892: restart: ! 893: mutex_lock(&all_psets_lock); ! 894: pset = (processor_set_t) queue_first(&all_psets); ! 895: while (!queue_end(&all_psets, (queue_entry_t) pset)) { ! 896: pset_lock(pset); ! 897: task = (task_t) queue_first(&pset->tasks); ! 898: while (!queue_end(&pset->tasks, (queue_entry_t) task)) { ! 899: task_lock(task); ! 900: thr_act = (thread_act_t) queue_first(&task->thr_acts); ! 901: while (!queue_end(&task->thr_acts, ! 902: (queue_entry_t) thr_act)) { ! 903: boolean_t swap_it; ! 904: thread_act_t act; ! 905: ! 906: thread = act_lock_thread(thr_act); ! 907: s = splsched(); ! 908: if (thread != THREAD_NULL) { ! 909: thread_lock(thread); ! 910: swap_it = TRUE; ! 911: for (act = thread->top_act; act; act = act->lower) ! 912: if (act->swap_state == TH_SW_UNSWAPPABLE) { ! 913: swap_it = FALSE; ! 914: break; ! 915: } ! 916: if (!swap_it) ! 917: ; ! 918: else if ( ! 919: /* don't swap real-time threads */ ! 920: (thread->policy & ! 921: POLICYCLASS_FIXEDPRI) != 0 || ! 922: (thread->state & ! 923: (TH_RUN|TH_SWAPPED_OUT)) != 0 || ! 924: thr_act->swap_state != TH_SW_IN || ! 925: (thread->state & TH_UNINT) != 0 || ! 926: (sched_tick-thread->sleep_stamp <= ! 927: maxslp) || ! 928: !thr_act->active) { ! 929: swap_it = FALSE; ! 930: } ! 931: } else { ! 932: if (! thread_swapout_empty_acts) { ! 933: splx(s); ! 934: act_unlock_thread(thr_act); ! 935: thr_act = (thread_act_t) ! 936: queue_next(&thr_act->thr_acts); ! 937: continue; ! 938: } ! 939: if (thr_act->swap_state == TH_SW_IN && ! 940: thr_act->active && ! 941: thr_act->thread_pool_next) { ! 942: swap_it = TRUE; ! 943: } else { ! 944: swap_it = FALSE; ! 945: } ! 946: } ! 947: if (swap_it) { ! 948: thr_act->swap_state = TH_SW_GOING_OUT; ! 949: if (thread != THREAD_NULL) { ! 950: if (thread->top_act == ! 951: thr_act) { ! 952: thread->state |= ! 953: TH_SWAPPED_OUT; ! 954: } ! 955: thread->ref_count++; ! 956: thread_unlock(thread); ! 957: } ! 958: act_locked_act_reference(thr_act); ! 959: (void) splx(s); ! 960: act_unlock_thread(thr_act); ! 961: task->ref_count++; ! 962: task_unlock(task); ! 963: pset->ref_count++; ! 964: pset_unlock(pset); ! 965: mutex_unlock(&all_psets_lock); ! 966: ! 967: thread_swapout(thr_act); /* swap it */ ! 968: ! 969: if (prev_thread != THREAD_NULL) { ! 970: thread_deallocate(prev_thread); ! 971: } ! 972: if (prev_act != THR_ACT_NULL) { ! 973: act_deallocate(prev_act); ! 974: } ! 975: if (prev_task != TASK_NULL) { ! 976: task_deallocate(prev_task); ! 977: } ! 978: if (prev_pset != PROCESSOR_SET_NULL) ! 979: pset_deallocate(prev_pset); ! 980: ! 981: prev_thread = thread; ! 982: prev_act = thr_act; ! 983: prev_task = task; ! 984: prev_pset = pset; ! 985: mutex_lock(&all_psets_lock); ! 986: pset_lock(pset); ! 987: task_lock(task); ! 988: thread = act_lock_thread(thr_act); ! 989: /* ! 990: * See if current act was terminated; if so, ! 991: * can't continue with current traversal -- ! 992: * we don't have a valid "next" pointer, and ! 993: * there's no certain way to acquire one. ! 994: * ! 995: * Also check for task having moved between ! 996: * psets; don't want to continue current ! 997: * traversal if it has. (The traversal would ! 998: * be valid in this case, just not complete.) ! 999: */ ! 1000: if (!thr_act->active || task->processor_set != pset) { ! 1001: act_unlock_thread(thr_act); ! 1002: task_unlock(task); ! 1003: pset_unlock(pset); ! 1004: mutex_unlock(&all_psets_lock); ! 1005: goto restart; ! 1006: } ! 1007: ! 1008: s = splsched(); ! 1009: assert(!prev_thread || thread == prev_thread); ! 1010: } else { ! 1011: if (thread != THREAD_NULL) ! 1012: thread_unlock(thread); ! 1013: } ! 1014: splx(s); ! 1015: act_unlock_thread(thr_act); ! 1016: thr_act = (thread_act_t) ! 1017: queue_next(&thr_act->thr_acts); ! 1018: } ! 1019: task_unlock(task); ! 1020: task = (task_t) queue_next(&task->pset_tasks); ! 1021: } ! 1022: pset_unlock(pset); ! 1023: pset = (processor_set_t) queue_next(&pset->all_psets); ! 1024: } ! 1025: mutex_unlock(&all_psets_lock); ! 1026: ! 1027: if (prev_thread != THREAD_NULL) { ! 1028: thread_deallocate(prev_thread); ! 1029: } ! 1030: if (prev_act != THR_ACT_NULL) { ! 1031: act_deallocate(prev_act); ! 1032: } ! 1033: if (prev_task != TASK_NULL) { ! 1034: task_deallocate(prev_task); ! 1035: } ! 1036: if (prev_pset != PROCESSOR_SET_NULL) ! 1037: pset_deallocate(prev_pset); ! 1038: } ! 1039: ! 1040: /* ! 1041: * swapout_thread: [exported] ! 1042: * ! 1043: * Executes as a separate kernel thread. This thread is periodically ! 1044: * woken up. When this happens, it initiates the scan for threads ! 1045: * to swap. ! 1046: */ ! 1047: void swapout_thread(void) ! 1048: { ! 1049: thread_swappable(current_act(), FALSE); ! 1050: stack_privilege(current_thread()); ! 1051: ! 1052: spllo(); ! 1053: while (TRUE) { ! 1054: swapout_scan(); ! 1055: ! 1056: assert_wait((event_t)&last_swap_tick, THREAD_UNINT); ! 1057: thread_block((void (*) (void)) 0); ! 1058: } ! 1059: } ! 1060: ! 1061: /* ! 1062: * Mark a thread as swappable or unswappable. May be called at ! 1063: * any time. No longer assumes thread is swapped in. Frees ! 1064: * kernel stack backing store when a kernel thread is made ! 1065: * unswappable. Panics if a kernel thread is subsequently made ! 1066: * swappable. ! 1067: */ ! 1068: void thread_swappable( ! 1069: thread_act_t thr_act, ! 1070: boolean_t is_swappable) ! 1071: { ! 1072: int s; ! 1073: thread_t thread; ! 1074: ! 1075: thread = act_lock_thread(thr_act); ! 1076: if (thread) { ! 1077: s = splsched(); ! 1078: thread_lock(thread); ! 1079: } ! 1080: if (is_swappable) { ! 1081: if (thr_act->swap_state == TH_SW_UNSWAPPABLE) { ! 1082: if (thr_act->task == kernel_task) { ! 1083: panic("thread_swappable"); ! 1084: } ! 1085: thr_act->swap_state = TH_SW_IN; ! 1086: } ! 1087: } ! 1088: else { ! 1089: switch(thr_act->swap_state) { ! 1090: case TH_SW_UNSWAPPABLE: ! 1091: /* ! 1092: * Thread is already unswappable, won't need to ! 1093: * free backing store. ! 1094: */ ! 1095: is_swappable = TRUE; ! 1096: break; ! 1097: ! 1098: case TH_SW_IN: ! 1099: thr_act->swap_state = TH_SW_UNSWAPPABLE; ! 1100: break; ! 1101: ! 1102: default: ! 1103: do { ! 1104: thread_swapin(thr_act, TRUE); ! 1105: assert_wait((event_t) &thr_act->swap_state, THREAD_UNINT); ! 1106: if (thread) { ! 1107: thread_unlock(thread); ! 1108: splx(s); ! 1109: } ! 1110: act_unlock_thread(thr_act); ! 1111: thread_block((void (*)(void))0); ! 1112: thread = act_lock_thread(thr_act); ! 1113: if (thread) { ! 1114: s = splsched(); ! 1115: thread_lock(thread); ! 1116: } ! 1117: } while (thr_act->swap_state != TH_SW_UNSWAPPABLE); ! 1118: break; ! 1119: } ! 1120: } ! 1121: if (thread) { ! 1122: thread_unlock(thread); ! 1123: splx(s); ! 1124: } ! 1125: act_unlock_thread(thr_act); ! 1126: ! 1127: /* ! 1128: * Deallocate kernel stack backing store for any ! 1129: * kernel thread made unswappable. ! 1130: */ ! 1131: if (!is_swappable && thr_act->task == kernel_task) { ! 1132: #if 0 ! 1133: stack_backing_free(thr_act->kernel_stack, KERNEL_STACK_SIZE); ! 1134: #endif ! 1135: } ! 1136: } ! 1137: ! 1138: int thread_swap_disable_swapins = 0; ! 1139: ! 1140: void ! 1141: thread_swap_disable( ! 1142: thread_act_t thr_act) ! 1143: { ! 1144: spl_t s; ! 1145: thread_t thread; ! 1146: ! 1147: /* ! 1148: * Note: we have to swapin the thread only to make sure ! 1149: * that its stacks are wired when we free them. ! 1150: */ ! 1151: thread = act_lock_thread(thr_act); ! 1152: if (thread) { ! 1153: s = splsched(); ! 1154: thread_lock(thread); ! 1155: } ! 1156: if ((thread_swap_unwire_stack || thread_swap_unwire_user_stack) && ! 1157: (thr_act->swap_state != TH_SW_UNSWAPPABLE)) { ! 1158: /* ! 1159: * Make the activation unswappable or it might get swapped ! 1160: * before we complete its termination. ! 1161: */ ! 1162: if (thr_act->swap_state == TH_SW_IN || ! 1163: thr_act->swap_state == (TH_SW_IN|TH_SW_TASK_SWAPPING)) { ! 1164: thr_act->swap_state = TH_SW_UNSWAPPABLE; ! 1165: if (thread) { ! 1166: thread_unlock(thread); ! 1167: splx(s); ! 1168: } ! 1169: act_unlock_thread(thr_act); ! 1170: } else { ! 1171: thr_act->swap_state |= TH_SW_MAKE_UNSWAPPABLE; ! 1172: if (thread) { ! 1173: thread_unlock(thread); ! 1174: splx(s); ! 1175: } ! 1176: act_unlock_thread(thr_act); ! 1177: thread_swap_disable_swapins++; ! 1178: thread_doswapin(thr_act); ! 1179: } ! 1180: ! 1181: assert(thr_act->swap_state == TH_SW_UNSWAPPABLE); ! 1182: } else { ! 1183: if (thread) { ! 1184: thread_unlock(thread); ! 1185: splx(s); ! 1186: } ! 1187: act_unlock_thread(thr_act); ! 1188: } ! 1189: } ! 1190: ! 1191: #endif /* UNUSED CODE */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.