![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_page.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_page.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Copyright (c) 2010-2014 Richard Braun.
3: *
4: * This program is free software: you can redistribute it and/or modify
5: * it under the terms of the GNU General Public License as published by
6: * the Free Software Foundation, either version 2 of the License, or
7: * (at your option) any later version.
8: *
9: * This program is distributed in the hope that it will be useful,
10: * but WITHOUT ANY WARRANTY; without even the implied warranty of
11: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12: * GNU General Public License for more details.
13: *
14: * You should have received a copy of the GNU General Public License
15: * along with this program. If not, see <http://www.gnu.org/licenses/>.
16: *
17: *
18: * This implementation uses the binary buddy system to manage its heap.
19: * Descriptions of the buddy system can be found in the following works :
20: * - "UNIX Internals: The New Frontiers", by Uresh Vahalia.
21: * - "Dynamic Storage Allocation: A Survey and Critical Review",
22: * by Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles.
23: *
24: * In addition, this allocator uses per-CPU pools of pages for order 0
25: * (i.e. single page) allocations. These pools act as caches (but are named
26: * differently to avoid confusion with CPU caches) that reduce contention on
27: * multiprocessor systems. When a pool is empty and cannot provide a page,
28: * it is filled by transferring multiple pages from the backend buddy system.
29: * The symmetric case is handled likewise.
1.1.1.2 ! root 30: *
! 31: * TODO Limit number of dirty pages, block allocations above a top limit.
1.1 root 32: */
33:
34: #include <string.h>
35: #include <kern/assert.h>
1.1.1.2 ! root 36: #include <kern/counters.h>
1.1 root 37: #include <kern/cpu_number.h>
38: #include <kern/debug.h>
39: #include <kern/list.h>
40: #include <kern/lock.h>
41: #include <kern/macros.h>
42: #include <kern/printf.h>
43: #include <kern/thread.h>
44: #include <mach/vm_param.h>
45: #include <machine/pmap.h>
46: #include <sys/types.h>
47: #include <vm/vm_page.h>
1.1.1.2 ! root 48: #include <vm/vm_pageout.h>
! 49:
! 50: #define DEBUG 0
1.1 root 51:
52: #define __init
53: #define __initdata
54: #define __read_mostly
55:
56: #define thread_pin()
57: #define thread_unpin()
58:
59: /*
60: * Number of free block lists per segment.
61: */
62: #define VM_PAGE_NR_FREE_LISTS 11
63:
64: /*
65: * The size of a CPU pool is computed by dividing the number of pages in its
66: * containing segment by this value.
67: */
68: #define VM_PAGE_CPU_POOL_RATIO 1024
69:
70: /*
71: * Maximum number of pages in a CPU pool.
72: */
73: #define VM_PAGE_CPU_POOL_MAX_SIZE 128
74:
75: /*
76: * The transfer size of a CPU pool is computed by dividing the pool size by
77: * this value.
78: */
79: #define VM_PAGE_CPU_POOL_TRANSFER_RATIO 2
80:
81: /*
82: * Per-processor cache of pages.
83: */
84: struct vm_page_cpu_pool {
85: simple_lock_data_t lock;
86: int size;
87: int transfer_size;
88: int nr_pages;
89: struct list pages;
90: } __aligned(CPU_L1_SIZE);
91:
92: /*
93: * Special order value for pages that aren't in a free list. Such pages are
94: * either allocated, or part of a free block of pages but not the head page.
95: */
96: #define VM_PAGE_ORDER_UNLISTED ((unsigned short)-1)
97:
98: /*
99: * Doubly-linked list of free blocks.
100: */
101: struct vm_page_free_list {
102: unsigned long size;
103: struct list blocks;
104: };
105:
106: /*
1.1.1.2 ! root 107: * XXX Because of a potential deadlock involving the default pager (see
! 108: * vm_map_lock()), it's currently impossible to reliably determine the
! 109: * minimum number of free pages required for successful pageout. Since
! 110: * that process is dependent on the amount of physical memory, we scale
! 111: * the minimum number of free pages from it, in the hope that memory
! 112: * exhaustion happens as rarely as possible...
! 113: */
! 114:
! 115: /*
! 116: * Ratio used to compute the minimum number of pages in a segment.
! 117: */
! 118: #define VM_PAGE_SEG_THRESHOLD_MIN_NUM 5
! 119: #define VM_PAGE_SEG_THRESHOLD_MIN_DENOM 100
! 120:
! 121: /*
! 122: * Number of pages reserved for privileged allocations in a segment.
! 123: */
! 124: #define VM_PAGE_SEG_THRESHOLD_MIN 500
! 125:
! 126: /*
! 127: * Ratio used to compute the threshold below which pageout is started.
! 128: */
! 129: #define VM_PAGE_SEG_THRESHOLD_LOW_NUM 6
! 130: #define VM_PAGE_SEG_THRESHOLD_LOW_DENOM 100
! 131:
! 132: /*
! 133: * Minimum value the low threshold can have for a segment.
! 134: */
! 135: #define VM_PAGE_SEG_THRESHOLD_LOW 600
! 136:
! 137: #if VM_PAGE_SEG_THRESHOLD_LOW <= VM_PAGE_SEG_THRESHOLD_MIN
! 138: #error VM_PAGE_SEG_THRESHOLD_LOW invalid
! 139: #endif /* VM_PAGE_SEG_THRESHOLD_LOW >= VM_PAGE_SEG_THRESHOLD_MIN */
! 140:
! 141: /*
! 142: * Ratio used to compute the threshold above which pageout is stopped.
! 143: */
! 144: #define VM_PAGE_SEG_THRESHOLD_HIGH_NUM 10
! 145: #define VM_PAGE_SEG_THRESHOLD_HIGH_DENOM 100
! 146:
! 147: /*
! 148: * Minimum value the high threshold can have for a segment.
! 149: */
! 150: #define VM_PAGE_SEG_THRESHOLD_HIGH 1000
! 151:
! 152: #if VM_PAGE_SEG_THRESHOLD_HIGH <= VM_PAGE_SEG_THRESHOLD_LOW
! 153: #error VM_PAGE_SEG_THRESHOLD_HIGH invalid
! 154: #endif /* VM_PAGE_SEG_THRESHOLD_HIGH <= VM_PAGE_SEG_THRESHOLD_LOW */
! 155:
! 156: /*
! 157: * Minimum number of pages allowed for a segment.
! 158: */
! 159: #define VM_PAGE_SEG_MIN_PAGES 2000
! 160:
! 161: #if VM_PAGE_SEG_MIN_PAGES <= VM_PAGE_SEG_THRESHOLD_HIGH
! 162: #error VM_PAGE_SEG_MIN_PAGES invalid
! 163: #endif /* VM_PAGE_SEG_MIN_PAGES <= VM_PAGE_SEG_THRESHOLD_HIGH */
! 164:
! 165: /*
! 166: * Ratio used to compute the threshold of active pages beyond which
! 167: * to refill the inactive queue.
! 168: */
! 169: #define VM_PAGE_HIGH_ACTIVE_PAGE_NUM 1
! 170: #define VM_PAGE_HIGH_ACTIVE_PAGE_DENOM 3
! 171:
! 172: /*
! 173: * Page cache queue.
! 174: *
! 175: * XXX The current implementation hardcodes a preference to evict external
! 176: * pages first and keep internal ones as much as possible. This is because
! 177: * the Hurd default pager implementation suffers from bugs that can easily
! 178: * cause the system to freeze.
! 179: */
! 180: struct vm_page_queue {
! 181: struct list internal_pages;
! 182: struct list external_pages;
! 183: };
! 184:
! 185: /*
1.1 root 186: * Segment name buffer size.
187: */
188: #define VM_PAGE_NAME_SIZE 16
189:
190: /*
191: * Segment of contiguous memory.
1.1.1.2 ! root 192: *
! 193: * XXX Per-segment locking is probably useless, since one or both of the
! 194: * page queues lock and the free page queue lock is held on any access.
! 195: * However it should first be made clear which lock protects access to
! 196: * which members of a segment.
1.1 root 197: */
198: struct vm_page_seg {
199: struct vm_page_cpu_pool cpu_pools[NCPUS];
200:
201: phys_addr_t start;
202: phys_addr_t end;
203: struct vm_page *pages;
204: struct vm_page *pages_end;
205: simple_lock_data_t lock;
206: struct vm_page_free_list free_lists[VM_PAGE_NR_FREE_LISTS];
207: unsigned long nr_free_pages;
1.1.1.2 ! root 208:
! 209: /* Free memory thresholds */
! 210: unsigned long min_free_pages; /* Privileged allocations only */
! 211: unsigned long low_free_pages; /* Pageout daemon starts scanning */
! 212: unsigned long high_free_pages; /* Pageout daemon stops scanning,
! 213: unprivileged allocations resume */
! 214:
! 215: /* Page cache related data */
! 216: struct vm_page_queue active_pages;
! 217: unsigned long nr_active_pages;
! 218: unsigned long high_active_pages;
! 219: struct vm_page_queue inactive_pages;
! 220: unsigned long nr_inactive_pages;
1.1 root 221: };
222:
223: /*
224: * Bootstrap information about a segment.
225: */
226: struct vm_page_boot_seg {
227: phys_addr_t start;
228: phys_addr_t end;
1.1.1.2 ! root 229: boolean_t heap_present;
1.1 root 230: phys_addr_t avail_start;
231: phys_addr_t avail_end;
232: };
233:
234: static int vm_page_is_ready __read_mostly;
235:
236: /*
237: * Segment table.
238: *
239: * The system supports a maximum of 4 segments :
240: * - DMA: suitable for DMA
241: * - DMA32: suitable for DMA when devices support 32-bits addressing
242: * - DIRECTMAP: direct physical mapping, allows direct access from
243: * the kernel with a simple offset translation
244: * - HIGHMEM: must be mapped before it can be accessed
245: *
246: * Segments are ordered by priority, 0 being the lowest priority. Their
247: * relative priorities are DMA < DMA32 < DIRECTMAP < HIGHMEM. Some segments
248: * may actually be aliases for others, e.g. if DMA is always possible from
249: * the direct physical mapping, DMA and DMA32 are aliases for DIRECTMAP,
250: * in which case the segment table contains DIRECTMAP and HIGHMEM only.
251: */
252: static struct vm_page_seg vm_page_segs[VM_PAGE_MAX_SEGS];
253:
254: /*
255: * Bootstrap segment table.
256: */
257: static struct vm_page_boot_seg vm_page_boot_segs[VM_PAGE_MAX_SEGS] __initdata;
258:
259: /*
260: * Number of loaded segments.
261: */
262: static unsigned int vm_page_segs_size __read_mostly;
263:
1.1.1.2 ! root 264: /*
! 265: * If true, unprivileged allocations are blocked, disregarding any other
! 266: * condition.
! 267: *
! 268: * This variable is also used to resume clients once pages are available.
! 269: *
! 270: * The free page queue lock must be held when accessing this variable.
! 271: */
! 272: static boolean_t vm_page_alloc_paused;
! 273:
1.1 root 274: static void __init
275: vm_page_init_pa(struct vm_page *page, unsigned short seg_index, phys_addr_t pa)
276: {
277: memset(page, 0, sizeof(*page));
278: vm_page_init(page); /* vm_resident members */
279: page->type = VM_PT_RESERVED;
280: page->seg_index = seg_index;
281: page->order = VM_PAGE_ORDER_UNLISTED;
282: page->priv = NULL;
283: page->phys_addr = pa;
284: }
285:
286: void
287: vm_page_set_type(struct vm_page *page, unsigned int order, unsigned short type)
288: {
289: unsigned int i, nr_pages;
290:
291: nr_pages = 1 << order;
292:
293: for (i = 0; i < nr_pages; i++)
294: page[i].type = type;
295: }
296:
1.1.1.2 ! root 297: static boolean_t
! 298: vm_page_pageable(const struct vm_page *page)
! 299: {
! 300: return (page->object != NULL)
! 301: && (page->wire_count == 0)
! 302: && (page->active || page->inactive);
! 303: }
! 304:
! 305: static boolean_t
! 306: vm_page_can_move(const struct vm_page *page)
! 307: {
! 308: /*
! 309: * This function is called on pages pulled from the page queues,
! 310: * implying they're pageable, which is why the wire count isn't
! 311: * checked here.
! 312: */
! 313:
! 314: return !page->busy
! 315: && !page->wanted
! 316: && !page->absent
! 317: && page->object->alive;
! 318: }
! 319:
! 320: static void
! 321: vm_page_remove_mappings(struct vm_page *page)
! 322: {
! 323: page->busy = TRUE;
! 324: pmap_page_protect(page->phys_addr, VM_PROT_NONE);
! 325:
! 326: if (!page->dirty) {
! 327: page->dirty = pmap_is_modified(page->phys_addr);
! 328: }
! 329: }
! 330:
1.1 root 331: static void __init
332: vm_page_free_list_init(struct vm_page_free_list *free_list)
333: {
334: free_list->size = 0;
335: list_init(&free_list->blocks);
336: }
337:
338: static inline void
339: vm_page_free_list_insert(struct vm_page_free_list *free_list,
340: struct vm_page *page)
341: {
342: assert(page->order == VM_PAGE_ORDER_UNLISTED);
343:
344: free_list->size++;
345: list_insert_head(&free_list->blocks, &page->node);
346: }
347:
348: static inline void
349: vm_page_free_list_remove(struct vm_page_free_list *free_list,
350: struct vm_page *page)
351: {
352: assert(page->order != VM_PAGE_ORDER_UNLISTED);
353:
354: free_list->size--;
355: list_remove(&page->node);
356: }
357:
358: static struct vm_page *
359: vm_page_seg_alloc_from_buddy(struct vm_page_seg *seg, unsigned int order)
360: {
361: struct vm_page_free_list *free_list = free_list;
362: struct vm_page *page, *buddy;
363: unsigned int i;
364:
365: assert(order < VM_PAGE_NR_FREE_LISTS);
366:
1.1.1.2 ! root 367: if (vm_page_alloc_paused && current_thread()
! 368: && !current_thread()->vm_privilege) {
! 369: return NULL;
! 370: } else if (seg->nr_free_pages <= seg->low_free_pages) {
! 371: vm_pageout_start();
! 372:
! 373: if ((seg->nr_free_pages <= seg->min_free_pages)
! 374: && current_thread() && !current_thread()->vm_privilege) {
! 375: vm_page_alloc_paused = TRUE;
! 376: return NULL;
! 377: }
! 378: }
! 379:
1.1 root 380: for (i = order; i < VM_PAGE_NR_FREE_LISTS; i++) {
381: free_list = &seg->free_lists[i];
382:
383: if (free_list->size != 0)
384: break;
385: }
386:
387: if (i == VM_PAGE_NR_FREE_LISTS)
388: return NULL;
389:
390: page = list_first_entry(&free_list->blocks, struct vm_page, node);
391: vm_page_free_list_remove(free_list, page);
392: page->order = VM_PAGE_ORDER_UNLISTED;
393:
394: while (i > order) {
395: i--;
396: buddy = &page[1 << i];
397: vm_page_free_list_insert(&seg->free_lists[i], buddy);
398: buddy->order = i;
399: }
400:
401: seg->nr_free_pages -= (1 << order);
1.1.1.2 ! root 402:
! 403: if (seg->nr_free_pages < seg->min_free_pages) {
! 404: vm_page_alloc_paused = TRUE;
! 405: }
! 406:
1.1 root 407: return page;
408: }
409:
410: static void
411: vm_page_seg_free_to_buddy(struct vm_page_seg *seg, struct vm_page *page,
412: unsigned int order)
413: {
414: struct vm_page *buddy;
415: phys_addr_t pa, buddy_pa;
416: unsigned int nr_pages;
417:
418: assert(page >= seg->pages);
419: assert(page < seg->pages_end);
420: assert(page->order == VM_PAGE_ORDER_UNLISTED);
421: assert(order < VM_PAGE_NR_FREE_LISTS);
422:
423: nr_pages = (1 << order);
424: pa = page->phys_addr;
425:
426: while (order < (VM_PAGE_NR_FREE_LISTS - 1)) {
427: buddy_pa = pa ^ vm_page_ptoa(1 << order);
428:
429: if ((buddy_pa < seg->start) || (buddy_pa >= seg->end))
430: break;
431:
432: buddy = &seg->pages[vm_page_atop(buddy_pa - seg->start)];
433:
434: if (buddy->order != order)
435: break;
436:
437: vm_page_free_list_remove(&seg->free_lists[order], buddy);
438: buddy->order = VM_PAGE_ORDER_UNLISTED;
439: order++;
440: pa &= -vm_page_ptoa(1 << order);
441: page = &seg->pages[vm_page_atop(pa - seg->start)];
442: }
443:
444: vm_page_free_list_insert(&seg->free_lists[order], page);
445: page->order = order;
446: seg->nr_free_pages += nr_pages;
447: }
448:
449: static void __init
450: vm_page_cpu_pool_init(struct vm_page_cpu_pool *cpu_pool, int size)
451: {
452: simple_lock_init(&cpu_pool->lock);
453: cpu_pool->size = size;
454: cpu_pool->transfer_size = (size + VM_PAGE_CPU_POOL_TRANSFER_RATIO - 1)
455: / VM_PAGE_CPU_POOL_TRANSFER_RATIO;
456: cpu_pool->nr_pages = 0;
457: list_init(&cpu_pool->pages);
458: }
459:
460: static inline struct vm_page_cpu_pool *
461: vm_page_cpu_pool_get(struct vm_page_seg *seg)
462: {
463: return &seg->cpu_pools[cpu_number()];
464: }
465:
466: static inline struct vm_page *
467: vm_page_cpu_pool_pop(struct vm_page_cpu_pool *cpu_pool)
468: {
469: struct vm_page *page;
470:
471: assert(cpu_pool->nr_pages != 0);
472: cpu_pool->nr_pages--;
473: page = list_first_entry(&cpu_pool->pages, struct vm_page, node);
474: list_remove(&page->node);
475: return page;
476: }
477:
478: static inline void
479: vm_page_cpu_pool_push(struct vm_page_cpu_pool *cpu_pool, struct vm_page *page)
480: {
481: assert(cpu_pool->nr_pages < cpu_pool->size);
482: cpu_pool->nr_pages++;
483: list_insert_head(&cpu_pool->pages, &page->node);
484: }
485:
486: static int
487: vm_page_cpu_pool_fill(struct vm_page_cpu_pool *cpu_pool,
488: struct vm_page_seg *seg)
489: {
490: struct vm_page *page;
491: int i;
492:
493: assert(cpu_pool->nr_pages == 0);
494:
495: simple_lock(&seg->lock);
496:
497: for (i = 0; i < cpu_pool->transfer_size; i++) {
498: page = vm_page_seg_alloc_from_buddy(seg, 0);
499:
500: if (page == NULL)
501: break;
502:
503: vm_page_cpu_pool_push(cpu_pool, page);
504: }
505:
506: simple_unlock(&seg->lock);
507:
508: return i;
509: }
510:
511: static void
512: vm_page_cpu_pool_drain(struct vm_page_cpu_pool *cpu_pool,
513: struct vm_page_seg *seg)
514: {
515: struct vm_page *page;
516: int i;
517:
518: assert(cpu_pool->nr_pages == cpu_pool->size);
519:
520: simple_lock(&seg->lock);
521:
522: for (i = cpu_pool->transfer_size; i > 0; i--) {
523: page = vm_page_cpu_pool_pop(cpu_pool);
524: vm_page_seg_free_to_buddy(seg, page, 0);
525: }
526:
527: simple_unlock(&seg->lock);
528: }
529:
1.1.1.2 ! root 530: static void
! 531: vm_page_queue_init(struct vm_page_queue *queue)
! 532: {
! 533: list_init(&queue->internal_pages);
! 534: list_init(&queue->external_pages);
! 535: }
! 536:
! 537: static void
! 538: vm_page_queue_push(struct vm_page_queue *queue, struct vm_page *page)
! 539: {
! 540: if (page->external) {
! 541: list_insert_tail(&queue->external_pages, &page->node);
! 542: } else {
! 543: list_insert_tail(&queue->internal_pages, &page->node);
! 544: }
! 545: }
! 546:
! 547: static void
! 548: vm_page_queue_remove(struct vm_page_queue *queue, struct vm_page *page)
! 549: {
! 550: (void)queue;
! 551: list_remove(&page->node);
! 552: }
! 553:
! 554: static struct vm_page *
! 555: vm_page_queue_first(struct vm_page_queue *queue, boolean_t external_only)
! 556: {
! 557: struct vm_page *page;
! 558:
! 559: if (!list_empty(&queue->external_pages)) {
! 560: page = list_first_entry(&queue->external_pages, struct vm_page, node);
! 561: return page;
! 562: }
! 563:
! 564: if (!external_only && !list_empty(&queue->internal_pages)) {
! 565: page = list_first_entry(&queue->internal_pages, struct vm_page, node);
! 566: return page;
! 567: }
! 568:
! 569: return NULL;
! 570: }
! 571:
! 572: static struct vm_page_seg *
! 573: vm_page_seg_get(unsigned short index)
! 574: {
! 575: assert(index < vm_page_segs_size);
! 576: return &vm_page_segs[index];
! 577: }
! 578:
! 579: static unsigned int
! 580: vm_page_seg_index(const struct vm_page_seg *seg)
! 581: {
! 582: unsigned int index;
! 583:
! 584: index = seg - vm_page_segs;
! 585: assert(index < vm_page_segs_size);
! 586: return index;
! 587: }
! 588:
1.1 root 589: static phys_addr_t __init
590: vm_page_seg_size(struct vm_page_seg *seg)
591: {
592: return seg->end - seg->start;
593: }
594:
595: static int __init
596: vm_page_seg_compute_pool_size(struct vm_page_seg *seg)
597: {
598: phys_addr_t size;
599:
600: size = vm_page_atop(vm_page_seg_size(seg)) / VM_PAGE_CPU_POOL_RATIO;
601:
602: if (size == 0)
603: size = 1;
604: else if (size > VM_PAGE_CPU_POOL_MAX_SIZE)
605: size = VM_PAGE_CPU_POOL_MAX_SIZE;
606:
607: return size;
608: }
609:
610: static void __init
1.1.1.2 ! root 611: vm_page_seg_compute_pageout_thresholds(struct vm_page_seg *seg)
! 612: {
! 613: unsigned long nr_pages;
! 614:
! 615: nr_pages = vm_page_atop(vm_page_seg_size(seg));
! 616:
! 617: if (nr_pages < VM_PAGE_SEG_MIN_PAGES) {
! 618: panic("vm_page: segment too small");
! 619: }
! 620:
! 621: seg->min_free_pages = nr_pages * VM_PAGE_SEG_THRESHOLD_MIN_NUM
! 622: / VM_PAGE_SEG_THRESHOLD_MIN_DENOM;
! 623:
! 624: if (seg->min_free_pages < VM_PAGE_SEG_THRESHOLD_MIN) {
! 625: seg->min_free_pages = VM_PAGE_SEG_THRESHOLD_MIN;
! 626: }
! 627:
! 628: seg->low_free_pages = nr_pages * VM_PAGE_SEG_THRESHOLD_LOW_NUM
! 629: / VM_PAGE_SEG_THRESHOLD_LOW_DENOM;
! 630:
! 631: if (seg->low_free_pages < VM_PAGE_SEG_THRESHOLD_LOW) {
! 632: seg->low_free_pages = VM_PAGE_SEG_THRESHOLD_LOW;
! 633: }
! 634:
! 635: seg->high_free_pages = nr_pages * VM_PAGE_SEG_THRESHOLD_HIGH_NUM
! 636: / VM_PAGE_SEG_THRESHOLD_HIGH_DENOM;
! 637:
! 638: if (seg->high_free_pages < VM_PAGE_SEG_THRESHOLD_HIGH) {
! 639: seg->high_free_pages = VM_PAGE_SEG_THRESHOLD_HIGH;
! 640: }
! 641: }
! 642:
! 643: static void __init
1.1 root 644: vm_page_seg_init(struct vm_page_seg *seg, phys_addr_t start, phys_addr_t end,
645: struct vm_page *pages)
646: {
647: phys_addr_t pa;
648: int pool_size;
649: unsigned int i;
650:
651: seg->start = start;
652: seg->end = end;
653: pool_size = vm_page_seg_compute_pool_size(seg);
654:
655: for (i = 0; i < ARRAY_SIZE(seg->cpu_pools); i++)
656: vm_page_cpu_pool_init(&seg->cpu_pools[i], pool_size);
657:
658: seg->pages = pages;
659: seg->pages_end = pages + vm_page_atop(vm_page_seg_size(seg));
660: simple_lock_init(&seg->lock);
661:
662: for (i = 0; i < ARRAY_SIZE(seg->free_lists); i++)
663: vm_page_free_list_init(&seg->free_lists[i]);
664:
665: seg->nr_free_pages = 0;
1.1.1.2 ! root 666:
! 667: vm_page_seg_compute_pageout_thresholds(seg);
! 668:
! 669: vm_page_queue_init(&seg->active_pages);
! 670: seg->nr_active_pages = 0;
! 671: vm_page_queue_init(&seg->inactive_pages);
! 672: seg->nr_inactive_pages = 0;
! 673:
! 674: i = vm_page_seg_index(seg);
1.1 root 675:
676: for (pa = seg->start; pa < seg->end; pa += PAGE_SIZE)
677: vm_page_init_pa(&pages[vm_page_atop(pa - seg->start)], i, pa);
678: }
679:
680: static struct vm_page *
681: vm_page_seg_alloc(struct vm_page_seg *seg, unsigned int order,
682: unsigned short type)
683: {
684: struct vm_page_cpu_pool *cpu_pool;
685: struct vm_page *page;
686: int filled;
687:
688: assert(order < VM_PAGE_NR_FREE_LISTS);
689:
690: if (order == 0) {
691: thread_pin();
692: cpu_pool = vm_page_cpu_pool_get(seg);
693: simple_lock(&cpu_pool->lock);
694:
695: if (cpu_pool->nr_pages == 0) {
696: filled = vm_page_cpu_pool_fill(cpu_pool, seg);
697:
698: if (!filled) {
699: simple_unlock(&cpu_pool->lock);
700: thread_unpin();
701: return NULL;
702: }
703: }
704:
705: page = vm_page_cpu_pool_pop(cpu_pool);
706: simple_unlock(&cpu_pool->lock);
707: thread_unpin();
708: } else {
709: simple_lock(&seg->lock);
710: page = vm_page_seg_alloc_from_buddy(seg, order);
711: simple_unlock(&seg->lock);
712:
713: if (page == NULL)
714: return NULL;
715: }
716:
717: assert(page->type == VM_PT_FREE);
718: vm_page_set_type(page, order, type);
719: return page;
720: }
721:
722: static void
723: vm_page_seg_free(struct vm_page_seg *seg, struct vm_page *page,
724: unsigned int order)
725: {
726: struct vm_page_cpu_pool *cpu_pool;
727:
728: assert(page->type != VM_PT_FREE);
729: assert(order < VM_PAGE_NR_FREE_LISTS);
730:
731: vm_page_set_type(page, order, VM_PT_FREE);
732:
733: if (order == 0) {
734: thread_pin();
735: cpu_pool = vm_page_cpu_pool_get(seg);
736: simple_lock(&cpu_pool->lock);
737:
738: if (cpu_pool->nr_pages == cpu_pool->size)
739: vm_page_cpu_pool_drain(cpu_pool, seg);
740:
741: vm_page_cpu_pool_push(cpu_pool, page);
742: simple_unlock(&cpu_pool->lock);
743: thread_unpin();
744: } else {
745: simple_lock(&seg->lock);
746: vm_page_seg_free_to_buddy(seg, page, order);
747: simple_unlock(&seg->lock);
748: }
749: }
750:
1.1.1.2 ! root 751: static void
! 752: vm_page_seg_add_active_page(struct vm_page_seg *seg, struct vm_page *page)
1.1 root 753: {
1.1.1.2 ! root 754: assert(page->object != NULL);
! 755: assert(page->seg_index == vm_page_seg_index(seg));
! 756: assert(page->type != VM_PT_FREE);
! 757: assert(page->order == VM_PAGE_ORDER_UNLISTED);
! 758: assert(!page->free && !page->active && !page->inactive);
! 759: page->active = TRUE;
! 760: page->reference = TRUE;
! 761: vm_page_queue_push(&seg->active_pages, page);
! 762: seg->nr_active_pages++;
! 763: vm_page_active_count++;
! 764: }
1.1 root 765:
1.1.1.2 ! root 766: static void
! 767: vm_page_seg_remove_active_page(struct vm_page_seg *seg, struct vm_page *page)
! 768: {
! 769: assert(page->object != NULL);
! 770: assert(page->seg_index == vm_page_seg_index(seg));
! 771: assert(page->type != VM_PT_FREE);
! 772: assert(page->order == VM_PAGE_ORDER_UNLISTED);
! 773: assert(!page->free && page->active && !page->inactive);
! 774: page->active = FALSE;
! 775: vm_page_queue_remove(&seg->active_pages, page);
! 776: seg->nr_active_pages--;
! 777: vm_page_active_count--;
! 778: }
1.1 root 779:
1.1.1.2 ! root 780: static void
! 781: vm_page_seg_add_inactive_page(struct vm_page_seg *seg, struct vm_page *page)
! 782: {
! 783: assert(page->object != NULL);
! 784: assert(page->seg_index == vm_page_seg_index(seg));
! 785: assert(page->type != VM_PT_FREE);
! 786: assert(page->order == VM_PAGE_ORDER_UNLISTED);
! 787: assert(!page->free && !page->active && !page->inactive);
! 788: page->inactive = TRUE;
! 789: vm_page_queue_push(&seg->inactive_pages, page);
! 790: seg->nr_inactive_pages++;
! 791: vm_page_inactive_count++;
1.1 root 792: }
793:
1.1.1.2 ! root 794: static void
! 795: vm_page_seg_remove_inactive_page(struct vm_page_seg *seg, struct vm_page *page)
1.1 root 796: {
1.1.1.2 ! root 797: assert(page->object != NULL);
! 798: assert(page->seg_index == vm_page_seg_index(seg));
! 799: assert(page->type != VM_PT_FREE);
! 800: assert(page->order == VM_PAGE_ORDER_UNLISTED);
! 801: assert(!page->free && !page->active && page->inactive);
! 802: page->inactive = FALSE;
! 803: vm_page_queue_remove(&seg->inactive_pages, page);
! 804: seg->nr_inactive_pages--;
! 805: vm_page_inactive_count--;
1.1 root 806: }
807:
1.1.1.2 ! root 808: /*
! 809: * Attempt to pull an active page.
! 810: *
! 811: * If successful, the object containing the page is locked.
! 812: */
! 813: static struct vm_page *
! 814: vm_page_seg_pull_active_page(struct vm_page_seg *seg, boolean_t external_only)
1.1 root 815: {
1.1.1.2 ! root 816: struct vm_page *page, *first;
! 817: boolean_t locked;
1.1 root 818:
1.1.1.2 ! root 819: first = NULL;
! 820:
! 821: for (;;) {
! 822: page = vm_page_queue_first(&seg->active_pages, external_only);
! 823:
! 824: if (page == NULL) {
! 825: break;
! 826: } else if (first == NULL) {
! 827: first = page;
! 828: } else if (first == page) {
! 829: break;
! 830: }
! 831:
! 832: vm_page_seg_remove_active_page(seg, page);
! 833: locked = vm_object_lock_try(page->object);
! 834:
! 835: if (!locked) {
! 836: vm_page_seg_add_active_page(seg, page);
! 837: continue;
! 838: }
! 839:
! 840: if (!vm_page_can_move(page)) {
! 841: vm_page_seg_add_active_page(seg, page);
! 842: vm_object_unlock(page->object);
! 843: continue;
! 844: }
! 845:
! 846: return page;
1.1 root 847: }
848:
1.1.1.2 ! root 849: return NULL;
1.1 root 850: }
851:
1.1.1.2 ! root 852: /*
! 853: * Attempt to pull an inactive page.
! 854: *
! 855: * If successful, the object containing the page is locked.
! 856: *
! 857: * XXX See vm_page_seg_pull_active_page (duplicated code).
! 858: */
! 859: static struct vm_page *
! 860: vm_page_seg_pull_inactive_page(struct vm_page_seg *seg, boolean_t external_only)
1.1 root 861: {
1.1.1.2 ! root 862: struct vm_page *page, *first;
! 863: boolean_t locked;
1.1 root 864:
1.1.1.2 ! root 865: first = NULL;
1.1 root 866:
1.1.1.2 ! root 867: for (;;) {
! 868: page = vm_page_queue_first(&seg->inactive_pages, external_only);
1.1 root 869:
1.1.1.2 ! root 870: if (page == NULL) {
! 871: break;
! 872: } else if (first == NULL) {
! 873: first = page;
! 874: } else if (first == page) {
! 875: break;
! 876: }
1.1 root 877:
1.1.1.2 ! root 878: vm_page_seg_remove_inactive_page(seg, page);
! 879: locked = vm_object_lock_try(page->object);
! 880:
! 881: if (!locked) {
! 882: vm_page_seg_add_inactive_page(seg, page);
! 883: continue;
! 884: }
! 885:
! 886: if (!vm_page_can_move(page)) {
! 887: vm_page_seg_add_inactive_page(seg, page);
! 888: vm_object_unlock(page->object);
1.1 root 889: continue;
1.1.1.2 ! root 890: }
1.1 root 891:
1.1.1.2 ! root 892: return page;
1.1 root 893: }
1.1.1.2 ! root 894:
! 895: return NULL;
1.1 root 896: }
897:
1.1.1.2 ! root 898: /*
! 899: * Attempt to pull a page cache page.
! 900: *
! 901: * If successful, the object containing the page is locked.
! 902: */
! 903: static struct vm_page *
! 904: vm_page_seg_pull_cache_page(struct vm_page_seg *seg,
! 905: boolean_t external_only,
! 906: boolean_t *was_active)
1.1 root 907: {
1.1.1.2 ! root 908: struct vm_page *page;
! 909:
! 910: page = vm_page_seg_pull_inactive_page(seg, external_only);
! 911:
! 912: if (page != NULL) {
! 913: *was_active = FALSE;
! 914: return page;
! 915: }
! 916:
! 917: page = vm_page_seg_pull_active_page(seg, external_only);
! 918:
! 919: if (page != NULL) {
! 920: *was_active = TRUE;
! 921: return page;
! 922: }
! 923:
! 924: return NULL;
1.1 root 925: }
926:
1.1.1.2 ! root 927: static boolean_t
! 928: vm_page_seg_page_available(const struct vm_page_seg *seg)
1.1 root 929: {
1.1.1.2 ! root 930: return (seg->nr_free_pages > seg->high_free_pages);
1.1 root 931: }
932:
1.1.1.2 ! root 933: static boolean_t
! 934: vm_page_seg_usable(const struct vm_page_seg *seg)
1.1 root 935: {
1.1.1.2 ! root 936: if ((seg->nr_active_pages + seg->nr_inactive_pages) == 0) {
! 937: /* Nothing to page out, assume segment is usable */
! 938: return TRUE;
1.1 root 939: }
940:
1.1.1.2 ! root 941: return (seg->nr_free_pages >= seg->high_free_pages);
1.1 root 942: }
943:
1.1.1.2 ! root 944: static void
! 945: vm_page_seg_double_lock(struct vm_page_seg *seg1, struct vm_page_seg *seg2)
! 946: {
! 947: assert(seg1 != seg2);
! 948:
! 949: if (seg1 < seg2) {
! 950: simple_lock(&seg1->lock);
! 951: simple_lock(&seg2->lock);
! 952: } else {
! 953: simple_lock(&seg2->lock);
! 954: simple_lock(&seg1->lock);
! 955: }
! 956: }
! 957:
! 958: static void
! 959: vm_page_seg_double_unlock(struct vm_page_seg *seg1, struct vm_page_seg *seg2)
! 960: {
! 961: simple_unlock(&seg1->lock);
! 962: simple_unlock(&seg2->lock);
! 963: }
! 964:
! 965: /*
! 966: * Attempt to balance a segment by moving one page to another segment.
! 967: *
! 968: * Return TRUE if a page was actually moved.
! 969: */
! 970: static boolean_t
! 971: vm_page_seg_balance_page(struct vm_page_seg *seg,
! 972: struct vm_page_seg *remote_seg)
! 973: {
! 974: struct vm_page *src, *dest;
! 975: vm_object_t object;
! 976: vm_offset_t offset;
! 977: boolean_t was_active;
! 978:
! 979: vm_page_lock_queues();
! 980: simple_lock(&vm_page_queue_free_lock);
! 981: vm_page_seg_double_lock(seg, remote_seg);
! 982:
! 983: if (vm_page_seg_usable(seg)
! 984: || !vm_page_seg_page_available(remote_seg)) {
! 985: goto error;
! 986: }
! 987:
! 988: src = vm_page_seg_pull_cache_page(seg, FALSE, &was_active);
! 989:
! 990: if (src == NULL) {
! 991: goto error;
! 992: }
! 993:
! 994: assert(src->object != NULL);
! 995: assert(!src->fictitious && !src->private);
! 996: assert(src->wire_count == 0);
! 997: assert(src->type != VM_PT_FREE);
! 998: assert(src->order == VM_PAGE_ORDER_UNLISTED);
! 999:
! 1000: dest = vm_page_seg_alloc_from_buddy(remote_seg, 0);
! 1001: assert(dest != NULL);
! 1002:
! 1003: vm_page_seg_double_unlock(seg, remote_seg);
! 1004: simple_unlock(&vm_page_queue_free_lock);
! 1005:
! 1006: if (!was_active && !src->reference && pmap_is_referenced(src->phys_addr)) {
! 1007: src->reference = TRUE;
! 1008: }
! 1009:
! 1010: object = src->object;
! 1011: offset = src->offset;
! 1012: vm_page_remove(src);
! 1013:
! 1014: vm_page_remove_mappings(src);
! 1015:
! 1016: vm_page_set_type(dest, 0, src->type);
! 1017: memcpy(&dest->vm_page_header, &src->vm_page_header,
! 1018: sizeof(*dest) - VM_PAGE_HEADER_SIZE);
! 1019: vm_page_copy(src, dest);
! 1020:
! 1021: if (!src->dirty) {
! 1022: pmap_clear_modify(dest->phys_addr);
! 1023: }
! 1024:
! 1025: dest->busy = FALSE;
! 1026:
! 1027: simple_lock(&vm_page_queue_free_lock);
! 1028: vm_page_init(src);
! 1029: src->free = TRUE;
! 1030: simple_lock(&seg->lock);
! 1031: vm_page_set_type(src, 0, VM_PT_FREE);
! 1032: vm_page_seg_free_to_buddy(seg, src, 0);
! 1033: simple_unlock(&seg->lock);
! 1034: simple_unlock(&vm_page_queue_free_lock);
! 1035:
! 1036: vm_page_insert(dest, object, offset);
! 1037: vm_object_unlock(object);
! 1038:
! 1039: if (was_active) {
! 1040: vm_page_activate(dest);
! 1041: } else {
! 1042: vm_page_deactivate(dest);
! 1043: }
! 1044:
! 1045: vm_page_unlock_queues();
! 1046:
! 1047: return TRUE;
! 1048:
! 1049: error:
! 1050: vm_page_seg_double_unlock(seg, remote_seg);
! 1051: simple_unlock(&vm_page_queue_free_lock);
! 1052: vm_page_unlock_queues();
! 1053: return FALSE;
! 1054: }
! 1055:
! 1056: static boolean_t
! 1057: vm_page_seg_balance(struct vm_page_seg *seg)
! 1058: {
! 1059: struct vm_page_seg *remote_seg;
! 1060: unsigned int i;
! 1061: boolean_t balanced;
! 1062:
! 1063: /*
! 1064: * It's important here that pages are moved to lower priority
! 1065: * segments first.
! 1066: */
! 1067:
! 1068: for (i = vm_page_segs_size - 1; i < vm_page_segs_size; i--) {
! 1069: remote_seg = vm_page_seg_get(i);
! 1070:
! 1071: if (remote_seg == seg) {
! 1072: continue;
! 1073: }
! 1074:
! 1075: balanced = vm_page_seg_balance_page(seg, remote_seg);
! 1076:
! 1077: if (balanced) {
! 1078: return TRUE;
! 1079: }
! 1080: }
! 1081:
! 1082: return FALSE;
! 1083: }
! 1084:
! 1085: static boolean_t
! 1086: vm_page_seg_evict(struct vm_page_seg *seg, boolean_t external_only,
! 1087: boolean_t alloc_paused)
! 1088: {
! 1089: struct vm_page *page;
! 1090: boolean_t reclaim, laundry;
! 1091: vm_object_t object;
! 1092: boolean_t was_active;
! 1093:
! 1094: page = NULL;
! 1095: object = NULL;
! 1096: laundry = FALSE;
! 1097:
! 1098: restart:
! 1099: vm_page_lock_queues();
! 1100: simple_lock(&seg->lock);
! 1101:
! 1102: if (page != NULL) {
! 1103: vm_object_lock(page->object);
! 1104: } else {
! 1105: page = vm_page_seg_pull_cache_page(seg, external_only, &was_active);
! 1106:
! 1107: if (page == NULL) {
! 1108: goto out;
! 1109: }
! 1110: }
! 1111:
! 1112: assert(page->object != NULL);
! 1113: assert(!page->fictitious && !page->private);
! 1114: assert(page->wire_count == 0);
! 1115: assert(page->type != VM_PT_FREE);
! 1116: assert(page->order == VM_PAGE_ORDER_UNLISTED);
! 1117:
! 1118: object = page->object;
! 1119:
! 1120: if (!was_active
! 1121: && (page->reference || pmap_is_referenced(page->phys_addr))) {
! 1122: vm_page_seg_add_active_page(seg, page);
! 1123: simple_unlock(&seg->lock);
! 1124: vm_object_unlock(object);
! 1125: vm_stat.reactivations++;
! 1126: current_task()->reactivations++;
! 1127: vm_page_unlock_queues();
! 1128: page = NULL;
! 1129: goto restart;
! 1130: }
! 1131:
! 1132: vm_page_remove_mappings(page);
! 1133:
! 1134: if (!page->dirty && !page->precious) {
! 1135: reclaim = TRUE;
! 1136: goto out;
! 1137: }
! 1138:
! 1139: reclaim = FALSE;
! 1140:
! 1141: /*
! 1142: * If we are very low on memory, then we can't rely on an external
! 1143: * pager to clean a dirty page, because external pagers are not
! 1144: * vm-privileged.
! 1145: *
! 1146: * The laundry bit tells vm_pageout_setup not to do any special
! 1147: * processing of this page since it's immediately going to be
! 1148: * double paged out to the default pager. The laundry bit is
! 1149: * reset and the page is inserted into an internal object by
! 1150: * vm_pageout_setup before the double paging pass.
! 1151: */
! 1152:
! 1153: assert(!page->laundry);
! 1154: assert(!(laundry && page->external));
! 1155:
! 1156: if (object->internal || !alloc_paused) {
! 1157: laundry = FALSE;
! 1158: } else {
! 1159: laundry = page->laundry = TRUE;
! 1160: }
! 1161:
! 1162: out:
! 1163: simple_unlock(&seg->lock);
! 1164:
! 1165: if (object == NULL) {
! 1166: vm_page_unlock_queues();
! 1167: return FALSE;
! 1168: }
! 1169:
! 1170: if (reclaim) {
! 1171: vm_page_free(page);
! 1172: vm_page_unlock_queues();
! 1173:
! 1174: if (vm_object_collectable(object)) {
! 1175: vm_object_collect(object);
! 1176: } else {
! 1177: vm_object_unlock(object);
! 1178: }
! 1179:
! 1180: return TRUE;
! 1181: }
! 1182:
! 1183: vm_page_unlock_queues();
! 1184:
! 1185: /*
! 1186: * If there is no memory object for the page, create one and hand it
! 1187: * to the default pager. First try to collapse, so we don't create
! 1188: * one unnecessarily.
! 1189: */
! 1190:
! 1191: if (!object->pager_initialized) {
! 1192: vm_object_collapse(object);
! 1193: }
! 1194:
! 1195: if (!object->pager_initialized) {
! 1196: vm_object_pager_create(object);
! 1197: }
! 1198:
! 1199: if (!object->pager_initialized) {
! 1200: panic("vm_page_seg_evict");
! 1201: }
! 1202:
! 1203: vm_pageout_page(page, FALSE, TRUE); /* flush it */
! 1204: vm_object_unlock(object);
! 1205:
! 1206: if (laundry) {
! 1207: goto restart;
! 1208: }
! 1209:
! 1210: return TRUE;
! 1211: }
! 1212:
! 1213: static void
! 1214: vm_page_seg_compute_high_active_page(struct vm_page_seg *seg)
! 1215: {
! 1216: unsigned long nr_pages;
! 1217:
! 1218: nr_pages = seg->nr_active_pages + seg->nr_inactive_pages;
! 1219: seg->high_active_pages = nr_pages * VM_PAGE_HIGH_ACTIVE_PAGE_NUM
! 1220: / VM_PAGE_HIGH_ACTIVE_PAGE_DENOM;
! 1221: }
! 1222:
! 1223: static void
! 1224: vm_page_seg_refill_inactive(struct vm_page_seg *seg)
! 1225: {
! 1226: struct vm_page *page;
! 1227:
! 1228: simple_lock(&seg->lock);
! 1229:
! 1230: vm_page_seg_compute_high_active_page(seg);
! 1231:
! 1232: while (seg->nr_active_pages > seg->high_active_pages) {
! 1233: page = vm_page_seg_pull_active_page(seg, FALSE);
! 1234:
! 1235: if (page == NULL) {
! 1236: break;
! 1237: }
! 1238:
! 1239: page->reference = FALSE;
! 1240: pmap_clear_reference(page->phys_addr);
! 1241: vm_page_seg_add_inactive_page(seg, page);
! 1242: vm_object_unlock(page->object);
! 1243: }
! 1244:
! 1245: simple_unlock(&seg->lock);
! 1246: }
! 1247:
! 1248: void __init
! 1249: vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
! 1250: {
! 1251: struct vm_page_boot_seg *seg;
! 1252:
! 1253: assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
! 1254: assert(vm_page_aligned(start));
! 1255: assert(vm_page_aligned(end));
! 1256: assert(start < end);
! 1257: assert(vm_page_segs_size < ARRAY_SIZE(vm_page_boot_segs));
! 1258:
! 1259: seg = &vm_page_boot_segs[seg_index];
! 1260: seg->start = start;
! 1261: seg->end = end;
! 1262: seg->heap_present = FALSE;
! 1263:
! 1264: #if DEBUG
! 1265: printf("vm_page: load: %s: %llx:%llx\n",
! 1266: vm_page_seg_name(seg_index),
! 1267: (unsigned long long)start, (unsigned long long)end);
! 1268: #endif
! 1269:
! 1270: vm_page_segs_size++;
! 1271: }
! 1272:
! 1273: void
! 1274: vm_page_load_heap(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
! 1275: {
! 1276: struct vm_page_boot_seg *seg;
! 1277:
! 1278: assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
! 1279: assert(vm_page_aligned(start));
! 1280: assert(vm_page_aligned(end));
! 1281:
! 1282: seg = &vm_page_boot_segs[seg_index];
! 1283:
! 1284: assert(seg->start <= start);
! 1285: assert(end <= seg-> end);
! 1286:
! 1287: seg->avail_start = start;
! 1288: seg->avail_end = end;
! 1289: seg->heap_present = TRUE;
! 1290:
! 1291: #if DEBUG
! 1292: printf("vm_page: heap: %s: %llx:%llx\n",
! 1293: vm_page_seg_name(seg_index),
! 1294: (unsigned long long)start, (unsigned long long)end);
! 1295: #endif
! 1296: }
! 1297:
! 1298: int
! 1299: vm_page_ready(void)
! 1300: {
! 1301: return vm_page_is_ready;
! 1302: }
! 1303:
! 1304: static unsigned int
! 1305: vm_page_select_alloc_seg(unsigned int selector)
! 1306: {
! 1307: unsigned int seg_index;
! 1308:
! 1309: switch (selector) {
! 1310: case VM_PAGE_SEL_DMA:
! 1311: seg_index = VM_PAGE_SEG_DMA;
! 1312: break;
! 1313: case VM_PAGE_SEL_DMA32:
! 1314: seg_index = VM_PAGE_SEG_DMA32;
! 1315: break;
! 1316: case VM_PAGE_SEL_DIRECTMAP:
! 1317: seg_index = VM_PAGE_SEG_DIRECTMAP;
! 1318: break;
! 1319: case VM_PAGE_SEL_HIGHMEM:
! 1320: seg_index = VM_PAGE_SEG_HIGHMEM;
! 1321: break;
! 1322: default:
! 1323: panic("vm_page: invalid selector");
! 1324: }
! 1325:
! 1326: return MIN(vm_page_segs_size - 1, seg_index);
! 1327: }
! 1328:
! 1329: static int __init
! 1330: vm_page_boot_seg_loaded(const struct vm_page_boot_seg *seg)
! 1331: {
! 1332: return (seg->end != 0);
! 1333: }
! 1334:
! 1335: static void __init
! 1336: vm_page_check_boot_segs(void)
! 1337: {
! 1338: unsigned int i;
! 1339: int expect_loaded;
! 1340:
! 1341: if (vm_page_segs_size == 0)
! 1342: panic("vm_page: no physical memory loaded");
! 1343:
! 1344: for (i = 0; i < ARRAY_SIZE(vm_page_boot_segs); i++) {
! 1345: expect_loaded = (i < vm_page_segs_size);
! 1346:
! 1347: if (vm_page_boot_seg_loaded(&vm_page_boot_segs[i]) == expect_loaded)
! 1348: continue;
! 1349:
! 1350: panic("vm_page: invalid boot segment table");
! 1351: }
! 1352: }
! 1353:
! 1354: static phys_addr_t __init
! 1355: vm_page_boot_seg_size(struct vm_page_boot_seg *seg)
! 1356: {
! 1357: return seg->end - seg->start;
! 1358: }
! 1359:
! 1360: static phys_addr_t __init
! 1361: vm_page_boot_seg_avail_size(struct vm_page_boot_seg *seg)
! 1362: {
! 1363: return seg->avail_end - seg->avail_start;
! 1364: }
! 1365:
! 1366: unsigned long __init
! 1367: vm_page_bootalloc(size_t size)
! 1368: {
! 1369: struct vm_page_boot_seg *seg;
! 1370: phys_addr_t pa;
! 1371: unsigned int i;
! 1372:
! 1373: for (i = vm_page_select_alloc_seg(VM_PAGE_SEL_DIRECTMAP);
! 1374: i < vm_page_segs_size;
! 1375: i--) {
! 1376: seg = &vm_page_boot_segs[i];
! 1377:
! 1378: if (size <= vm_page_boot_seg_avail_size(seg)) {
! 1379: pa = seg->avail_start;
! 1380: seg->avail_start += vm_page_round(size);
! 1381: return pa;
! 1382: }
! 1383: }
! 1384:
! 1385: panic("vm_page: no physical memory available");
! 1386: }
! 1387:
! 1388: void __init
! 1389: vm_page_setup(void)
1.1 root 1390: {
1391: struct vm_page_boot_seg *boot_seg;
1392: struct vm_page_seg *seg;
1393: struct vm_page *table, *page, *end;
1394: size_t nr_pages, table_size;
1395: unsigned long va;
1396: unsigned int i;
1397: phys_addr_t pa;
1398:
1399: vm_page_check_boot_segs();
1400:
1401: /*
1402: * Compute the page table size.
1403: */
1404: nr_pages = 0;
1405:
1406: for (i = 0; i < vm_page_segs_size; i++)
1407: nr_pages += vm_page_atop(vm_page_boot_seg_size(&vm_page_boot_segs[i]));
1408:
1409: table_size = vm_page_round(nr_pages * sizeof(struct vm_page));
1410: printf("vm_page: page table size: %lu entries (%luk)\n", nr_pages,
1411: table_size >> 10);
1412: table = (struct vm_page *)pmap_steal_memory(table_size);
1413: va = (unsigned long)table;
1414:
1415: /*
1416: * Initialize the segments, associating them to the page table. When
1417: * the segments are initialized, all their pages are set allocated.
1418: * Pages are then released, which populates the free lists.
1419: */
1420: for (i = 0; i < vm_page_segs_size; i++) {
1421: seg = &vm_page_segs[i];
1422: boot_seg = &vm_page_boot_segs[i];
1423: vm_page_seg_init(seg, boot_seg->start, boot_seg->end, table);
1424: page = seg->pages + vm_page_atop(boot_seg->avail_start
1425: - boot_seg->start);
1426: end = seg->pages + vm_page_atop(boot_seg->avail_end
1427: - boot_seg->start);
1428:
1429: while (page < end) {
1430: page->type = VM_PT_FREE;
1431: vm_page_seg_free_to_buddy(seg, page, 0);
1432: page++;
1433: }
1434:
1435: table += vm_page_atop(vm_page_seg_size(seg));
1436: }
1437:
1438: while (va < (unsigned long)table) {
1439: pa = pmap_extract(kernel_pmap, va);
1440: page = vm_page_lookup_pa(pa);
1441: assert((page != NULL) && (page->type == VM_PT_RESERVED));
1442: page->type = VM_PT_TABLE;
1443: va += PAGE_SIZE;
1444: }
1445:
1446: vm_page_is_ready = 1;
1447: }
1448:
1449: void __init
1450: vm_page_manage(struct vm_page *page)
1451: {
1452: assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
1453: assert(page->type == VM_PT_RESERVED);
1454:
1455: vm_page_set_type(page, 0, VM_PT_FREE);
1456: vm_page_seg_free_to_buddy(&vm_page_segs[page->seg_index], page, 0);
1457: }
1458:
1459: struct vm_page *
1460: vm_page_lookup_pa(phys_addr_t pa)
1461: {
1462: struct vm_page_seg *seg;
1463: unsigned int i;
1464:
1.1.1.2 ! root 1465: for (i = 0; i < vm_page_segs_size; i++) {
! 1466: seg = &vm_page_segs[i];
! 1467:
! 1468: if ((pa >= seg->start) && (pa < seg->end))
! 1469: return &seg->pages[vm_page_atop(pa - seg->start)];
! 1470: }
! 1471:
! 1472: return NULL;
! 1473: }
! 1474:
! 1475: static struct vm_page_seg *
! 1476: vm_page_lookup_seg(const struct vm_page *page)
! 1477: {
! 1478: struct vm_page_seg *seg;
! 1479: unsigned int i;
! 1480:
! 1481: for (i = 0; i < vm_page_segs_size; i++) {
! 1482: seg = &vm_page_segs[i];
! 1483:
! 1484: if ((page->phys_addr >= seg->start) && (page->phys_addr < seg->end)) {
! 1485: return seg;
! 1486: }
! 1487: }
! 1488:
! 1489: return NULL;
! 1490: }
! 1491:
! 1492: void vm_page_check(const struct vm_page *page)
! 1493: {
! 1494: if (page->fictitious) {
! 1495: if (page->private) {
! 1496: panic("vm_page: page both fictitious and private");
! 1497: }
! 1498:
! 1499: if (page->phys_addr != vm_page_fictitious_addr) {
! 1500: panic("vm_page: invalid fictitious page");
! 1501: }
! 1502: } else {
! 1503: struct vm_page_seg *seg;
! 1504:
! 1505: if (page->phys_addr == vm_page_fictitious_addr) {
! 1506: panic("vm_page: real page has fictitious address");
! 1507: }
1.1 root 1508:
1.1.1.2 ! root 1509: seg = vm_page_lookup_seg(page);
1.1 root 1510:
1.1.1.2 ! root 1511: if (seg == NULL) {
! 1512: if (!page->private) {
! 1513: panic("vm_page: page claims it's managed but not in any segment");
! 1514: }
! 1515: } else {
! 1516: if (page->private) {
! 1517: struct vm_page *real_page;
! 1518:
! 1519: if (vm_page_pageable(page)) {
! 1520: panic("vm_page: private page is pageable");
! 1521: }
! 1522:
! 1523: real_page = vm_page_lookup_pa(page->phys_addr);
! 1524:
! 1525: if (vm_page_pageable(real_page)) {
! 1526: panic("vm_page: page underlying private page is pageable");
! 1527: }
! 1528:
! 1529: if ((real_page->type == VM_PT_FREE)
! 1530: || (real_page->order != VM_PAGE_ORDER_UNLISTED)) {
! 1531: panic("vm_page: page underlying private pagei is free");
! 1532: }
! 1533: } else {
! 1534: unsigned int index;
! 1535:
! 1536: index = vm_page_seg_index(seg);
! 1537:
! 1538: if (index != page->seg_index) {
! 1539: panic("vm_page: page segment mismatch");
! 1540: }
! 1541: }
! 1542: }
! 1543: }
1.1 root 1544: }
1545:
1546: struct vm_page *
1547: vm_page_alloc_pa(unsigned int order, unsigned int selector, unsigned short type)
1548: {
1549: struct vm_page *page;
1550: unsigned int i;
1551:
1552: for (i = vm_page_select_alloc_seg(selector); i < vm_page_segs_size; i--) {
1553: page = vm_page_seg_alloc(&vm_page_segs[i], order, type);
1554:
1555: if (page != NULL)
1556: return page;
1557: }
1558:
1.1.1.2 ! root 1559: if (!current_thread() || current_thread()->vm_privilege)
! 1560: panic("vm_page: privileged thread unable to allocate page");
1.1 root 1561:
1562: return NULL;
1563: }
1564:
1565: void
1566: vm_page_free_pa(struct vm_page *page, unsigned int order)
1567: {
1568: assert(page != NULL);
1569: assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
1570:
1571: vm_page_seg_free(&vm_page_segs[page->seg_index], page, order);
1572: }
1573:
1574: const char *
1575: vm_page_seg_name(unsigned int seg_index)
1576: {
1577: /* Don't use a switch statement since segments can be aliased */
1578: if (seg_index == VM_PAGE_SEG_HIGHMEM)
1579: return "HIGHMEM";
1580: else if (seg_index == VM_PAGE_SEG_DIRECTMAP)
1581: return "DIRECTMAP";
1582: else if (seg_index == VM_PAGE_SEG_DMA32)
1583: return "DMA32";
1584: else if (seg_index == VM_PAGE_SEG_DMA)
1585: return "DMA";
1586: else
1587: panic("vm_page: invalid segment index");
1588: }
1589:
1590: void
1591: vm_page_info_all(void)
1592: {
1593: struct vm_page_seg *seg;
1594: unsigned long pages;
1595: unsigned int i;
1596:
1597: for (i = 0; i < vm_page_segs_size; i++) {
1598: seg = &vm_page_segs[i];
1599: pages = (unsigned long)(seg->pages_end - seg->pages);
1600: printf("vm_page: %s: pages: %lu (%luM), free: %lu (%luM)\n",
1601: vm_page_seg_name(i), pages, pages >> (20 - PAGE_SHIFT),
1602: seg->nr_free_pages, seg->nr_free_pages >> (20 - PAGE_SHIFT));
1.1.1.2 ! root 1603: printf("vm_page: %s: min:%lu low:%lu high:%lu\n",
! 1604: vm_page_seg_name(vm_page_seg_index(seg)),
! 1605: seg->min_free_pages, seg->low_free_pages, seg->high_free_pages);
! 1606: }
! 1607: }
! 1608:
! 1609: phys_addr_t
! 1610: vm_page_seg_end(unsigned int selector)
! 1611: {
! 1612: return vm_page_segs[vm_page_select_alloc_seg(selector)].end;
! 1613: }
! 1614:
! 1615: static unsigned long
! 1616: vm_page_boot_table_size(void)
! 1617: {
! 1618: unsigned long nr_pages;
! 1619: unsigned int i;
! 1620:
! 1621: nr_pages = 0;
! 1622:
! 1623: for (i = 0; i < vm_page_segs_size; i++) {
! 1624: nr_pages += vm_page_atop(vm_page_boot_seg_size(&vm_page_boot_segs[i]));
! 1625: }
! 1626:
! 1627: return nr_pages;
! 1628: }
! 1629:
! 1630: unsigned long
! 1631: vm_page_table_size(void)
! 1632: {
! 1633: unsigned long nr_pages;
! 1634: unsigned int i;
! 1635:
! 1636: if (!vm_page_is_ready) {
! 1637: return vm_page_boot_table_size();
! 1638: }
! 1639:
! 1640: nr_pages = 0;
! 1641:
! 1642: for (i = 0; i < vm_page_segs_size; i++) {
! 1643: nr_pages += vm_page_atop(vm_page_seg_size(&vm_page_segs[i]));
! 1644: }
! 1645:
! 1646: return nr_pages;
! 1647: }
! 1648:
! 1649: unsigned long
! 1650: vm_page_table_index(phys_addr_t pa)
! 1651: {
! 1652: struct vm_page_seg *seg;
! 1653: unsigned long index;
! 1654: unsigned int i;
! 1655:
! 1656: index = 0;
! 1657:
! 1658: for (i = 0; i < vm_page_segs_size; i++) {
! 1659: seg = &vm_page_segs[i];
! 1660:
! 1661: if ((pa >= seg->start) && (pa < seg->end)) {
! 1662: return index + vm_page_atop(pa - seg->start);
! 1663: }
! 1664:
! 1665: index += vm_page_atop(vm_page_seg_size(seg));
1.1 root 1666: }
1.1.1.2 ! root 1667:
! 1668: panic("vm_page: invalid physical address");
1.1 root 1669: }
1670:
1671: phys_addr_t
1672: vm_page_mem_size(void)
1673: {
1674: phys_addr_t total;
1675: unsigned int i;
1676:
1677: total = 0;
1678:
1679: for (i = 0; i < vm_page_segs_size; i++) {
1680: total += vm_page_seg_size(&vm_page_segs[i]);
1681: }
1682:
1683: return total;
1684: }
1685:
1686: unsigned long
1687: vm_page_mem_free(void)
1688: {
1689: unsigned long total;
1690: unsigned int i;
1691:
1692: total = 0;
1693:
1694: for (i = 0; i < vm_page_segs_size; i++) {
1695: total += vm_page_segs[i].nr_free_pages;
1696: }
1697:
1698: return total;
1699: }
1.1.1.2 ! root 1700:
! 1701: /*
! 1702: * Mark this page as wired down by yet another map, removing it
! 1703: * from paging queues as necessary.
! 1704: *
! 1705: * The page's object and the page queues must be locked.
! 1706: */
! 1707: void
! 1708: vm_page_wire(struct vm_page *page)
! 1709: {
! 1710: VM_PAGE_CHECK(page);
! 1711:
! 1712: if (page->wire_count == 0) {
! 1713: vm_page_queues_remove(page);
! 1714:
! 1715: if (!page->private && !page->fictitious) {
! 1716: vm_page_wire_count++;
! 1717: }
! 1718: }
! 1719:
! 1720: page->wire_count++;
! 1721: }
! 1722:
! 1723: /*
! 1724: * Release one wiring of this page, potentially enabling it to be paged again.
! 1725: *
! 1726: * The page's object and the page queues must be locked.
! 1727: */
! 1728: void
! 1729: vm_page_unwire(struct vm_page *page)
! 1730: {
! 1731: struct vm_page_seg *seg;
! 1732:
! 1733: VM_PAGE_CHECK(page);
! 1734:
! 1735: assert(page->wire_count != 0);
! 1736: page->wire_count--;
! 1737:
! 1738: if ((page->wire_count != 0)
! 1739: || page->fictitious
! 1740: || page->private) {
! 1741: return;
! 1742: }
! 1743:
! 1744: seg = vm_page_seg_get(page->seg_index);
! 1745:
! 1746: simple_lock(&seg->lock);
! 1747: vm_page_seg_add_active_page(seg, page);
! 1748: simple_unlock(&seg->lock);
! 1749:
! 1750: vm_page_wire_count--;
! 1751: }
! 1752:
! 1753: /*
! 1754: * Returns the given page to the inactive list, indicating that
! 1755: * no physical maps have access to this page.
! 1756: * [Used by the physical mapping system.]
! 1757: *
! 1758: * The page queues must be locked.
! 1759: */
! 1760: void
! 1761: vm_page_deactivate(struct vm_page *page)
! 1762: {
! 1763: struct vm_page_seg *seg;
! 1764:
! 1765: VM_PAGE_CHECK(page);
! 1766:
! 1767: /*
! 1768: * This page is no longer very interesting. If it was
! 1769: * interesting (active or inactive/referenced), then we
! 1770: * clear the reference bit and (re)enter it in the
! 1771: * inactive queue. Note wired pages should not have
! 1772: * their reference bit cleared.
! 1773: */
! 1774:
! 1775: if (page->active || (page->inactive && page->reference)) {
! 1776: if (!page->fictitious && !page->private && !page->absent) {
! 1777: pmap_clear_reference(page->phys_addr);
! 1778: }
! 1779:
! 1780: page->reference = FALSE;
! 1781: vm_page_queues_remove(page);
! 1782: }
! 1783:
! 1784: if ((page->wire_count == 0) && !page->fictitious
! 1785: && !page->private && !page->inactive) {
! 1786: seg = vm_page_seg_get(page->seg_index);
! 1787:
! 1788: simple_lock(&seg->lock);
! 1789: vm_page_seg_add_inactive_page(seg, page);
! 1790: simple_unlock(&seg->lock);
! 1791: }
! 1792: }
! 1793:
! 1794: /*
! 1795: * Put the specified page on the active list (if appropriate).
! 1796: *
! 1797: * The page queues must be locked.
! 1798: */
! 1799: void
! 1800: vm_page_activate(struct vm_page *page)
! 1801: {
! 1802: struct vm_page_seg *seg;
! 1803:
! 1804: VM_PAGE_CHECK(page);
! 1805:
! 1806: /*
! 1807: * Unconditionally remove so that, even if the page was already
! 1808: * active, it gets back to the end of the active queue.
! 1809: */
! 1810: vm_page_queues_remove(page);
! 1811:
! 1812: if ((page->wire_count == 0) && !page->fictitious && !page->private) {
! 1813: seg = vm_page_seg_get(page->seg_index);
! 1814:
! 1815: if (page->active)
! 1816: panic("vm_page_activate: already active");
! 1817:
! 1818: simple_lock(&seg->lock);
! 1819: vm_page_seg_add_active_page(seg, page);
! 1820: simple_unlock(&seg->lock);
! 1821: }
! 1822: }
! 1823:
! 1824: void
! 1825: vm_page_queues_remove(struct vm_page *page)
! 1826: {
! 1827: struct vm_page_seg *seg;
! 1828:
! 1829: assert(!page->active || !page->inactive);
! 1830:
! 1831: if (!page->active && !page->inactive) {
! 1832: return;
! 1833: }
! 1834:
! 1835: seg = vm_page_seg_get(page->seg_index);
! 1836:
! 1837: simple_lock(&seg->lock);
! 1838:
! 1839: if (page->active) {
! 1840: vm_page_seg_remove_active_page(seg, page);
! 1841: } else {
! 1842: vm_page_seg_remove_inactive_page(seg, page);
! 1843: }
! 1844:
! 1845: simple_unlock(&seg->lock);
! 1846: }
! 1847:
! 1848: /*
! 1849: * Check whether segments are all usable for unprivileged allocations.
! 1850: *
! 1851: * If all segments are usable, resume pending unprivileged allocations
! 1852: * and return TRUE.
! 1853: *
! 1854: * This function acquires vm_page_queue_free_lock, which is held on return.
! 1855: */
! 1856: static boolean_t
! 1857: vm_page_check_usable(void)
! 1858: {
! 1859: struct vm_page_seg *seg;
! 1860: boolean_t usable;
! 1861: unsigned int i;
! 1862:
! 1863: simple_lock(&vm_page_queue_free_lock);
! 1864:
! 1865: for (i = 0; i < vm_page_segs_size; i++) {
! 1866: seg = vm_page_seg_get(i);
! 1867:
! 1868: simple_lock(&seg->lock);
! 1869: usable = vm_page_seg_usable(seg);
! 1870: simple_unlock(&seg->lock);
! 1871:
! 1872: if (!usable) {
! 1873: return FALSE;
! 1874: }
! 1875: }
! 1876:
! 1877: vm_page_external_laundry_count = -1;
! 1878: vm_page_alloc_paused = FALSE;
! 1879: thread_wakeup(&vm_page_alloc_paused);
! 1880: return TRUE;
! 1881: }
! 1882:
! 1883: static boolean_t
! 1884: vm_page_may_balance(void)
! 1885: {
! 1886: struct vm_page_seg *seg;
! 1887: boolean_t page_available;
! 1888: unsigned int i;
! 1889:
! 1890: for (i = 0; i < vm_page_segs_size; i++) {
! 1891: seg = vm_page_seg_get(i);
! 1892:
! 1893: simple_lock(&seg->lock);
! 1894: page_available = vm_page_seg_page_available(seg);
! 1895: simple_unlock(&seg->lock);
! 1896:
! 1897: if (page_available) {
! 1898: return TRUE;
! 1899: }
! 1900: }
! 1901:
! 1902: return FALSE;
! 1903: }
! 1904:
! 1905: static boolean_t
! 1906: vm_page_balance_once(void)
! 1907: {
! 1908: boolean_t balanced;
! 1909: unsigned int i;
! 1910:
! 1911: /*
! 1912: * It's important here that pages are moved from higher priority
! 1913: * segments first.
! 1914: */
! 1915:
! 1916: for (i = 0; i < vm_page_segs_size; i++) {
! 1917: balanced = vm_page_seg_balance(vm_page_seg_get(i));
! 1918:
! 1919: if (balanced) {
! 1920: return TRUE;
! 1921: }
! 1922: }
! 1923:
! 1924: return FALSE;
! 1925: }
! 1926:
! 1927: boolean_t
! 1928: vm_page_balance(void)
! 1929: {
! 1930: boolean_t balanced;
! 1931:
! 1932: while (vm_page_may_balance()) {
! 1933: balanced = vm_page_balance_once();
! 1934:
! 1935: if (!balanced) {
! 1936: break;
! 1937: }
! 1938: }
! 1939:
! 1940: return vm_page_check_usable();
! 1941: }
! 1942:
! 1943: static boolean_t
! 1944: vm_page_evict_once(boolean_t external_only, boolean_t alloc_paused)
! 1945: {
! 1946: boolean_t evicted;
! 1947: unsigned int i;
! 1948:
! 1949: /*
! 1950: * It's important here that pages are evicted from lower priority
! 1951: * segments first.
! 1952: */
! 1953:
! 1954: for (i = vm_page_segs_size - 1; i < vm_page_segs_size; i--) {
! 1955: evicted = vm_page_seg_evict(vm_page_seg_get(i),
! 1956: external_only, alloc_paused);
! 1957:
! 1958: if (evicted) {
! 1959: return TRUE;
! 1960: }
! 1961: }
! 1962:
! 1963: return FALSE;
! 1964: }
! 1965:
! 1966: #define VM_PAGE_MAX_LAUNDRY 5
! 1967: #define VM_PAGE_MAX_EVICTIONS 5
! 1968:
! 1969: boolean_t
! 1970: vm_page_evict(boolean_t *should_wait)
! 1971: {
! 1972: boolean_t pause, evicted, external_only, alloc_paused;
! 1973: unsigned int i;
! 1974:
! 1975: *should_wait = TRUE;
! 1976: external_only = TRUE;
! 1977:
! 1978: simple_lock(&vm_page_queue_free_lock);
! 1979: vm_page_external_laundry_count = 0;
! 1980: alloc_paused = vm_page_alloc_paused;
! 1981: simple_unlock(&vm_page_queue_free_lock);
! 1982:
! 1983: again:
! 1984: vm_page_lock_queues();
! 1985: pause = (vm_page_laundry_count >= VM_PAGE_MAX_LAUNDRY);
! 1986: vm_page_unlock_queues();
! 1987:
! 1988: if (pause) {
! 1989: simple_lock(&vm_page_queue_free_lock);
! 1990: return FALSE;
! 1991: }
! 1992:
! 1993: for (i = 0; i < VM_PAGE_MAX_EVICTIONS; i++) {
! 1994: evicted = vm_page_evict_once(external_only, alloc_paused);
! 1995:
! 1996: if (!evicted) {
! 1997: break;
! 1998: }
! 1999: }
! 2000:
! 2001: simple_lock(&vm_page_queue_free_lock);
! 2002:
! 2003: /*
! 2004: * Keep in mind eviction may not cause pageouts, since non-precious
! 2005: * clean pages are simply released.
! 2006: */
! 2007: if ((vm_page_laundry_count == 0) && (vm_page_external_laundry_count == 0)) {
! 2008: /*
! 2009: * No pageout, but some clean pages were freed. Start a complete
! 2010: * scan again without waiting.
! 2011: */
! 2012: if (evicted) {
! 2013: *should_wait = FALSE;
! 2014: return FALSE;
! 2015: }
! 2016:
! 2017: /*
! 2018: * Eviction failed, consider pages from internal objects on the
! 2019: * next attempt.
! 2020: */
! 2021: if (external_only) {
! 2022: simple_unlock(&vm_page_queue_free_lock);
! 2023: external_only = FALSE;
! 2024: goto again;
! 2025: }
! 2026:
! 2027: /*
! 2028: * TODO Find out what could cause this and how to deal with it.
! 2029: * This will likely require an out-of-memory killer.
! 2030: */
! 2031: panic("vm_page: unable to recycle any page");
! 2032: }
! 2033:
! 2034: simple_unlock(&vm_page_queue_free_lock);
! 2035:
! 2036: return vm_page_check_usable();
! 2037: }
! 2038:
! 2039: void
! 2040: vm_page_refill_inactive(void)
! 2041: {
! 2042: unsigned int i;
! 2043:
! 2044: vm_page_lock_queues();
! 2045:
! 2046: for (i = 0; i < vm_page_segs_size; i++) {
! 2047: vm_page_seg_refill_inactive(vm_page_seg_get(i));
! 2048: }
! 2049:
! 2050: vm_page_unlock_queues();
! 2051: }
! 2052:
! 2053: void
! 2054: vm_page_wait(void (*continuation)(void))
! 2055: {
! 2056: assert(!current_thread()->vm_privilege);
! 2057:
! 2058: simple_lock(&vm_page_queue_free_lock);
! 2059:
! 2060: if (!vm_page_alloc_paused) {
! 2061: simple_unlock(&vm_page_queue_free_lock);
! 2062: return;
! 2063: }
! 2064:
! 2065: assert_wait(&vm_page_alloc_paused, FALSE);
! 2066:
! 2067: simple_unlock(&vm_page_queue_free_lock);
! 2068:
! 2069: if (continuation != 0) {
! 2070: counter(c_vm_page_wait_block_user++);
! 2071: thread_block(continuation);
! 2072: } else {
! 2073: counter(c_vm_page_wait_block_kernel++);
! 2074: thread_block((void (*)(void)) 0);
! 2075: }
! 2076: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.