Annotation of Gnu-Mach/vm/vm_page.c, revision 1.1.1.1

1.1       root        1: /*
                      2:  * Copyright (c) 2010-2014 Richard Braun.
                      3:  *
                      4:  * This program is free software: you can redistribute it and/or modify
                      5:  * it under the terms of the GNU General Public License as published by
                      6:  * the Free Software Foundation, either version 2 of the License, or
                      7:  * (at your option) any later version.
                      8:  *
                      9:  * This program is distributed in the hope that it will be useful,
                     10:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
                     11:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
                     12:  * GNU General Public License for more details.
                     13:  *
                     14:  * You should have received a copy of the GNU General Public License
                     15:  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     16:  *
                     17:  *
                     18:  * This implementation uses the binary buddy system to manage its heap.
                     19:  * Descriptions of the buddy system can be found in the following works :
                     20:  * - "UNIX Internals: The New Frontiers", by Uresh Vahalia.
                     21:  * - "Dynamic Storage Allocation: A Survey and Critical Review",
                     22:  *    by Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles.
                     23:  *
                     24:  * In addition, this allocator uses per-CPU pools of pages for order 0
                     25:  * (i.e. single page) allocations. These pools act as caches (but are named
                     26:  * differently to avoid confusion with CPU caches) that reduce contention on
                     27:  * multiprocessor systems. When a pool is empty and cannot provide a page,
                     28:  * it is filled by transferring multiple pages from the backend buddy system.
                     29:  * The symmetric case is handled likewise.
                     30:  */
                     31: 
                     32: #include <string.h>
                     33: #include <kern/assert.h>
                     34: #include <kern/cpu_number.h>
                     35: #include <kern/debug.h>
                     36: #include <kern/list.h>
                     37: #include <kern/lock.h>
                     38: #include <kern/macros.h>
                     39: #include <kern/printf.h>
                     40: #include <kern/thread.h>
                     41: #include <mach/vm_param.h>
                     42: #include <machine/pmap.h>
                     43: #include <sys/types.h>
                     44: #include <vm/vm_page.h>
                     45: 
                     46: #define __init
                     47: #define __initdata
                     48: #define __read_mostly
                     49: 
                     50: #define thread_pin()
                     51: #define thread_unpin()
                     52: 
                     53: /*
                     54:  * Number of free block lists per segment.
                     55:  */
                     56: #define VM_PAGE_NR_FREE_LISTS 11
                     57: 
                     58: /*
                     59:  * The size of a CPU pool is computed by dividing the number of pages in its
                     60:  * containing segment by this value.
                     61:  */
                     62: #define VM_PAGE_CPU_POOL_RATIO 1024
                     63: 
                     64: /*
                     65:  * Maximum number of pages in a CPU pool.
                     66:  */
                     67: #define VM_PAGE_CPU_POOL_MAX_SIZE 128
                     68: 
                     69: /*
                     70:  * The transfer size of a CPU pool is computed by dividing the pool size by
                     71:  * this value.
                     72:  */
                     73: #define VM_PAGE_CPU_POOL_TRANSFER_RATIO 2
                     74: 
                     75: /*
                     76:  * Per-processor cache of pages.
                     77:  */
                     78: struct vm_page_cpu_pool {
                     79:     simple_lock_data_t lock;
                     80:     int size;
                     81:     int transfer_size;
                     82:     int nr_pages;
                     83:     struct list pages;
                     84: } __aligned(CPU_L1_SIZE);
                     85: 
                     86: /*
                     87:  * Special order value for pages that aren't in a free list. Such pages are
                     88:  * either allocated, or part of a free block of pages but not the head page.
                     89:  */
                     90: #define VM_PAGE_ORDER_UNLISTED ((unsigned short)-1)
                     91: 
                     92: /*
                     93:  * Doubly-linked list of free blocks.
                     94:  */
                     95: struct vm_page_free_list {
                     96:     unsigned long size;
                     97:     struct list blocks;
                     98: };
                     99: 
                    100: /*
                    101:  * Segment name buffer size.
                    102:  */
                    103: #define VM_PAGE_NAME_SIZE 16
                    104: 
                    105: /*
                    106:  * Segment of contiguous memory.
                    107:  */
                    108: struct vm_page_seg {
                    109:     struct vm_page_cpu_pool cpu_pools[NCPUS];
                    110: 
                    111:     phys_addr_t start;
                    112:     phys_addr_t end;
                    113:     struct vm_page *pages;
                    114:     struct vm_page *pages_end;
                    115:     simple_lock_data_t lock;
                    116:     struct vm_page_free_list free_lists[VM_PAGE_NR_FREE_LISTS];
                    117:     unsigned long nr_free_pages;
                    118: };
                    119: 
                    120: /*
                    121:  * Bootstrap information about a segment.
                    122:  */
                    123: struct vm_page_boot_seg {
                    124:     phys_addr_t start;
                    125:     phys_addr_t end;
                    126:     phys_addr_t avail_start;
                    127:     phys_addr_t avail_end;
                    128: };
                    129: 
                    130: static int vm_page_is_ready __read_mostly;
                    131: 
                    132: /*
                    133:  * Segment table.
                    134:  *
                    135:  * The system supports a maximum of 4 segments :
                    136:  *  - DMA: suitable for DMA
                    137:  *  - DMA32: suitable for DMA when devices support 32-bits addressing
                    138:  *  - DIRECTMAP: direct physical mapping, allows direct access from
                    139:  *    the kernel with a simple offset translation
                    140:  *  - HIGHMEM: must be mapped before it can be accessed
                    141:  *
                    142:  * Segments are ordered by priority, 0 being the lowest priority. Their
                    143:  * relative priorities are DMA < DMA32 < DIRECTMAP < HIGHMEM. Some segments
                    144:  * may actually be aliases for others, e.g. if DMA is always possible from
                    145:  * the direct physical mapping, DMA and DMA32 are aliases for DIRECTMAP,
                    146:  * in which case the segment table contains DIRECTMAP and HIGHMEM only.
                    147:  */
                    148: static struct vm_page_seg vm_page_segs[VM_PAGE_MAX_SEGS];
                    149: 
                    150: /*
                    151:  * Bootstrap segment table.
                    152:  */
                    153: static struct vm_page_boot_seg vm_page_boot_segs[VM_PAGE_MAX_SEGS] __initdata;
                    154: 
                    155: /*
                    156:  * Number of loaded segments.
                    157:  */
                    158: static unsigned int vm_page_segs_size __read_mostly;
                    159: 
                    160: static void __init
                    161: vm_page_init_pa(struct vm_page *page, unsigned short seg_index, phys_addr_t pa)
                    162: {
                    163:     memset(page, 0, sizeof(*page));
                    164:     vm_page_init(page); /* vm_resident members */
                    165:     page->type = VM_PT_RESERVED;
                    166:     page->seg_index = seg_index;
                    167:     page->order = VM_PAGE_ORDER_UNLISTED;
                    168:     page->priv = NULL;
                    169:     page->phys_addr = pa;
                    170: }
                    171: 
                    172: void
                    173: vm_page_set_type(struct vm_page *page, unsigned int order, unsigned short type)
                    174: {
                    175:     unsigned int i, nr_pages;
                    176: 
                    177:     nr_pages = 1 << order;
                    178: 
                    179:     for (i = 0; i < nr_pages; i++)
                    180:         page[i].type = type;
                    181: }
                    182: 
                    183: static void __init
                    184: vm_page_free_list_init(struct vm_page_free_list *free_list)
                    185: {
                    186:     free_list->size = 0;
                    187:     list_init(&free_list->blocks);
                    188: }
                    189: 
                    190: static inline void
                    191: vm_page_free_list_insert(struct vm_page_free_list *free_list,
                    192:                          struct vm_page *page)
                    193: {
                    194:     assert(page->order == VM_PAGE_ORDER_UNLISTED);
                    195: 
                    196:     free_list->size++;
                    197:     list_insert_head(&free_list->blocks, &page->node);
                    198: }
                    199: 
                    200: static inline void
                    201: vm_page_free_list_remove(struct vm_page_free_list *free_list,
                    202:                          struct vm_page *page)
                    203: {
                    204:     assert(page->order != VM_PAGE_ORDER_UNLISTED);
                    205: 
                    206:     free_list->size--;
                    207:     list_remove(&page->node);
                    208: }
                    209: 
                    210: static struct vm_page *
                    211: vm_page_seg_alloc_from_buddy(struct vm_page_seg *seg, unsigned int order)
                    212: {
                    213:     struct vm_page_free_list *free_list = free_list;
                    214:     struct vm_page *page, *buddy;
                    215:     unsigned int i;
                    216: 
                    217:     assert(order < VM_PAGE_NR_FREE_LISTS);
                    218: 
                    219:     for (i = order; i < VM_PAGE_NR_FREE_LISTS; i++) {
                    220:         free_list = &seg->free_lists[i];
                    221: 
                    222:         if (free_list->size != 0)
                    223:             break;
                    224:     }
                    225: 
                    226:     if (i == VM_PAGE_NR_FREE_LISTS)
                    227:         return NULL;
                    228: 
                    229:     page = list_first_entry(&free_list->blocks, struct vm_page, node);
                    230:     vm_page_free_list_remove(free_list, page);
                    231:     page->order = VM_PAGE_ORDER_UNLISTED;
                    232: 
                    233:     while (i > order) {
                    234:         i--;
                    235:         buddy = &page[1 << i];
                    236:         vm_page_free_list_insert(&seg->free_lists[i], buddy);
                    237:         buddy->order = i;
                    238:     }
                    239: 
                    240:     seg->nr_free_pages -= (1 << order);
                    241:     return page;
                    242: }
                    243: 
                    244: static void
                    245: vm_page_seg_free_to_buddy(struct vm_page_seg *seg, struct vm_page *page,
                    246:                           unsigned int order)
                    247: {
                    248:     struct vm_page *buddy;
                    249:     phys_addr_t pa, buddy_pa;
                    250:     unsigned int nr_pages;
                    251: 
                    252:     assert(page >= seg->pages);
                    253:     assert(page < seg->pages_end);
                    254:     assert(page->order == VM_PAGE_ORDER_UNLISTED);
                    255:     assert(order < VM_PAGE_NR_FREE_LISTS);
                    256: 
                    257:     nr_pages = (1 << order);
                    258:     pa = page->phys_addr;
                    259: 
                    260:     while (order < (VM_PAGE_NR_FREE_LISTS - 1)) {
                    261:         buddy_pa = pa ^ vm_page_ptoa(1 << order);
                    262: 
                    263:         if ((buddy_pa < seg->start) || (buddy_pa >= seg->end))
                    264:             break;
                    265: 
                    266:         buddy = &seg->pages[vm_page_atop(buddy_pa - seg->start)];
                    267: 
                    268:         if (buddy->order != order)
                    269:             break;
                    270: 
                    271:         vm_page_free_list_remove(&seg->free_lists[order], buddy);
                    272:         buddy->order = VM_PAGE_ORDER_UNLISTED;
                    273:         order++;
                    274:         pa &= -vm_page_ptoa(1 << order);
                    275:         page = &seg->pages[vm_page_atop(pa - seg->start)];
                    276:     }
                    277: 
                    278:     vm_page_free_list_insert(&seg->free_lists[order], page);
                    279:     page->order = order;
                    280:     seg->nr_free_pages += nr_pages;
                    281: }
                    282: 
                    283: static void __init
                    284: vm_page_cpu_pool_init(struct vm_page_cpu_pool *cpu_pool, int size)
                    285: {
                    286:     simple_lock_init(&cpu_pool->lock);
                    287:     cpu_pool->size = size;
                    288:     cpu_pool->transfer_size = (size + VM_PAGE_CPU_POOL_TRANSFER_RATIO - 1)
                    289:                               / VM_PAGE_CPU_POOL_TRANSFER_RATIO;
                    290:     cpu_pool->nr_pages = 0;
                    291:     list_init(&cpu_pool->pages);
                    292: }
                    293: 
                    294: static inline struct vm_page_cpu_pool *
                    295: vm_page_cpu_pool_get(struct vm_page_seg *seg)
                    296: {
                    297:     return &seg->cpu_pools[cpu_number()];
                    298: }
                    299: 
                    300: static inline struct vm_page *
                    301: vm_page_cpu_pool_pop(struct vm_page_cpu_pool *cpu_pool)
                    302: {
                    303:     struct vm_page *page;
                    304: 
                    305:     assert(cpu_pool->nr_pages != 0);
                    306:     cpu_pool->nr_pages--;
                    307:     page = list_first_entry(&cpu_pool->pages, struct vm_page, node);
                    308:     list_remove(&page->node);
                    309:     return page;
                    310: }
                    311: 
                    312: static inline void
                    313: vm_page_cpu_pool_push(struct vm_page_cpu_pool *cpu_pool, struct vm_page *page)
                    314: {
                    315:     assert(cpu_pool->nr_pages < cpu_pool->size);
                    316:     cpu_pool->nr_pages++;
                    317:     list_insert_head(&cpu_pool->pages, &page->node);
                    318: }
                    319: 
                    320: static int
                    321: vm_page_cpu_pool_fill(struct vm_page_cpu_pool *cpu_pool,
                    322:                       struct vm_page_seg *seg)
                    323: {
                    324:     struct vm_page *page;
                    325:     int i;
                    326: 
                    327:     assert(cpu_pool->nr_pages == 0);
                    328: 
                    329:     simple_lock(&seg->lock);
                    330: 
                    331:     for (i = 0; i < cpu_pool->transfer_size; i++) {
                    332:         page = vm_page_seg_alloc_from_buddy(seg, 0);
                    333: 
                    334:         if (page == NULL)
                    335:             break;
                    336: 
                    337:         vm_page_cpu_pool_push(cpu_pool, page);
                    338:     }
                    339: 
                    340:     simple_unlock(&seg->lock);
                    341: 
                    342:     return i;
                    343: }
                    344: 
                    345: static void
                    346: vm_page_cpu_pool_drain(struct vm_page_cpu_pool *cpu_pool,
                    347:                        struct vm_page_seg *seg)
                    348: {
                    349:     struct vm_page *page;
                    350:     int i;
                    351: 
                    352:     assert(cpu_pool->nr_pages == cpu_pool->size);
                    353: 
                    354:     simple_lock(&seg->lock);
                    355: 
                    356:     for (i = cpu_pool->transfer_size; i > 0; i--) {
                    357:         page = vm_page_cpu_pool_pop(cpu_pool);
                    358:         vm_page_seg_free_to_buddy(seg, page, 0);
                    359:     }
                    360: 
                    361:     simple_unlock(&seg->lock);
                    362: }
                    363: 
                    364: static phys_addr_t __init
                    365: vm_page_seg_size(struct vm_page_seg *seg)
                    366: {
                    367:     return seg->end - seg->start;
                    368: }
                    369: 
                    370: static int __init
                    371: vm_page_seg_compute_pool_size(struct vm_page_seg *seg)
                    372: {
                    373:     phys_addr_t size;
                    374: 
                    375:     size = vm_page_atop(vm_page_seg_size(seg)) / VM_PAGE_CPU_POOL_RATIO;
                    376: 
                    377:     if (size == 0)
                    378:         size = 1;
                    379:     else if (size > VM_PAGE_CPU_POOL_MAX_SIZE)
                    380:         size = VM_PAGE_CPU_POOL_MAX_SIZE;
                    381: 
                    382:     return size;
                    383: }
                    384: 
                    385: static void __init
                    386: vm_page_seg_init(struct vm_page_seg *seg, phys_addr_t start, phys_addr_t end,
                    387:                  struct vm_page *pages)
                    388: {
                    389:     phys_addr_t pa;
                    390:     int pool_size;
                    391:     unsigned int i;
                    392: 
                    393:     seg->start = start;
                    394:     seg->end = end;
                    395:     pool_size = vm_page_seg_compute_pool_size(seg);
                    396: 
                    397:     for (i = 0; i < ARRAY_SIZE(seg->cpu_pools); i++)
                    398:         vm_page_cpu_pool_init(&seg->cpu_pools[i], pool_size);
                    399: 
                    400:     seg->pages = pages;
                    401:     seg->pages_end = pages + vm_page_atop(vm_page_seg_size(seg));
                    402:     simple_lock_init(&seg->lock);
                    403: 
                    404:     for (i = 0; i < ARRAY_SIZE(seg->free_lists); i++)
                    405:         vm_page_free_list_init(&seg->free_lists[i]);
                    406: 
                    407:     seg->nr_free_pages = 0;
                    408:     i = seg - vm_page_segs;
                    409: 
                    410:     for (pa = seg->start; pa < seg->end; pa += PAGE_SIZE)
                    411:         vm_page_init_pa(&pages[vm_page_atop(pa - seg->start)], i, pa);
                    412: }
                    413: 
                    414: static struct vm_page *
                    415: vm_page_seg_alloc(struct vm_page_seg *seg, unsigned int order,
                    416:                   unsigned short type)
                    417: {
                    418:     struct vm_page_cpu_pool *cpu_pool;
                    419:     struct vm_page *page;
                    420:     int filled;
                    421: 
                    422:     assert(order < VM_PAGE_NR_FREE_LISTS);
                    423: 
                    424:     if (order == 0) {
                    425:         thread_pin();
                    426:         cpu_pool = vm_page_cpu_pool_get(seg);
                    427:         simple_lock(&cpu_pool->lock);
                    428: 
                    429:         if (cpu_pool->nr_pages == 0) {
                    430:             filled = vm_page_cpu_pool_fill(cpu_pool, seg);
                    431: 
                    432:             if (!filled) {
                    433:                 simple_unlock(&cpu_pool->lock);
                    434:                 thread_unpin();
                    435:                 return NULL;
                    436:             }
                    437:         }
                    438: 
                    439:         page = vm_page_cpu_pool_pop(cpu_pool);
                    440:         simple_unlock(&cpu_pool->lock);
                    441:         thread_unpin();
                    442:     } else {
                    443:         simple_lock(&seg->lock);
                    444:         page = vm_page_seg_alloc_from_buddy(seg, order);
                    445:         simple_unlock(&seg->lock);
                    446: 
                    447:         if (page == NULL)
                    448:             return NULL;
                    449:     }
                    450: 
                    451:     assert(page->type == VM_PT_FREE);
                    452:     vm_page_set_type(page, order, type);
                    453:     return page;
                    454: }
                    455: 
                    456: static void
                    457: vm_page_seg_free(struct vm_page_seg *seg, struct vm_page *page,
                    458:                  unsigned int order)
                    459: {
                    460:     struct vm_page_cpu_pool *cpu_pool;
                    461: 
                    462:     assert(page->type != VM_PT_FREE);
                    463:     assert(order < VM_PAGE_NR_FREE_LISTS);
                    464: 
                    465:     vm_page_set_type(page, order, VM_PT_FREE);
                    466: 
                    467:     if (order == 0) {
                    468:         thread_pin();
                    469:         cpu_pool = vm_page_cpu_pool_get(seg);
                    470:         simple_lock(&cpu_pool->lock);
                    471: 
                    472:         if (cpu_pool->nr_pages == cpu_pool->size)
                    473:             vm_page_cpu_pool_drain(cpu_pool, seg);
                    474: 
                    475:         vm_page_cpu_pool_push(cpu_pool, page);
                    476:         simple_unlock(&cpu_pool->lock);
                    477:         thread_unpin();
                    478:     } else {
                    479:         simple_lock(&seg->lock);
                    480:         vm_page_seg_free_to_buddy(seg, page, order);
                    481:         simple_unlock(&seg->lock);
                    482:     }
                    483: }
                    484: 
                    485: void __init
                    486: vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end,
                    487:              phys_addr_t avail_start, phys_addr_t avail_end)
                    488: {
                    489:     struct vm_page_boot_seg *seg;
                    490: 
                    491:     assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
                    492:     assert(vm_page_aligned(start));
                    493:     assert(vm_page_aligned(end));
                    494:     assert(vm_page_aligned(avail_start));
                    495:     assert(vm_page_aligned(avail_end));
                    496:     assert(start < end);
                    497:     assert(start <= avail_start);
                    498:     assert(avail_end <= end);
                    499:     assert(vm_page_segs_size < ARRAY_SIZE(vm_page_boot_segs));
                    500: 
                    501:     seg = &vm_page_boot_segs[seg_index];
                    502:     seg->start = start;
                    503:     seg->end = end;
                    504:     seg->avail_start = avail_start;
                    505:     seg->avail_end = avail_end;
                    506:     vm_page_segs_size++;
                    507: }
                    508: 
                    509: int
                    510: vm_page_ready(void)
                    511: {
                    512:     return vm_page_is_ready;
                    513: }
                    514: 
                    515: static unsigned int
                    516: vm_page_select_alloc_seg(unsigned int selector)
                    517: {
                    518:     unsigned int seg_index;
                    519: 
                    520:     switch (selector) {
                    521:     case VM_PAGE_SEL_DMA:
                    522:         seg_index = VM_PAGE_SEG_DMA;
                    523:         break;
                    524:     case VM_PAGE_SEL_DMA32:
                    525:         seg_index = VM_PAGE_SEG_DMA32;
                    526:         break;
                    527:     case VM_PAGE_SEL_DIRECTMAP:
                    528:         seg_index = VM_PAGE_SEG_DIRECTMAP;
                    529:         break;
                    530:     case VM_PAGE_SEL_HIGHMEM:
                    531:         seg_index = VM_PAGE_SEG_HIGHMEM;
                    532:         break;
                    533:     default:
                    534:         panic("vm_page: invalid selector");
                    535:     }
                    536: 
                    537:     return MIN(vm_page_segs_size - 1, seg_index);
                    538: }
                    539: 
                    540: static int __init
                    541: vm_page_boot_seg_loaded(const struct vm_page_boot_seg *seg)
                    542: {
                    543:     return (seg->end != 0);
                    544: }
                    545: 
                    546: static void __init
                    547: vm_page_check_boot_segs(void)
                    548: {
                    549:     unsigned int i;
                    550:     int expect_loaded;
                    551: 
                    552:     if (vm_page_segs_size == 0)
                    553:         panic("vm_page: no physical memory loaded");
                    554: 
                    555:     for (i = 0; i < ARRAY_SIZE(vm_page_boot_segs); i++) {
                    556:         expect_loaded = (i < vm_page_segs_size);
                    557: 
                    558:         if (vm_page_boot_seg_loaded(&vm_page_boot_segs[i]) == expect_loaded)
                    559:             continue;
                    560: 
                    561:         panic("vm_page: invalid boot segment table");
                    562:     }
                    563: }
                    564: 
                    565: static phys_addr_t __init
                    566: vm_page_boot_seg_size(struct vm_page_boot_seg *seg)
                    567: {
                    568:     return seg->end - seg->start;
                    569: }
                    570: 
                    571: static phys_addr_t __init
                    572: vm_page_boot_seg_avail_size(struct vm_page_boot_seg *seg)
                    573: {
                    574:     return seg->avail_end - seg->avail_start;
                    575: }
                    576: 
                    577: unsigned long __init
                    578: vm_page_bootalloc(size_t size)
                    579: {
                    580:     struct vm_page_boot_seg *seg;
                    581:     phys_addr_t pa;
                    582:     unsigned int i;
                    583: 
                    584:     for (i = vm_page_select_alloc_seg(VM_PAGE_SEL_DIRECTMAP);
                    585:          i < vm_page_segs_size;
                    586:          i--) {
                    587:         seg = &vm_page_boot_segs[i];
                    588: 
                    589:         if (size <= vm_page_boot_seg_avail_size(seg)) {
                    590:             pa = seg->avail_start;
                    591:             seg->avail_start += vm_page_round(size);
                    592:             return pa;
                    593:         }
                    594:     }
                    595: 
                    596:     panic("vm_page: no physical memory available");
                    597: }
                    598: 
                    599: void __init
                    600: vm_page_setup(void)
                    601: {
                    602:     struct vm_page_boot_seg *boot_seg;
                    603:     struct vm_page_seg *seg;
                    604:     struct vm_page *table, *page, *end;
                    605:     size_t nr_pages, table_size;
                    606:     unsigned long va;
                    607:     unsigned int i;
                    608:     phys_addr_t pa;
                    609: 
                    610:     vm_page_check_boot_segs();
                    611: 
                    612:     /*
                    613:      * Compute the page table size.
                    614:      */
                    615:     nr_pages = 0;
                    616: 
                    617:     for (i = 0; i < vm_page_segs_size; i++)
                    618:         nr_pages += vm_page_atop(vm_page_boot_seg_size(&vm_page_boot_segs[i]));
                    619: 
                    620:     table_size = vm_page_round(nr_pages * sizeof(struct vm_page));
                    621:     printf("vm_page: page table size: %lu entries (%luk)\n", nr_pages,
                    622:            table_size >> 10);
                    623:     table = (struct vm_page *)pmap_steal_memory(table_size);
                    624:     va = (unsigned long)table;
                    625: 
                    626:     /*
                    627:      * Initialize the segments, associating them to the page table. When
                    628:      * the segments are initialized, all their pages are set allocated.
                    629:      * Pages are then released, which populates the free lists.
                    630:      */
                    631:     for (i = 0; i < vm_page_segs_size; i++) {
                    632:         seg = &vm_page_segs[i];
                    633:         boot_seg = &vm_page_boot_segs[i];
                    634:         vm_page_seg_init(seg, boot_seg->start, boot_seg->end, table);
                    635:         page = seg->pages + vm_page_atop(boot_seg->avail_start
                    636:                                          - boot_seg->start);
                    637:         end = seg->pages + vm_page_atop(boot_seg->avail_end
                    638:                                         - boot_seg->start);
                    639: 
                    640:         while (page < end) {
                    641:             page->type = VM_PT_FREE;
                    642:             vm_page_seg_free_to_buddy(seg, page, 0);
                    643:             page++;
                    644:         }
                    645: 
                    646:         table += vm_page_atop(vm_page_seg_size(seg));
                    647:     }
                    648: 
                    649:     while (va < (unsigned long)table) {
                    650:         pa = pmap_extract(kernel_pmap, va);
                    651:         page = vm_page_lookup_pa(pa);
                    652:         assert((page != NULL) && (page->type == VM_PT_RESERVED));
                    653:         page->type = VM_PT_TABLE;
                    654:         va += PAGE_SIZE;
                    655:     }
                    656: 
                    657:     vm_page_is_ready = 1;
                    658: }
                    659: 
                    660: void __init
                    661: vm_page_manage(struct vm_page *page)
                    662: {
                    663:     assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
                    664:     assert(page->type == VM_PT_RESERVED);
                    665: 
                    666:     vm_page_set_type(page, 0, VM_PT_FREE);
                    667:     vm_page_seg_free_to_buddy(&vm_page_segs[page->seg_index], page, 0);
                    668: }
                    669: 
                    670: struct vm_page *
                    671: vm_page_lookup_pa(phys_addr_t pa)
                    672: {
                    673:     struct vm_page_seg *seg;
                    674:     unsigned int i;
                    675: 
                    676:     for (i = 0; i < vm_page_segs_size; i++) {
                    677:         seg = &vm_page_segs[i];
                    678: 
                    679:         if ((pa >= seg->start) && (pa < seg->end))
                    680:             return &seg->pages[vm_page_atop(pa - seg->start)];
                    681:     }
                    682: 
                    683:     return NULL;
                    684: }
                    685: 
                    686: struct vm_page *
                    687: vm_page_alloc_pa(unsigned int order, unsigned int selector, unsigned short type)
                    688: {
                    689:     struct vm_page *page;
                    690:     unsigned int i;
                    691: 
                    692:     for (i = vm_page_select_alloc_seg(selector); i < vm_page_segs_size; i--) {
                    693:         page = vm_page_seg_alloc(&vm_page_segs[i], order, type);
                    694: 
                    695:         if (page != NULL)
                    696:             return page;
                    697:     }
                    698: 
                    699:     if (type == VM_PT_PMAP)
                    700:         panic("vm_page: unable to allocate pmap page");
                    701: 
                    702:     return NULL;
                    703: }
                    704: 
                    705: void
                    706: vm_page_free_pa(struct vm_page *page, unsigned int order)
                    707: {
                    708:     assert(page != NULL);
                    709:     assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
                    710: 
                    711:     vm_page_seg_free(&vm_page_segs[page->seg_index], page, order);
                    712: }
                    713: 
                    714: const char *
                    715: vm_page_seg_name(unsigned int seg_index)
                    716: {
                    717:     /* Don't use a switch statement since segments can be aliased */
                    718:     if (seg_index == VM_PAGE_SEG_HIGHMEM)
                    719:         return "HIGHMEM";
                    720:     else if (seg_index == VM_PAGE_SEG_DIRECTMAP)
                    721:         return "DIRECTMAP";
                    722:     else if (seg_index == VM_PAGE_SEG_DMA32)
                    723:         return "DMA32";
                    724:     else if (seg_index == VM_PAGE_SEG_DMA)
                    725:         return "DMA";
                    726:     else
                    727:         panic("vm_page: invalid segment index");
                    728: }
                    729: 
                    730: void
                    731: vm_page_info_all(void)
                    732: {
                    733:     struct vm_page_seg *seg;
                    734:     unsigned long pages;
                    735:     unsigned int i;
                    736: 
                    737:     for (i = 0; i < vm_page_segs_size; i++) {
                    738:         seg = &vm_page_segs[i];
                    739:         pages = (unsigned long)(seg->pages_end - seg->pages);
                    740:         printf("vm_page: %s: pages: %lu (%luM), free: %lu (%luM)\n",
                    741:                vm_page_seg_name(i), pages, pages >> (20 - PAGE_SHIFT),
                    742:                seg->nr_free_pages, seg->nr_free_pages >> (20 - PAGE_SHIFT));
                    743:     }
                    744: }
                    745: 
                    746: phys_addr_t
                    747: vm_page_mem_size(void)
                    748: {
                    749:     phys_addr_t total;
                    750:     unsigned int i;
                    751: 
                    752:     total = 0;
                    753: 
                    754:     for (i = 0; i < vm_page_segs_size; i++) {
                    755:         /* XXX */
                    756:         if (i > VM_PAGE_SEG_DIRECTMAP)
                    757:             continue;
                    758: 
                    759:         total += vm_page_seg_size(&vm_page_segs[i]);
                    760:     }
                    761: 
                    762:     return total;
                    763: }
                    764: 
                    765: unsigned long
                    766: vm_page_mem_free(void)
                    767: {
                    768:     unsigned long total;
                    769:     unsigned int i;
                    770: 
                    771:     total = 0;
                    772: 
                    773:     for (i = 0; i < vm_page_segs_size; i++) {
                    774:         /* XXX */
                    775:         if (i >  VM_PAGE_SEG_DIRECTMAP)
                    776:             continue;
                    777: 
                    778:         total += vm_page_segs[i].nr_free_pages;
                    779:     }
                    780: 
                    781:     return total;
                    782: }

unix.superglobalmegacorp.com

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.