--- Gnu-Mach/kern/slab.c 2020/09/02 04:49:56 1.1.1.3 +++ Gnu-Mach/kern/slab.c 2020/09/02 04:52:02 1.1.1.4 @@ -58,15 +58,10 @@ * over a hash table. Unlike a hash table, a BST provides a "lookup nearest" * operation, so obtaining the slab data (whether it is embedded in the slab or * off slab) from a buffer address simply consists of a "lookup nearest towards - * 0" tree search. Storing slabs instead of buffers also considerably reduces - * the number of elements to retain. Finally, a self-balancing tree is a true - * self-scaling data structure, whereas a hash table requires periodic - * maintenance and complete resizing, which is expensive. The only drawback is - * that releasing a buffer to the slab layer takes logarithmic time instead of - * constant time. But as the data set size is kept reasonable (because slabs - * are stored instead of buffers) and because the CPU pool layer services most - * requests, avoiding many accesses to the slab layer, it is considered an - * acceptable tradeoff. + * 0" tree search. Finally, a self-balancing tree is a true self-scaling data + * structure, whereas a hash table requires periodic maintenance and complete + * resizing, which is expensive. The only drawback is that releasing a buffer + * to the slab layer takes logarithmic time instead of constant time. * * This implementation uses per-cpu pools of objects, which service most * allocation requests. These pools act as caches (but are named differently @@ -87,6 +82,7 @@ #include #include #include +#include #include #include @@ -111,19 +107,6 @@ #define KMEM_ALIGN_MIN 8 /* - * Minimum number of buffers per slab. - * - * This value is ignored when the slab size exceeds a threshold. - */ -#define KMEM_MIN_BUFS_PER_SLAB 8 - -/* - * Special slab size beyond which the minimum number of buffers per slab is - * ignored when computing the slab size of a cache. - */ -#define KMEM_SLAB_SIZE_THRESHOLD (8 * PAGE_SIZE) - -/* * Special buffer size under which slab data is unconditionnally allocated * from its associated slab. */ @@ -163,11 +146,6 @@ #define KMEM_REDZONE_BYTE 0xbb /* - * Size of the VM submap from which default backend functions allocate. - */ -#define KMEM_MAP_SIZE (96 * 1024 * 1024) - -/* * Shift for the first kalloc cache size. */ #define KALLOC_FIRST_SHIFT 5 @@ -217,11 +195,17 @@ * * The flags don't change once set and can be tested without locking. */ -#define KMEM_CF_NO_CPU_POOL 0x01 /* CPU pool layer disabled */ -#define KMEM_CF_SLAB_EXTERNAL 0x02 /* Slab data is off slab */ -#define KMEM_CF_NO_RECLAIM 0x04 /* Slabs are not reclaimable */ -#define KMEM_CF_VERIFY 0x08 /* Debugging facilities enabled */ -#define KMEM_CF_DIRECT 0x10 /* No buf-to-slab tree lookup */ +#define KMEM_CF_SLAB_EXTERNAL 0x01 /* Slab data is off slab */ +#define KMEM_CF_PHYSMEM 0x02 /* Allocate from physical memory */ +#define KMEM_CF_DIRECT 0x04 /* Direct buf-to-slab translation + (implies !KMEM_CF_SLAB_EXTERNAL) */ +#define KMEM_CF_USE_TREE 0x08 /* Use red-black tree to track slab + data */ +#define KMEM_CF_USE_PAGE 0x10 /* Use page private data to track slab + data (implies KMEM_CF_SLAB_EXTERNAL + and KMEM_CF_PHYSMEM) */ +#define KMEM_CF_VERIFY 0x20 /* Debugging facilities enabled + (implies KMEM_CF_USE_TREE) */ /* * Options for kmem_cache_alloc_verify(). @@ -278,12 +262,6 @@ static unsigned int kmem_nr_caches; static simple_lock_data_t __attribute__((used)) kmem_cache_list_lock; /* - * VM submap for slab caches. - */ -static struct vm_map kmem_map_store; -vm_map_t kmem_map = &kmem_map_store; - -/* * Time of the last memory reclaim, in clock ticks. */ static unsigned long kmem_gc_last_tick; @@ -384,12 +362,49 @@ static inline void * kmem_bufctl_to_buf( return (void *)bufctl - cache->bufctl_dist; } -static vm_offset_t kmem_pagealloc(vm_size_t size) +static vm_offset_t +kmem_pagealloc_physmem(vm_size_t size) +{ + struct vm_page *page; + + assert(size == PAGE_SIZE); + + for (;;) { + page = vm_page_grab_contig(size, VM_PAGE_SEL_DIRECTMAP); + + if (page != NULL) + break; + + VM_PAGE_WAIT(NULL); + } + + return phystokv(vm_page_to_pa(page)); +} + +static void +kmem_pagefree_physmem(vm_offset_t addr, vm_size_t size) +{ + struct vm_page *page; + + assert(size == PAGE_SIZE); + page = vm_page_lookup_pa(kvtophys(addr)); + assert(page != NULL); + vm_page_free_contig(page, size); +} + +static vm_offset_t +kmem_pagealloc_virtual(vm_size_t size, vm_size_t align) { vm_offset_t addr; kern_return_t kr; - kr = kmem_alloc_wired(kmem_map, &addr, size); + assert(size > PAGE_SIZE); + size = vm_page_round(size); + + if (align <= PAGE_SIZE) + kr = kmem_alloc_wired(kernel_map, &addr, size); + else + kr = kmem_alloc_aligned(kernel_map, &addr, size); if (kr != KERN_SUCCESS) return 0; @@ -397,9 +412,29 @@ static vm_offset_t kmem_pagealloc(vm_siz return addr; } -static void kmem_pagefree(vm_offset_t ptr, vm_size_t size) +static void +kmem_pagefree_virtual(vm_offset_t addr, vm_size_t size) { - kmem_free(kmem_map, ptr, size); + assert(size > PAGE_SIZE); + size = vm_page_round(size); + kmem_free(kernel_map, addr, size); +} + +static vm_offset_t +kmem_pagealloc(vm_size_t size, vm_size_t align, int flags) +{ + assert(align <= size); + return (flags & KMEM_CF_PHYSMEM) + ? kmem_pagealloc_physmem(size) + : kmem_pagealloc_virtual(size, align); +} + +static void +kmem_pagefree(vm_offset_t addr, vm_size_t size, int flags) +{ + return (flags & KMEM_CF_PHYSMEM) + ? kmem_pagefree_physmem(addr, size) + : kmem_pagefree_virtual(addr, size); } static void kmem_slab_create_verify(struct kmem_slab *slab, @@ -434,28 +469,28 @@ static struct kmem_slab * kmem_slab_crea union kmem_bufctl *bufctl; size_t buf_size; unsigned long buffers; - void *slab_buf; + vm_offset_t slab_buf; - if (cache->slab_alloc_fn == NULL) - slab_buf = (void *)kmem_pagealloc(cache->slab_size); - else - slab_buf = (void *)cache->slab_alloc_fn(cache->slab_size); + slab_buf = kmem_pagealloc(cache->slab_size, cache->align, cache->flags); - if (slab_buf == NULL) + if (slab_buf == 0) return NULL; if (cache->flags & KMEM_CF_SLAB_EXTERNAL) { - assert(!(cache->flags & KMEM_CF_NO_RECLAIM)); slab = (struct kmem_slab *)kmem_cache_alloc(&kmem_slab_cache); if (slab == NULL) { - if (cache->slab_free_fn == NULL) - kmem_pagefree((vm_offset_t)slab_buf, cache->slab_size); - else - cache->slab_free_fn((vm_offset_t)slab_buf, cache->slab_size); - + kmem_pagefree(slab_buf, cache->slab_size, cache->flags); return NULL; } + + if (cache->flags & KMEM_CF_USE_PAGE) { + struct vm_page *page; + + page = vm_page_lookup_pa(kvtophys(slab_buf)); + assert(page != NULL); + vm_page_set_priv(page, slab); + } } else { slab = (struct kmem_slab *)(slab_buf + cache->slab_size) - 1; } @@ -464,7 +499,7 @@ static struct kmem_slab * kmem_slab_crea rbtree_node_init(&slab->tree_node); slab->nr_refs = 0; slab->first_free = NULL; - slab->addr = slab_buf + color; + slab->addr = (void *)(slab_buf + color); buf_size = cache->buf_size; bufctl = kmem_buf_to_bufctl(slab->addr, cache); @@ -518,25 +553,26 @@ static void kmem_slab_destroy(struct kme assert(slab->nr_refs == 0); assert(slab->first_free != NULL); - assert(!(cache->flags & KMEM_CF_NO_RECLAIM)); if (cache->flags & KMEM_CF_VERIFY) kmem_slab_destroy_verify(slab, cache); slab_buf = (vm_offset_t)P2ALIGN((unsigned long)slab->addr, PAGE_SIZE); - if (cache->slab_free_fn == NULL) - kmem_pagefree(slab_buf, cache->slab_size); - else - cache->slab_free_fn(slab_buf, cache->slab_size); + if (cache->flags & KMEM_CF_SLAB_EXTERNAL) { + if (cache->flags & KMEM_CF_USE_PAGE) { + struct vm_page *page; + + /* Not strictly needed, but let's increase safety */ + page = vm_page_lookup_pa(kvtophys(slab_buf)); + assert(page != NULL); + vm_page_set_priv(page, NULL); + } - if (cache->flags & KMEM_CF_SLAB_EXTERNAL) kmem_cache_free(&kmem_slab_cache, (vm_offset_t)slab); -} + } -static inline int kmem_slab_use_tree(int flags) -{ - return !(flags & KMEM_CF_DIRECT) || (flags & KMEM_CF_VERIFY); + kmem_pagefree(slab_buf, cache->slab_size, cache->flags); } static inline int kmem_slab_cmp_lookup(const void *addr, @@ -694,82 +730,81 @@ static void kmem_cache_error(struct kmem } /* - * Compute an appropriate slab size for the given cache. + * Compute properties such as slab size for the given cache. * * Once the slab size is known, this function sets the related properties - * (buffers per slab and maximum color). It can also set the KMEM_CF_DIRECT - * and/or KMEM_CF_SLAB_EXTERNAL flags depending on the resulting layout. + * (buffers per slab and maximum color). It can also set some KMEM_CF_xxx + * flags depending on the resulting layout. */ -static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags) +static void kmem_cache_compute_properties(struct kmem_cache *cache, int flags) { - size_t i, buffers, buf_size, slab_size, free_slab_size, optimal_size = 0; - size_t waste, waste_min; - int embed, optimal_embed = 0; - - buf_size = cache->buf_size; + size_t size, waste; + int embed; - if (buf_size < KMEM_BUF_SIZE_THRESHOLD) + if (cache->buf_size < KMEM_BUF_SIZE_THRESHOLD) flags |= KMEM_CACHE_NOOFFSLAB; - i = 0; - waste_min = (size_t)-1; - - do { - i++; - slab_size = P2ROUND(i * buf_size, PAGE_SIZE); - free_slab_size = slab_size; - - if (flags & KMEM_CACHE_NOOFFSLAB) - free_slab_size -= sizeof(struct kmem_slab); - - buffers = free_slab_size / buf_size; - waste = free_slab_size % buf_size; - - if (buffers > i) - i = buffers; + cache->slab_size = PAGE_SIZE; + for (;;) { if (flags & KMEM_CACHE_NOOFFSLAB) embed = 1; - else if (sizeof(struct kmem_slab) <= waste) { - embed = 1; - waste -= sizeof(struct kmem_slab); - } else { - embed = 0; + else { + waste = cache->slab_size % cache->buf_size; + embed = (sizeof(struct kmem_slab) <= waste); } - if (waste <= waste_min) { - waste_min = waste; - optimal_size = slab_size; - optimal_embed = embed; - } - } while ((buffers < KMEM_MIN_BUFS_PER_SLAB) - && (slab_size < KMEM_SLAB_SIZE_THRESHOLD)); + size = cache->slab_size; - assert(optimal_size > 0); - assert(!(flags & KMEM_CACHE_NOOFFSLAB) || optimal_embed); + if (embed) + size -= sizeof(struct kmem_slab); - cache->slab_size = optimal_size; - slab_size = cache->slab_size - (optimal_embed - ? sizeof(struct kmem_slab) - : 0); - cache->bufs_per_slab = slab_size / buf_size; - cache->color_max = slab_size % buf_size; + if (size >= cache->buf_size) + break; + + cache->slab_size += PAGE_SIZE; + } + + cache->bufs_per_slab = size / cache->buf_size; + cache->color_max = size % cache->buf_size; if (cache->color_max >= PAGE_SIZE) - cache->color_max = PAGE_SIZE - 1; + cache->color_max = 0; + + if (!embed) + cache->flags |= KMEM_CF_SLAB_EXTERNAL; + + if ((flags & KMEM_CACHE_PHYSMEM) || (cache->slab_size == PAGE_SIZE)) { + cache->flags |= KMEM_CF_PHYSMEM; - if (optimal_embed) { + /* + * Avoid using larger-than-page slabs backed by the direct physical + * mapping to completely prevent physical memory fragmentation from + * making slab allocations fail. + */ + if (cache->slab_size != PAGE_SIZE) + panic("slab: invalid cache parameters"); + } + + if (cache->flags & KMEM_CF_VERIFY) + cache->flags |= KMEM_CF_USE_TREE; + + if (cache->flags & KMEM_CF_SLAB_EXTERNAL) { + if (cache->flags & KMEM_CF_PHYSMEM) + cache->flags |= KMEM_CF_USE_PAGE; + else + cache->flags |= KMEM_CF_USE_TREE; + } else { if (cache->slab_size == PAGE_SIZE) cache->flags |= KMEM_CF_DIRECT; - } else { - cache->flags |= KMEM_CF_SLAB_EXTERNAL; + else + cache->flags |= KMEM_CF_USE_TREE; } } void kmem_cache_init(struct kmem_cache *cache, const char *name, - size_t obj_size, size_t align, kmem_cache_ctor_t ctor, - kmem_slab_alloc_fn_t slab_alloc_fn, - kmem_slab_free_fn_t slab_free_fn, int flags) + size_t obj_size, size_t align, + kmem_cache_ctor_t ctor, int flags) { #if SLAB_USE_CPU_POOLS struct kmem_cpu_pool_type *cpu_pool_type; @@ -783,15 +818,6 @@ void kmem_cache_init(struct kmem_cache * cache->flags = 0; #endif /* SLAB_VERIFY */ - if (flags & KMEM_CACHE_NOCPUPOOL) - cache->flags |= KMEM_CF_NO_CPU_POOL; - - if (flags & KMEM_CACHE_NORECLAIM) { - assert(slab_free_fn == NULL); - flags |= KMEM_CACHE_NOOFFSLAB; - cache->flags |= KMEM_CF_NO_RECLAIM; - } - if (flags & KMEM_CACHE_VERIFY) cache->flags |= KMEM_CF_VERIFY; @@ -800,7 +826,6 @@ void kmem_cache_init(struct kmem_cache * assert(obj_size > 0); assert(ISP2(align)); - assert(align < PAGE_SIZE); buf_size = P2ROUND(obj_size, align); @@ -819,8 +844,6 @@ void kmem_cache_init(struct kmem_cache * cache->nr_slabs = 0; cache->nr_free_slabs = 0; cache->ctor = ctor; - cache->slab_alloc_fn = slab_alloc_fn; - cache->slab_free_fn = slab_free_fn; strncpy(cache->name, name, sizeof(cache->name)); cache->name[sizeof(cache->name) - 1] = '\0'; cache->buftag_dist = 0; @@ -835,7 +858,7 @@ void kmem_cache_init(struct kmem_cache * cache->buf_size = buf_size; } - kmem_cache_compute_sizes(cache, flags); + kmem_cache_compute_properties(cache, flags); #if SLAB_USE_CPU_POOLS for (cpu_pool_type = kmem_cpu_pool_types; @@ -908,9 +931,6 @@ static void kmem_cache_reap(struct kmem_ struct list dead_slabs; unsigned long nr_free_slabs; - if (cache->flags & KMEM_CF_NO_RECLAIM) - return; - simple_lock(&cache->lock); list_set_head(&dead_slabs, &cache->free_slabs); list_init(&cache->free_slabs); @@ -971,7 +991,7 @@ static void * kmem_cache_alloc_from_slab cache->nr_free_slabs--; } - if ((slab->nr_refs == 1) && kmem_slab_use_tree(cache->flags)) + if ((slab->nr_refs == 1) && (cache->flags & KMEM_CF_USE_TREE)) rbtree_insert(&cache->active_slabs, &slab->tree_node, kmem_slab_cmp_insert); @@ -992,17 +1012,26 @@ static void kmem_cache_free_to_slab(stru assert(cache->slab_size == PAGE_SIZE); slab = (struct kmem_slab *)P2END((unsigned long)buf, cache->slab_size) - 1; + } else if (cache->flags & KMEM_CF_USE_PAGE) { + struct vm_page *page; + + page = vm_page_lookup_pa(kvtophys((vm_offset_t)buf)); + assert(page != NULL); + slab = vm_page_get_priv(page); } else { struct rbtree_node *node; + assert(cache->flags & KMEM_CF_USE_TREE); node = rbtree_lookup_nearest(&cache->active_slabs, buf, kmem_slab_cmp_lookup, RBTREE_LEFT); assert(node != NULL); slab = rbtree_entry(node, struct kmem_slab, tree_node); - assert((unsigned long)buf < (P2ALIGN((unsigned long)slab->addr - + cache->slab_size, PAGE_SIZE))); } + assert((unsigned long)buf >= (unsigned long)slab->addr); + assert(((unsigned long)buf + cache->buf_size) + <= vm_page_trunc((unsigned long)slab->addr + cache->slab_size)); + assert(slab->nr_refs >= 1); assert(slab->nr_refs <= cache->bufs_per_slab); bufctl = kmem_buf_to_bufctl(buf, cache); @@ -1014,7 +1043,7 @@ static void kmem_cache_free_to_slab(stru if (slab->nr_refs == 0) { /* The slab has become free */ - if (kmem_slab_use_tree(cache->flags)) + if (cache->flags & KMEM_CF_USE_TREE) rbtree_remove(&cache->active_slabs, &slab->tree_node); if (cache->bufs_per_slab > 1) @@ -1135,6 +1164,8 @@ static void kmem_cache_free_verify(struc unsigned char *redzone_byte; unsigned long slabend; + assert(cache->flags & KMEM_CF_USE_TREE); + simple_lock(&cache->lock); node = rbtree_lookup_nearest(&cache->active_slabs, buf, kmem_slab_cmp_lookup, RBTREE_LEFT); @@ -1280,16 +1311,12 @@ void slab_bootstrap(void) void slab_init(void) { - vm_offset_t min, max; - #if SLAB_USE_CPU_POOLS struct kmem_cpu_pool_type *cpu_pool_type; char name[KMEM_CACHE_NAME_SIZE]; size_t i, size; #endif /* SLAB_USE_CPU_POOLS */ - kmem_submap(kmem_map, kernel_map, &min, &max, KMEM_MAP_SIZE, FALSE); - #if SLAB_USE_CPU_POOLS for (i = 0; i < ARRAY_SIZE(kmem_cpu_pool_types); i++) { cpu_pool_type = &kmem_cpu_pool_types[i]; @@ -1297,7 +1324,7 @@ void slab_init(void) sprintf(name, "kmem_cpu_array_%d", cpu_pool_type->array_size); size = sizeof(void *) * cpu_pool_type->array_size; kmem_cache_init(cpu_pool_type->array_cache, name, size, - cpu_pool_type->array_align, NULL, NULL, NULL, 0); + cpu_pool_type->array_align, NULL, 0); } #endif /* SLAB_USE_CPU_POOLS */ @@ -1305,25 +1332,7 @@ void slab_init(void) * Prevent off slab data for the slab cache to avoid infinite recursion. */ kmem_cache_init(&kmem_slab_cache, "kmem_slab", sizeof(struct kmem_slab), - 0, NULL, NULL, NULL, KMEM_CACHE_NOOFFSLAB); -} - -static vm_offset_t kalloc_pagealloc(vm_size_t size) -{ - vm_offset_t addr; - kern_return_t kr; - - kr = kmem_alloc_wired(kmem_map, &addr, size); - - if (kr != KERN_SUCCESS) - return 0; - - return addr; -} - -static void kalloc_pagefree(vm_offset_t ptr, vm_size_t size) -{ - kmem_free(kmem_map, ptr, size); + 0, NULL, KMEM_CACHE_NOOFFSLAB); } void kalloc_init(void) @@ -1335,8 +1344,7 @@ void kalloc_init(void) for (i = 0; i < ARRAY_SIZE(kalloc_caches); i++) { sprintf(name, "kalloc_%lu", size); - kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL, - kalloc_pagealloc, kalloc_pagefree, 0); + kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL, 0); size <<= 1; } } @@ -1387,8 +1395,9 @@ vm_offset_t kalloc(vm_size_t size) if ((buf != 0) && (cache->flags & KMEM_CF_VERIFY)) kalloc_verify(cache, buf, size); - } else - buf = (void *)kalloc_pagealloc(size); + } else { + buf = (void *)kmem_pagealloc_virtual(size, 0); + } return (vm_offset_t)buf; } @@ -1429,7 +1438,7 @@ void kfree(vm_offset_t data, vm_size_t s kmem_cache_free(cache, data); } else { - kalloc_pagefree(data, size); + kmem_pagefree_virtual(data, size); } } @@ -1529,16 +1538,7 @@ kern_return_t host_slab_info(host_t host list_for_each_entry(&kmem_cache_list, cache, node) { simple_lock(&cache->lock); - info[i].flags = ((cache->flags & KMEM_CF_NO_CPU_POOL) - ? CACHE_FLAGS_NO_CPU_POOL : 0) - | ((cache->flags & KMEM_CF_SLAB_EXTERNAL) - ? CACHE_FLAGS_SLAB_EXTERNAL : 0) - | ((cache->flags & KMEM_CF_NO_RECLAIM) - ? CACHE_FLAGS_NO_RECLAIM : 0) - | ((cache->flags & KMEM_CF_VERIFY) - ? CACHE_FLAGS_VERIFY : 0) - | ((cache->flags & KMEM_CF_DIRECT) - ? CACHE_FLAGS_DIRECT : 0); + info[i].flags = cache->flags; #if SLAB_USE_CPU_POOLS info[i].cpu_pool_size = cache->cpu_pool_type->array_size; #else /* SLAB_USE_CPU_POOLS */