![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_resident.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52: /*
53: * File: vm/vm_page.c
54: * Author: Avadis Tevanian, Jr., Michael Wayne Young
55: *
56: * Resident memory management module.
57: */
58:
59: #include <mach/vm_prot.h>
60: #include <mach/vm_statistics.h>
61: #include <kern/counters.h>
62: #include <kern/sched_prim.h>
63: #include <kern/task.h>
64: #include <kern/thread.h>
65: #include <kern/zalloc.h>
66: #include <kern/xpr.h>
67: #include <vm/pmap.h>
68: #include <vm/vm_init.h>
69: #include <vm/vm_map.h>
70: #include <vm/vm_page.h>
71: #include <vm/vm_pageout.h>
72: #include <vm/vm_kern.h> /* kernel_memory_allocate() */
73: #include <kern/misc_protos.h>
74: #include <zone_debug.h>
75: #include <vm/cpm.h>
76:
77: /*
78: * Associated with page of user-allocatable memory is a
79: * page structure.
80: */
81:
82: /*
83: * These variables record the values returned by vm_page_bootstrap,
84: * for debugging purposes. The implementation of pmap_steal_memory
85: * and pmap_startup here also uses them internally.
86: */
87:
88: vm_offset_t virtual_space_start;
89: vm_offset_t virtual_space_end;
90: int vm_page_pages;
91:
92: /*
93: * The vm_page_lookup() routine, which provides for fast
94: * (virtual memory object, offset) to page lookup, employs
95: * the following hash table. The vm_page_{insert,remove}
96: * routines install and remove associations in the table.
97: * [This table is often called the virtual-to-physical,
98: * or VP, table.]
99: */
100: typedef struct {
101: vm_page_t pages;
102: #if MACH_PAGE_HASH_STATS
103: int cur_count; /* current count */
104: int hi_count; /* high water mark */
105: #endif /* MACH_PAGE_HASH_STATS */
106: } vm_page_bucket_t;
107:
108: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
109: unsigned int vm_page_bucket_count = 0; /* How big is array? */
110: unsigned int vm_page_hash_mask; /* Mask for hash function */
111: unsigned int vm_page_hash_shift; /* Shift for hash function */
112: decl_simple_lock_data(,vm_page_bucket_lock)
113:
114: #if MACH_PAGE_HASH_STATS
115: /* This routine is only for debug. It is intended to be called by
116: * hand by a developer using a kernel debugger. This routine prints
117: * out vm_page_hash table statistics to the kernel debug console.
118: */
119: void
120: hash_debug(void)
121: {
122: int i;
123: int numbuckets = 0;
124: int highsum = 0;
125: int maxdepth = 0;
126:
127: for (i = 0; i < vm_page_bucket_count; i++) {
128: if (vm_page_buckets[i].hi_count) {
129: numbuckets++;
130: highsum += vm_page_buckets[i].hi_count;
131: if (vm_page_buckets[i].hi_count > maxdepth)
132: maxdepth = vm_page_buckets[i].hi_count;
133: }
134: }
135: printf("Total number of buckets: %d\n", vm_page_bucket_count);
136: printf("Number used buckets: %d = %d%%\n",
137: numbuckets, 100*numbuckets/vm_page_bucket_count);
138: printf("Number unused buckets: %d = %d%%\n",
139: vm_page_bucket_count - numbuckets,
140: 100*(vm_page_bucket_count-numbuckets)/vm_page_bucket_count);
141: printf("Sum of bucket max depth: %d\n", highsum);
142: printf("Average bucket depth: %d.%2d\n",
143: highsum/vm_page_bucket_count,
144: highsum%vm_page_bucket_count);
145: printf("Maximum bucket depth: %d\n", maxdepth);
146: }
147: #endif /* MACH_PAGE_HASH_STATS */
148:
149: /*
150: * The virtual page size is currently implemented as a runtime
151: * variable, but is constant once initialized using vm_set_page_size.
152: * This initialization must be done in the machine-dependent
153: * bootstrap sequence, before calling other machine-independent
154: * initializations.
155: *
156: * All references to the virtual page size outside this
157: * module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT
158: * constants.
159: */
160: #ifndef PAGE_SIZE_FIXED
161: vm_size_t page_size = 4096;
162: vm_size_t page_mask = 4095;
163: int page_shift = 12;
164: #endif /* PAGE_SIZE_FIXED */
165:
166: /*
167: * Resident page structures are initialized from
168: * a template (see vm_page_alloc).
169: *
170: * When adding a new field to the virtual memory
171: * object structure, be sure to add initialization
172: * (see vm_page_bootstrap).
173: */
174: struct vm_page vm_page_template;
175:
176: /*
177: * Resident pages that represent real memory
178: * are allocated from a free list.
179: */
180: vm_page_t vm_page_queue_free;
181: vm_page_t vm_page_queue_fictitious;
182: decl_mutex_data(,vm_page_queue_free_lock)
183: unsigned int vm_page_free_wanted;
184: int vm_page_free_count;
185: int vm_page_fictitious_count;
186:
187: unsigned int vm_page_free_count_minimum; /* debugging */
188:
189: /*
190: * Occasionally, the virtual memory system uses
191: * resident page structures that do not refer to
192: * real pages, for example to leave a page with
193: * important state information in the VP table.
194: *
195: * These page structures are allocated the way
196: * most other kernel structures are.
197: */
198: zone_t vm_page_zone;
199: decl_mutex_data(,vm_page_alloc_lock)
200:
201: /*
202: * Fictitious pages don't have a physical address,
203: * but we must initialize phys_addr to something.
204: * For debugging, this should be a strange value
205: * that the pmap module can recognize in assertions.
206: */
207: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
208:
209: /*
210: * Resident page structures are also chained on
211: * queues that are used by the page replacement
212: * system (pageout daemon). These queues are
213: * defined here, but are shared by the pageout
214: * module.
215: */
216: queue_head_t vm_page_queue_active;
217: queue_head_t vm_page_queue_inactive;
218: decl_mutex_data(,vm_page_queue_lock)
219: int vm_page_active_count;
220: int vm_page_inactive_count;
221: int vm_page_wire_count;
222: int vm_page_gobble_count = 0;
223: int vm_page_wire_count_warning = 0;
224: int vm_page_gobble_count_warning = 0;
225:
226: /* the following fields are protected by the vm_page_queue_lock */
227: queue_head_t vm_page_queue_limbo;
228: int vm_page_limbo_count = 0; /* total pages in limbo */
229: int vm_page_limbo_real_count = 0; /* real pages in limbo */
230: int vm_page_pin_count = 0; /* number of pinned pages */
231:
232: decl_simple_lock_data(,vm_page_preppin_lock)
233:
234: /*
235: * Several page replacement parameters are also
236: * shared with this module, so that page allocation
237: * (done here in vm_page_alloc) can trigger the
238: * pageout daemon.
239: */
240: int vm_page_free_target = 0;
241: int vm_page_free_min = 0;
242: int vm_page_inactive_target = 0;
243: int vm_page_free_reserved = 0;
244: int vm_page_laundry_count = 0;
245:
246: /*
247: * The VM system has a couple of heuristics for deciding
248: * that pages are "uninteresting" and should be placed
249: * on the inactive queue as likely candidates for replacement.
250: * These variables let the heuristics be controlled at run-time
251: * to make experimentation easier.
252: */
253:
254: boolean_t vm_page_deactivate_hint = TRUE;
255:
256: /*
257: * Definition for automatic physical memory reservation. Declared in
258: * vm_page.h, defined here. See vm_page.h for details. There's an
259: * extra zero entry to allow the code to compile if there are no requests
260: * for physical memory allocation; pmem_reserve_ctl_size is decremented
261: * by one to compensate.
262: */
263: #include <flipc.h>
264: #if FLIPC
265: #include <flipc/flipc_usermsg.h>
266: #endif
267:
268: struct pmem_reserve pmem_reserve_ctl_array[] = {
269: #if FLIPC
270: { &flipc_cb_length, (vm_offset_t *) &flipc_cb_base },
271: #endif
272: { 0, (vm_offset_t *) 0 }
273: };
274:
275: struct pmem_reserve *pmem_reserve_ctl = &pmem_reserve_ctl_array[0];
276:
277: int pmem_reserve_ctl_size =
278: (sizeof(pmem_reserve_ctl_array) / sizeof(struct pmem_reserve)) - 1;
279:
280: /*
281: * vm_set_page_size:
282: *
283: * Sets the page size, perhaps based upon the memory
284: * size. Must be called before any use of page-size
285: * dependent functions.
286: *
287: * Sets page_shift and page_mask from page_size.
288: */
289: void
290: vm_set_page_size(void)
291: {
292: #ifndef PAGE_SIZE_FIXED
293: page_mask = page_size - 1;
294:
295: if ((page_mask & page_size) != 0)
296: panic("vm_set_page_size: page size not a power of two");
297:
298: for (page_shift = 0; ; page_shift++)
299: if ((1 << page_shift) == page_size)
300: break;
301: #endif /* PAGE_SIZE_FIXED */
302: }
303:
304: /*
305: * vm_page_bootstrap:
306: *
307: * Initializes the resident memory module.
308: *
309: * Allocates memory for the page cells, and
310: * for the object/offset-to-page hash table headers.
311: * Each page cell is initialized and placed on the free list.
312: * Returns the range of available kernel virtual memory.
313: */
314:
315: void
316: vm_page_bootstrap(
317: vm_offset_t *startp,
318: vm_offset_t *endp)
319: {
320: register vm_page_t m;
321: int i;
322: unsigned int log1;
323: unsigned int log2;
324: unsigned int size;
325:
326: /*
327: * Initialize the vm_page template.
328: */
329:
330: m = &vm_page_template;
331: m->object = VM_OBJECT_NULL; /* reset later */
332: m->offset = 0; /* reset later */
333: m->wire_count = 0;
334:
335: m->inactive = FALSE;
336: m->active = FALSE;
337: m->laundry = FALSE;
338: m->free = FALSE;
339: m->reference = FALSE;
340: m->pageout = FALSE;
341: m->list_req_pending = FALSE;
342:
343: m->busy = TRUE;
344: m->wanted = FALSE;
345: m->tabled = FALSE;
346: m->fictitious = FALSE;
347: m->private = FALSE;
348: m->absent = FALSE;
349: m->error = FALSE;
350: m->dirty = FALSE;
351: m->cleaning = FALSE;
352: m->precious = FALSE;
353: m->clustered = FALSE;
354: m->lock_supplied = FALSE;
355: m->unusual = FALSE;
356: m->restart = FALSE;
357: m->limbo = FALSE;
358:
359: m->phys_addr = 0; /* reset later */
360:
361: m->page_lock = VM_PROT_NONE;
362: m->unlock_request = VM_PROT_NONE;
363: m->page_error = KERN_SUCCESS;
364:
365: /*
366: * Initialize the page queues.
367: */
368:
369: mutex_init(&vm_page_queue_free_lock, ETAP_VM_PAGEQ_FREE);
370: mutex_init(&vm_page_queue_lock, ETAP_VM_PAGEQ);
371: simple_lock_init(&vm_page_preppin_lock, ETAP_VM_PREPPIN);
372:
373: vm_page_queue_free = VM_PAGE_NULL;
374: vm_page_queue_fictitious = VM_PAGE_NULL;
375: queue_init(&vm_page_queue_active);
376: queue_init(&vm_page_queue_inactive);
377: queue_init(&vm_page_queue_limbo);
378:
379: vm_page_free_wanted = 0;
380:
381: /*
382: * Steal memory for the map and zone subsystems.
383: */
384:
385: vm_map_steal_memory();
386: zone_steal_memory();
387:
388: /*
389: * Allocate (and initialize) the virtual-to-physical
390: * table hash buckets.
391: *
392: * The number of buckets should be a power of two to
393: * get a good hash function. The following computation
394: * chooses the first power of two that is greater
395: * than the number of physical pages in the system.
396: */
397:
398: simple_lock_init(&vm_page_bucket_lock, ETAP_VM_BUCKET);
399:
400: if (vm_page_bucket_count == 0) {
401: unsigned int npages = pmap_free_pages();
402:
403: vm_page_bucket_count = 1;
404: while (vm_page_bucket_count < npages)
405: vm_page_bucket_count <<= 1;
406: }
407:
408: vm_page_hash_mask = vm_page_bucket_count - 1;
409:
410: /*
411: * Calculate object shift value for hashing algorithm:
412: * O = log2(sizeof(struct vm_object))
413: * B = log2(vm_page_bucket_count)
414: * hash shifts the object left by
415: * B/2 - O
416: */
417: size = vm_page_bucket_count;
418: for (log1 = 0; size > 1; log1++)
419: size /= 2;
420: size = sizeof(struct vm_object);
421: for (log2 = 0; size > 1; log2++)
422: size /= 2;
423: vm_page_hash_shift = log1/2 - log2 + 1;
424:
425: if (vm_page_hash_mask & vm_page_bucket_count)
426: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
427:
428: vm_page_buckets = (vm_page_bucket_t *)
429: pmap_steal_memory(vm_page_bucket_count *
430: sizeof(vm_page_bucket_t));
431:
432: for (i = 0; i < vm_page_bucket_count; i++) {
433: register vm_page_bucket_t *bucket = &vm_page_buckets[i];
434:
435: bucket->pages = VM_PAGE_NULL;
436: #if MACH_PAGE_HASH_STATS
437: bucket->cur_count = 0;
438: bucket->hi_count = 0;
439: #endif /* MACH_PAGE_HASH_STATS */
440: }
441:
442: /*
443: * Machine-dependent code allocates the resident page table.
444: * It uses vm_page_init to initialize the page frames.
445: * The code also returns to us the virtual space available
446: * to the kernel. We don't trust the pmap module
447: * to get the alignment right.
448: */
449:
450: pmap_startup(&virtual_space_start, &virtual_space_end);
451: virtual_space_start = round_page(virtual_space_start);
452: virtual_space_end = trunc_page(virtual_space_end);
453:
454: *startp = virtual_space_start;
455: *endp = virtual_space_end;
456:
457: /*
458: * Compute the initial "wire" count.
459: * Up until now, the pages which have been set aside are not under
460: * the VM system's control, so although they aren't explicitly
461: * wired, they nonetheless can't be moved. At this moment,
462: * all VM managed pages are "free", courtesy of pmap_startup.
463: */
464: vm_page_wire_count = atop(mem_size) - vm_page_free_count; /* initial value */
465:
466: printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);
467: vm_page_free_count_minimum = vm_page_free_count;
468: }
469:
470: #ifndef MACHINE_PAGES
471: /*
472: * We implement pmap_steal_memory and pmap_startup with the help
473: * of two simpler functions, pmap_virtual_space and pmap_next_page.
474: */
475:
476: vm_offset_t
477: pmap_steal_memory(
478: vm_size_t size)
479: {
480: vm_offset_t addr, vaddr, paddr;
481:
482: /*
483: * We round the size to a round multiple.
484: */
485:
486: size = (size + sizeof (void *) - 1) &~ (sizeof (void *) - 1);
487:
488: /*
489: * If this is the first call to pmap_steal_memory,
490: * we have to initialize ourself.
491: */
492:
493: if (virtual_space_start == virtual_space_end) {
494: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
495:
496: /*
497: * The initial values must be aligned properly, and
498: * we don't trust the pmap module to do it right.
499: */
500:
501: virtual_space_start = round_page(virtual_space_start);
502: virtual_space_end = trunc_page(virtual_space_end);
503: }
504:
505: /*
506: * Allocate virtual memory for this request.
507: */
508:
509: addr = virtual_space_start;
510: virtual_space_start += size;
511:
512: kprintf("pmap_steal_memory: %08X - %08X; size=%08X\n", addr, virtual_space_start, size); /* (TEST/DEBUG) */
513:
514: /*
515: * Allocate and map physical pages to back new virtual pages.
516: */
517:
518: for (vaddr = round_page(addr);
519: vaddr < addr + size;
520: vaddr += PAGE_SIZE) {
521: if (!pmap_next_page(&paddr))
522: panic("pmap_steal_memory");
523:
524: /*
525: * XXX Logically, these mappings should be wired,
526: * but some pmap modules barf if they are.
527: */
528:
529: pmap_enter(kernel_pmap, vaddr, paddr,
530: VM_PROT_READ|VM_PROT_WRITE, FALSE);
531: /*
532: * Account for newly stolen memory
533: */
534: vm_page_wire_count++;
535:
536: }
537:
538: return addr;
539: }
540:
541: void
542: pmap_startup(
543: vm_offset_t *startp,
544: vm_offset_t *endp)
545: {
546: unsigned int i, npages, pages_initialized;
547: vm_page_t pages;
548: vm_offset_t paddr;
549:
550: /*
551: * We calculate how many page frames we will have
552: * and then allocate the page structures in one chunk.
553: */
554:
555: npages = ((PAGE_SIZE * pmap_free_pages() +
556: (round_page(virtual_space_start) - virtual_space_start)) /
557: (PAGE_SIZE + sizeof *pages));
558:
559: pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
560:
561: /*
562: * Initialize the page frames.
563: */
564:
565: for (i = 0, pages_initialized = 0; i < npages; i++) {
566: if (!pmap_next_page(&paddr))
567: break;
568:
569: vm_page_init(&pages[i], paddr);
570: vm_page_pages++;
571: pages_initialized++;
572: }
573:
574: /*
575: * Release pages in reverse order so that physical pages
576: * initially get allocated in ascending addresses. This keeps
577: * the devices (which must address physical memory) happy if
578: * they require several consecutive pages.
579: */
580:
581: for (i = pages_initialized; i > 0; i--) {
582: vm_page_release(&pages[i - 1]);
583: }
584:
585: /*
586: * We have to re-align virtual_space_start,
587: * because pmap_steal_memory has been using it.
588: */
589:
590: virtual_space_start = round_page(virtual_space_start);
591:
592: *startp = virtual_space_start;
593: *endp = virtual_space_end;
594: }
595: #endif /* MACHINE_PAGES */
596:
597: /*
598: * Routine: vm_page_module_init
599: * Purpose:
600: * Second initialization pass, to be done after
601: * the basic VM system is ready.
602: */
603: void
604: vm_page_module_init(void)
605: {
606: vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
607: 0, PAGE_SIZE, "vm pages");
608:
609: #if ZONE_DEBUG
610: zone_debug_disable(vm_page_zone);
611: #endif /* ZONE_DEBUG */
612:
613: zone_change(vm_page_zone, Z_EXPAND, FALSE);
614: zone_change(vm_page_zone, Z_EXHAUST, TRUE);
615: zone_change(vm_page_zone, Z_FOREIGN, TRUE);
616:
617: /*
618: * Adjust zone statistics to account for the real pages allocated
619: * in vm_page_create(). [Q: is this really what we want?]
620: */
621: vm_page_zone->count += vm_page_pages;
622: vm_page_zone->cur_size += vm_page_pages * vm_page_zone->elem_size;
623:
624: mutex_init(&vm_page_alloc_lock, ETAP_VM_PAGE_ALLOC);
625: }
626:
627: /*
628: * Routine: vm_page_create
629: * Purpose:
630: * After the VM system is up, machine-dependent code
631: * may stumble across more physical memory. For example,
632: * memory that it was reserving for a frame buffer.
633: * vm_page_create turns this memory into available pages.
634: */
635:
636: void
637: vm_page_create(
638: vm_offset_t start,
639: vm_offset_t end)
640: {
641: vm_offset_t paddr;
642: vm_page_t m;
643:
644: for (paddr = round_page(start);
645: paddr < trunc_page(end);
646: paddr += PAGE_SIZE) {
647: while ((m = (vm_page_t) vm_page_grab_fictitious())
648: == VM_PAGE_NULL)
649: vm_page_more_fictitious();
650:
651: vm_page_init(m, paddr);
652: vm_page_pages++;
653: vm_page_release(m);
654: }
655: }
656:
657: /*
658: * vm_page_hash:
659: *
660: * Distributes the object/offset key pair among hash buckets.
661: *
662: * NOTE: To get a good hash function, the bucket count should
663: * be a power of two.
664: */
665: #define vm_page_hash(object, offset) (\
666: ( ((natural_t)(vm_offset_t)object<<vm_page_hash_shift) + (natural_t)atop(offset))\
667: & vm_page_hash_mask)
668:
669: /*
670: * vm_page_insert: [ internal use only ]
671: *
672: * Inserts the given mem entry into the object/object-page
673: * table and object list.
674: *
675: * The object must be locked.
676: */
677:
678: void
679: vm_page_insert(
680: register vm_page_t mem,
681: register vm_object_t object,
682: register vm_offset_t offset)
683: {
684: register vm_page_bucket_t *bucket;
685:
686: XPR(XPR_VM_PAGE,
687: "vm_page_insert, object 0x%X offset 0x%X page 0x%X\n",
688: (integer_t)object, (integer_t)offset, (integer_t)mem, 0,0);
689:
690: VM_PAGE_CHECK(mem);
691:
692: if (mem->tabled)
693: panic("vm_page_insert");
694:
695: assert(!object->internal || offset < object->size);
696:
697: /* only insert "pageout" pages into "pageout" objects,
698: * and normal pages into normal objects */
699: assert(object->pageout == mem->pageout);
700:
701: /*
702: * Record the object/offset pair in this page
703: */
704:
705: mem->object = object;
706: mem->offset = offset;
707:
708: /*
709: * Insert it into the object_object/offset hash table
710: */
711:
712: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
713: simple_lock(&vm_page_bucket_lock);
714: mem->next = bucket->pages;
715: bucket->pages = mem;
716: #if MACH_PAGE_HASH_STATS
717: if (++bucket->cur_count > bucket->hi_count)
718: bucket->hi_count = bucket->cur_count;
719: #endif /* MACH_PAGE_HASH_STATS */
720: simple_unlock(&vm_page_bucket_lock);
721:
722: /*
723: * Now link into the object's list of backed pages.
724: */
725:
726: queue_enter(&object->memq, mem, vm_page_t, listq);
727: mem->tabled = TRUE;
728:
729: /*
730: * Show that the object has one more resident page.
731: */
732:
733: object->resident_page_count++;
734: }
735:
736: /*
737: * vm_page_replace:
738: *
739: * Exactly like vm_page_insert, except that we first
740: * remove any existing page at the given offset in object.
741: *
742: * The object and page queues must be locked.
743: */
744:
745: void
746: vm_page_replace(
747: register vm_page_t mem,
748: register vm_object_t object,
749: register vm_offset_t offset)
750: {
751: register vm_page_bucket_t *bucket;
752:
753: VM_PAGE_CHECK(mem);
754:
755: if (mem->tabled)
756: panic("vm_page_replace");
757:
758: /*
759: * Record the object/offset pair in this page
760: */
761:
762: mem->object = object;
763: mem->offset = offset;
764:
765: /*
766: * Insert it into the object_object/offset hash table,
767: * replacing any page that might have been there.
768: */
769:
770: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
771: simple_lock(&vm_page_bucket_lock);
772: if (bucket->pages) {
773: vm_page_t *mp = &bucket->pages;
774: register vm_page_t m = *mp;
775: do {
776: if (m->object == object && m->offset == offset) {
777: /*
778: * Remove page from bucket and from object,
779: * and return it to the free list.
780: */
781: *mp = m->next;
782: queue_remove(&object->memq, m, vm_page_t,
783: listq);
784: m->tabled = FALSE;
785: object->resident_page_count--;
786:
787: /*
788: * Return page to the free list.
789: * Note the page is not tabled now, so this
790: * won't self-deadlock on the bucket lock.
791: */
792:
793: vm_page_free(m);
794: break;
795: }
796: mp = &m->next;
797: } while (m = *mp);
798: mem->next = bucket->pages;
799: } else {
800: mem->next = VM_PAGE_NULL;
801: }
802: bucket->pages = mem;
803: simple_unlock(&vm_page_bucket_lock);
804:
805: /*
806: * Now link into the object's list of backed pages.
807: */
808:
809: queue_enter(&object->memq, mem, vm_page_t, listq);
810: mem->tabled = TRUE;
811:
812: /*
813: * And show that the object has one more resident
814: * page.
815: */
816:
817: object->resident_page_count++;
818: }
819:
820: /*
821: * vm_page_remove: [ internal use only ]
822: *
823: * Removes the given mem entry from the object/offset-page
824: * table and the object page list.
825: *
826: * The object and page must be locked.
827: */
828:
829: void
830: vm_page_remove(
831: register vm_page_t mem)
832: {
833: register vm_page_bucket_t *bucket;
834: register vm_page_t this;
835:
836: XPR(XPR_VM_PAGE,
837: "vm_page_remove, object 0x%X offset 0x%X page 0x%X\n",
838: (integer_t)mem->object, (integer_t)mem->offset,
839: (integer_t)mem, 0,0);
840:
841: assert(mem->tabled);
842: assert(!mem->cleaning);
843: VM_PAGE_CHECK(mem);
844:
845: /*
846: * Remove from the object_object/offset hash table
847: */
848:
849: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
850: simple_lock(&vm_page_bucket_lock);
851: if ((this = bucket->pages) == mem) {
852: /* optimize for common case */
853:
854: bucket->pages = mem->next;
855: } else {
856: register vm_page_t *prev;
857:
858: for (prev = &this->next;
859: (this = *prev) != mem;
860: prev = &this->next)
861: continue;
862: *prev = this->next;
863: }
864: #if MACH_PAGE_HASH_STATS
865: bucket->cur_count--;
866: #endif /* MACH_PAGE_HASH_STATS */
867: simple_unlock(&vm_page_bucket_lock);
868:
869: /*
870: * Now remove from the object's list of backed pages.
871: */
872:
873: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
874:
875: /*
876: * And show that the object has one fewer resident
877: * page.
878: */
879:
880: mem->object->resident_page_count--;
881:
882: mem->tabled = FALSE;
883: mem->object = VM_OBJECT_NULL;
884: mem->offset = 0;
885: }
886:
887: /*
888: * vm_page_lookup:
889: *
890: * Returns the page associated with the object/offset
891: * pair specified; if none is found, VM_PAGE_NULL is returned.
892: *
893: * The object must be locked. No side effects.
894: */
895:
896: vm_page_t
897: vm_page_lookup(
898: register vm_object_t object,
899: register vm_offset_t offset)
900: {
901: register vm_page_t mem;
902: register vm_page_bucket_t *bucket;
903:
904: /*
905: * Search the hash table for this object/offset pair
906: */
907:
908: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
909:
910: simple_lock(&vm_page_bucket_lock);
911: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
912: VM_PAGE_CHECK(mem);
913: if ((mem->object == object) && (mem->offset == offset))
914: break;
915: }
916: simple_unlock(&vm_page_bucket_lock);
917: return(mem);
918: }
919:
920: /*
921: * vm_page_rename:
922: *
923: * Move the given memory entry from its
924: * current object to the specified target object/offset.
925: *
926: * The object must be locked.
927: */
928: void
929: vm_page_rename(
930: register vm_page_t mem,
931: register vm_object_t new_object,
932: vm_offset_t new_offset)
933: {
934: assert(mem->object != new_object);
935: /*
936: * Changes to mem->object require the page lock because
937: * the pageout daemon uses that lock to get the object.
938: */
939:
940: XPR(XPR_VM_PAGE,
941: "vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n",
942: (integer_t)new_object, (integer_t)new_offset,
943: (integer_t)mem, 0,0);
944:
945: vm_page_lock_queues();
946: vm_page_remove(mem);
947: vm_page_insert(mem, new_object, new_offset);
948: vm_page_unlock_queues();
949: }
950:
951: /*
952: * vm_page_init:
953: *
954: * Initialize the fields in a new page.
955: * This takes a structure with random values and initializes it
956: * so that it can be given to vm_page_release or vm_page_insert.
957: */
958: void
959: vm_page_init(
960: vm_page_t mem,
961: vm_offset_t phys_addr)
962: {
963: *mem = vm_page_template;
964: mem->phys_addr = phys_addr;
965: }
966:
967: /*
968: * vm_page_grab_fictitious:
969: *
970: * Remove a fictitious page from the free list.
971: * Returns VM_PAGE_NULL if there are no free pages.
972: */
973: int c_vm_page_grab_fictitious = 0;
974: int c_vm_page_release_fictitious = 0;
975: int c_vm_page_more_fictitious = 0;
976:
977: vm_page_t
978: vm_page_grab_fictitious(void)
979: {
980: register vm_page_t m;
981:
982: m = (vm_page_t)zget(vm_page_zone);
983: if (m) {
984: m->free = FALSE;
985: #if MACH_ASSERT || ZONE_DEBUG
986: vm_page_init(m, vm_page_fictitious_addr);
987: m->fictitious = TRUE;
988: #endif /* MACH_ASSERT || ZONE_DEBUG */
989: }
990:
991: c_vm_page_grab_fictitious++;
992: return m;
993: }
994:
995: /*
996: * vm_page_release_fictitious:
997: *
998: * Release a fictitious page to the free list.
999: */
1000:
1001: void
1002: vm_page_release_fictitious(
1003: register vm_page_t m)
1004: {
1005: assert(!m->free);
1006: assert(m->busy);
1007: assert(m->fictitious);
1008: assert(m->phys_addr == vm_page_fictitious_addr);
1009:
1010: c_vm_page_release_fictitious++;
1011:
1012: if (m->free)
1013: panic("vm_page_release_fictitious");
1014: m->free = TRUE;
1015: zfree(vm_page_zone, (vm_offset_t)m);
1016: }
1017:
1018: /*
1019: * vm_page_more_fictitious:
1020: *
1021: * Add more fictitious pages to the free list.
1022: * Allowed to block. This routine is way intimate
1023: * with the zones code, for several reasons:
1024: * 1. we need to carve some page structures out of physical
1025: * memory before zones work, so they _cannot_ come from
1026: * the zone_map.
1027: * 2. the zone needs to be collectable in order to prevent
1028: * growth without bound. These structures are used by
1029: * the device pager (by the hundreds and thousands), as
1030: * private pages for pageout, and as blocking pages for
1031: * pagein. Temporary bursts in demand should not result in
1032: * permanent allocation of a resource.
1033: * 3. To smooth allocation humps, we allocate single pages
1034: * with kernel_memory_allocate(), and cram them into the
1035: * zone. This also allows us to initialize the vm_page_t's
1036: * on the way into the zone, so that zget() always returns
1037: * an initialized structure. The zone free element pointer
1038: * and the free page pointer are both the first item in the
1039: * vm_page_t.
1040: * 4. By having the pages in the zone pre-initialized, we need
1041: * not keep 2 levels of lists. The garbage collector simply
1042: * scans our list, and reduces physical memory usage as it
1043: * sees fit.
1044: */
1045:
1046: void vm_page_more_fictitious(void)
1047: {
1048: extern vm_map_t zone_map;
1049: register vm_page_t m;
1050: vm_offset_t addr;
1051: kern_return_t retval;
1052: int i;
1053:
1054: c_vm_page_more_fictitious++;
1055:
1056: /* this may free up some fictitious pages */
1057: cleanup_limbo_queue();
1058:
1059: /*
1060: * Allocate a single page from the zone_map. Do not wait if no physical
1061: * pages are immediately available, and do not zero the space. We need
1062: * our own blocking lock here to prevent having multiple,
1063: * simultaneous requests from piling up on the zone_map lock. Exactly
1064: * one (of our) threads should be potentially waiting on the map lock.
1065: * If winner is not vm-privileged, then the page allocation will fail,
1066: * and it will temporarily block here in the vm_page_wait().
1067: */
1068: mutex_lock(&vm_page_alloc_lock);
1069: /*
1070: * If another thread allocated space, just bail out now.
1071: */
1072: if (zone_free_count(vm_page_zone) > 5) {
1073: /*
1074: * The number "5" is a small number that is larger than the
1075: * number of fictitious pages that any single caller will
1076: * attempt to allocate. Otherwise, a thread will attempt to
1077: * acquire a fictitious page (vm_page_grab_fictitious), fail,
1078: * release all of the resources and locks already acquired,
1079: * and then call this routine. This routine finds the pages
1080: * that the caller released, so fails to allocate new space.
1081: * The process repeats infinitely. The largest known number
1082: * of fictitious pages required in this manner is 2. 5 is
1083: * simply a somewhat larger number.
1084: */
1085: mutex_unlock(&vm_page_alloc_lock);
1086: return;
1087: }
1088:
1089: if ((retval = kernel_memory_allocate(zone_map,
1090: &addr, PAGE_SIZE, VM_PROT_ALL,
1091: KMA_KOBJECT|KMA_NOPAGEWAIT)) != KERN_SUCCESS) {
1092: /*
1093: * No page was available. Tell the pageout daemon, drop the
1094: * lock to give another thread a chance at it, and
1095: * wait for the pageout daemon to make progress.
1096: */
1097: mutex_unlock(&vm_page_alloc_lock);
1098: vm_page_wait();
1099: return;
1100: }
1101: /*
1102: * Initialize as many vm_page_t's as will fit on this page. This
1103: * depends on the zone code disturbing ONLY the first item of
1104: * each zone element.
1105: */
1106: m = (vm_page_t)addr;
1107: for (i = PAGE_SIZE/sizeof(struct vm_page); i > 0; i--) {
1108: vm_page_init(m, vm_page_fictitious_addr);
1109: m->fictitious = TRUE;
1110: m++;
1111: }
1112: zcram(vm_page_zone, addr, PAGE_SIZE);
1113: mutex_unlock(&vm_page_alloc_lock);
1114: }
1115:
1116: /*
1117: * vm_page_convert:
1118: *
1119: * Attempt to convert a fictitious page into a real page.
1120: */
1121:
1122: boolean_t
1123: vm_page_convert(
1124: register vm_page_t m)
1125: {
1126: register vm_page_t real_m;
1127:
1128: assert(m->busy);
1129: assert(m->fictitious);
1130: assert(!m->dirty);
1131:
1132: real_m = vm_page_grab();
1133: if (real_m == VM_PAGE_NULL)
1134: return FALSE;
1135:
1136: m->phys_addr = real_m->phys_addr;
1137: m->fictitious = FALSE;
1138:
1139: vm_page_lock_queues();
1140: if (m->active)
1141: vm_page_active_count++;
1142: else if (m->inactive)
1143: vm_page_inactive_count++;
1144: vm_page_unlock_queues();
1145:
1146: real_m->phys_addr = vm_page_fictitious_addr;
1147: real_m->fictitious = TRUE;
1148:
1149: vm_page_release_fictitious(real_m);
1150: return TRUE;
1151: }
1152:
1153: /*
1154: * vm_pool_low():
1155: *
1156: * Return true if it is not likely that a non-vm_privileged thread
1157: * can get memory without blocking. Advisory only, since the
1158: * situation may change under us.
1159: */
1160: int
1161: vm_pool_low(void)
1162: {
1163: /* No locking, at worst we will fib. */
1164: return( vm_page_free_count < vm_page_free_reserved );
1165: }
1166:
1167: /*
1168: * vm_page_grab:
1169: *
1170: * Remove a page from the free list.
1171: * Returns VM_PAGE_NULL if the free list is too small.
1172: */
1173:
1174: unsigned long vm_page_grab_count = 0; /* measure demand */
1175:
1176: vm_page_t
1177: vm_page_grab(void)
1178: {
1179: register vm_page_t mem;
1180:
1181: mutex_lock(&vm_page_queue_free_lock);
1182: vm_page_grab_count++;
1183:
1184: /*
1185: * Optionally produce warnings if the wire or gobble
1186: * counts exceed some threshold.
1187: */
1188: if (vm_page_wire_count_warning > 0
1189: && vm_page_wire_count >= vm_page_wire_count_warning) {
1190: printf("mk: vm_page_grab(): high wired page count of %d\n",
1191: vm_page_wire_count);
1192: assert(vm_page_wire_count < vm_page_wire_count_warning);
1193: }
1194: if (vm_page_gobble_count_warning > 0
1195: && vm_page_gobble_count >= vm_page_gobble_count_warning) {
1196: printf("mk: vm_page_grab(): high gobbled page count of %d\n",
1197: vm_page_gobble_count);
1198: assert(vm_page_gobble_count < vm_page_gobble_count_warning);
1199: }
1200:
1201: /*
1202: * Only let privileged threads (involved in pageout)
1203: * dip into the reserved pool.
1204: */
1205:
1206: if ((vm_page_free_count < vm_page_free_reserved) &&
1207: !current_thread()->vm_privilege) {
1208: mutex_unlock(&vm_page_queue_free_lock);
1209: mem = VM_PAGE_NULL;
1210: goto wakeup_pageout;
1211: }
1212:
1213: while (vm_page_queue_free == VM_PAGE_NULL) {
1214: printf("vm_page_grab: no free pages, trouble expected...\n");
1215: mutex_unlock(&vm_page_queue_free_lock);
1216: VM_PAGE_WAIT();
1217: mutex_lock(&vm_page_queue_free_lock);
1218: }
1219:
1220: if (--vm_page_free_count < vm_page_free_count_minimum)
1221: vm_page_free_count_minimum = vm_page_free_count;
1222: mem = vm_page_queue_free;
1223: vm_page_queue_free = (vm_page_t) mem->pageq.next;
1224: mem->free = FALSE;
1225: mutex_unlock(&vm_page_queue_free_lock);
1226:
1227: /*
1228: * Decide if we should poke the pageout daemon.
1229: * We do this if the free count is less than the low
1230: * water mark, or if the free count is less than the high
1231: * water mark (but above the low water mark) and the inactive
1232: * count is less than its target.
1233: *
1234: * We don't have the counts locked ... if they change a little,
1235: * it doesn't really matter.
1236: */
1237:
1238: wakeup_pageout:
1239: if ((vm_page_free_count < vm_page_free_min) ||
1240: ((vm_page_free_count < vm_page_free_target) &&
1241: (vm_page_inactive_count < vm_page_inactive_target)))
1242: thread_wakeup((event_t) &vm_page_free_wanted);
1243:
1244: // dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 4); /* (TEST/DEBUG) */
1245:
1246: return mem;
1247: }
1248:
1249: /*
1250: * vm_page_release:
1251: *
1252: * Return a page to the free list.
1253: */
1254:
1255: void
1256: vm_page_release(
1257: register vm_page_t mem)
1258: {
1259: assert(!mem->private && !mem->fictitious);
1260:
1261: // dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 5); /* (TEST/DEBUG) */
1262:
1263: mutex_lock(&vm_page_queue_free_lock);
1264: if (mem->free)
1265: panic("vm_page_release");
1266: mem->free = TRUE;
1267: mem->pageq.next = (queue_entry_t) vm_page_queue_free;
1268: vm_page_queue_free = mem;
1269: vm_page_free_count++;
1270:
1271: /*
1272: * Check if we should wake up someone waiting for page.
1273: * But don't bother waking them unless they can allocate.
1274: *
1275: * We wakeup only one thread, to prevent starvation.
1276: * Because the scheduling system handles wait queues FIFO,
1277: * if we wakeup all waiting threads, one greedy thread
1278: * can starve multiple niceguy threads. When the threads
1279: * all wakeup, the greedy threads runs first, grabs the page,
1280: * and waits for another page. It will be the first to run
1281: * when the next page is freed.
1282: *
1283: * However, there is a slight danger here.
1284: * The thread we wake might not use the free page.
1285: * Then the other threads could wait indefinitely
1286: * while the page goes unused. To forestall this,
1287: * the pageout daemon will keep making free pages
1288: * as long as vm_page_free_wanted is non-zero.
1289: */
1290:
1291: if ((vm_page_free_wanted > 0) &&
1292: (vm_page_free_count >= vm_page_free_reserved)) {
1293: vm_page_free_wanted--;
1294: thread_wakeup_one((event_t) &vm_page_free_count);
1295: }
1296:
1297: mutex_unlock(&vm_page_queue_free_lock);
1298: }
1299:
1300: /*
1301: * Release a page to the limbo list.
1302: * Put real pages at the head of the queue, fictitious at the tail.
1303: * Page queues must be locked.
1304: */
1305: void
1306: vm_page_release_limbo(
1307: register vm_page_t m)
1308: {
1309: assert(m->limbo);
1310: vm_page_limbo_count++;
1311: if (m->fictitious) {
1312: queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq);
1313: } else {
1314: vm_page_limbo_real_count++;
1315: queue_enter_first(&vm_page_queue_limbo, m, vm_page_t, pageq);
1316: }
1317: }
1318:
1319: /*
1320: * Exchange a real page in limbo (limbo_m) with a fictitious page (new_m).
1321: * The end result is that limbo_m is fictitious and still in limbo, and new_m
1322: * is the real page. The prep and pin counts remain with the page in limbo
1323: * although they will be briefly cleared by vm_page_init. This is OK since
1324: * there will be no interrupt-level interactions (the page is in limbo) and
1325: * vm_page_unprep must lock the page queues before changing the prep count.
1326: *
1327: * Page queues must be locked, and limbo_m must have been removed from its
1328: * object.
1329: */
1330: void
1331: vm_page_limbo_exchange(
1332: register vm_page_t limbo_m,
1333: register vm_page_t new_m)
1334: {
1335: assert(limbo_m->limbo && !limbo_m->fictitious);
1336: assert(!limbo_m->tabled);
1337: assert(new_m->fictitious);
1338:
1339: *new_m = *limbo_m;
1340: vm_page_init(limbo_m, vm_page_fictitious_addr);
1341:
1342: limbo_m->fictitious = TRUE;
1343: limbo_m->limbo = TRUE;
1344: new_m->limbo = FALSE;
1345:
1346: limbo_m->prep_pin_count = new_m->prep_pin_count;
1347: new_m->prep_pin_count = 0;
1348: }
1349:
1350: /*
1351: * vm_page_wait:
1352: *
1353: * Wait for a page to become available.
1354: * If there are plenty of free pages, then we don't sleep.
1355: */
1356:
1357: void
1358: vm_page_wait( void )
1359: {
1360: /*
1361: * We can't use vm_page_free_reserved to make this
1362: * determination. Consider: some thread might
1363: * need to allocate two pages. The first allocation
1364: * succeeds, the second fails. After the first page is freed,
1365: * a call to vm_page_wait must really block.
1366: */
1367:
1368: mutex_lock(&vm_page_queue_free_lock);
1369: if (vm_page_free_count < vm_page_free_target) {
1370: if (vm_page_free_wanted++ == 0)
1371: thread_wakeup((event_t)&vm_page_free_wanted);
1372: assert_wait((event_t)&vm_page_free_count, THREAD_UNINT);
1373: mutex_unlock(&vm_page_queue_free_lock);
1374: counter(c_vm_page_wait_block++);
1375: thread_block((void (*)(void))0);
1376: } else
1377: mutex_unlock(&vm_page_queue_free_lock);
1378: }
1379:
1380: /*
1381: * vm_page_alloc:
1382: *
1383: * Allocate and return a memory cell associated
1384: * with this VM object/offset pair.
1385: *
1386: * Object must be locked.
1387: */
1388:
1389: vm_page_t
1390: vm_page_alloc(
1391: vm_object_t object,
1392: vm_offset_t offset)
1393: {
1394: register vm_page_t mem;
1395:
1396: mem = vm_page_grab();
1397: if (mem == VM_PAGE_NULL)
1398: return VM_PAGE_NULL;
1399:
1400: vm_page_insert(mem, object, offset);
1401:
1402: return(mem);
1403: }
1404:
1405: int c_limbo_page_free = 0; /* debugging */
1406: int c_limbo_convert = 0; /* debugging */
1407: counter(unsigned int c_laundry_pages_freed = 0;)
1408:
1409: int vm_pagein_cluster_unused = 0;
1410: boolean_t vm_page_free_verify = FALSE;
1411: /*
1412: * vm_page_free:
1413: *
1414: * Returns the given page to the free list,
1415: * disassociating it with any VM object.
1416: *
1417: * Object and page queues must be locked prior to entry.
1418: */
1419: void
1420: vm_page_free(
1421: register vm_page_t mem)
1422: {
1423: vm_object_t object = mem->object;
1424:
1425: assert(!mem->free);
1426: assert(!mem->cleaning);
1427: assert(!mem->pageout);
1428: assert(!vm_page_free_verify || pmap_verify_free(mem->phys_addr));
1429:
1430: if (mem->tabled)
1431: vm_page_remove(mem); /* clears tabled, object, offset */
1432: VM_PAGE_QUEUES_REMOVE(mem); /* clears active or inactive */
1433:
1434: if (mem->clustered) {
1435: mem->clustered = FALSE;
1436: vm_pagein_cluster_unused++;
1437: }
1438:
1439: if (mem->wire_count) {
1440: if (!mem->private && !mem->fictitious)
1441: vm_page_wire_count--;
1442: mem->wire_count = 0;
1443: assert(!mem->gobbled);
1444: } else if (mem->gobbled) {
1445: if (!mem->private && !mem->fictitious)
1446: vm_page_wire_count--;
1447: vm_page_gobble_count--;
1448: }
1449: mem->gobbled = FALSE;
1450:
1451: if (mem->laundry) {
1452: extern int vm_page_laundry_min;
1453: vm_page_laundry_count--;
1454: mem->laundry = FALSE; /* laundry is now clear */
1455: counter(++c_laundry_pages_freed);
1456: if (vm_page_laundry_count < vm_page_laundry_min) {
1457: vm_page_laundry_min = 0;
1458: thread_wakeup((event_t) &vm_page_laundry_count);
1459: }
1460: }
1461:
1462: mem->discard_request = FALSE;
1463:
1464: PAGE_WAKEUP(mem); /* clears wanted */
1465:
1466: if (mem->absent)
1467: vm_object_absent_release(object);
1468:
1469: if (mem->limbo) {
1470: /*
1471: * The pageout daemon put this page into limbo and then freed
1472: * it. The page has already been removed from the object and
1473: * queues, so any attempt to look it up will fail. Put it
1474: * on the limbo queue; the pageout daemon will convert it to a
1475: * fictitious page and/or free the real one later.
1476: */
1477: /* assert that it came from pageout daemon (how?) */
1478: assert(!mem->fictitious && !mem->absent);
1479: c_limbo_page_free++;
1480: vm_page_release_limbo(mem);
1481: return;
1482: }
1483: assert(mem->prep_pin_count == 0);
1484:
1485: /* Some of these may be unnecessary */
1486: mem->page_lock = 0;
1487: mem->unlock_request = 0;
1488: mem->busy = TRUE;
1489: mem->absent = FALSE;
1490: mem->error = FALSE;
1491: mem->dirty = FALSE;
1492: mem->precious = FALSE;
1493: mem->reference = FALSE;
1494:
1495: mem->page_error = KERN_SUCCESS;
1496:
1497: if (mem->private) {
1498: mem->private = FALSE;
1499: mem->fictitious = TRUE;
1500: mem->phys_addr = vm_page_fictitious_addr;
1501: }
1502: if (mem->fictitious) {
1503: vm_page_release_fictitious(mem);
1504: } else {
1505: vm_page_init(mem, mem->phys_addr);
1506: vm_page_release(mem);
1507: }
1508: }
1509:
1510: /*
1511: * vm_page_wire:
1512: *
1513: * Mark this page as wired down by yet
1514: * another map, removing it from paging queues
1515: * as necessary.
1516: *
1517: * The page's object and the page queues must be locked.
1518: */
1519: void
1520: vm_page_wire(
1521: register vm_page_t mem)
1522: {
1523:
1524: // dbgLog(current_act(), mem->offset, mem->object, 1); /* (TEST/DEBUG) */
1525:
1526: VM_PAGE_CHECK(mem);
1527:
1528: if (mem->wire_count == 0) {
1529: VM_PAGE_QUEUES_REMOVE(mem);
1530: if (!mem->private && !mem->fictitious && !mem->gobbled)
1531: vm_page_wire_count++;
1532: if (mem->gobbled)
1533: vm_page_gobble_count--;
1534: mem->gobbled = FALSE;
1535: }
1536: assert(!mem->gobbled);
1537: mem->wire_count++;
1538: }
1539:
1540: /*
1541: * vm_page_gobble:
1542: *
1543: * Mark this page as consumed by the vm/ipc/xmm subsystems.
1544: *
1545: * Called only for freshly vm_page_grab()ed pages - w/ nothing locked.
1546: */
1547: void
1548: vm_page_gobble(
1549: register vm_page_t mem)
1550: {
1551: vm_page_lock_queues();
1552: VM_PAGE_CHECK(mem);
1553:
1554: assert(!mem->gobbled);
1555: assert(mem->wire_count == 0);
1556:
1557: if (!mem->gobbled && mem->wire_count == 0) {
1558: if (!mem->private && !mem->fictitious)
1559: vm_page_wire_count++;
1560: }
1561: vm_page_gobble_count++;
1562: mem->gobbled = TRUE;
1563: vm_page_unlock_queues();
1564: }
1565:
1566: /*
1567: * vm_page_unwire:
1568: *
1569: * Release one wiring of this page, potentially
1570: * enabling it to be paged again.
1571: *
1572: * The page's object and the page queues must be locked.
1573: */
1574: void
1575: vm_page_unwire(
1576: register vm_page_t mem)
1577: {
1578:
1579: // dbgLog(current_act(), mem->offset, mem->object, 0); /* (TEST/DEBUG) */
1580:
1581: VM_PAGE_CHECK(mem);
1582: assert(mem->wire_count > 0);
1583:
1584: if (--mem->wire_count == 0) {
1585: assert(!mem->private && !mem->fictitious);
1586: vm_page_wire_count--;
1587: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1588: vm_page_active_count++;
1589: mem->active = TRUE;
1590: mem->reference = TRUE;
1591: }
1592: }
1593:
1594: /*
1595: * vm_page_deactivate:
1596: *
1597: * Returns the given page to the inactive list,
1598: * indicating that no physical maps have access
1599: * to this page. [Used by the physical mapping system.]
1600: *
1601: * The page queues must be locked.
1602: */
1603: void
1604: vm_page_deactivate(
1605: register vm_page_t m)
1606: {
1607: VM_PAGE_CHECK(m);
1608:
1609: // dbgLog(m->phys_addr, vm_page_free_count, vm_page_wire_count, 6); /* (TEST/DEBUG) */
1610:
1611: /*
1612: * This page is no longer very interesting. If it was
1613: * interesting (active or inactive/referenced), then we
1614: * clear the reference bit and (re)enter it in the
1615: * inactive queue. Note wired pages should not have
1616: * their reference bit cleared.
1617: */
1618: if (m->gobbled) { /* can this happen? */
1619: assert(m->wire_count == 0);
1620: if (!m->private && !m->fictitious)
1621: vm_page_wire_count--;
1622: vm_page_gobble_count--;
1623: m->gobbled = FALSE;
1624: }
1625: if (m->private || (m->wire_count != 0))
1626: return;
1627: if (m->active || (m->inactive && m->reference)) {
1628: if (!m->fictitious && !m->absent)
1629: pmap_clear_reference(m->phys_addr);
1630: m->reference = FALSE;
1631: VM_PAGE_QUEUES_REMOVE(m);
1632: }
1633: if (m->wire_count == 0 && !m->inactive) {
1634: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1635: m->inactive = TRUE;
1636: if (!m->fictitious)
1637: vm_page_inactive_count++;
1638: }
1639: }
1640:
1641: /*
1642: * vm_page_activate:
1643: *
1644: * Put the specified page on the active list (if appropriate).
1645: *
1646: * The page queues must be locked.
1647: */
1648:
1649: void
1650: vm_page_activate(
1651: register vm_page_t m)
1652: {
1653: VM_PAGE_CHECK(m);
1654:
1655: if (m->gobbled) {
1656: assert(m->wire_count == 0);
1657: if (!m->private && !m->fictitious)
1658: vm_page_wire_count--;
1659: vm_page_gobble_count--;
1660: m->gobbled = FALSE;
1661: }
1662: if (m->private)
1663: return;
1664:
1665: if (m->inactive) {
1666: queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
1667: if (!m->fictitious)
1668: vm_page_inactive_count--;
1669: m->inactive = FALSE;
1670: }
1671: if (m->wire_count == 0) {
1672: if (m->active)
1673: panic("vm_page_activate: already active");
1674:
1675: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1676: m->active = TRUE;
1677: m->reference = TRUE;
1678: if (!m->fictitious)
1679: vm_page_active_count++;
1680: }
1681: }
1682:
1683: /*
1684: * vm_page_part_zero_fill:
1685: *
1686: * Zero-fill a part of the page.
1687: */
1688: void
1689: vm_page_part_zero_fill(
1690: vm_page_t m,
1691: vm_offset_t m_pa,
1692: vm_size_t len)
1693: {
1694: VM_PAGE_CHECK(m);
1695:
1696: pmap_zero_part_page(m->phys_addr, m_pa, len);
1697: }
1698:
1699: /*
1700: * vm_page_zero_fill:
1701: *
1702: * Zero-fill the specified page.
1703: */
1704: void
1705: vm_page_zero_fill(
1706: vm_page_t m)
1707: {
1708: XPR(XPR_VM_PAGE,
1709: "vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n",
1710: (integer_t)m->object, (integer_t)m->offset, (integer_t)m, 0,0);
1711:
1712: VM_PAGE_CHECK(m);
1713:
1714: pmap_zero_page(m->phys_addr);
1715: }
1716:
1717: /*
1718: * vm_page_part_copy:
1719: *
1720: * copy part of one page to another
1721: */
1722:
1723: void
1724: vm_page_part_copy(
1725: vm_page_t src_m,
1726: vm_offset_t src_pa,
1727: vm_page_t dst_m,
1728: vm_offset_t dst_pa,
1729: vm_size_t len)
1730: {
1731: VM_PAGE_CHECK(src_m);
1732: VM_PAGE_CHECK(dst_m);
1733:
1734: pmap_copy_part_page(src_m->phys_addr, src_pa,
1735: dst_m->phys_addr, dst_pa, len);
1736: }
1737:
1738: /*
1739: * vm_page_copy:
1740: *
1741: * Copy one page to another
1742: */
1743:
1744: void
1745: vm_page_copy(
1746: vm_page_t src_m,
1747: vm_page_t dest_m)
1748: {
1749: XPR(XPR_VM_PAGE,
1750: "vm_page_copy, object 0x%X offset 0x%X to object 0x%X offset 0x%X\n",
1751: (integer_t)src_m->object, src_m->offset,
1752: (integer_t)dest_m->object, dest_m->offset,
1753: 0);
1754:
1755: VM_PAGE_CHECK(src_m);
1756: VM_PAGE_CHECK(dest_m);
1757:
1758: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1759: }
1760:
1761: /*
1762: * Limbo pages are placed on the limbo queue to await their prep count
1763: * going to zero. A page is put into limbo by the pageout daemon. If the
1764: * page is real, then the pageout daemon did not need to page out the page,
1765: * it just freed it. When the prep_pin_count is zero the page can be freed.
1766: * Real pages with a non-zero prep count are converted to fictitious pages
1767: * so that the memory can be reclaimed; the fictitious page will remain on
1768: * the limbo queue until its prep count reaches zero.
1769: *
1770: * cleanup_limbo_queue is called by vm_page_more_fictitious and the pageout
1771: * daemon since it can free both real and fictitious pages.
1772: * It returns the number of fictitious pages freed.
1773: */
1774: void
1775: cleanup_limbo_queue(void)
1776: {
1777: register vm_page_t free_m, m;
1778: vm_offset_t phys_addr;
1779:
1780: vm_page_lock_queues();
1781: assert(vm_page_limbo_count >= vm_page_limbo_real_count);
1782:
1783: /*
1784: * first free up all pages with prep/pin counts of zero. This
1785: * may free both real and fictitious pages, which may be needed
1786: * later to convert real ones.
1787: */
1788: m = (vm_page_t)queue_first(&vm_page_queue_limbo);
1789: while (!queue_end(&vm_page_queue_limbo, (queue_entry_t)m)) {
1790: if (m->prep_pin_count == 0) {
1791: free_m = m;
1792: m = (vm_page_t)queue_next(&m->pageq);
1793: queue_remove(&vm_page_queue_limbo, free_m, vm_page_t,
1794: pageq);
1795: vm_page_limbo_count--;
1796: if (!free_m->fictitious)
1797: vm_page_limbo_real_count--;
1798: free_m->limbo = FALSE;
1799: vm_page_free(free_m);
1800: assert(vm_page_limbo_count >= 0);
1801: assert(vm_page_limbo_real_count >= 0);
1802: } else {
1803: m = (vm_page_t)queue_next(&m->pageq);
1804: }
1805: }
1806:
1807: /*
1808: * now convert any remaining real pages to fictitious and free the
1809: * real ones.
1810: */
1811: while (vm_page_limbo_real_count > 0) {
1812: queue_remove_first(&vm_page_queue_limbo, m, vm_page_t, pageq);
1813: assert(!m->fictitious);
1814: assert(m->limbo);
1815:
1816: /*
1817: * Try to get a fictitious page. If impossible,
1818: * requeue the real one and give up.
1819: */
1820: free_m = vm_page_grab_fictitious();
1821: if (free_m == VM_PAGE_NULL) {
1822: queue_enter_first(&vm_page_queue_limbo, m, vm_page_t,
1823: pageq);
1824: break;
1825: }
1826: c_limbo_convert++;
1827: vm_page_limbo_exchange(m, free_m);
1828: assert(m->limbo && m->fictitious);
1829: assert(!free_m->limbo && !free_m->fictitious);
1830: queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq);
1831: vm_page_free(free_m);
1832: vm_page_limbo_real_count--;
1833: }
1834:
1835: vm_page_unlock_queues();
1836: }
1837:
1838: /*
1839: * Increment prep_count on a page.
1840: * Must be called in thread context. Page must not disappear: object
1841: * must be locked.
1842: */
1843: kern_return_t
1844: vm_page_prep(
1845: register vm_page_t m)
1846: {
1847: kern_return_t retval = KERN_SUCCESS;
1848:
1849: assert(m != VM_PAGE_NULL);
1850: vm_page_lock_queues();
1851: if (!m->busy && !m->error && !m->fictitious && !m->absent) {
1852: if (m->prep_pin_count != 0) {
1853: vm_page_pin_lock();
1854: m->prep_count++;
1855: vm_page_pin_unlock();
1856: } else {
1857: m->prep_count++;
1858: }
1859: assert(m->prep_count != 0); /* check for wraparound */
1860: } else {
1861: retval = KERN_FAILURE;
1862: }
1863: vm_page_unlock_queues();
1864: return retval;
1865: }
1866:
1867:
1868: /*
1869: * Pin a page (increment pin count).
1870: * Must have been previously prepped.
1871: *
1872: * MUST BE CALLED AT SPLVM.
1873: *
1874: * May be called from thread or interrupt context.
1875: * If page is in "limbo" it cannot be pinned.
1876: */
1877: kern_return_t
1878: vm_page_pin(
1879: register vm_page_t m)
1880: {
1881: kern_return_t retval = KERN_SUCCESS;
1882:
1883: assert(m != VM_PAGE_NULL);
1884: vm_page_pin_lock();
1885: if (m->limbo || m->prep_count == 0) {
1886: retval = KERN_FAILURE;
1887: } else {
1888: assert(!m->fictitious);
1889: if (m->pin_count == 0)
1890: vm_page_pin_count++;
1891: m->pin_count++;
1892: }
1893: vm_page_pin_unlock();
1894: return retval;
1895: }
1896:
1897:
1898: /*
1899: * Unprep a page (decrement prep count).
1900: * Must have been previously prepped.
1901: * Called to decrement prep count after an attempt to pin failed.
1902: * Must be called from thread context.
1903: */
1904: kern_return_t
1905: vm_page_unprep(
1906: register vm_page_t m)
1907: {
1908: kern_return_t retval = KERN_SUCCESS;
1909:
1910: assert(m != VM_PAGE_NULL);
1911: vm_page_lock_queues();
1912: vm_page_pin_lock();
1913: assert(m->prep_count != 0);
1914: if (m->prep_count == 0)
1915: retval = KERN_FAILURE; /* shouldn't happen */
1916: else
1917: m->prep_count--;
1918: vm_page_pin_unlock();
1919: vm_page_unlock_queues();
1920: return retval;
1921: }
1922:
1923:
1924: /*
1925: * Unpin a page: decrement pin AND prep counts.
1926: * Must have been previously prepped AND pinned.
1927: *
1928: * MUST BE CALLED AT SPLVM.
1929: *
1930: * May be called from thread or interrupt context.
1931: */
1932: kern_return_t
1933: vm_page_unpin(
1934: register vm_page_t m)
1935: {
1936: kern_return_t retval = KERN_SUCCESS;
1937:
1938: assert(m != VM_PAGE_NULL);
1939: vm_page_pin_lock();
1940: assert(m->prep_count != 0 && m->pin_count != 0);
1941: assert(m->prep_count >= m->pin_count);
1942: assert(!m->limbo && !m->fictitious);
1943: if (m->prep_count != 0 && m->pin_count != 0) {
1944: m->prep_count--;
1945: m->pin_count--;
1946: if (m->pin_count == 0)
1947: vm_page_pin_count--;
1948: } else {
1949: retval = KERN_FAILURE; /* shouldn't happen */
1950: }
1951: vm_page_pin_unlock();
1952: return retval;
1953: }
1954:
1955: /*
1956: * Currently, this is a primitive allocator that grabs
1957: * free pages from the system, sorts them by physical
1958: * address, then searches for a region large enough to
1959: * satisfy the user's request.
1960: *
1961: * Additional levels of effort:
1962: * + steal clean active/inactive pages
1963: * + force pageouts of dirty pages
1964: * + maintain a map of available physical
1965: * memory
1966: */
1967:
1968: #define SET_NEXT_PAGE(m,n) ((m)->pageq.next = (struct queue_entry *) (n))
1969:
1970: #if MACH_ASSERT
1971: int vm_page_verify_contiguous(
1972: vm_page_t pages,
1973: unsigned int npages);
1974: #endif /* MACH_ASSERT */
1975:
1976: cpm_counter(unsigned int vpfls_pages_handled = 0;)
1977: cpm_counter(unsigned int vpfls_head_insertions = 0;)
1978: cpm_counter(unsigned int vpfls_tail_insertions = 0;)
1979: cpm_counter(unsigned int vpfls_general_insertions = 0;)
1980: cpm_counter(unsigned int vpfc_failed = 0;)
1981: cpm_counter(unsigned int vpfc_satisfied = 0;)
1982:
1983: /*
1984: * Sort free list by ascending physical address,
1985: * using a not-particularly-bright sort algorithm.
1986: * Caller holds vm_page_queue_free_lock.
1987: */
1988: static void
1989: vm_page_free_list_sort(void)
1990: {
1991: vm_page_t sort_list;
1992: vm_page_t sort_list_end;
1993: vm_page_t m, m1, *prev, next_m;
1994: vm_offset_t addr;
1995: #if MACH_ASSERT
1996: unsigned int npages;
1997: int old_free_count;
1998: #endif /* MACH_ASSERT */
1999:
2000: #if MACH_ASSERT
2001: /*
2002: * Verify pages in the free list..
2003: */
2004: npages = 0;
2005: for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m))
2006: ++npages;
2007: if (npages != vm_page_free_count)
2008: panic("vm_sort_free_list: prelim: npages %d free_count %d",
2009: npages, vm_page_free_count);
2010: old_free_count = vm_page_free_count;
2011: #endif /* MACH_ASSERT */
2012:
2013: sort_list = sort_list_end = vm_page_queue_free;
2014: m = NEXT_PAGE(vm_page_queue_free);
2015: SET_NEXT_PAGE(vm_page_queue_free, VM_PAGE_NULL);
2016: cpm_counter(vpfls_pages_handled = 0);
2017: while (m != VM_PAGE_NULL) {
2018: cpm_counter(++vpfls_pages_handled);
2019: next_m = NEXT_PAGE(m);
2020: if (m->phys_addr < sort_list->phys_addr) {
2021: cpm_counter(++vpfls_head_insertions);
2022: SET_NEXT_PAGE(m, sort_list);
2023: sort_list = m;
2024: } else if (m->phys_addr > sort_list_end->phys_addr) {
2025: cpm_counter(++vpfls_tail_insertions);
2026: SET_NEXT_PAGE(sort_list_end, m);
2027: SET_NEXT_PAGE(m, VM_PAGE_NULL);
2028: sort_list_end = m;
2029: } else {
2030: cpm_counter(++vpfls_general_insertions);
2031: /* general sorted list insertion */
2032: prev = &sort_list;
2033: for (m1=sort_list; m1!=VM_PAGE_NULL; m1=NEXT_PAGE(m1)) {
2034: if (m1->phys_addr > m->phys_addr) {
2035: if (*prev != m1)
2036: panic("vm_sort_free_list: ugh");
2037: SET_NEXT_PAGE(m, *prev);
2038: *prev = m;
2039: break;
2040: }
2041: prev = (vm_page_t *) &m1->pageq.next;
2042: }
2043: }
2044: m = next_m;
2045: }
2046:
2047: #if MACH_ASSERT
2048: /*
2049: * Verify that pages are sorted into ascending order.
2050: */
2051: for (m = sort_list, npages = 0; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
2052: if (m != sort_list &&
2053: m->phys_addr <= addr) {
2054: printf("m 0x%x addr 0x%x\n", m, addr);
2055: panic("vm_sort_free_list");
2056: }
2057: addr = m->phys_addr;
2058: ++npages;
2059: }
2060: if (old_free_count != vm_page_free_count)
2061: panic("vm_sort_free_list: old_free %d free_count %d",
2062: old_free_count, vm_page_free_count);
2063: if (npages != vm_page_free_count)
2064: panic("vm_sort_free_list: npages %d free_count %d",
2065: npages, vm_page_free_count);
2066: #endif /* MACH_ASSERT */
2067:
2068: vm_page_queue_free = sort_list;
2069: }
2070:
2071:
2072: #if MACH_ASSERT
2073: /*
2074: * Check that the list of pages is ordered by
2075: * ascending physical address and has no holes.
2076: */
2077: int
2078: vm_page_verify_contiguous(
2079: vm_page_t pages,
2080: unsigned int npages)
2081: {
2082: register vm_page_t m;
2083: unsigned int page_count;
2084: vm_offset_t prev_addr;
2085:
2086: prev_addr = pages->phys_addr;
2087: page_count = 1;
2088: for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
2089: if (m->phys_addr != prev_addr + page_size) {
2090: printf("m 0x%x prev_addr 0x%x, current addr 0x%x\n",
2091: m, prev_addr, m->phys_addr);
2092: printf("pages 0x%x page_count %d\n", pages, page_count);
2093: panic("vm_page_verify_contiguous: not contiguous!");
2094: }
2095: prev_addr = m->phys_addr;
2096: ++page_count;
2097: }
2098: if (page_count != npages) {
2099: printf("pages 0x%x actual count 0x%x but requested 0x%x\n",
2100: pages, page_count, npages);
2101: panic("vm_page_verify_contiguous: count error");
2102: }
2103: return 1;
2104: }
2105: #endif /* MACH_ASSERT */
2106:
2107:
2108: /*
2109: * Find a region large enough to contain at least npages
2110: * of contiguous physical memory.
2111: *
2112: * Requirements:
2113: * - Called while holding vm_page_queue_free_lock.
2114: * - Doesn't respect vm_page_free_reserved; caller
2115: * must not ask for more pages than are legal to grab.
2116: *
2117: * Returns a pointer to a list of gobbled pages or VM_PAGE_NULL.
2118: *
2119: */
2120: static vm_page_t
2121: vm_page_find_contiguous(
2122: int npages)
2123: {
2124: vm_page_t m, *contig_prev, *prev_ptr;
2125: vm_offset_t prev_addr;
2126: unsigned int contig_npages;
2127: vm_page_t list;
2128:
2129: if (npages < 1)
2130: return VM_PAGE_NULL;
2131:
2132: prev_addr = vm_page_queue_free->phys_addr - (page_size + 1);
2133: prev_ptr = &vm_page_queue_free;
2134: for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
2135:
2136: if (m->phys_addr != prev_addr + page_size) {
2137: /*
2138: * Whoops! Pages aren't contiguous. Start over.
2139: */
2140: contig_npages = 0;
2141: contig_prev = prev_ptr;
2142: }
2143:
2144: if (++contig_npages == npages) {
2145: /*
2146: * Chop these pages out of the free list.
2147: * Mark them all as gobbled.
2148: */
2149: list = *contig_prev;
2150: *contig_prev = NEXT_PAGE(m);
2151: SET_NEXT_PAGE(m, VM_PAGE_NULL);
2152: for (m = list; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
2153: assert(m->free);
2154: assert(!m->wanted);
2155: m->free = FALSE;
2156: m->gobbled = TRUE;
2157: }
2158: vm_page_free_count -= npages;
2159: if (vm_page_free_count < vm_page_free_count_minimum)
2160: vm_page_free_count_minimum = vm_page_free_count;
2161: vm_page_gobble_count += npages;
2162: cpm_counter(++vpfc_satisfied);
2163: assert(vm_page_verify_contiguous(list, contig_npages));
2164: return list;
2165: }
2166:
2167: assert(contig_npages < npages);
2168: prev_ptr = (vm_page_t *) &m->pageq.next;
2169: prev_addr = m->phys_addr;
2170: }
2171: cpm_counter(++vpfc_failed);
2172: return VM_PAGE_NULL;
2173: }
2174:
2175: /*
2176: * Allocate a list of contiguous, wired pages.
2177: */
2178: kern_return_t
2179: cpm_allocate(
2180: vm_size_t size,
2181: vm_page_t *list,
2182: boolean_t wire)
2183: {
2184: register vm_page_t m;
2185: vm_page_t *first_contig;
2186: vm_page_t free_list, pages;
2187: unsigned int npages, n1pages;
2188: int vm_pages_available;
2189:
2190: if (size % page_size != 0)
2191: return KERN_INVALID_ARGUMENT;
2192:
2193: vm_page_lock_queues();
2194: mutex_lock(&vm_page_queue_free_lock);
2195:
2196: /*
2197: * Should also take active and inactive pages
2198: * into account... One day...
2199: */
2200: vm_pages_available = vm_page_free_count - vm_page_free_reserved;
2201:
2202: if (size > vm_pages_available * page_size) {
2203: mutex_unlock(&vm_page_queue_free_lock);
2204: return KERN_RESOURCE_SHORTAGE;
2205: }
2206:
2207: vm_page_free_list_sort();
2208:
2209: npages = size / page_size;
2210:
2211: /*
2212: * Obtain a pointer to a subset of the free
2213: * list large enough to satisfy the request;
2214: * the region will be physically contiguous.
2215: */
2216: pages = vm_page_find_contiguous(npages);
2217: if (pages == VM_PAGE_NULL) {
2218: mutex_unlock(&vm_page_queue_free_lock);
2219: return KERN_NO_SPACE;
2220: }
2221:
2222: mutex_unlock(&vm_page_queue_free_lock);
2223:
2224: /*
2225: * Walk the returned list, wiring the pages.
2226: */
2227: if (wire == TRUE)
2228: for (m = pages; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
2229: /*
2230: * Essentially inlined vm_page_wire.
2231: */
2232: assert(!m->active);
2233: assert(!m->inactive);
2234: assert(!m->private);
2235: assert(!m->fictitious);
2236: assert(m->wire_count == 0);
2237: assert(m->gobbled);
2238: m->gobbled = FALSE;
2239: m->wire_count++;
2240: ++vm_page_wire_count;
2241: --vm_page_gobble_count;
2242: }
2243: vm_page_unlock_queues();
2244:
2245: /*
2246: * The CPM pages should now be available and
2247: * ordered by ascending physical address.
2248: */
2249: assert(vm_page_verify_contiguous(pages, npages));
2250:
2251: *list = pages;
2252: return KERN_SUCCESS;
2253: }
2254:
2255:
2256: #include <mach_vm_debug.h>
2257: #if MACH_VM_DEBUG
2258:
2259: #include <mach_debug/hash_info.h>
2260: #include <vm/vm_debug.h>
2261:
2262: /*
2263: * Routine: vm_page_info
2264: * Purpose:
2265: * Return information about the global VP table.
2266: * Fills the buffer with as much information as possible
2267: * and returns the desired size of the buffer.
2268: * Conditions:
2269: * Nothing locked. The caller should provide
2270: * possibly-pageable memory.
2271: */
2272:
2273: unsigned int
2274: vm_page_info(
2275: hash_info_bucket_t *info,
2276: unsigned int count)
2277: {
2278: int i;
2279:
2280: if (vm_page_bucket_count < count)
2281: count = vm_page_bucket_count;
2282:
2283: for (i = 0; i < count; i++) {
2284: vm_page_bucket_t *bucket = &vm_page_buckets[i];
2285: unsigned int bucket_count = 0;
2286: vm_page_t m;
2287:
2288: simple_lock(&vm_page_bucket_lock);
2289: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
2290: bucket_count++;
2291: simple_unlock(&vm_page_bucket_lock);
2292:
2293: /* don't touch pageable memory while holding locks */
2294: info[i].hib_count = bucket_count;
2295: }
2296:
2297: return vm_page_bucket_count;
2298: }
2299: #endif /* MACH_VM_DEBUG */
2300:
2301: #include <mach_kdb.h>
2302: #if MACH_KDB
2303:
2304: #include <ddb/db_output.h>
2305: #include <vm/vm_print.h>
2306: #define printf kdbprintf
2307:
2308: /*
2309: * Routine: vm_page_print [exported]
2310: */
2311: void
2312: vm_page_print(
2313: vm_page_t p)
2314: {
2315: extern db_indent;
2316:
2317: iprintf("page 0x%x\n", p);
2318:
2319: db_indent += 2;
2320:
2321: iprintf("object=0x%x", p->object);
2322: printf(", offset=0x%x", p->offset);
2323: printf(", wire_count=%d", p->wire_count);
2324: printf(", prep_count=%d", p->prep_count);
2325: printf(", pin_count=%d\n", p->pin_count);
2326:
2327: iprintf("%sinactive, %sactive, %sgobbled, %slaundry, %sfree, %sref, %sdiscard\n",
2328: (p->inactive ? "" : "!"),
2329: (p->active ? "" : "!"),
2330: (p->gobbled ? "" : "!"),
2331: (p->laundry ? "" : "!"),
2332: (p->free ? "" : "!"),
2333: (p->reference ? "" : "!"),
2334: (p->discard_request ? "" : "!"));
2335: iprintf("%sbusy, %swanted, %stabled, %sfictitious, %sprivate, %sprecious\n",
2336: (p->busy ? "" : "!"),
2337: (p->wanted ? "" : "!"),
2338: (p->tabled ? "" : "!"),
2339: (p->fictitious ? "" : "!"),
2340: (p->private ? "" : "!"),
2341: (p->precious ? "" : "!"));
2342: iprintf("%sabsent, %serror, %sdirty, %scleaning, %spageout, %sclustered\n",
2343: (p->absent ? "" : "!"),
2344: (p->error ? "" : "!"),
2345: (p->dirty ? "" : "!"),
2346: (p->cleaning ? "" : "!"),
2347: (p->pageout ? "" : "!"),
2348: (p->clustered ? "" : "!"));
2349: iprintf("%slock_supplied, %soverwriting, %srestart, %sunusual, %slimbo\n",
2350: (p->lock_supplied ? "" : "!"),
2351: (p->overwriting ? "" : "!"),
2352: (p->restart ? "" : "!"),
2353: (p->unusual ? "" : "!"),
2354: (p->limbo ? "" : "!"));
2355:
2356: iprintf("phys_addr=0x%x", p->phys_addr);
2357: printf(", page_error=0x%x", p->page_error);
2358: printf(", page_lock=0x%x", p->page_lock);
2359: printf(", unlock_request=%d\n", p->unlock_request);
2360:
2361: db_indent -= 2;
2362: }
2363: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.