![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_resident.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_page.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Resident memory management module.
34: */
1.1.1.3 ! root 35:
! 36: #include <kern/printf.h>
! 37: #include <string.h>
1.1 root 38:
39: #include <mach/vm_prot.h>
40: #include <kern/counters.h>
1.1.1.3 ! root 41: #include <kern/debug.h>
1.1 root 42: #include <kern/sched_prim.h>
43: #include <kern/task.h>
44: #include <kern/thread.h>
45: #include <mach/vm_statistics.h>
1.1.1.3 ! root 46: #include <machine/vm_param.h>
1.1 root 47: #include <kern/xpr.h>
1.1.1.3 ! root 48: #include <kern/slab.h>
1.1 root 49: #include <vm/pmap.h>
50: #include <vm/vm_map.h>
51: #include <vm/vm_page.h>
52: #include <vm/vm_pageout.h>
53: #include <vm/vm_kern.h>
54:
55: #if MACH_VM_DEBUG
56: #include <mach/kern_return.h>
57: #include <mach_debug/hash_info.h>
58: #include <vm/vm_user.h>
59: #endif
60:
1.1.1.3 ! root 61: #if MACH_KDB
! 62: #include <ddb/db_output.h>
! 63: #include <vm/vm_print.h>
! 64: #endif /* MACH_KDB */
! 65:
1.1 root 66:
67: /*
68: * Associated with eacn page of user-allocatable memory is a
69: * page structure.
70: */
71:
72: /*
73: * These variables record the values returned by vm_page_bootstrap,
74: * for debugging purposes. The implementation of pmap_steal_memory
75: * and pmap_startup here also uses them internally.
76: */
77:
78: vm_offset_t virtual_space_start;
79: vm_offset_t virtual_space_end;
80:
81: /*
82: * The vm_page_lookup() routine, which provides for fast
83: * (virtual memory object, offset) to page lookup, employs
84: * the following hash table. The vm_page_{insert,remove}
85: * routines install and remove associations in the table.
86: * [This table is often called the virtual-to-physical,
87: * or VP, table.]
88: */
89: typedef struct {
90: decl_simple_lock_data(,lock)
91: vm_page_t pages;
92: } vm_page_bucket_t;
93:
94: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
95: unsigned int vm_page_bucket_count = 0; /* How big is array? */
96: unsigned int vm_page_hash_mask; /* Mask for hash function */
97:
98: /*
99: * Resident page structures are initialized from
100: * a template (see vm_page_alloc).
101: *
102: * When adding a new field to the virtual memory
103: * object structure, be sure to add initialization
104: * (see vm_page_bootstrap).
105: */
106: struct vm_page vm_page_template;
107:
108: /*
109: * Resident pages that represent real memory
110: * are allocated from a free list.
111: */
112: vm_page_t vm_page_queue_free;
113: vm_page_t vm_page_queue_fictitious;
114: decl_simple_lock_data(,vm_page_queue_free_lock)
115: unsigned int vm_page_free_wanted;
116: int vm_page_free_count;
117: int vm_page_fictitious_count;
1.1.1.2 root 118: int vm_page_external_count;
1.1 root 119:
120: unsigned int vm_page_free_count_minimum; /* debugging */
121:
122: /*
123: * Occasionally, the virtual memory system uses
124: * resident page structures that do not refer to
125: * real pages, for example to leave a page with
126: * important state information in the VP table.
127: *
128: * These page structures are allocated the way
129: * most other kernel structures are.
130: */
1.1.1.3 ! root 131: struct kmem_cache vm_page_cache;
1.1 root 132:
133: /*
134: * Fictitious pages don't have a physical address,
135: * but we must initialize phys_addr to something.
136: * For debugging, this should be a strange value
137: * that the pmap module can recognize in assertions.
138: */
139: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
140:
141: /*
142: * Resident page structures are also chained on
143: * queues that are used by the page replacement
144: * system (pageout daemon). These queues are
145: * defined here, but are shared by the pageout
146: * module.
147: */
148: queue_head_t vm_page_queue_active;
149: queue_head_t vm_page_queue_inactive;
150: decl_simple_lock_data(,vm_page_queue_lock)
151: int vm_page_active_count;
152: int vm_page_inactive_count;
153: int vm_page_wire_count;
154:
155: /*
156: * Several page replacement parameters are also
157: * shared with this module, so that page allocation
158: * (done here in vm_page_alloc) can trigger the
159: * pageout daemon.
160: */
161: int vm_page_free_target = 0;
162: int vm_page_free_min = 0;
163: int vm_page_inactive_target = 0;
164: int vm_page_free_reserved = 0;
165: int vm_page_laundry_count = 0;
1.1.1.2 root 166: int vm_page_external_limit = 0;
167:
1.1 root 168:
169: /*
170: * The VM system has a couple of heuristics for deciding
171: * that pages are "uninteresting" and should be placed
172: * on the inactive queue as likely candidates for replacement.
173: * These variables let the heuristics be controlled at run-time
174: * to make experimentation easier.
175: */
176:
177: boolean_t vm_page_deactivate_behind = TRUE;
178: boolean_t vm_page_deactivate_hint = TRUE;
179:
180: /*
181: * vm_page_bootstrap:
182: *
183: * Initializes the resident memory module.
184: *
185: * Allocates memory for the page cells, and
186: * for the object/offset-to-page hash table headers.
187: * Each page cell is initialized and placed on the free list.
188: * Returns the range of available kernel virtual memory.
189: */
190:
191: void vm_page_bootstrap(
192: vm_offset_t *startp,
193: vm_offset_t *endp)
194: {
195: register vm_page_t m;
196: int i;
197:
198: /*
199: * Initialize the vm_page template.
200: */
201:
202: m = &vm_page_template;
203: m->object = VM_OBJECT_NULL; /* reset later */
204: m->offset = 0; /* reset later */
205: m->wire_count = 0;
206:
207: m->inactive = FALSE;
208: m->active = FALSE;
209: m->laundry = FALSE;
210: m->free = FALSE;
1.1.1.2 root 211: m->external = FALSE;
1.1 root 212:
213: m->busy = TRUE;
214: m->wanted = FALSE;
215: m->tabled = FALSE;
216: m->fictitious = FALSE;
217: m->private = FALSE;
218: m->absent = FALSE;
219: m->error = FALSE;
220: m->dirty = FALSE;
221: m->precious = FALSE;
222: m->reference = FALSE;
223:
224: m->phys_addr = 0; /* reset later */
225:
226: m->page_lock = VM_PROT_NONE;
227: m->unlock_request = VM_PROT_NONE;
228:
229: /*
230: * Initialize the page queues.
231: */
232:
233: simple_lock_init(&vm_page_queue_free_lock);
234: simple_lock_init(&vm_page_queue_lock);
235:
236: vm_page_queue_free = VM_PAGE_NULL;
237: vm_page_queue_fictitious = VM_PAGE_NULL;
238: queue_init(&vm_page_queue_active);
239: queue_init(&vm_page_queue_inactive);
240:
241: vm_page_free_wanted = 0;
242:
243: /*
1.1.1.3 ! root 244: * Steal memory for the kernel map entries.
1.1 root 245: */
246:
247: kentry_data = pmap_steal_memory(kentry_data_size);
248:
249: /*
250: * Allocate (and initialize) the virtual-to-physical
251: * table hash buckets.
252: *
253: * The number of buckets should be a power of two to
254: * get a good hash function. The following computation
255: * chooses the first power of two that is greater
256: * than the number of physical pages in the system.
257: */
258:
259: if (vm_page_bucket_count == 0) {
260: unsigned int npages = pmap_free_pages();
261:
262: vm_page_bucket_count = 1;
263: while (vm_page_bucket_count < npages)
264: vm_page_bucket_count <<= 1;
265: }
266:
267: vm_page_hash_mask = vm_page_bucket_count - 1;
268:
269: if (vm_page_hash_mask & vm_page_bucket_count)
270: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
271:
272: vm_page_buckets = (vm_page_bucket_t *)
273: pmap_steal_memory(vm_page_bucket_count *
274: sizeof(vm_page_bucket_t));
275:
276: for (i = 0; i < vm_page_bucket_count; i++) {
277: register vm_page_bucket_t *bucket = &vm_page_buckets[i];
278:
279: bucket->pages = VM_PAGE_NULL;
280: simple_lock_init(&bucket->lock);
281: }
282:
283: /*
284: * Machine-dependent code allocates the resident page table.
285: * It uses vm_page_init to initialize the page frames.
286: * The code also returns to us the virtual space available
287: * to the kernel. We don't trust the pmap module
288: * to get the alignment right.
289: */
290:
291: pmap_startup(&virtual_space_start, &virtual_space_end);
292: virtual_space_start = round_page(virtual_space_start);
293: virtual_space_end = trunc_page(virtual_space_end);
294:
295: *startp = virtual_space_start;
296: *endp = virtual_space_end;
297:
298: /* printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
299: vm_page_free_count_minimum = vm_page_free_count;
300: }
301:
302: #ifndef MACHINE_PAGES
303: /*
304: * We implement pmap_steal_memory and pmap_startup with the help
305: * of two simpler functions, pmap_virtual_space and pmap_next_page.
306: */
307:
308: vm_offset_t pmap_steal_memory(
309: vm_size_t size)
310: {
311: vm_offset_t addr, vaddr, paddr;
312:
313: /*
314: * We round the size to an integer multiple.
315: */
316:
317: size = (size + 3) &~ 3;
318:
319: /*
320: * If this is the first call to pmap_steal_memory,
321: * we have to initialize ourself.
322: */
323:
324: if (virtual_space_start == virtual_space_end) {
325: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
326:
327: /*
328: * The initial values must be aligned properly, and
329: * we don't trust the pmap module to do it right.
330: */
331:
332: virtual_space_start = round_page(virtual_space_start);
333: virtual_space_end = trunc_page(virtual_space_end);
334: }
335:
336: /*
337: * Allocate virtual memory for this request.
338: */
339:
340: addr = virtual_space_start;
341: virtual_space_start += size;
342:
343: /*
344: * Allocate and map physical pages to back new virtual pages.
345: */
346:
347: for (vaddr = round_page(addr);
348: vaddr < addr + size;
349: vaddr += PAGE_SIZE) {
350: if (!pmap_next_page(&paddr))
351: panic("pmap_steal_memory");
352:
353: /*
354: * XXX Logically, these mappings should be wired,
355: * but some pmap modules barf if they are.
356: */
357:
358: pmap_enter(kernel_pmap, vaddr, paddr,
359: VM_PROT_READ|VM_PROT_WRITE, FALSE);
360: }
361:
362: return addr;
363: }
364:
365: void pmap_startup(
366: vm_offset_t *startp,
367: vm_offset_t *endp)
368: {
369: unsigned int i, npages, pages_initialized;
370: vm_page_t pages;
371: vm_offset_t paddr;
372:
373: /*
374: * We calculate how many page frames we will have
375: * and then allocate the page structures in one chunk.
376: */
377:
378: npages = ((PAGE_SIZE * pmap_free_pages() +
379: (round_page(virtual_space_start) - virtual_space_start)) /
380: (PAGE_SIZE + sizeof *pages));
381:
382: pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
383:
384: /*
385: * Initialize the page frames.
386: */
387:
388: for (i = 0, pages_initialized = 0; i < npages; i++) {
389: if (!pmap_next_page(&paddr))
390: break;
391:
392: vm_page_init(&pages[i], paddr);
393: pages_initialized++;
394: }
1.1.1.3 ! root 395: i = 0;
! 396: while (pmap_next_page(&paddr))
! 397: i++;
! 398: if (i)
! 399: printf("%u memory page(s) left away\n", i);
1.1 root 400:
401: /*
402: * Release pages in reverse order so that physical pages
403: * initially get allocated in ascending addresses. This keeps
404: * the devices (which must address physical memory) happy if
405: * they require several consecutive pages.
406: */
407:
408: for (i = pages_initialized; i > 0; i--) {
1.1.1.2 root 409: vm_page_release(&pages[i - 1], FALSE);
1.1 root 410: }
411:
412: /*
413: * We have to re-align virtual_space_start,
414: * because pmap_steal_memory has been using it.
415: */
416:
417: virtual_space_start = round_page(virtual_space_start);
418:
419: *startp = virtual_space_start;
420: *endp = virtual_space_end;
421: }
422: #endif /* MACHINE_PAGES */
423:
424: /*
425: * Routine: vm_page_module_init
426: * Purpose:
427: * Second initialization pass, to be done after
428: * the basic VM system is ready.
429: */
430: void vm_page_module_init(void)
431: {
1.1.1.3 ! root 432: kmem_cache_init(&vm_page_cache, "vm_page", sizeof(struct vm_page), 0,
! 433: NULL, NULL, NULL, 0);
1.1 root 434: }
435:
436: /*
437: * Routine: vm_page_create
438: * Purpose:
439: * After the VM system is up, machine-dependent code
440: * may stumble across more physical memory. For example,
441: * memory that it was reserving for a frame buffer.
442: * vm_page_create turns this memory into available pages.
443: */
444:
445: void vm_page_create(
446: vm_offset_t start,
447: vm_offset_t end)
448: {
449: vm_offset_t paddr;
450: vm_page_t m;
451:
452: for (paddr = round_page(start);
453: paddr < trunc_page(end);
454: paddr += PAGE_SIZE) {
1.1.1.3 ! root 455: m = (vm_page_t) kmem_cache_alloc(&vm_page_cache);
1.1 root 456: if (m == VM_PAGE_NULL)
457: panic("vm_page_create");
458:
459: vm_page_init(m, paddr);
1.1.1.2 root 460: vm_page_release(m, FALSE);
1.1 root 461: }
462: }
463:
464: /*
465: * vm_page_hash:
466: *
467: * Distributes the object/offset key pair among hash buckets.
468: *
469: * NOTE: To get a good hash function, the bucket count should
470: * be a power of two.
471: */
472: #define vm_page_hash(object, offset) \
473: (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
474: & vm_page_hash_mask)
475:
476: /*
477: * vm_page_insert: [ internal use only ]
478: *
479: * Inserts the given mem entry into the object/object-page
480: * table and object list.
481: *
482: * The object and page must be locked.
483: */
484:
485: void vm_page_insert(
486: register vm_page_t mem,
487: register vm_object_t object,
488: register vm_offset_t offset)
489: {
490: register vm_page_bucket_t *bucket;
491:
492: VM_PAGE_CHECK(mem);
493:
494: if (mem->tabled)
495: panic("vm_page_insert");
496:
497: /*
498: * Record the object/offset pair in this page
499: */
500:
501: mem->object = object;
502: mem->offset = offset;
503:
504: /*
505: * Insert it into the object_object/offset hash table
506: */
507:
508: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
509: simple_lock(&bucket->lock);
510: mem->next = bucket->pages;
511: bucket->pages = mem;
512: simple_unlock(&bucket->lock);
513:
514: /*
515: * Now link into the object's list of backed pages.
516: */
517:
518: queue_enter(&object->memq, mem, vm_page_t, listq);
519: mem->tabled = TRUE;
520:
521: /*
522: * Show that the object has one more resident page.
523: */
524:
525: object->resident_page_count++;
1.1.1.3 ! root 526: assert(object->resident_page_count >= 0);
! 527:
! 528: if (object->can_persist && (object->ref_count == 0))
! 529: vm_object_cached_pages_update(1);
1.1 root 530:
531: /*
532: * Detect sequential access and inactivate previous page.
533: * We ignore busy pages.
534: */
535:
536: if (vm_page_deactivate_behind &&
537: (offset == object->last_alloc + PAGE_SIZE)) {
538: vm_page_t last_mem;
539:
540: last_mem = vm_page_lookup(object, object->last_alloc);
541: if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
542: vm_page_deactivate(last_mem);
543: }
544: object->last_alloc = offset;
545: }
546:
547: /*
548: * vm_page_replace:
549: *
550: * Exactly like vm_page_insert, except that we first
551: * remove any existing page at the given offset in object
552: * and we don't do deactivate-behind.
553: *
554: * The object and page must be locked.
555: */
556:
557: void vm_page_replace(
558: register vm_page_t mem,
559: register vm_object_t object,
560: register vm_offset_t offset)
561: {
562: register vm_page_bucket_t *bucket;
563:
564: VM_PAGE_CHECK(mem);
565:
566: if (mem->tabled)
567: panic("vm_page_replace");
568:
569: /*
570: * Record the object/offset pair in this page
571: */
572:
573: mem->object = object;
574: mem->offset = offset;
575:
576: /*
577: * Insert it into the object_object/offset hash table,
578: * replacing any page that might have been there.
579: */
580:
581: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
582: simple_lock(&bucket->lock);
583: if (bucket->pages) {
584: vm_page_t *mp = &bucket->pages;
585: register vm_page_t m = *mp;
586: do {
587: if (m->object == object && m->offset == offset) {
588: /*
589: * Remove page from bucket and from object,
590: * and return it to the free list.
591: */
592: *mp = m->next;
593: queue_remove(&object->memq, m, vm_page_t,
594: listq);
595: m->tabled = FALSE;
596: object->resident_page_count--;
597:
1.1.1.3 ! root 598: if (object->can_persist
! 599: && (object->ref_count == 0))
! 600: vm_object_cached_pages_update(-1);
! 601:
1.1 root 602: /*
603: * Return page to the free list.
604: * Note the page is not tabled now, so this
605: * won't self-deadlock on the bucket lock.
606: */
607:
608: vm_page_free(m);
609: break;
610: }
611: mp = &m->next;
612: } while ((m = *mp) != 0);
613: mem->next = bucket->pages;
614: } else {
615: mem->next = VM_PAGE_NULL;
616: }
617: bucket->pages = mem;
618: simple_unlock(&bucket->lock);
619:
620: /*
621: * Now link into the object's list of backed pages.
622: */
623:
624: queue_enter(&object->memq, mem, vm_page_t, listq);
625: mem->tabled = TRUE;
626:
627: /*
628: * And show that the object has one more resident
629: * page.
630: */
631:
632: object->resident_page_count++;
1.1.1.3 ! root 633: assert(object->resident_page_count >= 0);
! 634:
! 635: if (object->can_persist && (object->ref_count == 0))
! 636: vm_object_cached_pages_update(1);
1.1 root 637: }
638:
639: /*
640: * vm_page_remove: [ internal use only ]
641: *
642: * Removes the given mem entry from the object/offset-page
643: * table and the object page list.
644: *
645: * The object and page must be locked.
646: */
647:
648: void vm_page_remove(
649: register vm_page_t mem)
650: {
651: register vm_page_bucket_t *bucket;
652: register vm_page_t this;
653:
654: assert(mem->tabled);
655: VM_PAGE_CHECK(mem);
656:
657: /*
658: * Remove from the object_object/offset hash table
659: */
660:
661: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
662: simple_lock(&bucket->lock);
663: if ((this = bucket->pages) == mem) {
664: /* optimize for common case */
665:
666: bucket->pages = mem->next;
667: } else {
668: register vm_page_t *prev;
669:
670: for (prev = &this->next;
671: (this = *prev) != mem;
672: prev = &this->next)
673: continue;
674: *prev = this->next;
675: }
676: simple_unlock(&bucket->lock);
677:
678: /*
679: * Now remove from the object's list of backed pages.
680: */
681:
682: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
683:
684: /*
685: * And show that the object has one fewer resident
686: * page.
687: */
688:
689: mem->object->resident_page_count--;
690:
691: mem->tabled = FALSE;
1.1.1.3 ! root 692:
! 693: if (mem->object->can_persist && (mem->object->ref_count == 0))
! 694: vm_object_cached_pages_update(-1);
1.1 root 695: }
696:
697: /*
698: * vm_page_lookup:
699: *
700: * Returns the page associated with the object/offset
701: * pair specified; if none is found, VM_PAGE_NULL is returned.
702: *
703: * The object must be locked. No side effects.
704: */
705:
706: vm_page_t vm_page_lookup(
707: register vm_object_t object,
708: register vm_offset_t offset)
709: {
710: register vm_page_t mem;
711: register vm_page_bucket_t *bucket;
712:
713: /*
714: * Search the hash table for this object/offset pair
715: */
716:
717: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
718:
719: simple_lock(&bucket->lock);
720: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
721: VM_PAGE_CHECK(mem);
722: if ((mem->object == object) && (mem->offset == offset))
723: break;
724: }
725: simple_unlock(&bucket->lock);
726: return mem;
727: }
728:
729: /*
730: * vm_page_rename:
731: *
732: * Move the given memory entry from its
733: * current object to the specified target object/offset.
734: *
735: * The object must be locked.
736: */
737: void vm_page_rename(
738: register vm_page_t mem,
739: register vm_object_t new_object,
740: vm_offset_t new_offset)
741: {
742: /*
743: * Changes to mem->object require the page lock because
744: * the pageout daemon uses that lock to get the object.
745: */
746:
747: vm_page_lock_queues();
748: vm_page_remove(mem);
749: vm_page_insert(mem, new_object, new_offset);
750: vm_page_unlock_queues();
751: }
752:
753: /*
754: * vm_page_init:
755: *
756: * Initialize the fields in a new page.
757: * This takes a structure with random values and initializes it
758: * so that it can be given to vm_page_release or vm_page_insert.
759: */
760: void vm_page_init(
761: vm_page_t mem,
762: vm_offset_t phys_addr)
763: {
764: *mem = vm_page_template;
765: mem->phys_addr = phys_addr;
766: }
767:
768: /*
769: * vm_page_grab_fictitious:
770: *
771: * Remove a fictitious page from the free list.
772: * Returns VM_PAGE_NULL if there are no free pages.
773: */
774:
775: vm_page_t vm_page_grab_fictitious(void)
776: {
777: register vm_page_t m;
778:
779: simple_lock(&vm_page_queue_free_lock);
780: m = vm_page_queue_fictitious;
781: if (m != VM_PAGE_NULL) {
782: vm_page_fictitious_count--;
783: vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
784: m->free = FALSE;
785: }
786: simple_unlock(&vm_page_queue_free_lock);
787:
788: return m;
789: }
790:
791: /*
792: * vm_page_release_fictitious:
793: *
794: * Release a fictitious page to the free list.
795: */
796:
797: void vm_page_release_fictitious(
798: register vm_page_t m)
799: {
800: simple_lock(&vm_page_queue_free_lock);
801: if (m->free)
802: panic("vm_page_release_fictitious");
803: m->free = TRUE;
804: m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
805: vm_page_queue_fictitious = m;
806: vm_page_fictitious_count++;
807: simple_unlock(&vm_page_queue_free_lock);
808: }
809:
810: /*
811: * vm_page_more_fictitious:
812: *
813: * Add more fictitious pages to the free list.
814: * Allowed to block.
815: */
816:
817: int vm_page_fictitious_quantum = 5;
818:
819: void vm_page_more_fictitious(void)
820: {
821: register vm_page_t m;
822: int i;
823:
824: for (i = 0; i < vm_page_fictitious_quantum; i++) {
1.1.1.3 ! root 825: m = (vm_page_t) kmem_cache_alloc(&vm_page_cache);
1.1 root 826: if (m == VM_PAGE_NULL)
827: panic("vm_page_more_fictitious");
828:
829: vm_page_init(m, vm_page_fictitious_addr);
830: m->fictitious = TRUE;
831: vm_page_release_fictitious(m);
832: }
833: }
834:
835: /*
836: * vm_page_convert:
837: *
838: * Attempt to convert a fictitious page into a real page.
839: */
840:
841: boolean_t vm_page_convert(
1.1.1.2 root 842: register vm_page_t m,
843: boolean_t external)
1.1 root 844: {
845: register vm_page_t real_m;
846:
1.1.1.2 root 847: real_m = vm_page_grab(external);
1.1 root 848: if (real_m == VM_PAGE_NULL)
849: return FALSE;
850:
851: m->phys_addr = real_m->phys_addr;
852: m->fictitious = FALSE;
853:
854: real_m->phys_addr = vm_page_fictitious_addr;
855: real_m->fictitious = TRUE;
856:
857: vm_page_release_fictitious(real_m);
858: return TRUE;
859: }
860:
861: /*
862: * vm_page_grab:
863: *
864: * Remove a page from the free list.
865: * Returns VM_PAGE_NULL if the free list is too small.
866: */
867:
1.1.1.2 root 868: vm_page_t vm_page_grab(
869: boolean_t external)
1.1 root 870: {
871: register vm_page_t mem;
872:
873: simple_lock(&vm_page_queue_free_lock);
874:
875: /*
876: * Only let privileged threads (involved in pageout)
1.1.1.2 root 877: * dip into the reserved pool or exceed the limit
878: * for externally-managed pages.
1.1 root 879: */
880:
1.1.1.2 root 881: if (((vm_page_free_count < vm_page_free_reserved)
882: || (external
883: && (vm_page_external_count > vm_page_external_limit)))
884: && !current_thread()->vm_privilege) {
1.1 root 885: simple_unlock(&vm_page_queue_free_lock);
886: return VM_PAGE_NULL;
887: }
888:
889: if (vm_page_queue_free == VM_PAGE_NULL)
890: panic("vm_page_grab");
891:
892: if (--vm_page_free_count < vm_page_free_count_minimum)
893: vm_page_free_count_minimum = vm_page_free_count;
1.1.1.2 root 894: if (external)
895: vm_page_external_count++;
1.1 root 896: mem = vm_page_queue_free;
897: vm_page_queue_free = (vm_page_t) mem->pageq.next;
898: mem->free = FALSE;
1.1.1.2 root 899: mem->extcounted = mem->external = external;
1.1 root 900: simple_unlock(&vm_page_queue_free_lock);
901:
902: /*
903: * Decide if we should poke the pageout daemon.
904: * We do this if the free count is less than the low
905: * water mark, or if the free count is less than the high
906: * water mark (but above the low water mark) and the inactive
907: * count is less than its target.
908: *
909: * We don't have the counts locked ... if they change a little,
910: * it doesn't really matter.
911: */
912:
913: if ((vm_page_free_count < vm_page_free_min) ||
914: ((vm_page_free_count < vm_page_free_target) &&
915: (vm_page_inactive_count < vm_page_inactive_target)))
916: thread_wakeup((event_t) &vm_page_free_wanted);
917:
918: return mem;
919: }
920:
1.1.1.2 root 921: vm_offset_t vm_page_grab_phys_addr()
1.1 root 922: {
1.1.1.2 root 923: vm_page_t p = vm_page_grab(FALSE);
1.1 root 924: if (p == VM_PAGE_NULL)
925: return -1;
926: else
927: return p->phys_addr;
928: }
929:
930: /*
931: * vm_page_grab_contiguous_pages:
932: *
933: * Take N pages off the free list, the pages should
934: * cover a contiguous range of physical addresses.
935: * [Used by device drivers to cope with DMA limitations]
936: *
937: * Returns the page descriptors in ascending order, or
938: * Returns KERN_RESOURCE_SHORTAGE if it could not.
939: */
940:
941: /* Biggest phys page number for the pages we handle in VM */
942:
943: vm_size_t vm_page_big_pagenum = 0; /* Set this before call! */
944:
945: kern_return_t
946: vm_page_grab_contiguous_pages(
947: int npages,
948: vm_page_t pages[],
1.1.1.2 root 949: natural_t *bits,
950: boolean_t external)
1.1 root 951: {
952: register int first_set;
953: int size, alloc_size;
954: kern_return_t ret;
1.1.1.3 ! root 955: vm_page_t mem, *prevmemp;
1.1 root 956:
957: #ifndef NBBY
958: #define NBBY 8 /* size in bits of sizeof()`s unity */
959: #endif
960:
961: #define NBPEL (sizeof(natural_t)*NBBY)
962:
963: size = (vm_page_big_pagenum + NBPEL - 1)
964: & ~(NBPEL - 1); /* in bits */
965:
966: size = size / NBBY; /* in bytes */
967:
968: /*
969: * If we are called before the VM system is fully functional
970: * the invoker must provide us with the work space. [one bit
971: * per page starting at phys 0 and up to vm_page_big_pagenum]
972: */
973: if (bits == 0) {
974: alloc_size = round_page(size);
975: if (kmem_alloc_wired(kernel_map,
976: (vm_offset_t *)&bits,
977: alloc_size)
978: != KERN_SUCCESS)
979: return KERN_RESOURCE_SHORTAGE;
980: } else
981: alloc_size = 0;
982:
1.1.1.3 ! root 983: memset(bits, 0, size);
1.1 root 984:
985: /*
986: * A very large granularity call, its rare so that is ok
987: */
988: simple_lock(&vm_page_queue_free_lock);
989:
990: /*
991: * Do not dip into the reserved pool.
992: */
993:
1.1.1.2 root 994: if ((vm_page_free_count < vm_page_free_reserved)
995: || (vm_page_external_count >= vm_page_external_limit)) {
1.1.1.3 ! root 996: printf_once("no more room for vm_page_grab_contiguous_pages");
1.1 root 997: simple_unlock(&vm_page_queue_free_lock);
998: return KERN_RESOURCE_SHORTAGE;
999: }
1000:
1001: /*
1002: * First pass through, build a big bit-array of
1003: * the pages that are free. It is not going to
1004: * be too large anyways, in 4k we can fit info
1005: * for 32k pages.
1006: */
1007: mem = vm_page_queue_free;
1008: while (mem) {
1009: register int word_index, bit_index;
1010:
1011: bit_index = (mem->phys_addr >> PAGE_SHIFT);
1012: word_index = bit_index / NBPEL;
1013: bit_index = bit_index - (word_index * NBPEL);
1014: bits[word_index] |= 1 << bit_index;
1015:
1016: mem = (vm_page_t) mem->pageq.next;
1017: }
1018:
1019: /*
1020: * Second loop. Scan the bit array for NPAGES
1021: * contiguous bits. That gives us, if any,
1022: * the range of pages we will be grabbing off
1023: * the free list.
1024: */
1025: {
1026: register int bits_so_far = 0, i;
1027:
1028: first_set = 0;
1029:
1030: for (i = 0; i < size; i += sizeof(natural_t)) {
1031:
1032: register natural_t v = bits[i / sizeof(natural_t)];
1033: register int bitpos;
1034:
1035: /*
1036: * Bitscan this one word
1037: */
1038: if (v) {
1039: /*
1040: * keep counting them beans ?
1041: */
1042: bitpos = 0;
1043:
1044: if (bits_so_far) {
1045: count_ones:
1046: while (v & 1) {
1047: bitpos++;
1048: /*
1049: * got enough beans ?
1050: */
1051: if (++bits_so_far == npages)
1052: goto found_em;
1053: v >>= 1;
1054: }
1055: /* if we are being lucky, roll again */
1056: if (bitpos == NBPEL)
1057: continue;
1058: }
1059:
1060: /*
1061: * search for beans here
1062: */
1063: bits_so_far = 0;
1064: while ((bitpos < NBPEL) && ((v & 1) == 0)) {
1065: bitpos++;
1066: v >>= 1;
1067: }
1068: if (v & 1) {
1069: first_set = (i * NBBY) + bitpos;
1070: goto count_ones;
1071: }
1072: }
1073: /*
1074: * No luck
1075: */
1076: bits_so_far = 0;
1077: }
1078: }
1079:
1080: /*
1081: * We could not find enough contiguous pages.
1082: */
1083: simple_unlock(&vm_page_queue_free_lock);
1084:
1.1.1.3 ! root 1085: printf_once("no contiguous room for vm_page_grab_contiguous_pages");
1.1 root 1086: ret = KERN_RESOURCE_SHORTAGE;
1087: goto out;
1088:
1089: /*
1090: * Final pass. Now we know which pages we want.
1091: * Scan the list until we find them all, grab
1092: * pages as we go. FIRST_SET tells us where
1093: * in the bit-array our pages start.
1094: */
1095: found_em:
1096: vm_page_free_count -= npages;
1097: if (vm_page_free_count < vm_page_free_count_minimum)
1098: vm_page_free_count_minimum = vm_page_free_count;
1.1.1.2 root 1099: if (external)
1100: vm_page_external_count += npages;
1.1 root 1101: {
1102: register vm_offset_t first_phys, last_phys;
1103:
1104: /* cache values for compare */
1105: first_phys = first_set << PAGE_SHIFT;
1106: last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
1107:
1108: /* running pointers */
1109: mem = vm_page_queue_free;
1.1.1.3 ! root 1110: prevmemp = &vm_page_queue_free;
1.1 root 1111:
1112: while (mem) {
1113:
1114: register vm_offset_t addr;
1115:
1116: addr = mem->phys_addr;
1117:
1118: if ((addr >= first_phys) &&
1119: (addr < last_phys)) {
1.1.1.3 ! root 1120: *prevmemp = (vm_page_t) mem->pageq.next;
1.1 root 1121: pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
1122: mem->free = FALSE;
1.1.1.2 root 1123: mem->extcounted = mem->external = external;
1.1 root 1124: /*
1125: * Got them all ?
1126: */
1127: if (--npages == 0) break;
1128: } else
1.1.1.3 ! root 1129: prevmemp = (vm_page_t *) &mem->pageq.next;
1.1 root 1130:
1131: mem = (vm_page_t) mem->pageq.next;
1132: }
1133: }
1134:
1135: simple_unlock(&vm_page_queue_free_lock);
1136:
1137: /*
1138: * Decide if we should poke the pageout daemon.
1139: * We do this if the free count is less than the low
1140: * water mark, or if the free count is less than the high
1141: * water mark (but above the low water mark) and the inactive
1142: * count is less than its target.
1143: *
1144: * We don't have the counts locked ... if they change a little,
1145: * it doesn't really matter.
1146: */
1147:
1148: if ((vm_page_free_count < vm_page_free_min) ||
1149: ((vm_page_free_count < vm_page_free_target) &&
1150: (vm_page_inactive_count < vm_page_inactive_target)))
1151: thread_wakeup(&vm_page_free_wanted);
1152:
1153: ret = KERN_SUCCESS;
1154: out:
1155: if (alloc_size)
1156: kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
1157:
1158: return ret;
1159: }
1160:
1161: /*
1162: * vm_page_release:
1163: *
1164: * Return a page to the free list.
1165: */
1166:
1167: void vm_page_release(
1.1.1.2 root 1168: register vm_page_t mem,
1169: boolean_t external)
1.1 root 1170: {
1171: simple_lock(&vm_page_queue_free_lock);
1172: if (mem->free)
1173: panic("vm_page_release");
1174: mem->free = TRUE;
1175: mem->pageq.next = (queue_entry_t) vm_page_queue_free;
1176: vm_page_queue_free = mem;
1177: vm_page_free_count++;
1.1.1.2 root 1178: if (external)
1179: vm_page_external_count--;
1.1 root 1180:
1181: /*
1182: * Check if we should wake up someone waiting for page.
1183: * But don't bother waking them unless they can allocate.
1184: *
1185: * We wakeup only one thread, to prevent starvation.
1186: * Because the scheduling system handles wait queues FIFO,
1187: * if we wakeup all waiting threads, one greedy thread
1188: * can starve multiple niceguy threads. When the threads
1189: * all wakeup, the greedy threads runs first, grabs the page,
1190: * and waits for another page. It will be the first to run
1191: * when the next page is freed.
1192: *
1193: * However, there is a slight danger here.
1194: * The thread we wake might not use the free page.
1195: * Then the other threads could wait indefinitely
1196: * while the page goes unused. To forestall this,
1197: * the pageout daemon will keep making free pages
1198: * as long as vm_page_free_wanted is non-zero.
1199: */
1200:
1201: if ((vm_page_free_wanted > 0) &&
1202: (vm_page_free_count >= vm_page_free_reserved)) {
1203: vm_page_free_wanted--;
1204: thread_wakeup_one((event_t) &vm_page_free_count);
1205: }
1206:
1207: simple_unlock(&vm_page_queue_free_lock);
1208: }
1209:
1210: /*
1211: * vm_page_wait:
1212: *
1213: * Wait for a page to become available.
1214: * If there are plenty of free pages, then we don't sleep.
1215: */
1216:
1217: void vm_page_wait(
1218: void (*continuation)(void))
1219: {
1220:
1221: /*
1222: * We can't use vm_page_free_reserved to make this
1223: * determination. Consider: some thread might
1224: * need to allocate two pages. The first allocation
1225: * succeeds, the second fails. After the first page is freed,
1226: * a call to vm_page_wait must really block.
1227: */
1228:
1229: simple_lock(&vm_page_queue_free_lock);
1.1.1.2 root 1230: if ((vm_page_free_count < vm_page_free_target)
1231: || (vm_page_external_count > vm_page_external_limit)) {
1.1 root 1232: if (vm_page_free_wanted++ == 0)
1233: thread_wakeup((event_t)&vm_page_free_wanted);
1234: assert_wait((event_t)&vm_page_free_count, FALSE);
1235: simple_unlock(&vm_page_queue_free_lock);
1236: if (continuation != 0) {
1237: counter(c_vm_page_wait_block_user++);
1238: thread_block(continuation);
1239: } else {
1240: counter(c_vm_page_wait_block_kernel++);
1241: thread_block((void (*)(void)) 0);
1242: }
1243: } else
1244: simple_unlock(&vm_page_queue_free_lock);
1245: }
1246:
1247: /*
1248: * vm_page_alloc:
1249: *
1250: * Allocate and return a memory cell associated
1251: * with this VM object/offset pair.
1252: *
1253: * Object must be locked.
1254: */
1255:
1256: vm_page_t vm_page_alloc(
1257: vm_object_t object,
1258: vm_offset_t offset)
1259: {
1260: register vm_page_t mem;
1261:
1.1.1.2 root 1262: mem = vm_page_grab(!object->internal);
1.1 root 1263: if (mem == VM_PAGE_NULL)
1264: return VM_PAGE_NULL;
1265:
1266: vm_page_lock_queues();
1267: vm_page_insert(mem, object, offset);
1268: vm_page_unlock_queues();
1269:
1270: return mem;
1271: }
1272:
1273: /*
1274: * vm_page_free:
1275: *
1276: * Returns the given page to the free list,
1277: * disassociating it with any VM object.
1278: *
1279: * Object and page queues must be locked prior to entry.
1280: */
1281: void vm_page_free(
1282: register vm_page_t mem)
1283: {
1284: if (mem->free)
1285: panic("vm_page_free");
1286:
1287: if (mem->tabled)
1288: vm_page_remove(mem);
1289: VM_PAGE_QUEUES_REMOVE(mem);
1290:
1291: if (mem->wire_count != 0) {
1292: if (!mem->private && !mem->fictitious)
1293: vm_page_wire_count--;
1294: mem->wire_count = 0;
1295: }
1296:
1297: if (mem->laundry) {
1298: vm_page_laundry_count--;
1299: mem->laundry = FALSE;
1300: }
1301:
1302: PAGE_WAKEUP_DONE(mem);
1303:
1304: if (mem->absent)
1305: vm_object_absent_release(mem->object);
1306:
1307: /*
1308: * XXX The calls to vm_page_init here are
1309: * really overkill.
1310: */
1311:
1312: if (mem->private || mem->fictitious) {
1313: vm_page_init(mem, vm_page_fictitious_addr);
1314: mem->fictitious = TRUE;
1315: vm_page_release_fictitious(mem);
1316: } else {
1.1.1.2 root 1317: int external = mem->external && mem->extcounted;
1.1 root 1318: vm_page_init(mem, mem->phys_addr);
1.1.1.2 root 1319: vm_page_release(mem, external);
1.1 root 1320: }
1321: }
1322:
1323: /*
1324: * vm_page_wire:
1325: *
1326: * Mark this page as wired down by yet
1327: * another map, removing it from paging queues
1328: * as necessary.
1329: *
1330: * The page's object and the page queues must be locked.
1331: */
1332: void vm_page_wire(
1333: register vm_page_t mem)
1334: {
1335: VM_PAGE_CHECK(mem);
1336:
1337: if (mem->wire_count == 0) {
1338: VM_PAGE_QUEUES_REMOVE(mem);
1339: if (!mem->private && !mem->fictitious)
1340: vm_page_wire_count++;
1341: }
1342: mem->wire_count++;
1343: }
1344:
1345: /*
1346: * vm_page_unwire:
1347: *
1348: * Release one wiring of this page, potentially
1349: * enabling it to be paged again.
1350: *
1351: * The page's object and the page queues must be locked.
1352: */
1353: void vm_page_unwire(
1354: register vm_page_t mem)
1355: {
1356: VM_PAGE_CHECK(mem);
1357:
1358: if (--mem->wire_count == 0) {
1359: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1360: vm_page_active_count++;
1361: mem->active = TRUE;
1362: if (!mem->private && !mem->fictitious)
1363: vm_page_wire_count--;
1364: }
1365: }
1366:
1367: /*
1368: * vm_page_deactivate:
1369: *
1370: * Returns the given page to the inactive list,
1371: * indicating that no physical maps have access
1372: * to this page. [Used by the physical mapping system.]
1373: *
1374: * The page queues must be locked.
1375: */
1376: void vm_page_deactivate(
1377: register vm_page_t m)
1378: {
1379: VM_PAGE_CHECK(m);
1380:
1381: /*
1382: * This page is no longer very interesting. If it was
1383: * interesting (active or inactive/referenced), then we
1384: * clear the reference bit and (re)enter it in the
1385: * inactive queue. Note wired pages should not have
1386: * their reference bit cleared.
1387: */
1388:
1389: if (m->active || (m->inactive && m->reference)) {
1390: if (!m->fictitious && !m->absent)
1391: pmap_clear_reference(m->phys_addr);
1392: m->reference = FALSE;
1393: VM_PAGE_QUEUES_REMOVE(m);
1394: }
1395: if (m->wire_count == 0 && !m->inactive) {
1396: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1397: m->inactive = TRUE;
1398: vm_page_inactive_count++;
1399: }
1400: }
1401:
1402: /*
1403: * vm_page_activate:
1404: *
1405: * Put the specified page on the active list (if appropriate).
1406: *
1407: * The page queues must be locked.
1408: */
1409:
1410: void vm_page_activate(
1411: register vm_page_t m)
1412: {
1413: VM_PAGE_CHECK(m);
1414:
1415: if (m->inactive) {
1416: queue_remove(&vm_page_queue_inactive, m, vm_page_t,
1417: pageq);
1418: vm_page_inactive_count--;
1419: m->inactive = FALSE;
1420: }
1421: if (m->wire_count == 0) {
1422: if (m->active)
1423: panic("vm_page_activate: already active");
1424:
1425: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1426: m->active = TRUE;
1427: vm_page_active_count++;
1428: }
1429: }
1430:
1431: /*
1432: * vm_page_zero_fill:
1433: *
1434: * Zero-fill the specified page.
1435: */
1436: void vm_page_zero_fill(
1437: vm_page_t m)
1438: {
1439: VM_PAGE_CHECK(m);
1440:
1441: pmap_zero_page(m->phys_addr);
1442: }
1443:
1444: /*
1445: * vm_page_copy:
1446: *
1447: * Copy one page to another
1448: */
1449:
1450: void vm_page_copy(
1451: vm_page_t src_m,
1452: vm_page_t dest_m)
1453: {
1454: VM_PAGE_CHECK(src_m);
1455: VM_PAGE_CHECK(dest_m);
1456:
1457: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1458: }
1459:
1460: #if MACH_VM_DEBUG
1461: /*
1462: * Routine: vm_page_info
1463: * Purpose:
1464: * Return information about the global VP table.
1465: * Fills the buffer with as much information as possible
1466: * and returns the desired size of the buffer.
1467: * Conditions:
1468: * Nothing locked. The caller should provide
1469: * possibly-pageable memory.
1470: */
1471:
1472: unsigned int
1473: vm_page_info(
1474: hash_info_bucket_t *info,
1475: unsigned int count)
1476: {
1477: int i;
1478:
1479: if (vm_page_bucket_count < count)
1480: count = vm_page_bucket_count;
1481:
1482: for (i = 0; i < count; i++) {
1483: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1484: unsigned int bucket_count = 0;
1485: vm_page_t m;
1486:
1487: simple_lock(&bucket->lock);
1488: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
1489: bucket_count++;
1490: simple_unlock(&bucket->lock);
1491:
1492: /* don't touch pageable memory while holding locks */
1493: info[i].hib_count = bucket_count;
1494: }
1495:
1496: return vm_page_bucket_count;
1497: }
1498: #endif /* MACH_VM_DEBUG */
1499:
1.1.1.3 ! root 1500:
1.1 root 1501: #if MACH_KDB
1502: #define printf kdbprintf
1503:
1504: /*
1505: * Routine: vm_page_print [exported]
1506: */
1507: void vm_page_print(p)
1508: vm_page_t p;
1509: {
1510: iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
1511: printf(" offset 0x%X", (vm_offset_t) p->offset);
1512: printf("wire_count %d,", p->wire_count);
1513: printf(" %s",
1514: (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
1515: printf("%s",
1516: (p->free ? " free" : ""));
1517: printf("%s ",
1518: (p->laundry ? " laundry" : ""));
1519: printf("%s",
1520: (p->dirty ? "dirty" : "clean"));
1521: printf("%s",
1522: (p->busy ? " busy" : ""));
1523: printf("%s",
1524: (p->absent ? " absent" : ""));
1525: printf("%s",
1526: (p->error ? " error" : ""));
1527: printf("%s",
1528: (p->fictitious ? " fictitious" : ""));
1529: printf("%s",
1530: (p->private ? " private" : ""));
1531: printf("%s",
1532: (p->wanted ? " wanted" : ""));
1533: printf("%s,",
1534: (p->tabled ? "" : "not_tabled"));
1535: printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
1536: (vm_offset_t) p->phys_addr,
1537: (vm_offset_t) p->page_lock,
1538: (vm_offset_t) p->unlock_request);
1539: }
1540: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.