![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_resident.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_page.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Resident memory management module.
34: */
35: #include <cpus.h>
36:
37: #include <mach/vm_prot.h>
38: #include <kern/counters.h>
39: #include <kern/sched_prim.h>
40: #include <kern/task.h>
41: #include <kern/thread.h>
42: #include <mach/vm_statistics.h>
43: #include "vm_param.h"
44: #include <kern/xpr.h>
45: #include <kern/zalloc.h>
46: #include <vm/pmap.h>
47: #include <vm/vm_map.h>
48: #include <vm/vm_page.h>
49: #include <vm/vm_pageout.h>
50: #include <vm/vm_kern.h>
51:
52: #include <mach_vm_debug.h>
53: #if MACH_VM_DEBUG
54: #include <mach/kern_return.h>
55: #include <mach_debug/hash_info.h>
56: #include <vm/vm_user.h>
57: #endif
58:
59: /* in zalloc.c XXX */
60: extern vm_offset_t zdata;
61: extern vm_size_t zdata_size;
62:
63: /*
64: * Associated with eacn page of user-allocatable memory is a
65: * page structure.
66: */
67:
68: /*
69: * These variables record the values returned by vm_page_bootstrap,
70: * for debugging purposes. The implementation of pmap_steal_memory
71: * and pmap_startup here also uses them internally.
72: */
73:
74: vm_offset_t virtual_space_start;
75: vm_offset_t virtual_space_end;
76:
77: /*
78: * The vm_page_lookup() routine, which provides for fast
79: * (virtual memory object, offset) to page lookup, employs
80: * the following hash table. The vm_page_{insert,remove}
81: * routines install and remove associations in the table.
82: * [This table is often called the virtual-to-physical,
83: * or VP, table.]
84: */
85: typedef struct {
86: decl_simple_lock_data(,lock)
87: vm_page_t pages;
88: } vm_page_bucket_t;
89:
90: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
91: unsigned int vm_page_bucket_count = 0; /* How big is array? */
92: unsigned int vm_page_hash_mask; /* Mask for hash function */
93:
94: /*
95: * Resident page structures are initialized from
96: * a template (see vm_page_alloc).
97: *
98: * When adding a new field to the virtual memory
99: * object structure, be sure to add initialization
100: * (see vm_page_bootstrap).
101: */
102: struct vm_page vm_page_template;
103:
104: /*
105: * Resident pages that represent real memory
106: * are allocated from a free list.
107: */
108: vm_page_t vm_page_queue_free;
109: vm_page_t vm_page_queue_fictitious;
110: decl_simple_lock_data(,vm_page_queue_free_lock)
111: unsigned int vm_page_free_wanted;
112: int vm_page_free_count;
113: int vm_page_fictitious_count;
1.1.1.2 ! root 114: int vm_page_external_count;
1.1 root 115:
116: unsigned int vm_page_free_count_minimum; /* debugging */
117:
118: /*
119: * Occasionally, the virtual memory system uses
120: * resident page structures that do not refer to
121: * real pages, for example to leave a page with
122: * important state information in the VP table.
123: *
124: * These page structures are allocated the way
125: * most other kernel structures are.
126: */
127: zone_t vm_page_zone;
128:
129: /*
130: * Fictitious pages don't have a physical address,
131: * but we must initialize phys_addr to something.
132: * For debugging, this should be a strange value
133: * that the pmap module can recognize in assertions.
134: */
135: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
136:
137: /*
138: * Resident page structures are also chained on
139: * queues that are used by the page replacement
140: * system (pageout daemon). These queues are
141: * defined here, but are shared by the pageout
142: * module.
143: */
144: queue_head_t vm_page_queue_active;
145: queue_head_t vm_page_queue_inactive;
146: decl_simple_lock_data(,vm_page_queue_lock)
147: int vm_page_active_count;
148: int vm_page_inactive_count;
149: int vm_page_wire_count;
150:
151: /*
152: * Several page replacement parameters are also
153: * shared with this module, so that page allocation
154: * (done here in vm_page_alloc) can trigger the
155: * pageout daemon.
156: */
157: int vm_page_free_target = 0;
158: int vm_page_free_min = 0;
159: int vm_page_inactive_target = 0;
160: int vm_page_free_reserved = 0;
161: int vm_page_laundry_count = 0;
1.1.1.2 ! root 162: int vm_page_external_limit = 0;
! 163:
1.1 root 164:
165: /*
166: * The VM system has a couple of heuristics for deciding
167: * that pages are "uninteresting" and should be placed
168: * on the inactive queue as likely candidates for replacement.
169: * These variables let the heuristics be controlled at run-time
170: * to make experimentation easier.
171: */
172:
173: boolean_t vm_page_deactivate_behind = TRUE;
174: boolean_t vm_page_deactivate_hint = TRUE;
175:
176: /*
177: * vm_page_bootstrap:
178: *
179: * Initializes the resident memory module.
180: *
181: * Allocates memory for the page cells, and
182: * for the object/offset-to-page hash table headers.
183: * Each page cell is initialized and placed on the free list.
184: * Returns the range of available kernel virtual memory.
185: */
186:
187: void vm_page_bootstrap(
188: vm_offset_t *startp,
189: vm_offset_t *endp)
190: {
191: register vm_page_t m;
192: int i;
193:
194: /*
195: * Initialize the vm_page template.
196: */
197:
198: m = &vm_page_template;
199: m->object = VM_OBJECT_NULL; /* reset later */
200: m->offset = 0; /* reset later */
201: m->wire_count = 0;
202:
203: m->inactive = FALSE;
204: m->active = FALSE;
205: m->laundry = FALSE;
206: m->free = FALSE;
1.1.1.2 ! root 207: m->external = FALSE;
1.1 root 208:
209: m->busy = TRUE;
210: m->wanted = FALSE;
211: m->tabled = FALSE;
212: m->fictitious = FALSE;
213: m->private = FALSE;
214: m->absent = FALSE;
215: m->error = FALSE;
216: m->dirty = FALSE;
217: m->precious = FALSE;
218: m->reference = FALSE;
219:
220: m->phys_addr = 0; /* reset later */
221:
222: m->page_lock = VM_PROT_NONE;
223: m->unlock_request = VM_PROT_NONE;
224:
225: /*
226: * Initialize the page queues.
227: */
228:
229: simple_lock_init(&vm_page_queue_free_lock);
230: simple_lock_init(&vm_page_queue_lock);
231:
232: vm_page_queue_free = VM_PAGE_NULL;
233: vm_page_queue_fictitious = VM_PAGE_NULL;
234: queue_init(&vm_page_queue_active);
235: queue_init(&vm_page_queue_inactive);
236:
237: vm_page_free_wanted = 0;
238:
239: /*
240: * Steal memory for the zone system.
241: */
242:
243: kentry_data_size = kentry_count * sizeof(struct vm_map_entry);
244: kentry_data = pmap_steal_memory(kentry_data_size);
245:
246: zdata = pmap_steal_memory(zdata_size);
247:
248: /*
249: * Allocate (and initialize) the virtual-to-physical
250: * table hash buckets.
251: *
252: * The number of buckets should be a power of two to
253: * get a good hash function. The following computation
254: * chooses the first power of two that is greater
255: * than the number of physical pages in the system.
256: */
257:
258: if (vm_page_bucket_count == 0) {
259: unsigned int npages = pmap_free_pages();
260:
261: vm_page_bucket_count = 1;
262: while (vm_page_bucket_count < npages)
263: vm_page_bucket_count <<= 1;
264: }
265:
266: vm_page_hash_mask = vm_page_bucket_count - 1;
267:
268: if (vm_page_hash_mask & vm_page_bucket_count)
269: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
270:
271: vm_page_buckets = (vm_page_bucket_t *)
272: pmap_steal_memory(vm_page_bucket_count *
273: sizeof(vm_page_bucket_t));
274:
275: for (i = 0; i < vm_page_bucket_count; i++) {
276: register vm_page_bucket_t *bucket = &vm_page_buckets[i];
277:
278: bucket->pages = VM_PAGE_NULL;
279: simple_lock_init(&bucket->lock);
280: }
281:
282: /*
283: * Machine-dependent code allocates the resident page table.
284: * It uses vm_page_init to initialize the page frames.
285: * The code also returns to us the virtual space available
286: * to the kernel. We don't trust the pmap module
287: * to get the alignment right.
288: */
289:
290: pmap_startup(&virtual_space_start, &virtual_space_end);
291: virtual_space_start = round_page(virtual_space_start);
292: virtual_space_end = trunc_page(virtual_space_end);
293:
294: *startp = virtual_space_start;
295: *endp = virtual_space_end;
296:
297: /* printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
298: vm_page_free_count_minimum = vm_page_free_count;
299: }
300:
301: #ifndef MACHINE_PAGES
302: /*
303: * We implement pmap_steal_memory and pmap_startup with the help
304: * of two simpler functions, pmap_virtual_space and pmap_next_page.
305: */
306:
307: vm_offset_t pmap_steal_memory(
308: vm_size_t size)
309: {
310: vm_offset_t addr, vaddr, paddr;
311:
312: /*
313: * We round the size to an integer multiple.
314: */
315:
316: size = (size + 3) &~ 3;
317:
318: /*
319: * If this is the first call to pmap_steal_memory,
320: * we have to initialize ourself.
321: */
322:
323: if (virtual_space_start == virtual_space_end) {
324: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
325:
326: /*
327: * The initial values must be aligned properly, and
328: * we don't trust the pmap module to do it right.
329: */
330:
331: virtual_space_start = round_page(virtual_space_start);
332: virtual_space_end = trunc_page(virtual_space_end);
333: }
334:
335: /*
336: * Allocate virtual memory for this request.
337: */
338:
339: addr = virtual_space_start;
340: virtual_space_start += size;
341:
342: /*
343: * Allocate and map physical pages to back new virtual pages.
344: */
345:
346: for (vaddr = round_page(addr);
347: vaddr < addr + size;
348: vaddr += PAGE_SIZE) {
349: if (!pmap_next_page(&paddr))
350: panic("pmap_steal_memory");
351:
352: /*
353: * XXX Logically, these mappings should be wired,
354: * but some pmap modules barf if they are.
355: */
356:
357: pmap_enter(kernel_pmap, vaddr, paddr,
358: VM_PROT_READ|VM_PROT_WRITE, FALSE);
359: }
360:
361: return addr;
362: }
363:
364: void pmap_startup(
365: vm_offset_t *startp,
366: vm_offset_t *endp)
367: {
368: unsigned int i, npages, pages_initialized;
369: vm_page_t pages;
370: vm_offset_t paddr;
371:
372: /*
373: * We calculate how many page frames we will have
374: * and then allocate the page structures in one chunk.
375: */
376:
377: npages = ((PAGE_SIZE * pmap_free_pages() +
378: (round_page(virtual_space_start) - virtual_space_start)) /
379: (PAGE_SIZE + sizeof *pages));
380:
381: pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
382:
383: /*
384: * Initialize the page frames.
385: */
386:
387: for (i = 0, pages_initialized = 0; i < npages; i++) {
388: if (!pmap_next_page(&paddr))
389: break;
390:
391: vm_page_init(&pages[i], paddr);
392: pages_initialized++;
393: }
394:
395: /*
396: * Release pages in reverse order so that physical pages
397: * initially get allocated in ascending addresses. This keeps
398: * the devices (which must address physical memory) happy if
399: * they require several consecutive pages.
400: */
401:
402: for (i = pages_initialized; i > 0; i--) {
1.1.1.2 ! root 403: vm_page_release(&pages[i - 1], FALSE);
1.1 root 404: }
405:
406: /*
407: * We have to re-align virtual_space_start,
408: * because pmap_steal_memory has been using it.
409: */
410:
411: virtual_space_start = round_page(virtual_space_start);
412:
413: *startp = virtual_space_start;
414: *endp = virtual_space_end;
415: }
416: #endif /* MACHINE_PAGES */
417:
418: /*
419: * Routine: vm_page_module_init
420: * Purpose:
421: * Second initialization pass, to be done after
422: * the basic VM system is ready.
423: */
424: void vm_page_module_init(void)
425: {
426: vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
427: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
428: PAGE_SIZE,
429: 0, "vm pages");
430: }
431:
432: /*
433: * Routine: vm_page_create
434: * Purpose:
435: * After the VM system is up, machine-dependent code
436: * may stumble across more physical memory. For example,
437: * memory that it was reserving for a frame buffer.
438: * vm_page_create turns this memory into available pages.
439: */
440:
441: void vm_page_create(
442: vm_offset_t start,
443: vm_offset_t end)
444: {
445: vm_offset_t paddr;
446: vm_page_t m;
447:
448: for (paddr = round_page(start);
449: paddr < trunc_page(end);
450: paddr += PAGE_SIZE) {
451: m = (vm_page_t) zalloc(vm_page_zone);
452: if (m == VM_PAGE_NULL)
453: panic("vm_page_create");
454:
455: vm_page_init(m, paddr);
1.1.1.2 ! root 456: vm_page_release(m, FALSE);
1.1 root 457: }
458: }
459:
460: /*
461: * vm_page_hash:
462: *
463: * Distributes the object/offset key pair among hash buckets.
464: *
465: * NOTE: To get a good hash function, the bucket count should
466: * be a power of two.
467: */
468: #define vm_page_hash(object, offset) \
469: (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
470: & vm_page_hash_mask)
471:
472: /*
473: * vm_page_insert: [ internal use only ]
474: *
475: * Inserts the given mem entry into the object/object-page
476: * table and object list.
477: *
478: * The object and page must be locked.
479: */
480:
481: void vm_page_insert(
482: register vm_page_t mem,
483: register vm_object_t object,
484: register vm_offset_t offset)
485: {
486: register vm_page_bucket_t *bucket;
487:
488: VM_PAGE_CHECK(mem);
489:
490: if (mem->tabled)
491: panic("vm_page_insert");
492:
493: /*
494: * Record the object/offset pair in this page
495: */
496:
497: mem->object = object;
498: mem->offset = offset;
499:
500: /*
501: * Insert it into the object_object/offset hash table
502: */
503:
504: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
505: simple_lock(&bucket->lock);
506: mem->next = bucket->pages;
507: bucket->pages = mem;
508: simple_unlock(&bucket->lock);
509:
510: /*
511: * Now link into the object's list of backed pages.
512: */
513:
514: queue_enter(&object->memq, mem, vm_page_t, listq);
515: mem->tabled = TRUE;
516:
517: /*
518: * Show that the object has one more resident page.
519: */
520:
521: object->resident_page_count++;
522:
523: /*
524: * Detect sequential access and inactivate previous page.
525: * We ignore busy pages.
526: */
527:
528: if (vm_page_deactivate_behind &&
529: (offset == object->last_alloc + PAGE_SIZE)) {
530: vm_page_t last_mem;
531:
532: last_mem = vm_page_lookup(object, object->last_alloc);
533: if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
534: vm_page_deactivate(last_mem);
535: }
536: object->last_alloc = offset;
537: }
538:
539: /*
540: * vm_page_replace:
541: *
542: * Exactly like vm_page_insert, except that we first
543: * remove any existing page at the given offset in object
544: * and we don't do deactivate-behind.
545: *
546: * The object and page must be locked.
547: */
548:
549: void vm_page_replace(
550: register vm_page_t mem,
551: register vm_object_t object,
552: register vm_offset_t offset)
553: {
554: register vm_page_bucket_t *bucket;
555:
556: VM_PAGE_CHECK(mem);
557:
558: if (mem->tabled)
559: panic("vm_page_replace");
560:
561: /*
562: * Record the object/offset pair in this page
563: */
564:
565: mem->object = object;
566: mem->offset = offset;
567:
568: /*
569: * Insert it into the object_object/offset hash table,
570: * replacing any page that might have been there.
571: */
572:
573: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
574: simple_lock(&bucket->lock);
575: if (bucket->pages) {
576: vm_page_t *mp = &bucket->pages;
577: register vm_page_t m = *mp;
578: do {
579: if (m->object == object && m->offset == offset) {
580: /*
581: * Remove page from bucket and from object,
582: * and return it to the free list.
583: */
584: *mp = m->next;
585: queue_remove(&object->memq, m, vm_page_t,
586: listq);
587: m->tabled = FALSE;
588: object->resident_page_count--;
589:
590: /*
591: * Return page to the free list.
592: * Note the page is not tabled now, so this
593: * won't self-deadlock on the bucket lock.
594: */
595:
596: vm_page_free(m);
597: break;
598: }
599: mp = &m->next;
600: } while ((m = *mp) != 0);
601: mem->next = bucket->pages;
602: } else {
603: mem->next = VM_PAGE_NULL;
604: }
605: bucket->pages = mem;
606: simple_unlock(&bucket->lock);
607:
608: /*
609: * Now link into the object's list of backed pages.
610: */
611:
612: queue_enter(&object->memq, mem, vm_page_t, listq);
613: mem->tabled = TRUE;
614:
615: /*
616: * And show that the object has one more resident
617: * page.
618: */
619:
620: object->resident_page_count++;
621: }
622:
623: /*
624: * vm_page_remove: [ internal use only ]
625: *
626: * Removes the given mem entry from the object/offset-page
627: * table and the object page list.
628: *
629: * The object and page must be locked.
630: */
631:
632: void vm_page_remove(
633: register vm_page_t mem)
634: {
635: register vm_page_bucket_t *bucket;
636: register vm_page_t this;
637:
638: assert(mem->tabled);
639: VM_PAGE_CHECK(mem);
640:
641: /*
642: * Remove from the object_object/offset hash table
643: */
644:
645: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
646: simple_lock(&bucket->lock);
647: if ((this = bucket->pages) == mem) {
648: /* optimize for common case */
649:
650: bucket->pages = mem->next;
651: } else {
652: register vm_page_t *prev;
653:
654: for (prev = &this->next;
655: (this = *prev) != mem;
656: prev = &this->next)
657: continue;
658: *prev = this->next;
659: }
660: simple_unlock(&bucket->lock);
661:
662: /*
663: * Now remove from the object's list of backed pages.
664: */
665:
666: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
667:
668: /*
669: * And show that the object has one fewer resident
670: * page.
671: */
672:
673: mem->object->resident_page_count--;
674:
675: mem->tabled = FALSE;
676: }
677:
678: /*
679: * vm_page_lookup:
680: *
681: * Returns the page associated with the object/offset
682: * pair specified; if none is found, VM_PAGE_NULL is returned.
683: *
684: * The object must be locked. No side effects.
685: */
686:
687: vm_page_t vm_page_lookup(
688: register vm_object_t object,
689: register vm_offset_t offset)
690: {
691: register vm_page_t mem;
692: register vm_page_bucket_t *bucket;
693:
694: /*
695: * Search the hash table for this object/offset pair
696: */
697:
698: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
699:
700: simple_lock(&bucket->lock);
701: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
702: VM_PAGE_CHECK(mem);
703: if ((mem->object == object) && (mem->offset == offset))
704: break;
705: }
706: simple_unlock(&bucket->lock);
707: return mem;
708: }
709:
710: /*
711: * vm_page_rename:
712: *
713: * Move the given memory entry from its
714: * current object to the specified target object/offset.
715: *
716: * The object must be locked.
717: */
718: void vm_page_rename(
719: register vm_page_t mem,
720: register vm_object_t new_object,
721: vm_offset_t new_offset)
722: {
723: /*
724: * Changes to mem->object require the page lock because
725: * the pageout daemon uses that lock to get the object.
726: */
727:
728: vm_page_lock_queues();
729: vm_page_remove(mem);
730: vm_page_insert(mem, new_object, new_offset);
731: vm_page_unlock_queues();
732: }
733:
734: /*
735: * vm_page_init:
736: *
737: * Initialize the fields in a new page.
738: * This takes a structure with random values and initializes it
739: * so that it can be given to vm_page_release or vm_page_insert.
740: */
741: void vm_page_init(
742: vm_page_t mem,
743: vm_offset_t phys_addr)
744: {
745: *mem = vm_page_template;
746: mem->phys_addr = phys_addr;
747: }
748:
749: /*
750: * vm_page_grab_fictitious:
751: *
752: * Remove a fictitious page from the free list.
753: * Returns VM_PAGE_NULL if there are no free pages.
754: */
755:
756: vm_page_t vm_page_grab_fictitious(void)
757: {
758: register vm_page_t m;
759:
760: simple_lock(&vm_page_queue_free_lock);
761: m = vm_page_queue_fictitious;
762: if (m != VM_PAGE_NULL) {
763: vm_page_fictitious_count--;
764: vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
765: m->free = FALSE;
766: }
767: simple_unlock(&vm_page_queue_free_lock);
768:
769: return m;
770: }
771:
772: /*
773: * vm_page_release_fictitious:
774: *
775: * Release a fictitious page to the free list.
776: */
777:
778: void vm_page_release_fictitious(
779: register vm_page_t m)
780: {
781: simple_lock(&vm_page_queue_free_lock);
782: if (m->free)
783: panic("vm_page_release_fictitious");
784: m->free = TRUE;
785: m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
786: vm_page_queue_fictitious = m;
787: vm_page_fictitious_count++;
788: simple_unlock(&vm_page_queue_free_lock);
789: }
790:
791: /*
792: * vm_page_more_fictitious:
793: *
794: * Add more fictitious pages to the free list.
795: * Allowed to block.
796: */
797:
798: int vm_page_fictitious_quantum = 5;
799:
800: void vm_page_more_fictitious(void)
801: {
802: register vm_page_t m;
803: int i;
804:
805: for (i = 0; i < vm_page_fictitious_quantum; i++) {
806: m = (vm_page_t) zalloc(vm_page_zone);
807: if (m == VM_PAGE_NULL)
808: panic("vm_page_more_fictitious");
809:
810: vm_page_init(m, vm_page_fictitious_addr);
811: m->fictitious = TRUE;
812: vm_page_release_fictitious(m);
813: }
814: }
815:
816: /*
817: * vm_page_convert:
818: *
819: * Attempt to convert a fictitious page into a real page.
820: */
821:
822: boolean_t vm_page_convert(
1.1.1.2 ! root 823: register vm_page_t m,
! 824: boolean_t external)
1.1 root 825: {
826: register vm_page_t real_m;
827:
1.1.1.2 ! root 828: real_m = vm_page_grab(external);
1.1 root 829: if (real_m == VM_PAGE_NULL)
830: return FALSE;
831:
832: m->phys_addr = real_m->phys_addr;
833: m->fictitious = FALSE;
834:
835: real_m->phys_addr = vm_page_fictitious_addr;
836: real_m->fictitious = TRUE;
837:
838: vm_page_release_fictitious(real_m);
839: return TRUE;
840: }
841:
842: /*
843: * vm_page_grab:
844: *
845: * Remove a page from the free list.
846: * Returns VM_PAGE_NULL if the free list is too small.
847: */
848:
1.1.1.2 ! root 849: vm_page_t vm_page_grab(
! 850: boolean_t external)
1.1 root 851: {
852: register vm_page_t mem;
853:
854: simple_lock(&vm_page_queue_free_lock);
855:
856: /*
857: * Only let privileged threads (involved in pageout)
1.1.1.2 ! root 858: * dip into the reserved pool or exceed the limit
! 859: * for externally-managed pages.
1.1 root 860: */
861:
1.1.1.2 ! root 862: if (((vm_page_free_count < vm_page_free_reserved)
! 863: || (external
! 864: && (vm_page_external_count > vm_page_external_limit)))
! 865: && !current_thread()->vm_privilege) {
1.1 root 866: simple_unlock(&vm_page_queue_free_lock);
867: return VM_PAGE_NULL;
868: }
869:
870: if (vm_page_queue_free == VM_PAGE_NULL)
871: panic("vm_page_grab");
872:
873: if (--vm_page_free_count < vm_page_free_count_minimum)
874: vm_page_free_count_minimum = vm_page_free_count;
1.1.1.2 ! root 875: if (external)
! 876: vm_page_external_count++;
1.1 root 877: mem = vm_page_queue_free;
878: vm_page_queue_free = (vm_page_t) mem->pageq.next;
879: mem->free = FALSE;
1.1.1.2 ! root 880: mem->extcounted = mem->external = external;
1.1 root 881: simple_unlock(&vm_page_queue_free_lock);
882:
883: /*
884: * Decide if we should poke the pageout daemon.
885: * We do this if the free count is less than the low
886: * water mark, or if the free count is less than the high
887: * water mark (but above the low water mark) and the inactive
888: * count is less than its target.
889: *
890: * We don't have the counts locked ... if they change a little,
891: * it doesn't really matter.
892: */
893:
894: if ((vm_page_free_count < vm_page_free_min) ||
895: ((vm_page_free_count < vm_page_free_target) &&
896: (vm_page_inactive_count < vm_page_inactive_target)))
897: thread_wakeup((event_t) &vm_page_free_wanted);
898:
899: return mem;
900: }
901:
1.1.1.2 ! root 902: vm_offset_t vm_page_grab_phys_addr()
1.1 root 903: {
1.1.1.2 ! root 904: vm_page_t p = vm_page_grab(FALSE);
1.1 root 905: if (p == VM_PAGE_NULL)
906: return -1;
907: else
908: return p->phys_addr;
909: }
910:
911: /*
912: * vm_page_grab_contiguous_pages:
913: *
914: * Take N pages off the free list, the pages should
915: * cover a contiguous range of physical addresses.
916: * [Used by device drivers to cope with DMA limitations]
917: *
918: * Returns the page descriptors in ascending order, or
919: * Returns KERN_RESOURCE_SHORTAGE if it could not.
920: */
921:
922: /* Biggest phys page number for the pages we handle in VM */
923:
924: vm_size_t vm_page_big_pagenum = 0; /* Set this before call! */
925:
926: kern_return_t
927: vm_page_grab_contiguous_pages(
928: int npages,
929: vm_page_t pages[],
1.1.1.2 ! root 930: natural_t *bits,
! 931: boolean_t external)
1.1 root 932: {
933: register int first_set;
934: int size, alloc_size;
935: kern_return_t ret;
936: vm_page_t mem, prevmem;
937:
938: #ifndef NBBY
939: #define NBBY 8 /* size in bits of sizeof()`s unity */
940: #endif
941:
942: #define NBPEL (sizeof(natural_t)*NBBY)
943:
944: size = (vm_page_big_pagenum + NBPEL - 1)
945: & ~(NBPEL - 1); /* in bits */
946:
947: size = size / NBBY; /* in bytes */
948:
949: /*
950: * If we are called before the VM system is fully functional
951: * the invoker must provide us with the work space. [one bit
952: * per page starting at phys 0 and up to vm_page_big_pagenum]
953: */
954: if (bits == 0) {
955: alloc_size = round_page(size);
956: if (kmem_alloc_wired(kernel_map,
957: (vm_offset_t *)&bits,
958: alloc_size)
959: != KERN_SUCCESS)
960: return KERN_RESOURCE_SHORTAGE;
961: } else
962: alloc_size = 0;
963:
964: bzero(bits, size);
965:
966: /*
967: * A very large granularity call, its rare so that is ok
968: */
969: simple_lock(&vm_page_queue_free_lock);
970:
971: /*
972: * Do not dip into the reserved pool.
973: */
974:
1.1.1.2 ! root 975: if ((vm_page_free_count < vm_page_free_reserved)
! 976: || (vm_page_external_count >= vm_page_external_limit)) {
1.1 root 977: simple_unlock(&vm_page_queue_free_lock);
978: return KERN_RESOURCE_SHORTAGE;
979: }
980:
981: /*
982: * First pass through, build a big bit-array of
983: * the pages that are free. It is not going to
984: * be too large anyways, in 4k we can fit info
985: * for 32k pages.
986: */
987: mem = vm_page_queue_free;
988: while (mem) {
989: register int word_index, bit_index;
990:
991: bit_index = (mem->phys_addr >> PAGE_SHIFT);
992: word_index = bit_index / NBPEL;
993: bit_index = bit_index - (word_index * NBPEL);
994: bits[word_index] |= 1 << bit_index;
995:
996: mem = (vm_page_t) mem->pageq.next;
997: }
998:
999: /*
1000: * Second loop. Scan the bit array for NPAGES
1001: * contiguous bits. That gives us, if any,
1002: * the range of pages we will be grabbing off
1003: * the free list.
1004: */
1005: {
1006: register int bits_so_far = 0, i;
1007:
1008: first_set = 0;
1009:
1010: for (i = 0; i < size; i += sizeof(natural_t)) {
1011:
1012: register natural_t v = bits[i / sizeof(natural_t)];
1013: register int bitpos;
1014:
1015: /*
1016: * Bitscan this one word
1017: */
1018: if (v) {
1019: /*
1020: * keep counting them beans ?
1021: */
1022: bitpos = 0;
1023:
1024: if (bits_so_far) {
1025: count_ones:
1026: while (v & 1) {
1027: bitpos++;
1028: /*
1029: * got enough beans ?
1030: */
1031: if (++bits_so_far == npages)
1032: goto found_em;
1033: v >>= 1;
1034: }
1035: /* if we are being lucky, roll again */
1036: if (bitpos == NBPEL)
1037: continue;
1038: }
1039:
1040: /*
1041: * search for beans here
1042: */
1043: bits_so_far = 0;
1044: count_zeroes:
1045: while ((bitpos < NBPEL) && ((v & 1) == 0)) {
1046: bitpos++;
1047: v >>= 1;
1048: }
1049: if (v & 1) {
1050: first_set = (i * NBBY) + bitpos;
1051: goto count_ones;
1052: }
1053: }
1054: /*
1055: * No luck
1056: */
1057: bits_so_far = 0;
1058: }
1059: }
1060:
1061: /*
1062: * We could not find enough contiguous pages.
1063: */
1064: not_found_em:
1065: simple_unlock(&vm_page_queue_free_lock);
1066:
1067: ret = KERN_RESOURCE_SHORTAGE;
1068: goto out;
1069:
1070: /*
1071: * Final pass. Now we know which pages we want.
1072: * Scan the list until we find them all, grab
1073: * pages as we go. FIRST_SET tells us where
1074: * in the bit-array our pages start.
1075: */
1076: found_em:
1077: vm_page_free_count -= npages;
1078: if (vm_page_free_count < vm_page_free_count_minimum)
1079: vm_page_free_count_minimum = vm_page_free_count;
1.1.1.2 ! root 1080: if (external)
! 1081: vm_page_external_count += npages;
1.1 root 1082: {
1083: register vm_offset_t first_phys, last_phys;
1084:
1085: /* cache values for compare */
1086: first_phys = first_set << PAGE_SHIFT;
1087: last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
1088:
1089: /* running pointers */
1090: mem = vm_page_queue_free;
1091: prevmem = VM_PAGE_NULL;
1092:
1093: while (mem) {
1094:
1095: register vm_offset_t addr;
1096:
1097: addr = mem->phys_addr;
1098:
1099: if ((addr >= first_phys) &&
1100: (addr < last_phys)) {
1101: if (prevmem)
1102: prevmem->pageq.next = mem->pageq.next;
1103: pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
1104: mem->free = FALSE;
1.1.1.2 ! root 1105: mem->extcounted = mem->external = external;
1.1 root 1106: /*
1107: * Got them all ?
1108: */
1109: if (--npages == 0) break;
1110: } else
1111: prevmem = mem;
1112:
1113: mem = (vm_page_t) mem->pageq.next;
1114: }
1115: }
1116:
1117: simple_unlock(&vm_page_queue_free_lock);
1118:
1119: /*
1120: * Decide if we should poke the pageout daemon.
1121: * We do this if the free count is less than the low
1122: * water mark, or if the free count is less than the high
1123: * water mark (but above the low water mark) and the inactive
1124: * count is less than its target.
1125: *
1126: * We don't have the counts locked ... if they change a little,
1127: * it doesn't really matter.
1128: */
1129:
1130: if ((vm_page_free_count < vm_page_free_min) ||
1131: ((vm_page_free_count < vm_page_free_target) &&
1132: (vm_page_inactive_count < vm_page_inactive_target)))
1133: thread_wakeup(&vm_page_free_wanted);
1134:
1135: ret = KERN_SUCCESS;
1136: out:
1137: if (alloc_size)
1138: kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
1139:
1140: return ret;
1141: }
1142:
1143: /*
1144: * vm_page_release:
1145: *
1146: * Return a page to the free list.
1147: */
1148:
1149: void vm_page_release(
1.1.1.2 ! root 1150: register vm_page_t mem,
! 1151: boolean_t external)
1.1 root 1152: {
1153: simple_lock(&vm_page_queue_free_lock);
1154: if (mem->free)
1155: panic("vm_page_release");
1156: mem->free = TRUE;
1157: mem->pageq.next = (queue_entry_t) vm_page_queue_free;
1158: vm_page_queue_free = mem;
1159: vm_page_free_count++;
1.1.1.2 ! root 1160: if (external)
! 1161: vm_page_external_count--;
1.1 root 1162:
1163: /*
1164: * Check if we should wake up someone waiting for page.
1165: * But don't bother waking them unless they can allocate.
1166: *
1167: * We wakeup only one thread, to prevent starvation.
1168: * Because the scheduling system handles wait queues FIFO,
1169: * if we wakeup all waiting threads, one greedy thread
1170: * can starve multiple niceguy threads. When the threads
1171: * all wakeup, the greedy threads runs first, grabs the page,
1172: * and waits for another page. It will be the first to run
1173: * when the next page is freed.
1174: *
1175: * However, there is a slight danger here.
1176: * The thread we wake might not use the free page.
1177: * Then the other threads could wait indefinitely
1178: * while the page goes unused. To forestall this,
1179: * the pageout daemon will keep making free pages
1180: * as long as vm_page_free_wanted is non-zero.
1181: */
1182:
1183: if ((vm_page_free_wanted > 0) &&
1184: (vm_page_free_count >= vm_page_free_reserved)) {
1185: vm_page_free_wanted--;
1186: thread_wakeup_one((event_t) &vm_page_free_count);
1187: }
1188:
1189: simple_unlock(&vm_page_queue_free_lock);
1190: }
1191:
1192: /*
1193: * vm_page_wait:
1194: *
1195: * Wait for a page to become available.
1196: * If there are plenty of free pages, then we don't sleep.
1197: */
1198:
1199: void vm_page_wait(
1200: void (*continuation)(void))
1201: {
1202:
1203: #ifndef CONTINUATIONS
1204: assert (continuation == 0);
1205: #endif
1206:
1207: /*
1208: * We can't use vm_page_free_reserved to make this
1209: * determination. Consider: some thread might
1210: * need to allocate two pages. The first allocation
1211: * succeeds, the second fails. After the first page is freed,
1212: * a call to vm_page_wait must really block.
1213: */
1214:
1215: simple_lock(&vm_page_queue_free_lock);
1.1.1.2 ! root 1216: if ((vm_page_free_count < vm_page_free_target)
! 1217: || (vm_page_external_count > vm_page_external_limit)) {
1.1 root 1218: if (vm_page_free_wanted++ == 0)
1219: thread_wakeup((event_t)&vm_page_free_wanted);
1220: assert_wait((event_t)&vm_page_free_count, FALSE);
1221: simple_unlock(&vm_page_queue_free_lock);
1222: if (continuation != 0) {
1223: counter(c_vm_page_wait_block_user++);
1224: thread_block(continuation);
1225: } else {
1226: counter(c_vm_page_wait_block_kernel++);
1227: thread_block((void (*)(void)) 0);
1228: }
1229: } else
1230: simple_unlock(&vm_page_queue_free_lock);
1231: }
1232:
1233: /*
1234: * vm_page_alloc:
1235: *
1236: * Allocate and return a memory cell associated
1237: * with this VM object/offset pair.
1238: *
1239: * Object must be locked.
1240: */
1241:
1242: vm_page_t vm_page_alloc(
1243: vm_object_t object,
1244: vm_offset_t offset)
1245: {
1246: register vm_page_t mem;
1247:
1.1.1.2 ! root 1248: mem = vm_page_grab(!object->internal);
1.1 root 1249: if (mem == VM_PAGE_NULL)
1250: return VM_PAGE_NULL;
1251:
1252: vm_page_lock_queues();
1253: vm_page_insert(mem, object, offset);
1254: vm_page_unlock_queues();
1255:
1256: return mem;
1257: }
1258:
1259: /*
1260: * vm_page_free:
1261: *
1262: * Returns the given page to the free list,
1263: * disassociating it with any VM object.
1264: *
1265: * Object and page queues must be locked prior to entry.
1266: */
1267: void vm_page_free(
1268: register vm_page_t mem)
1269: {
1270: if (mem->free)
1271: panic("vm_page_free");
1272:
1273: if (mem->tabled)
1274: vm_page_remove(mem);
1275: VM_PAGE_QUEUES_REMOVE(mem);
1276:
1277: if (mem->wire_count != 0) {
1278: if (!mem->private && !mem->fictitious)
1279: vm_page_wire_count--;
1280: mem->wire_count = 0;
1281: }
1282:
1283: if (mem->laundry) {
1284: vm_page_laundry_count--;
1285: mem->laundry = FALSE;
1286: }
1287:
1288: PAGE_WAKEUP_DONE(mem);
1289:
1290: if (mem->absent)
1291: vm_object_absent_release(mem->object);
1292:
1293: /*
1294: * XXX The calls to vm_page_init here are
1295: * really overkill.
1296: */
1297:
1298: if (mem->private || mem->fictitious) {
1299: vm_page_init(mem, vm_page_fictitious_addr);
1300: mem->fictitious = TRUE;
1301: vm_page_release_fictitious(mem);
1302: } else {
1.1.1.2 ! root 1303: int external = mem->external && mem->extcounted;
1.1 root 1304: vm_page_init(mem, mem->phys_addr);
1.1.1.2 ! root 1305: vm_page_release(mem, external);
1.1 root 1306: }
1307: }
1308:
1309: /*
1310: * vm_page_wire:
1311: *
1312: * Mark this page as wired down by yet
1313: * another map, removing it from paging queues
1314: * as necessary.
1315: *
1316: * The page's object and the page queues must be locked.
1317: */
1318: void vm_page_wire(
1319: register vm_page_t mem)
1320: {
1321: VM_PAGE_CHECK(mem);
1322:
1323: if (mem->wire_count == 0) {
1324: VM_PAGE_QUEUES_REMOVE(mem);
1325: if (!mem->private && !mem->fictitious)
1326: vm_page_wire_count++;
1327: }
1328: mem->wire_count++;
1329: }
1330:
1331: /*
1332: * vm_page_unwire:
1333: *
1334: * Release one wiring of this page, potentially
1335: * enabling it to be paged again.
1336: *
1337: * The page's object and the page queues must be locked.
1338: */
1339: void vm_page_unwire(
1340: register vm_page_t mem)
1341: {
1342: VM_PAGE_CHECK(mem);
1343:
1344: if (--mem->wire_count == 0) {
1345: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1346: vm_page_active_count++;
1347: mem->active = TRUE;
1348: if (!mem->private && !mem->fictitious)
1349: vm_page_wire_count--;
1350: }
1351: }
1352:
1353: /*
1354: * vm_page_deactivate:
1355: *
1356: * Returns the given page to the inactive list,
1357: * indicating that no physical maps have access
1358: * to this page. [Used by the physical mapping system.]
1359: *
1360: * The page queues must be locked.
1361: */
1362: void vm_page_deactivate(
1363: register vm_page_t m)
1364: {
1365: VM_PAGE_CHECK(m);
1366:
1367: /*
1368: * This page is no longer very interesting. If it was
1369: * interesting (active or inactive/referenced), then we
1370: * clear the reference bit and (re)enter it in the
1371: * inactive queue. Note wired pages should not have
1372: * their reference bit cleared.
1373: */
1374:
1375: if (m->active || (m->inactive && m->reference)) {
1376: if (!m->fictitious && !m->absent)
1377: pmap_clear_reference(m->phys_addr);
1378: m->reference = FALSE;
1379: VM_PAGE_QUEUES_REMOVE(m);
1380: }
1381: if (m->wire_count == 0 && !m->inactive) {
1382: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1383: m->inactive = TRUE;
1384: vm_page_inactive_count++;
1385: }
1386: }
1387:
1388: /*
1389: * vm_page_activate:
1390: *
1391: * Put the specified page on the active list (if appropriate).
1392: *
1393: * The page queues must be locked.
1394: */
1395:
1396: void vm_page_activate(
1397: register vm_page_t m)
1398: {
1399: VM_PAGE_CHECK(m);
1400:
1401: if (m->inactive) {
1402: queue_remove(&vm_page_queue_inactive, m, vm_page_t,
1403: pageq);
1404: vm_page_inactive_count--;
1405: m->inactive = FALSE;
1406: }
1407: if (m->wire_count == 0) {
1408: if (m->active)
1409: panic("vm_page_activate: already active");
1410:
1411: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1412: m->active = TRUE;
1413: vm_page_active_count++;
1414: }
1415: }
1416:
1417: /*
1418: * vm_page_zero_fill:
1419: *
1420: * Zero-fill the specified page.
1421: */
1422: void vm_page_zero_fill(
1423: vm_page_t m)
1424: {
1425: VM_PAGE_CHECK(m);
1426:
1427: pmap_zero_page(m->phys_addr);
1428: }
1429:
1430: /*
1431: * vm_page_copy:
1432: *
1433: * Copy one page to another
1434: */
1435:
1436: void vm_page_copy(
1437: vm_page_t src_m,
1438: vm_page_t dest_m)
1439: {
1440: VM_PAGE_CHECK(src_m);
1441: VM_PAGE_CHECK(dest_m);
1442:
1443: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1444: }
1445:
1446: #if MACH_VM_DEBUG
1447: /*
1448: * Routine: vm_page_info
1449: * Purpose:
1450: * Return information about the global VP table.
1451: * Fills the buffer with as much information as possible
1452: * and returns the desired size of the buffer.
1453: * Conditions:
1454: * Nothing locked. The caller should provide
1455: * possibly-pageable memory.
1456: */
1457:
1458: unsigned int
1459: vm_page_info(
1460: hash_info_bucket_t *info,
1461: unsigned int count)
1462: {
1463: int i;
1464:
1465: if (vm_page_bucket_count < count)
1466: count = vm_page_bucket_count;
1467:
1468: for (i = 0; i < count; i++) {
1469: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1470: unsigned int bucket_count = 0;
1471: vm_page_t m;
1472:
1473: simple_lock(&bucket->lock);
1474: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
1475: bucket_count++;
1476: simple_unlock(&bucket->lock);
1477:
1478: /* don't touch pageable memory while holding locks */
1479: info[i].hib_count = bucket_count;
1480: }
1481:
1482: return vm_page_bucket_count;
1483: }
1484: #endif /* MACH_VM_DEBUG */
1485:
1486: #include <mach_kdb.h>
1487: #if MACH_KDB
1488: #define printf kdbprintf
1489:
1490: /*
1491: * Routine: vm_page_print [exported]
1492: */
1493: void vm_page_print(p)
1494: vm_page_t p;
1495: {
1496: iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
1497: printf(" offset 0x%X", (vm_offset_t) p->offset);
1498: printf("wire_count %d,", p->wire_count);
1499: printf(" %s",
1500: (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
1501: printf("%s",
1502: (p->free ? " free" : ""));
1503: printf("%s ",
1504: (p->laundry ? " laundry" : ""));
1505: printf("%s",
1506: (p->dirty ? "dirty" : "clean"));
1507: printf("%s",
1508: (p->busy ? " busy" : ""));
1509: printf("%s",
1510: (p->absent ? " absent" : ""));
1511: printf("%s",
1512: (p->error ? " error" : ""));
1513: printf("%s",
1514: (p->fictitious ? " fictitious" : ""));
1515: printf("%s",
1516: (p->private ? " private" : ""));
1517: printf("%s",
1518: (p->wanted ? " wanted" : ""));
1519: printf("%s,",
1520: (p->tabled ? "" : "not_tabled"));
1521: printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
1522: (vm_offset_t) p->phys_addr,
1523: (vm_offset_t) p->page_lock,
1524: (vm_offset_t) p->unlock_request);
1525: }
1526: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.