![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_resident.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_page.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Resident memory management module.
34: */
35: #include <cpus.h>
36:
37: #include <mach/vm_prot.h>
38: #include <kern/counters.h>
39: #include <kern/sched_prim.h>
40: #include <kern/task.h>
41: #include <kern/thread.h>
42: #include <mach/vm_statistics.h>
43: #include "vm_param.h"
44: #include <kern/zalloc.h>
45: #include <vm/pmap.h>
46: #include <vm/vm_map.h>
47: #include <vm/vm_page.h>
48: #include <vm/vm_pageout.h>
49: #include <vm/vm_kern.h>
50:
51: #include <mach_vm_debug.h>
52: #if MACH_VM_DEBUG
53: #include <mach/kern_return.h>
54: #include <mach_debug/hash_info.h>
55: #include <vm/vm_user.h>
56: #endif
57:
58: /*
59: * Associated with eacn page of user-allocatable memory is a
60: * page structure.
61: */
62:
63: /*
64: * These variables record the values returned by vm_page_bootstrap,
65: * for debugging purposes. The implementation of pmap_steal_memory
66: * and pmap_startup here also uses them internally.
67: */
68:
69: vm_offset_t virtual_space_start;
70: vm_offset_t virtual_space_end;
71:
72: /*
73: * The vm_page_lookup() routine, which provides for fast
74: * (virtual memory object, offset) to page lookup, employs
75: * the following hash table. The vm_page_{insert,remove}
76: * routines install and remove associations in the table.
77: * [This table is often called the virtual-to-physical,
78: * or VP, table.]
79: */
80: typedef struct {
81: decl_simple_lock_data(,lock)
82: vm_page_t pages;
83: } vm_page_bucket_t;
84:
85: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
86: unsigned int vm_page_bucket_count = 0; /* How big is array? */
87: unsigned int vm_page_hash_mask; /* Mask for hash function */
88:
89: /*
90: * Resident page structures are initialized from
91: * a template (see vm_page_alloc).
92: *
93: * When adding a new field to the virtual memory
94: * object structure, be sure to add initialization
95: * (see vm_page_bootstrap).
96: */
97: struct vm_page vm_page_template;
98:
99: /*
100: * Resident pages that represent real memory
101: * are allocated from a free list.
102: */
103: vm_page_t vm_page_queue_free;
104: vm_page_t vm_page_queue_fictitious;
105: decl_simple_lock_data(,vm_page_queue_free_lock)
106: unsigned int vm_page_free_wanted;
107: int vm_page_queue_free_count;
108: int vm_page_unqueued_count;
109: int vm_page_fictitious_count;
110: int vm_page_external_count;
111:
112: unsigned int vm_page_free_count_minimum; /* debugging */
113:
114: /*
115: * Occasionally, the virtual memory system uses
116: * resident page structures that do not refer to
117: * real pages, for example to leave a page with
118: * important state information in the VP table.
119: *
120: * These page structures are allocated the way
121: * most other kernel structures are.
122: */
123: zone_t vm_page_zone;
124:
125: /*
126: * Fictitious pages don't have a physical address,
127: * but we must initialize phys_addr to something.
128: * For debugging, this should be a strange value
129: * that the pmap module can recognize in assertions.
130: */
131: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
132:
133: /*
134: * Resident page structures are also chained on
135: * queues that are used by the page replacement
136: * system (pageout daemon). These queues are
137: * defined here, but are shared by the pageout
138: * module.
139: */
140: queue_head_t vm_page_queue_active;
141: queue_head_t vm_page_queue_inactive;
142: decl_simple_lock_data(,vm_page_queue_lock)
143: int vm_page_active_count;
144: int vm_page_inactive_count;
145: int vm_page_wire_count;
146:
147: /*
148: * Several page replacement parameters are also
149: * shared with this module, so that page allocation
150: * (done here in vm_page_alloc) can trigger the
151: * pageout daemon.
152: */
153: int vm_page_free_target = 0;
154: int vm_page_free_min = 0;
155: int vm_page_inactive_target = 0;
156: int vm_page_free_reserved = 0;
157: int vm_page_laundry_count = 0;
158: int vm_page_external_limit = 0;
159:
160:
161: /*
162: * The VM system has a couple of heuristics for deciding
163: * that pages are "uninteresting" and should be placed
164: * on the inactive queue as likely candidates for replacement.
165: * These variables let the heuristics be controlled at run-time
166: * to make experimentation easier.
167: */
168:
169: boolean_t vm_page_deactivate_behind = TRUE;
170: boolean_t vm_page_deactivate_hint = TRUE;
171:
172: /*
173: * vm_page_bootstrap:
174: *
175: * Initializes the resident memory module.
176: *
177: * Allocates memory for the page cells, and
178: * for the object/offset-to-page hash table headers.
179: * Each page cell is initialized and placed on the free list.
180: * Returns the range of available kernel virtual memory.
181: */
182:
183: void vm_page_bootstrap(
184: vm_offset_t *startp,
185: vm_offset_t *endp)
186: {
187: register vm_page_t m;
188: int i;
189:
190: /*
191: * Initialize the vm_page template.
192: */
193:
194: m = &vm_page_template;
195: m->object = VM_OBJECT_NULL; /* reset later */
196: m->offset = 0; /* reset later */
197: m->wire_count = 0;
198:
199: m->inactive = FALSE;
200: m->active = FALSE;
201: m->laundry = FALSE;
202: m->free = FALSE;
203: m->external = FALSE;
204:
205: m->busy = TRUE;
206: m->wanted = FALSE;
207: m->tabled = FALSE;
208: m->fictitious = FALSE;
209: m->private = FALSE;
210: m->absent = FALSE;
211: m->error = FALSE;
212: m->dirty = FALSE;
213: m->precious = FALSE;
214: m->reference = FALSE;
215:
216: m->phys_addr = 0; /* reset later */
217:
218: m->page_lock = VM_PROT_NONE;
219: m->unlock_request = VM_PROT_NONE;
220:
221: /*
222: * Initialize the page queues.
223: */
224:
225: simple_lock_init(&vm_page_queue_free_lock);
226: simple_lock_init(&vm_page_queue_lock);
227:
228: vm_page_queue_free = VM_PAGE_NULL;
229: vm_page_queue_fictitious = VM_PAGE_NULL;
230: queue_init(&vm_page_queue_active);
231: queue_init(&vm_page_queue_inactive);
232:
233: vm_page_free_wanted = 0;
234:
235: /*
236: * Allocate (and initialize) the virtual-to-physical
237: * table hash buckets.
238: *
239: * The number of buckets should be a power of two to
240: * get a good hash function. The following computation
241: * chooses the first power of two that is greater
242: * than the number of physical pages in the system.
243: */
244:
245: if (vm_page_bucket_count == 0) {
246: unsigned int npages = pmap_free_pages();
247:
248: vm_page_bucket_count = 1;
249: while (vm_page_bucket_count < npages)
250: vm_page_bucket_count <<= 1;
251: }
252:
253: vm_page_hash_mask = vm_page_bucket_count - 1;
254:
255: if (vm_page_hash_mask & vm_page_bucket_count)
256: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
257:
258: vm_page_buckets = (vm_page_bucket_t *)
259: pmap_steal_memory(vm_page_bucket_count *
260: sizeof(vm_page_bucket_t));
261:
262: for (i = 0; i < vm_page_bucket_count; i++) {
263: register vm_page_bucket_t *bucket = &vm_page_buckets[i];
264:
265: bucket->pages = VM_PAGE_NULL;
266: simple_lock_init(&bucket->lock);
267: }
268:
269: /*
270: * Steal memory for the kentry zone.
271: */
272:
273: kentry_data_size = kentry_count * sizeof(struct vm_map_entry);
274: kentry_data = pmap_steal_memory(kentry_data_size);
275:
276: /*
277: * Machine-dependent code allocates the resident page table.
278: * It uses vm_page_init to initialize the page frames.
279: * The code also returns to us the virtual space available
280: * to the kernel. We don't trust the pmap module
281: * to get the alignment right.
282: */
283:
284: /* pmap_startup is responsible for stealing memory for
285: the zone system and putting it in zdata. */
286: pmap_startup(&virtual_space_start, &virtual_space_end);
287: virtual_space_start = round_page(virtual_space_start);
288: virtual_space_end = trunc_page(virtual_space_end);
289:
290: *startp = virtual_space_start;
291: *endp = virtual_space_end;
292:
293: /* printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
294: vm_page_free_count_minimum = vm_page_free_count;
295: }
296:
297: #ifndef MACHINE_PAGES
298: /*
299: * We implement pmap_steal_memory and pmap_startup with the help
300: * of two simpler functions, pmap_virtual_space and pmap_next_page.
301: */
302:
303: vm_offset_t pmap_steal_memory(
304: vm_size_t size)
305: {
306: vm_offset_t addr, vaddr, paddr;
307:
308: /*
309: * We round the size to an integer multiple.
310: */
311:
312: size = (size + 3) &~ 3;
313:
314: /*
315: * If this is the first call to pmap_steal_memory,
316: * we have to initialize ourself.
317: */
318:
319: if (virtual_space_start == virtual_space_end) {
320: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
321:
322: /*
323: * The initial values must be aligned properly, and
324: * we don't trust the pmap module to do it right.
325: */
326:
327: virtual_space_start = round_page(virtual_space_start);
328: virtual_space_end = trunc_page(virtual_space_end);
329: }
330:
331: /*
332: * Allocate virtual memory for this request.
333: */
334:
335: addr = virtual_space_start;
336: virtual_space_start += size;
337:
338: /*
339: * Allocate and map physical pages to back new virtual pages.
340: */
341:
342: for (vaddr = round_page(addr);
343: vaddr < addr + size;
344: vaddr += PAGE_SIZE) {
345: paddr = pmap_grab_page();
346:
347: /*
348: * XXX Logically, these mappings should be wired,
349: * but some pmap modules barf if they are.
350: */
351:
352: pmap_enter(kernel_pmap, vaddr, paddr,
353: VM_PROT_READ|VM_PROT_WRITE, FALSE);
354: }
355:
356: return addr;
357: }
358:
359: #if 0
360: void pmap_startup(
361: vm_offset_t *startp,
362: vm_offset_t *endp)
363: {
364: unsigned int i, npages, pages_initialized;
365: vm_page_t pages;
366: vm_offset_t paddr;
367:
368: /*
369: * We calculate how many page frames we will have
370: * and then allocate the page structures in one chunk.
371: */
372:
373: npages = ((PAGE_SIZE * pmap_free_pages() +
374: (round_page(virtual_space_start) - virtual_space_start)) /
375: (PAGE_SIZE + sizeof *pages));
376:
377: pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
378:
379: /*
380: * Initialize the page frames.
381: */
382:
383: for (i = 0, pages_initialized = 0; i < npages; i++) {
384: if (!pmap_next_page(&paddr))
385: break;
386:
387: vm_page_init(&pages[i], paddr);
388: pages_initialized++;
389: }
390:
391: /*
392: * Release pages in reverse order so that physical pages
393: * initially get allocated in ascending addresses. This keeps
394: * the devices (which must address physical memory) happy if
395: * they require several consecutive pages.
396: */
397:
398: for (i = pages_initialized; i > 0; i--) {
399: vm_page_release(&pages[i - 1], FALSE);
400: }
401:
402: /*
403: * We have to re-align virtual_space_start,
404: * because pmap_steal_memory has been using it.
405: */
406:
407: virtual_space_start = round_page(virtual_space_start);
408:
409: *startp = virtual_space_start;
410: *endp = virtual_space_end;
411: }
412: #endif
413: #endif /* MACHINE_PAGES */
414:
415: /*
416: * Routine: vm_page_module_init
417: * Purpose:
418: * Second initialization pass, to be done after
419: * the basic VM system is ready.
420: */
421: void vm_page_module_init(void)
422: {
423: vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
424: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
425: PAGE_SIZE,
426: /*
427: * This zone can be collectable, since we
428: * dynamically add and remove pages from the
429: * pool as the needs of drivers for contiguous
430: * memory changes.
431: */
432: ZONE_COLLECTABLE, "vm pages");
433: }
434:
435: /*
436: * Routine: vm_page_create
437: * Purpose:
438: * After the VM system is up, machine-dependent code
439: * may stumble across more physical memory. For example,
440: * memory that it was reserving for a frame buffer.
441: * vm_page_create turns this memory into available pages.
442: */
443:
444: void vm_page_create(
445: vm_offset_t start,
446: vm_offset_t end)
447: {
448: vm_offset_t paddr;
449: vm_page_t m;
450:
451: for (paddr = round_page(start);
452: paddr < trunc_page(end);
453: paddr += PAGE_SIZE) {
454: m = (vm_page_t) zalloc(vm_page_zone);
455: if (m == VM_PAGE_NULL)
456: panic("vm_page_create");
457:
458: vm_page_init(m, paddr);
459: vm_page_release(m, FALSE);
460: }
461: }
462:
463: /*
464: * vm_page_hash:
465: *
466: * Distributes the object/offset key pair among hash buckets.
467: *
468: * NOTE: To get a good hash function, the bucket count should
469: * be a power of two.
470: */
471: #define vm_page_hash(object, offset) \
472: (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
473: & vm_page_hash_mask)
474:
475: /*
476: * vm_page_insert: [ internal use only ]
477: *
478: * Inserts the given mem entry into the object/object-page
479: * table and object list.
480: *
481: * The object and page must be locked.
482: */
483:
484: void vm_page_insert(
485: register vm_page_t mem,
486: register vm_object_t object,
487: register vm_offset_t offset)
488: {
489: register vm_page_bucket_t *bucket;
490:
491: VM_PAGE_CHECK(mem);
492:
493: if (mem->tabled)
494: panic("vm_page_insert");
495:
496: /*
497: * Record the object/offset pair in this page
498: */
499:
500: mem->object = object;
501: mem->offset = offset;
502:
503: /*
504: * Insert it into the object_object/offset hash table
505: */
506:
507: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
508: simple_lock(&bucket->lock);
509: mem->next = bucket->pages;
510: bucket->pages = mem;
511: simple_unlock(&bucket->lock);
512:
513: /*
514: * Now link into the object's list of backed pages.
515: */
516:
517: queue_enter(&object->memq, mem, vm_page_t, listq);
518: mem->tabled = TRUE;
519:
520: /*
521: * Show that the object has one more resident page.
522: */
523:
524: object->resident_page_count++;
525:
526: /*
527: * Detect sequential access and inactivate previous page.
528: * We ignore busy pages.
529: */
530:
531: if (vm_page_deactivate_behind &&
532: (offset == object->last_alloc + PAGE_SIZE)) {
533: vm_page_t last_mem;
534:
535: last_mem = vm_page_lookup(object, object->last_alloc);
536: if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
537: vm_page_deactivate(last_mem);
538: }
539: object->last_alloc = offset;
540: }
541:
542: /*
543: * vm_page_replace:
544: *
545: * Exactly like vm_page_insert, except that we first
546: * remove any existing page at the given offset in object
547: * and we don't do deactivate-behind.
548: *
549: * The object and page must be locked.
550: */
551:
552: void vm_page_replace(
553: register vm_page_t mem,
554: register vm_object_t object,
555: register vm_offset_t offset)
556: {
557: register vm_page_bucket_t *bucket;
558:
559: VM_PAGE_CHECK(mem);
560:
561: if (mem->tabled)
562: panic("vm_page_replace");
563:
564: /*
565: * Record the object/offset pair in this page
566: */
567:
568: mem->object = object;
569: mem->offset = offset;
570:
571: /*
572: * Insert it into the object_object/offset hash table,
573: * replacing any page that might have been there.
574: */
575:
576: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
577: simple_lock(&bucket->lock);
578: if (bucket->pages) {
579: vm_page_t *mp = &bucket->pages;
580: register vm_page_t m = *mp;
581: do {
582: if (m->object == object && m->offset == offset) {
583: /*
584: * Remove page from bucket and from object,
585: * and return it to the free list.
586: */
587: *mp = m->next;
588: queue_remove(&object->memq, m, vm_page_t,
589: listq);
590: m->tabled = FALSE;
591: object->resident_page_count--;
592:
593: /*
594: * Return page to the free list.
595: * Note the page is not tabled now, so this
596: * won't self-deadlock on the bucket lock.
597: */
598:
599: vm_page_free(m);
600: break;
601: }
602: mp = &m->next;
603: } while ((m = *mp) != 0);
604: mem->next = bucket->pages;
605: } else {
606: mem->next = VM_PAGE_NULL;
607: }
608: bucket->pages = mem;
609: simple_unlock(&bucket->lock);
610:
611: /*
612: * Now link into the object's list of backed pages.
613: */
614:
615: queue_enter(&object->memq, mem, vm_page_t, listq);
616: mem->tabled = TRUE;
617:
618: /*
619: * And show that the object has one more resident
620: * page.
621: */
622:
623: object->resident_page_count++;
624: }
625:
626: /*
627: * vm_page_remove: [ internal use only ]
628: *
629: * Removes the given mem entry from the object/offset-page
630: * table and the object page list.
631: *
632: * The object and page must be locked.
633: */
634:
635: void vm_page_remove(
636: register vm_page_t mem)
637: {
638: register vm_page_bucket_t *bucket;
639: register vm_page_t this;
640:
641: assert(mem->tabled);
642: VM_PAGE_CHECK(mem);
643:
644: /*
645: * Remove from the object_object/offset hash table
646: */
647:
648: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
649: simple_lock(&bucket->lock);
650: if ((this = bucket->pages) == mem) {
651: /* optimize for common case */
652:
653: bucket->pages = mem->next;
654: } else {
655: register vm_page_t *prev;
656:
657: for (prev = &this->next;
658: (this = *prev) != mem;
659: prev = &this->next)
660: continue;
661: *prev = this->next;
662: }
663: simple_unlock(&bucket->lock);
664:
665: /*
666: * Now remove from the object's list of backed pages.
667: */
668:
669: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
670:
671: /*
672: * And show that the object has one fewer resident
673: * page.
674: */
675:
676: mem->object->resident_page_count--;
677:
678: mem->tabled = FALSE;
679: }
680:
681: /*
682: * vm_page_lookup:
683: *
684: * Returns the page associated with the object/offset
685: * pair specified; if none is found, VM_PAGE_NULL is returned.
686: *
687: * The object must be locked. No side effects.
688: */
689:
690: vm_page_t vm_page_lookup(
691: register vm_object_t object,
692: register vm_offset_t offset)
693: {
694: register vm_page_t mem;
695: register vm_page_bucket_t *bucket;
696:
697: /*
698: * Search the hash table for this object/offset pair
699: */
700:
701: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
702:
703: simple_lock(&bucket->lock);
704: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
705: VM_PAGE_CHECK(mem);
706: if ((mem->object == object) && (mem->offset == offset))
707: break;
708: }
709: simple_unlock(&bucket->lock);
710: return mem;
711: }
712:
713: /*
714: * vm_page_rename:
715: *
716: * Move the given memory entry from its
717: * current object to the specified target object/offset.
718: *
719: * The object must be locked.
720: */
721: void vm_page_rename(
722: register vm_page_t mem,
723: register vm_object_t new_object,
724: vm_offset_t new_offset)
725: {
726: /*
727: * Changes to mem->object require the page lock because
728: * the pageout daemon uses that lock to get the object.
729: */
730:
731: vm_page_lock_queues();
732: vm_page_remove(mem);
733: vm_page_insert(mem, new_object, new_offset);
734: vm_page_unlock_queues();
735: }
736:
737: /*
738: * vm_page_init:
739: *
740: * Initialize the fields in a new page.
741: * This takes a structure with random values and initializes it
742: * so that it can be given to vm_page_release or vm_page_insert.
743: */
744: void vm_page_init(
745: vm_page_t mem,
746: vm_offset_t phys_addr)
747: {
748: *mem = vm_page_template;
749: mem->phys_addr = phys_addr;
750: }
751:
752: /*
753: * vm_page_grab_fictitious:
754: *
755: * Remove a fictitious page from the free list.
756: * Returns VM_PAGE_NULL if there are no free pages.
757: */
758:
759: vm_page_t vm_page_grab_fictitious(void)
760: {
761: register vm_page_t m;
762:
763: simple_lock(&vm_page_queue_free_lock);
764: m = vm_page_queue_fictitious;
765: if (m != VM_PAGE_NULL) {
766: vm_page_fictitious_count--;
767: vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
768: m->free = FALSE;
769: }
770: simple_unlock(&vm_page_queue_free_lock);
771:
772: return m;
773: }
774:
775: /*
776: * vm_page_release_fictitious:
777: *
778: * Release a fictitious page to the free list.
779: */
780:
781: void vm_page_release_fictitious(
782: register vm_page_t m)
783: {
784: simple_lock(&vm_page_queue_free_lock);
785: if (m->free)
786: panic("vm_page_release_fictitious");
787: m->free = TRUE;
788: m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
789: vm_page_queue_fictitious = m;
790: vm_page_fictitious_count++;
791: simple_unlock(&vm_page_queue_free_lock);
792: }
793:
794: /*
795: * vm_page_more_fictitious:
796: *
797: * Add more fictitious pages to the free list.
798: * Allowed to block.
799: */
800:
801: int vm_page_fictitious_quantum = 5;
802:
803: void vm_page_more_fictitious(void)
804: {
805: register vm_page_t m;
806: int i;
807:
808: for (i = 0; i < vm_page_fictitious_quantum; i++) {
809: m = (vm_page_t) zalloc(vm_page_zone);
810: if (m == VM_PAGE_NULL)
811: panic("vm_page_more_fictitious");
812:
813: vm_page_init(m, vm_page_fictitious_addr);
814: m->fictitious = TRUE;
815: vm_page_release_fictitious(m);
816: }
817: }
818:
819: /*
820: * vm_page_convert:
821: *
822: * Attempt to convert a fictitious page into a real page.
823: */
824:
825: boolean_t vm_page_convert(
826: register vm_page_t m,
827: boolean_t external)
828: {
829: register vm_page_t real_m;
830:
831: real_m = vm_page_grab(external);
832: if (real_m == VM_PAGE_NULL)
833: return FALSE;
834:
835: m->phys_addr = real_m->phys_addr;
836: m->fictitious = FALSE;
837:
838: real_m->phys_addr = vm_page_fictitious_addr;
839: real_m->fictitious = TRUE;
840:
841: vm_page_release_fictitious(real_m);
842: return TRUE;
843: }
844:
845: /*
846: * vm_page_grab:
847: *
848: * Remove a page from the free list.
849: * Returns VM_PAGE_NULL if the free list is too small.
850: */
851:
852: vm_page_t vm_page_grab(
853: boolean_t external)
854: {
855: register vm_page_t mem;
856:
857: simple_lock(&vm_page_queue_free_lock);
858:
859: /*
860: * Only let privileged threads (involved in pageout)
861: * dip into the reserved pool or exceed the limit
862: * for externally-managed pages.
863: */
864:
865: if (((vm_page_free_count < vm_page_free_reserved)
866: || (external
867: && (vm_page_external_count > vm_page_external_limit)))
868: && !current_thread()->vm_privilege) {
869: simple_unlock(&vm_page_queue_free_lock);
870: return VM_PAGE_NULL;
871: }
872:
873: if (vm_page_queue_free == VM_PAGE_NULL) {
874: vm_page_grab_oskit_page();
875: assert (vm_page_queue_free != VM_PAGE_NULL);
876: }
877:
878: --vm_page_queue_free_count;
879: if (vm_page_free_count < vm_page_free_count_minimum)
880: vm_page_free_count_minimum = vm_page_free_count;
881: if (external)
882: vm_page_external_count++;
883: mem = vm_page_queue_free;
884: vm_page_queue_free = (vm_page_t) mem->pageq.next;
885: mem->free = FALSE;
886: mem->extcounted = mem->external = external;
887: simple_unlock(&vm_page_queue_free_lock);
888:
889: /*
890: * Decide if we should poke the pageout daemon.
891: * We do this if the free count is less than the low
892: * water mark, or if the free count is less than the high
893: * water mark (but above the low water mark) and the inactive
894: * count is less than its target.
895: *
896: * We don't have the counts locked ... if they change a little,
897: * it doesn't really matter.
898: */
899:
900: if ((vm_page_free_count < vm_page_free_min) ||
901: ((vm_page_free_count < vm_page_free_target) &&
902: (vm_page_inactive_count < vm_page_inactive_target)) ||
903: vm_page_queue_free_count < vm_page_unqueued_count)
904: thread_wakeup((event_t) &vm_page_free_wanted);
905:
906: return mem;
907: }
908:
909: #if 0
910: vm_offset_t vm_page_grab_phys_addr()
911: {
912: vm_page_t p = vm_page_grab(FALSE);
913: if (p == VM_PAGE_NULL)
914: return -1;
915: else
916: return p->phys_addr;
917: }
918:
919: /*
920: * vm_page_grab_contiguous_pages:
921: *
922: * Take N pages off the free list, the pages should
923: * cover a contiguous range of physical addresses.
924: * [Used by device drivers to cope with DMA limitations]
925: *
926: * Returns the page descriptors in ascending order, or
927: * Returns KERN_RESOURCE_SHORTAGE if it could not.
928: */
929:
930: /* Biggest phys page number for the pages we handle in VM */
931:
932: vm_size_t vm_page_big_pagenum = 0; /* Set this before call! */
933:
934: kern_return_t
935: vm_page_grab_contiguous_pages(
936: int npages,
937: vm_page_t pages[],
938: natural_t *bits,
939: boolean_t external)
940: {
941: register int first_set;
942: int size, alloc_size;
943: kern_return_t ret;
944: vm_page_t mem, prevmem;
945:
946: #ifndef NBBY
947: #define NBBY 8 /* size in bits of sizeof()`s unity */
948: #endif
949:
950: #define NBPEL (sizeof(natural_t)*NBBY)
951:
952: size = (vm_page_big_pagenum + NBPEL - 1)
953: & ~(NBPEL - 1); /* in bits */
954:
955: size = size / NBBY; /* in bytes */
956:
957: /*
958: * If we are called before the VM system is fully functional
959: * the invoker must provide us with the work space. [one bit
960: * per page starting at phys 0 and up to vm_page_big_pagenum]
961: */
962: if (bits == 0) {
963: alloc_size = round_page(size);
964: if (kmem_alloc_wired(kernel_map,
965: (vm_offset_t *)&bits,
966: alloc_size)
967: != KERN_SUCCESS)
968: return KERN_RESOURCE_SHORTAGE;
969: } else
970: alloc_size = 0;
971:
972: bzero(bits, size);
973:
974: /*
975: * A very large granularity call, its rare so that is ok
976: */
977: simple_lock(&vm_page_queue_free_lock);
978:
979: /*
980: * Do not dip into the reserved pool.
981: */
982:
983: if ((vm_page_free_count < vm_page_free_reserved)
984: || (vm_page_external_count >= vm_page_external_limit)) {
985: simple_unlock(&vm_page_queue_free_lock);
986: return KERN_RESOURCE_SHORTAGE;
987: }
988:
989: /*
990: * First pass through, build a big bit-array of
991: * the pages that are free. It is not going to
992: * be too large anyways, in 4k we can fit info
993: * for 32k pages.
994: */
995: mem = vm_page_queue_free;
996: while (mem) {
997: register int word_index, bit_index;
998:
999: bit_index = (mem->phys_addr >> PAGE_SHIFT);
1000: word_index = bit_index / NBPEL;
1001: bit_index = bit_index - (word_index * NBPEL);
1002: bits[word_index] |= 1 << bit_index;
1003:
1004: mem = (vm_page_t) mem->pageq.next;
1005: }
1006:
1007: /*
1008: * Second loop. Scan the bit array for NPAGES
1009: * contiguous bits. That gives us, if any,
1010: * the range of pages we will be grabbing off
1011: * the free list.
1012: */
1013: {
1014: register int bits_so_far = 0, i;
1015:
1016: first_set = 0;
1017:
1018: for (i = 0; i < size; i += sizeof(natural_t)) {
1019:
1020: register natural_t v = bits[i / sizeof(natural_t)];
1021: register int bitpos;
1022:
1023: /*
1024: * Bitscan this one word
1025: */
1026: if (v) {
1027: /*
1028: * keep counting them beans ?
1029: */
1030: bitpos = 0;
1031:
1032: if (bits_so_far) {
1033: count_ones:
1034: while (v & 1) {
1035: bitpos++;
1036: /*
1037: * got enough beans ?
1038: */
1039: if (++bits_so_far == npages)
1040: goto found_em;
1041: v >>= 1;
1042: }
1043: /* if we are being lucky, roll again */
1044: if (bitpos == NBPEL)
1045: continue;
1046: }
1047:
1048: /*
1049: * search for beans here
1050: */
1051: bits_so_far = 0;
1052: count_zeroes:
1053: while ((bitpos < NBPEL) && ((v & 1) == 0)) {
1054: bitpos++;
1055: v >>= 1;
1056: }
1057: if (v & 1) {
1058: first_set = (i * NBBY) + bitpos;
1059: goto count_ones;
1060: }
1061: }
1062: /*
1063: * No luck
1064: */
1065: bits_so_far = 0;
1066: }
1067: }
1068:
1069: /*
1070: * We could not find enough contiguous pages.
1071: */
1072: not_found_em:
1073: simple_unlock(&vm_page_queue_free_lock);
1074:
1075: ret = KERN_RESOURCE_SHORTAGE;
1076: goto out;
1077:
1078: /*
1079: * Final pass. Now we know which pages we want.
1080: * Scan the list until we find them all, grab
1081: * pages as we go. FIRST_SET tells us where
1082: * in the bit-array our pages start.
1083: */
1084: found_em:
1085: vm_page_free_count -= npages;
1086: if (vm_page_free_count < vm_page_free_count_minimum)
1087: vm_page_free_count_minimum = vm_page_free_count;
1088: if (external)
1089: vm_page_external_count += npages;
1090: {
1091: register vm_offset_t first_phys, last_phys;
1092:
1093: /* cache values for compare */
1094: first_phys = first_set << PAGE_SHIFT;
1095: last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
1096:
1097: /* running pointers */
1098: mem = vm_page_queue_free;
1099: prevmem = VM_PAGE_NULL;
1100:
1101: while (mem) {
1102:
1103: register vm_offset_t addr;
1104:
1105: addr = mem->phys_addr;
1106:
1107: if ((addr >= first_phys) &&
1108: (addr < last_phys)) {
1109: if (prevmem)
1110: prevmem->pageq.next = mem->pageq.next;
1111: pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
1112: mem->free = FALSE;
1113: mem->extcounted = mem->external = external;
1114: /*
1115: * Got them all ?
1116: */
1117: if (--npages == 0) break;
1118: } else
1119: prevmem = mem;
1120:
1121: mem = (vm_page_t) mem->pageq.next;
1122: }
1123: }
1124:
1125: simple_unlock(&vm_page_queue_free_lock);
1126:
1127: /*
1128: * Decide if we should poke the pageout daemon.
1129: * We do this if the free count is less than the low
1130: * water mark, or if the free count is less than the high
1131: * water mark (but above the low water mark) and the inactive
1132: * count is less than its target.
1133: *
1134: * We don't have the counts locked ... if they change a little,
1135: * it doesn't really matter.
1136: */
1137:
1138: if ((vm_page_free_count < vm_page_free_min) ||
1139: ((vm_page_free_count < vm_page_free_target) &&
1140: (vm_page_inactive_count < vm_page_inactive_target)))
1141: thread_wakeup(&vm_page_free_wanted);
1142:
1143: ret = KERN_SUCCESS;
1144: out:
1145: if (alloc_size)
1146: kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
1147:
1148: return ret;
1149: }
1150: #endif
1151:
1152: /*
1153: * vm_page_release:
1154: *
1155: * Return a page to the free list.
1156: */
1157:
1158: void vm_page_release(
1159: register vm_page_t mem,
1160: boolean_t external)
1161: {
1162: simple_lock(&vm_page_queue_free_lock);
1163: if (mem->free)
1164: panic("vm_page_release");
1165: mem->free = TRUE;
1166: mem->pageq.next = (queue_entry_t) vm_page_queue_free;
1167: vm_page_queue_free = mem;
1168: vm_page_queue_free_count++;
1169: if (external)
1170: vm_page_external_count--;
1171:
1172: /*
1173: * Check if we should wake up someone waiting for page.
1174: * But don't bother waking them unless they can allocate.
1175: *
1176: * We wakeup only one thread, to prevent starvation.
1177: * Because the scheduling system handles wait queues FIFO,
1178: * if we wakeup all waiting threads, one greedy thread
1179: * can starve multiple niceguy threads. When the threads
1180: * all wakeup, the greedy threads runs first, grabs the page,
1181: * and waits for another page. It will be the first to run
1182: * when the next page is freed.
1183: *
1184: * However, there is a slight danger here.
1185: * The thread we wake might not use the free page.
1186: * Then the other threads could wait indefinitely
1187: * while the page goes unused. To forestall this,
1188: * the pageout daemon will keep making free pages
1189: * as long as vm_page_free_wanted is non-zero.
1190: */
1191:
1192: if ((vm_page_free_wanted > 0) &&
1193: (vm_page_free_count >= vm_page_free_reserved)) {
1194: vm_page_free_wanted--;
1195: thread_wakeup_one((event_t) &vm_page_queue_free_count);
1196: }
1197:
1198: simple_unlock(&vm_page_queue_free_lock);
1199: }
1200:
1201: /*
1202: * vm_page_wait:
1203: *
1204: * Wait for a page to become available.
1205: * If there are plenty of free pages, then we don't sleep.
1206: */
1207:
1208: void vm_page_wait(
1209: void (*continuation)(void))
1210: {
1211:
1212: #ifndef CONTINUATIONS
1213: assert (continuation == 0);
1214: #endif
1215:
1216: /*
1217: * We can't use vm_page_free_reserved to make this
1218: * determination. Consider: some thread might
1219: * need to allocate two pages. The first allocation
1220: * succeeds, the second fails. After the first page is freed,
1221: * a call to vm_page_wait must really block.
1222: */
1223:
1224: simple_lock(&vm_page_queue_free_lock);
1225: if ((vm_page_free_count < vm_page_free_target)
1226: || (vm_page_external_count > vm_page_external_limit)) {
1227: if (vm_page_free_wanted++ == 0)
1228: thread_wakeup((event_t)&vm_page_free_wanted);
1229: assert_wait((event_t)&vm_page_queue_free_count, FALSE);
1230: simple_unlock(&vm_page_queue_free_lock);
1231: if (continuation != 0) {
1232: counter(c_vm_page_wait_block_user++);
1233: thread_block(continuation);
1234: } else {
1235: counter(c_vm_page_wait_block_kernel++);
1236: thread_block((void (*)(void)) 0);
1237: }
1238: } else
1239: simple_unlock(&vm_page_queue_free_lock);
1240: }
1241:
1242: /*
1243: * vm_page_alloc:
1244: *
1245: * Allocate and return a memory cell associated
1246: * with this VM object/offset pair.
1247: *
1248: * Object must be locked.
1249: */
1250:
1251: vm_page_t vm_page_alloc(
1252: vm_object_t object,
1253: vm_offset_t offset)
1254: {
1255: register vm_page_t mem;
1256:
1257: mem = vm_page_grab(!object->internal);
1258: if (mem == VM_PAGE_NULL)
1259: return VM_PAGE_NULL;
1260:
1261: vm_page_lock_queues();
1262: vm_page_insert(mem, object, offset);
1263: vm_page_unlock_queues();
1264:
1265: return mem;
1266: }
1267:
1268: /*
1269: * vm_page_free:
1270: *
1271: * Returns the given page to the free list,
1272: * disassociating it with any VM object.
1273: *
1274: * Object and page queues must be locked prior to entry.
1275: */
1276: void vm_page_free(
1277: register vm_page_t mem)
1278: {
1279: if (mem->free)
1280: panic("vm_page_free");
1281:
1282: if (mem->tabled)
1283: vm_page_remove(mem);
1284: VM_PAGE_QUEUES_REMOVE(mem);
1285:
1286: if (mem->wire_count != 0) {
1287: if (!mem->private && !mem->fictitious)
1288: vm_page_wire_count--;
1289: mem->wire_count = 0;
1290: }
1291:
1292: if (mem->laundry) {
1293: vm_page_laundry_count--;
1294: mem->laundry = FALSE;
1295: }
1296:
1297: PAGE_WAKEUP_DONE(mem);
1298:
1299: if (mem->absent)
1300: vm_object_absent_release(mem->object);
1301:
1302: /*
1303: * XXX The calls to vm_page_init here are
1304: * really overkill.
1305: */
1306:
1307: if (mem->private || mem->fictitious) {
1308: vm_page_init(mem, vm_page_fictitious_addr);
1309: mem->fictitious = TRUE;
1310: vm_page_release_fictitious(mem);
1311: } else {
1312: int external = mem->external && mem->extcounted;
1313: vm_page_init(mem, mem->phys_addr);
1314: vm_page_release(mem, external);
1315: }
1316: }
1317:
1318: /*
1319: * vm_page_wire:
1320: *
1321: * Mark this page as wired down by yet
1322: * another map, removing it from paging queues
1323: * as necessary.
1324: *
1325: * The page's object and the page queues must be locked.
1326: */
1327: void vm_page_wire(
1328: register vm_page_t mem)
1329: {
1330: VM_PAGE_CHECK(mem);
1331:
1332: if (mem->wire_count == 0) {
1333: VM_PAGE_QUEUES_REMOVE(mem);
1334: if (!mem->private && !mem->fictitious)
1335: vm_page_wire_count++;
1336: }
1337: mem->wire_count++;
1338: }
1339:
1340: /*
1341: * vm_page_unwire:
1342: *
1343: * Release one wiring of this page, potentially
1344: * enabling it to be paged again.
1345: *
1346: * The page's object and the page queues must be locked.
1347: */
1348: void vm_page_unwire(
1349: register vm_page_t mem)
1350: {
1351: VM_PAGE_CHECK(mem);
1352:
1353: if (--mem->wire_count == 0) {
1354: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1355: vm_page_active_count++;
1356: mem->active = TRUE;
1357: if (!mem->private && !mem->fictitious)
1358: vm_page_wire_count--;
1359: }
1360: }
1361:
1362: /*
1363: * vm_page_deactivate:
1364: *
1365: * Returns the given page to the inactive list,
1366: * indicating that no physical maps have access
1367: * to this page. [Used by the physical mapping system.]
1368: *
1369: * The page queues must be locked.
1370: */
1371: void vm_page_deactivate(
1372: register vm_page_t m)
1373: {
1374: VM_PAGE_CHECK(m);
1375:
1376: /*
1377: * This page is no longer very interesting. If it was
1378: * interesting (active or inactive/referenced), then we
1379: * clear the reference bit and (re)enter it in the
1380: * inactive queue. Note wired pages should not have
1381: * their reference bit cleared.
1382: */
1383:
1384: if (m->active || (m->inactive && m->reference)) {
1385: if (!m->fictitious && !m->absent)
1386: pmap_clear_reference(m->phys_addr);
1387: m->reference = FALSE;
1388: VM_PAGE_QUEUES_REMOVE(m);
1389: }
1390: if (m->wire_count == 0 && !m->inactive) {
1391: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1392: m->inactive = TRUE;
1393: vm_page_inactive_count++;
1394: }
1395: }
1396:
1397: /*
1398: * vm_page_activate:
1399: *
1400: * Put the specified page on the active list (if appropriate).
1401: *
1402: * The page queues must be locked.
1403: */
1404:
1405: void vm_page_activate(
1406: register vm_page_t m)
1407: {
1408: VM_PAGE_CHECK(m);
1409:
1410: if (m->inactive) {
1411: queue_remove(&vm_page_queue_inactive, m, vm_page_t,
1412: pageq);
1413: vm_page_inactive_count--;
1414: m->inactive = FALSE;
1415: }
1416: if (m->wire_count == 0) {
1417: if (m->active)
1418: panic("vm_page_activate: already active");
1419:
1420: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1421: m->active = TRUE;
1422: vm_page_active_count++;
1423: }
1424: }
1425:
1426: /*
1427: * vm_page_zero_fill:
1428: *
1429: * Zero-fill the specified page.
1430: */
1431: void vm_page_zero_fill(
1432: vm_page_t m)
1433: {
1434: VM_PAGE_CHECK(m);
1435:
1436: pmap_zero_page(m->phys_addr);
1437: }
1438:
1439: /*
1440: * vm_page_copy:
1441: *
1442: * Copy one page to another
1443: */
1444:
1445: void vm_page_copy(
1446: vm_page_t src_m,
1447: vm_page_t dest_m)
1448: {
1449: VM_PAGE_CHECK(src_m);
1450: VM_PAGE_CHECK(dest_m);
1451:
1452: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1453: }
1454:
1455: #if MACH_VM_DEBUG
1456: /*
1457: * Routine: vm_page_info
1458: * Purpose:
1459: * Return information about the global VP table.
1460: * Fills the buffer with as much information as possible
1461: * and returns the desired size of the buffer.
1462: * Conditions:
1463: * Nothing locked. The caller should provide
1464: * possibly-pageable memory.
1465: */
1466:
1467: unsigned int
1468: vm_page_info(
1469: hash_info_bucket_t *info,
1470: unsigned int count)
1471: {
1472: int i;
1473:
1474: if (vm_page_bucket_count < count)
1475: count = vm_page_bucket_count;
1476:
1477: for (i = 0; i < count; i++) {
1478: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1479: unsigned int bucket_count = 0;
1480: vm_page_t m;
1481:
1482: simple_lock(&bucket->lock);
1483: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
1484: bucket_count++;
1485: simple_unlock(&bucket->lock);
1486:
1487: /* don't touch pageable memory while holding locks */
1488: info[i].hib_count = bucket_count;
1489: }
1490:
1491: return vm_page_bucket_count;
1492: }
1493: #endif /* MACH_VM_DEBUG */
1494:
1495: #include <mach_kdb.h>
1496: #if MACH_KDB
1497: #define printf kdbprintf
1498:
1499: /*
1500: * Routine: vm_page_print [exported]
1501: */
1502: void vm_page_print(p)
1503: vm_page_t p;
1504: {
1505: iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
1506: printf(" offset 0x%X", (vm_offset_t) p->offset);
1507: printf("wire_count %d,", p->wire_count);
1508: printf(" %s",
1509: (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
1510: printf("%s",
1511: (p->free ? " free" : ""));
1512: printf("%s ",
1513: (p->laundry ? " laundry" : ""));
1514: printf("%s",
1515: (p->dirty ? "dirty" : "clean"));
1516: printf("%s",
1517: (p->busy ? " busy" : ""));
1518: printf("%s",
1519: (p->absent ? " absent" : ""));
1520: printf("%s",
1521: (p->error ? " error" : ""));
1522: printf("%s",
1523: (p->fictitious ? " fictitious" : ""));
1524: printf("%s",
1525: (p->private ? " private" : ""));
1526: printf("%s",
1527: (p->wanted ? " wanted" : ""));
1528: printf("%s,",
1529: (p->tabled ? "" : "not_tabled"));
1530: printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
1531: (vm_offset_t) p->phys_addr,
1532: (vm_offset_t) p->page_lock,
1533: (vm_offset_t) p->unlock_request);
1534: }
1535: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.