![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_resident.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_page.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Resident memory management module.
34: */
35: #include <cpus.h>
36:
37: #include <mach/vm_prot.h>
38: #include <kern/counters.h>
39: #include <kern/sched_prim.h>
40: #include <kern/task.h>
41: #include <kern/thread.h>
42: #include <mach/vm_statistics.h>
43: #include "vm_param.h"
44: #include <kern/xpr.h>
45: #include <kern/zalloc.h>
46: #include <vm/pmap.h>
47: #include <vm/vm_map.h>
48: #include <vm/vm_page.h>
49: #include <vm/vm_pageout.h>
50: #include <vm/vm_kern.h>
51:
52: #include <mach_vm_debug.h>
53: #if MACH_VM_DEBUG
54: #include <mach/kern_return.h>
55: #include <mach_debug/hash_info.h>
56: #include <vm/vm_user.h>
57: #endif
58:
59: /* in zalloc.c XXX */
60: extern vm_offset_t zdata;
61: extern vm_size_t zdata_size;
62:
63: /*
64: * Associated with eacn page of user-allocatable memory is a
65: * page structure.
66: */
67:
68: /*
69: * These variables record the values returned by vm_page_bootstrap,
70: * for debugging purposes. The implementation of pmap_steal_memory
71: * and pmap_startup here also uses them internally.
72: */
73:
74: vm_offset_t virtual_space_start;
75: vm_offset_t virtual_space_end;
76:
77: /*
78: * The vm_page_lookup() routine, which provides for fast
79: * (virtual memory object, offset) to page lookup, employs
80: * the following hash table. The vm_page_{insert,remove}
81: * routines install and remove associations in the table.
82: * [This table is often called the virtual-to-physical,
83: * or VP, table.]
84: */
85: typedef struct {
86: decl_simple_lock_data(,lock)
87: vm_page_t pages;
88: } vm_page_bucket_t;
89:
90: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
91: unsigned int vm_page_bucket_count = 0; /* How big is array? */
92: unsigned int vm_page_hash_mask; /* Mask for hash function */
93:
94: /*
95: * Resident page structures are initialized from
96: * a template (see vm_page_alloc).
97: *
98: * When adding a new field to the virtual memory
99: * object structure, be sure to add initialization
100: * (see vm_page_bootstrap).
101: */
102: struct vm_page vm_page_template;
103:
104: /*
105: * Resident pages that represent real memory
106: * are allocated from a free list.
107: */
108: vm_page_t vm_page_queue_free;
109: vm_page_t vm_page_queue_fictitious;
110: decl_simple_lock_data(,vm_page_queue_free_lock)
111: unsigned int vm_page_free_wanted;
112: int vm_page_free_count;
113: int vm_page_fictitious_count;
114:
115: unsigned int vm_page_free_count_minimum; /* debugging */
116:
117: /*
118: * Occasionally, the virtual memory system uses
119: * resident page structures that do not refer to
120: * real pages, for example to leave a page with
121: * important state information in the VP table.
122: *
123: * These page structures are allocated the way
124: * most other kernel structures are.
125: */
126: zone_t vm_page_zone;
127:
128: /*
129: * Fictitious pages don't have a physical address,
130: * but we must initialize phys_addr to something.
131: * For debugging, this should be a strange value
132: * that the pmap module can recognize in assertions.
133: */
134: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
135:
136: /*
137: * Resident page structures are also chained on
138: * queues that are used by the page replacement
139: * system (pageout daemon). These queues are
140: * defined here, but are shared by the pageout
141: * module.
142: */
143: queue_head_t vm_page_queue_active;
144: queue_head_t vm_page_queue_inactive;
145: decl_simple_lock_data(,vm_page_queue_lock)
146: int vm_page_active_count;
147: int vm_page_inactive_count;
148: int vm_page_wire_count;
149:
150: /*
151: * Several page replacement parameters are also
152: * shared with this module, so that page allocation
153: * (done here in vm_page_alloc) can trigger the
154: * pageout daemon.
155: */
156: int vm_page_free_target = 0;
157: int vm_page_free_min = 0;
158: int vm_page_inactive_target = 0;
159: int vm_page_free_reserved = 0;
160: int vm_page_laundry_count = 0;
161:
162: /*
163: * The VM system has a couple of heuristics for deciding
164: * that pages are "uninteresting" and should be placed
165: * on the inactive queue as likely candidates for replacement.
166: * These variables let the heuristics be controlled at run-time
167: * to make experimentation easier.
168: */
169:
170: boolean_t vm_page_deactivate_behind = TRUE;
171: boolean_t vm_page_deactivate_hint = TRUE;
172:
173: /*
174: * vm_page_bootstrap:
175: *
176: * Initializes the resident memory module.
177: *
178: * Allocates memory for the page cells, and
179: * for the object/offset-to-page hash table headers.
180: * Each page cell is initialized and placed on the free list.
181: * Returns the range of available kernel virtual memory.
182: */
183:
184: void vm_page_bootstrap(
185: vm_offset_t *startp,
186: vm_offset_t *endp)
187: {
188: register vm_page_t m;
189: int i;
190:
191: /*
192: * Initialize the vm_page template.
193: */
194:
195: m = &vm_page_template;
196: m->object = VM_OBJECT_NULL; /* reset later */
197: m->offset = 0; /* reset later */
198: m->wire_count = 0;
199:
200: m->inactive = FALSE;
201: m->active = FALSE;
202: m->laundry = FALSE;
203: m->free = FALSE;
204:
205: m->busy = TRUE;
206: m->wanted = FALSE;
207: m->tabled = FALSE;
208: m->fictitious = FALSE;
209: m->private = FALSE;
210: m->absent = FALSE;
211: m->error = FALSE;
212: m->dirty = FALSE;
213: m->precious = FALSE;
214: m->reference = FALSE;
215:
216: m->phys_addr = 0; /* reset later */
217:
218: m->page_lock = VM_PROT_NONE;
219: m->unlock_request = VM_PROT_NONE;
220:
221: /*
222: * Initialize the page queues.
223: */
224:
225: simple_lock_init(&vm_page_queue_free_lock);
226: simple_lock_init(&vm_page_queue_lock);
227:
228: vm_page_queue_free = VM_PAGE_NULL;
229: vm_page_queue_fictitious = VM_PAGE_NULL;
230: queue_init(&vm_page_queue_active);
231: queue_init(&vm_page_queue_inactive);
232:
233: vm_page_free_wanted = 0;
234:
235: /*
236: * Steal memory for the zone system.
237: */
238:
239: kentry_data_size = kentry_count * sizeof(struct vm_map_entry);
240: kentry_data = pmap_steal_memory(kentry_data_size);
241:
242: zdata = pmap_steal_memory(zdata_size);
243:
244: /*
245: * Allocate (and initialize) the virtual-to-physical
246: * table hash buckets.
247: *
248: * The number of buckets should be a power of two to
249: * get a good hash function. The following computation
250: * chooses the first power of two that is greater
251: * than the number of physical pages in the system.
252: */
253:
254: if (vm_page_bucket_count == 0) {
255: unsigned int npages = pmap_free_pages();
256:
257: vm_page_bucket_count = 1;
258: while (vm_page_bucket_count < npages)
259: vm_page_bucket_count <<= 1;
260: }
261:
262: vm_page_hash_mask = vm_page_bucket_count - 1;
263:
264: if (vm_page_hash_mask & vm_page_bucket_count)
265: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
266:
267: vm_page_buckets = (vm_page_bucket_t *)
268: pmap_steal_memory(vm_page_bucket_count *
269: sizeof(vm_page_bucket_t));
270:
271: for (i = 0; i < vm_page_bucket_count; i++) {
272: register vm_page_bucket_t *bucket = &vm_page_buckets[i];
273:
274: bucket->pages = VM_PAGE_NULL;
275: simple_lock_init(&bucket->lock);
276: }
277:
278: /*
279: * Machine-dependent code allocates the resident page table.
280: * It uses vm_page_init to initialize the page frames.
281: * The code also returns to us the virtual space available
282: * to the kernel. We don't trust the pmap module
283: * to get the alignment right.
284: */
285:
286: pmap_startup(&virtual_space_start, &virtual_space_end);
287: virtual_space_start = round_page(virtual_space_start);
288: virtual_space_end = trunc_page(virtual_space_end);
289:
290: *startp = virtual_space_start;
291: *endp = virtual_space_end;
292:
293: /* printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);*/
294: vm_page_free_count_minimum = vm_page_free_count;
295: }
296:
297: #ifndef MACHINE_PAGES
298: /*
299: * We implement pmap_steal_memory and pmap_startup with the help
300: * of two simpler functions, pmap_virtual_space and pmap_next_page.
301: */
302:
303: vm_offset_t pmap_steal_memory(
304: vm_size_t size)
305: {
306: vm_offset_t addr, vaddr, paddr;
307:
308: /*
309: * We round the size to an integer multiple.
310: */
311:
312: size = (size + 3) &~ 3;
313:
314: /*
315: * If this is the first call to pmap_steal_memory,
316: * we have to initialize ourself.
317: */
318:
319: if (virtual_space_start == virtual_space_end) {
320: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
321:
322: /*
323: * The initial values must be aligned properly, and
324: * we don't trust the pmap module to do it right.
325: */
326:
327: virtual_space_start = round_page(virtual_space_start);
328: virtual_space_end = trunc_page(virtual_space_end);
329: }
330:
331: /*
332: * Allocate virtual memory for this request.
333: */
334:
335: addr = virtual_space_start;
336: virtual_space_start += size;
337:
338: /*
339: * Allocate and map physical pages to back new virtual pages.
340: */
341:
342: for (vaddr = round_page(addr);
343: vaddr < addr + size;
344: vaddr += PAGE_SIZE) {
345: if (!pmap_next_page(&paddr))
346: panic("pmap_steal_memory");
347:
348: /*
349: * XXX Logically, these mappings should be wired,
350: * but some pmap modules barf if they are.
351: */
352:
353: pmap_enter(kernel_pmap, vaddr, paddr,
354: VM_PROT_READ|VM_PROT_WRITE, FALSE);
355: }
356:
357: return addr;
358: }
359:
360: void pmap_startup(
361: vm_offset_t *startp,
362: vm_offset_t *endp)
363: {
364: unsigned int i, npages, pages_initialized;
365: vm_page_t pages;
366: vm_offset_t paddr;
367:
368: /*
369: * We calculate how many page frames we will have
370: * and then allocate the page structures in one chunk.
371: */
372:
373: npages = ((PAGE_SIZE * pmap_free_pages() +
374: (round_page(virtual_space_start) - virtual_space_start)) /
375: (PAGE_SIZE + sizeof *pages));
376:
377: pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
378:
379: /*
380: * Initialize the page frames.
381: */
382:
383: for (i = 0, pages_initialized = 0; i < npages; i++) {
384: if (!pmap_next_page(&paddr))
385: break;
386:
387: vm_page_init(&pages[i], paddr);
388: pages_initialized++;
389: }
390:
391: /*
392: * Release pages in reverse order so that physical pages
393: * initially get allocated in ascending addresses. This keeps
394: * the devices (which must address physical memory) happy if
395: * they require several consecutive pages.
396: */
397:
398: for (i = pages_initialized; i > 0; i--) {
399: vm_page_release(&pages[i - 1]);
400: }
401:
402: /*
403: * We have to re-align virtual_space_start,
404: * because pmap_steal_memory has been using it.
405: */
406:
407: virtual_space_start = round_page(virtual_space_start);
408:
409: *startp = virtual_space_start;
410: *endp = virtual_space_end;
411: }
412: #endif /* MACHINE_PAGES */
413:
414: /*
415: * Routine: vm_page_module_init
416: * Purpose:
417: * Second initialization pass, to be done after
418: * the basic VM system is ready.
419: */
420: void vm_page_module_init(void)
421: {
422: vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
423: VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
424: PAGE_SIZE,
425: 0, "vm pages");
426: }
427:
428: /*
429: * Routine: vm_page_create
430: * Purpose:
431: * After the VM system is up, machine-dependent code
432: * may stumble across more physical memory. For example,
433: * memory that it was reserving for a frame buffer.
434: * vm_page_create turns this memory into available pages.
435: */
436:
437: void vm_page_create(
438: vm_offset_t start,
439: vm_offset_t end)
440: {
441: vm_offset_t paddr;
442: vm_page_t m;
443:
444: for (paddr = round_page(start);
445: paddr < trunc_page(end);
446: paddr += PAGE_SIZE) {
447: m = (vm_page_t) zalloc(vm_page_zone);
448: if (m == VM_PAGE_NULL)
449: panic("vm_page_create");
450:
451: vm_page_init(m, paddr);
452: vm_page_release(m);
453: }
454: }
455:
456: /*
457: * vm_page_hash:
458: *
459: * Distributes the object/offset key pair among hash buckets.
460: *
461: * NOTE: To get a good hash function, the bucket count should
462: * be a power of two.
463: */
464: #define vm_page_hash(object, offset) \
465: (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
466: & vm_page_hash_mask)
467:
468: /*
469: * vm_page_insert: [ internal use only ]
470: *
471: * Inserts the given mem entry into the object/object-page
472: * table and object list.
473: *
474: * The object and page must be locked.
475: */
476:
477: void vm_page_insert(
478: register vm_page_t mem,
479: register vm_object_t object,
480: register vm_offset_t offset)
481: {
482: register vm_page_bucket_t *bucket;
483:
484: VM_PAGE_CHECK(mem);
485:
486: if (mem->tabled)
487: panic("vm_page_insert");
488:
489: /*
490: * Record the object/offset pair in this page
491: */
492:
493: mem->object = object;
494: mem->offset = offset;
495:
496: /*
497: * Insert it into the object_object/offset hash table
498: */
499:
500: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
501: simple_lock(&bucket->lock);
502: mem->next = bucket->pages;
503: bucket->pages = mem;
504: simple_unlock(&bucket->lock);
505:
506: /*
507: * Now link into the object's list of backed pages.
508: */
509:
510: queue_enter(&object->memq, mem, vm_page_t, listq);
511: mem->tabled = TRUE;
512:
513: /*
514: * Show that the object has one more resident page.
515: */
516:
517: object->resident_page_count++;
518:
519: /*
520: * Detect sequential access and inactivate previous page.
521: * We ignore busy pages.
522: */
523:
524: if (vm_page_deactivate_behind &&
525: (offset == object->last_alloc + PAGE_SIZE)) {
526: vm_page_t last_mem;
527:
528: last_mem = vm_page_lookup(object, object->last_alloc);
529: if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
530: vm_page_deactivate(last_mem);
531: }
532: object->last_alloc = offset;
533: }
534:
535: /*
536: * vm_page_replace:
537: *
538: * Exactly like vm_page_insert, except that we first
539: * remove any existing page at the given offset in object
540: * and we don't do deactivate-behind.
541: *
542: * The object and page must be locked.
543: */
544:
545: void vm_page_replace(
546: register vm_page_t mem,
547: register vm_object_t object,
548: register vm_offset_t offset)
549: {
550: register vm_page_bucket_t *bucket;
551:
552: VM_PAGE_CHECK(mem);
553:
554: if (mem->tabled)
555: panic("vm_page_replace");
556:
557: /*
558: * Record the object/offset pair in this page
559: */
560:
561: mem->object = object;
562: mem->offset = offset;
563:
564: /*
565: * Insert it into the object_object/offset hash table,
566: * replacing any page that might have been there.
567: */
568:
569: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
570: simple_lock(&bucket->lock);
571: if (bucket->pages) {
572: vm_page_t *mp = &bucket->pages;
573: register vm_page_t m = *mp;
574: do {
575: if (m->object == object && m->offset == offset) {
576: /*
577: * Remove page from bucket and from object,
578: * and return it to the free list.
579: */
580: *mp = m->next;
581: queue_remove(&object->memq, m, vm_page_t,
582: listq);
583: m->tabled = FALSE;
584: object->resident_page_count--;
585:
586: /*
587: * Return page to the free list.
588: * Note the page is not tabled now, so this
589: * won't self-deadlock on the bucket lock.
590: */
591:
592: vm_page_free(m);
593: break;
594: }
595: mp = &m->next;
596: } while ((m = *mp) != 0);
597: mem->next = bucket->pages;
598: } else {
599: mem->next = VM_PAGE_NULL;
600: }
601: bucket->pages = mem;
602: simple_unlock(&bucket->lock);
603:
604: /*
605: * Now link into the object's list of backed pages.
606: */
607:
608: queue_enter(&object->memq, mem, vm_page_t, listq);
609: mem->tabled = TRUE;
610:
611: /*
612: * And show that the object has one more resident
613: * page.
614: */
615:
616: object->resident_page_count++;
617: }
618:
619: /*
620: * vm_page_remove: [ internal use only ]
621: *
622: * Removes the given mem entry from the object/offset-page
623: * table and the object page list.
624: *
625: * The object and page must be locked.
626: */
627:
628: void vm_page_remove(
629: register vm_page_t mem)
630: {
631: register vm_page_bucket_t *bucket;
632: register vm_page_t this;
633:
634: assert(mem->tabled);
635: VM_PAGE_CHECK(mem);
636:
637: /*
638: * Remove from the object_object/offset hash table
639: */
640:
641: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
642: simple_lock(&bucket->lock);
643: if ((this = bucket->pages) == mem) {
644: /* optimize for common case */
645:
646: bucket->pages = mem->next;
647: } else {
648: register vm_page_t *prev;
649:
650: for (prev = &this->next;
651: (this = *prev) != mem;
652: prev = &this->next)
653: continue;
654: *prev = this->next;
655: }
656: simple_unlock(&bucket->lock);
657:
658: /*
659: * Now remove from the object's list of backed pages.
660: */
661:
662: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
663:
664: /*
665: * And show that the object has one fewer resident
666: * page.
667: */
668:
669: mem->object->resident_page_count--;
670:
671: mem->tabled = FALSE;
672: }
673:
674: /*
675: * vm_page_lookup:
676: *
677: * Returns the page associated with the object/offset
678: * pair specified; if none is found, VM_PAGE_NULL is returned.
679: *
680: * The object must be locked. No side effects.
681: */
682:
683: vm_page_t vm_page_lookup(
684: register vm_object_t object,
685: register vm_offset_t offset)
686: {
687: register vm_page_t mem;
688: register vm_page_bucket_t *bucket;
689:
690: /*
691: * Search the hash table for this object/offset pair
692: */
693:
694: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
695:
696: simple_lock(&bucket->lock);
697: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
698: VM_PAGE_CHECK(mem);
699: if ((mem->object == object) && (mem->offset == offset))
700: break;
701: }
702: simple_unlock(&bucket->lock);
703: return mem;
704: }
705:
706: /*
707: * vm_page_rename:
708: *
709: * Move the given memory entry from its
710: * current object to the specified target object/offset.
711: *
712: * The object must be locked.
713: */
714: void vm_page_rename(
715: register vm_page_t mem,
716: register vm_object_t new_object,
717: vm_offset_t new_offset)
718: {
719: /*
720: * Changes to mem->object require the page lock because
721: * the pageout daemon uses that lock to get the object.
722: */
723:
724: vm_page_lock_queues();
725: vm_page_remove(mem);
726: vm_page_insert(mem, new_object, new_offset);
727: vm_page_unlock_queues();
728: }
729:
730: /*
731: * vm_page_init:
732: *
733: * Initialize the fields in a new page.
734: * This takes a structure with random values and initializes it
735: * so that it can be given to vm_page_release or vm_page_insert.
736: */
737: void vm_page_init(
738: vm_page_t mem,
739: vm_offset_t phys_addr)
740: {
741: *mem = vm_page_template;
742: mem->phys_addr = phys_addr;
743: }
744:
745: /*
746: * vm_page_grab_fictitious:
747: *
748: * Remove a fictitious page from the free list.
749: * Returns VM_PAGE_NULL if there are no free pages.
750: */
751:
752: vm_page_t vm_page_grab_fictitious(void)
753: {
754: register vm_page_t m;
755:
756: simple_lock(&vm_page_queue_free_lock);
757: m = vm_page_queue_fictitious;
758: if (m != VM_PAGE_NULL) {
759: vm_page_fictitious_count--;
760: vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
761: m->free = FALSE;
762: }
763: simple_unlock(&vm_page_queue_free_lock);
764:
765: return m;
766: }
767:
768: /*
769: * vm_page_release_fictitious:
770: *
771: * Release a fictitious page to the free list.
772: */
773:
774: void vm_page_release_fictitious(
775: register vm_page_t m)
776: {
777: simple_lock(&vm_page_queue_free_lock);
778: if (m->free)
779: panic("vm_page_release_fictitious");
780: m->free = TRUE;
781: m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
782: vm_page_queue_fictitious = m;
783: vm_page_fictitious_count++;
784: simple_unlock(&vm_page_queue_free_lock);
785: }
786:
787: /*
788: * vm_page_more_fictitious:
789: *
790: * Add more fictitious pages to the free list.
791: * Allowed to block.
792: */
793:
794: int vm_page_fictitious_quantum = 5;
795:
796: void vm_page_more_fictitious(void)
797: {
798: register vm_page_t m;
799: int i;
800:
801: for (i = 0; i < vm_page_fictitious_quantum; i++) {
802: m = (vm_page_t) zalloc(vm_page_zone);
803: if (m == VM_PAGE_NULL)
804: panic("vm_page_more_fictitious");
805:
806: vm_page_init(m, vm_page_fictitious_addr);
807: m->fictitious = TRUE;
808: vm_page_release_fictitious(m);
809: }
810: }
811:
812: /*
813: * vm_page_convert:
814: *
815: * Attempt to convert a fictitious page into a real page.
816: */
817:
818: boolean_t vm_page_convert(
819: register vm_page_t m)
820: {
821: register vm_page_t real_m;
822:
823: real_m = vm_page_grab();
824: if (real_m == VM_PAGE_NULL)
825: return FALSE;
826:
827: m->phys_addr = real_m->phys_addr;
828: m->fictitious = FALSE;
829:
830: real_m->phys_addr = vm_page_fictitious_addr;
831: real_m->fictitious = TRUE;
832:
833: vm_page_release_fictitious(real_m);
834: return TRUE;
835: }
836:
837: /*
838: * vm_page_grab:
839: *
840: * Remove a page from the free list.
841: * Returns VM_PAGE_NULL if the free list is too small.
842: */
843:
844: vm_page_t vm_page_grab(void)
845: {
846: register vm_page_t mem;
847:
848: simple_lock(&vm_page_queue_free_lock);
849:
850: /*
851: * Only let privileged threads (involved in pageout)
852: * dip into the reserved pool.
853: */
854:
855: if ((vm_page_free_count < vm_page_free_reserved) &&
856: !current_thread()->vm_privilege) {
857: simple_unlock(&vm_page_queue_free_lock);
858: return VM_PAGE_NULL;
859: }
860:
861: if (vm_page_queue_free == VM_PAGE_NULL)
862: panic("vm_page_grab");
863:
864: if (--vm_page_free_count < vm_page_free_count_minimum)
865: vm_page_free_count_minimum = vm_page_free_count;
866: mem = vm_page_queue_free;
867: vm_page_queue_free = (vm_page_t) mem->pageq.next;
868: mem->free = FALSE;
869: simple_unlock(&vm_page_queue_free_lock);
870:
871: /*
872: * Decide if we should poke the pageout daemon.
873: * We do this if the free count is less than the low
874: * water mark, or if the free count is less than the high
875: * water mark (but above the low water mark) and the inactive
876: * count is less than its target.
877: *
878: * We don't have the counts locked ... if they change a little,
879: * it doesn't really matter.
880: */
881:
882: if ((vm_page_free_count < vm_page_free_min) ||
883: ((vm_page_free_count < vm_page_free_target) &&
884: (vm_page_inactive_count < vm_page_inactive_target)))
885: thread_wakeup((event_t) &vm_page_free_wanted);
886:
887: return mem;
888: }
889:
890: vm_offset_t vm_page_grab_phys_addr(void)
891: {
892: vm_page_t p = vm_page_grab();
893: if (p == VM_PAGE_NULL)
894: return -1;
895: else
896: return p->phys_addr;
897: }
898:
899: /*
900: * vm_page_grab_contiguous_pages:
901: *
902: * Take N pages off the free list, the pages should
903: * cover a contiguous range of physical addresses.
904: * [Used by device drivers to cope with DMA limitations]
905: *
906: * Returns the page descriptors in ascending order, or
907: * Returns KERN_RESOURCE_SHORTAGE if it could not.
908: */
909:
910: /* Biggest phys page number for the pages we handle in VM */
911:
912: vm_size_t vm_page_big_pagenum = 0; /* Set this before call! */
913:
914: kern_return_t
915: vm_page_grab_contiguous_pages(
916: int npages,
917: vm_page_t pages[],
918: natural_t *bits)
919: {
920: register int first_set;
921: int size, alloc_size;
922: kern_return_t ret;
923: vm_page_t mem, prevmem;
924:
925: #ifndef NBBY
926: #define NBBY 8 /* size in bits of sizeof()`s unity */
927: #endif
928:
929: #define NBPEL (sizeof(natural_t)*NBBY)
930:
931: size = (vm_page_big_pagenum + NBPEL - 1)
932: & ~(NBPEL - 1); /* in bits */
933:
934: size = size / NBBY; /* in bytes */
935:
936: /*
937: * If we are called before the VM system is fully functional
938: * the invoker must provide us with the work space. [one bit
939: * per page starting at phys 0 and up to vm_page_big_pagenum]
940: */
941: if (bits == 0) {
942: alloc_size = round_page(size);
943: if (kmem_alloc_wired(kernel_map,
944: (vm_offset_t *)&bits,
945: alloc_size)
946: != KERN_SUCCESS)
947: return KERN_RESOURCE_SHORTAGE;
948: } else
949: alloc_size = 0;
950:
951: bzero(bits, size);
952:
953: /*
954: * A very large granularity call, its rare so that is ok
955: */
956: simple_lock(&vm_page_queue_free_lock);
957:
958: /*
959: * Do not dip into the reserved pool.
960: */
961:
962: if (vm_page_free_count < vm_page_free_reserved) {
963: simple_unlock(&vm_page_queue_free_lock);
964: return KERN_RESOURCE_SHORTAGE;
965: }
966:
967: /*
968: * First pass through, build a big bit-array of
969: * the pages that are free. It is not going to
970: * be too large anyways, in 4k we can fit info
971: * for 32k pages.
972: */
973: mem = vm_page_queue_free;
974: while (mem) {
975: register int word_index, bit_index;
976:
977: bit_index = (mem->phys_addr >> PAGE_SHIFT);
978: word_index = bit_index / NBPEL;
979: bit_index = bit_index - (word_index * NBPEL);
980: bits[word_index] |= 1 << bit_index;
981:
982: mem = (vm_page_t) mem->pageq.next;
983: }
984:
985: /*
986: * Second loop. Scan the bit array for NPAGES
987: * contiguous bits. That gives us, if any,
988: * the range of pages we will be grabbing off
989: * the free list.
990: */
991: {
992: register int bits_so_far = 0, i;
993:
994: first_set = 0;
995:
996: for (i = 0; i < size; i += sizeof(natural_t)) {
997:
998: register natural_t v = bits[i / sizeof(natural_t)];
999: register int bitpos;
1000:
1001: /*
1002: * Bitscan this one word
1003: */
1004: if (v) {
1005: /*
1006: * keep counting them beans ?
1007: */
1008: bitpos = 0;
1009:
1010: if (bits_so_far) {
1011: count_ones:
1012: while (v & 1) {
1013: bitpos++;
1014: /*
1015: * got enough beans ?
1016: */
1017: if (++bits_so_far == npages)
1018: goto found_em;
1019: v >>= 1;
1020: }
1021: /* if we are being lucky, roll again */
1022: if (bitpos == NBPEL)
1023: continue;
1024: }
1025:
1026: /*
1027: * search for beans here
1028: */
1029: bits_so_far = 0;
1030: count_zeroes:
1031: while ((bitpos < NBPEL) && ((v & 1) == 0)) {
1032: bitpos++;
1033: v >>= 1;
1034: }
1035: if (v & 1) {
1036: first_set = (i * NBBY) + bitpos;
1037: goto count_ones;
1038: }
1039: }
1040: /*
1041: * No luck
1042: */
1043: bits_so_far = 0;
1044: }
1045: }
1046:
1047: /*
1048: * We could not find enough contiguous pages.
1049: */
1050: not_found_em:
1051: simple_unlock(&vm_page_queue_free_lock);
1052:
1053: ret = KERN_RESOURCE_SHORTAGE;
1054: goto out;
1055:
1056: /*
1057: * Final pass. Now we know which pages we want.
1058: * Scan the list until we find them all, grab
1059: * pages as we go. FIRST_SET tells us where
1060: * in the bit-array our pages start.
1061: */
1062: found_em:
1063: vm_page_free_count -= npages;
1064: if (vm_page_free_count < vm_page_free_count_minimum)
1065: vm_page_free_count_minimum = vm_page_free_count;
1066:
1067: {
1068: register vm_offset_t first_phys, last_phys;
1069:
1070: /* cache values for compare */
1071: first_phys = first_set << PAGE_SHIFT;
1072: last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
1073:
1074: /* running pointers */
1075: mem = vm_page_queue_free;
1076: prevmem = VM_PAGE_NULL;
1077:
1078: while (mem) {
1079:
1080: register vm_offset_t addr;
1081:
1082: addr = mem->phys_addr;
1083:
1084: if ((addr >= first_phys) &&
1085: (addr < last_phys)) {
1086: if (prevmem)
1087: prevmem->pageq.next = mem->pageq.next;
1088: pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
1089: mem->free = FALSE;
1090: /*
1091: * Got them all ?
1092: */
1093: if (--npages == 0) break;
1094: } else
1095: prevmem = mem;
1096:
1097: mem = (vm_page_t) mem->pageq.next;
1098: }
1099: }
1100:
1101: simple_unlock(&vm_page_queue_free_lock);
1102:
1103: /*
1104: * Decide if we should poke the pageout daemon.
1105: * We do this if the free count is less than the low
1106: * water mark, or if the free count is less than the high
1107: * water mark (but above the low water mark) and the inactive
1108: * count is less than its target.
1109: *
1110: * We don't have the counts locked ... if they change a little,
1111: * it doesn't really matter.
1112: */
1113:
1114: if ((vm_page_free_count < vm_page_free_min) ||
1115: ((vm_page_free_count < vm_page_free_target) &&
1116: (vm_page_inactive_count < vm_page_inactive_target)))
1117: thread_wakeup(&vm_page_free_wanted);
1118:
1119: ret = KERN_SUCCESS;
1120: out:
1121: if (alloc_size)
1122: kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
1123:
1124: return ret;
1125: }
1126:
1127: /*
1128: * vm_page_release:
1129: *
1130: * Return a page to the free list.
1131: */
1132:
1133: void vm_page_release(
1134: register vm_page_t mem)
1135: {
1136: simple_lock(&vm_page_queue_free_lock);
1137: if (mem->free)
1138: panic("vm_page_release");
1139: mem->free = TRUE;
1140: mem->pageq.next = (queue_entry_t) vm_page_queue_free;
1141: vm_page_queue_free = mem;
1142: vm_page_free_count++;
1143:
1144: /*
1145: * Check if we should wake up someone waiting for page.
1146: * But don't bother waking them unless they can allocate.
1147: *
1148: * We wakeup only one thread, to prevent starvation.
1149: * Because the scheduling system handles wait queues FIFO,
1150: * if we wakeup all waiting threads, one greedy thread
1151: * can starve multiple niceguy threads. When the threads
1152: * all wakeup, the greedy threads runs first, grabs the page,
1153: * and waits for another page. It will be the first to run
1154: * when the next page is freed.
1155: *
1156: * However, there is a slight danger here.
1157: * The thread we wake might not use the free page.
1158: * Then the other threads could wait indefinitely
1159: * while the page goes unused. To forestall this,
1160: * the pageout daemon will keep making free pages
1161: * as long as vm_page_free_wanted is non-zero.
1162: */
1163:
1164: if ((vm_page_free_wanted > 0) &&
1165: (vm_page_free_count >= vm_page_free_reserved)) {
1166: vm_page_free_wanted--;
1167: thread_wakeup_one((event_t) &vm_page_free_count);
1168: }
1169:
1170: simple_unlock(&vm_page_queue_free_lock);
1171: }
1172:
1173: /*
1174: * vm_page_wait:
1175: *
1176: * Wait for a page to become available.
1177: * If there are plenty of free pages, then we don't sleep.
1178: */
1179:
1180: void vm_page_wait(
1181: void (*continuation)(void))
1182: {
1183:
1184: #ifndef CONTINUATIONS
1185: assert (continuation == 0);
1186: #endif
1187:
1188: /*
1189: * We can't use vm_page_free_reserved to make this
1190: * determination. Consider: some thread might
1191: * need to allocate two pages. The first allocation
1192: * succeeds, the second fails. After the first page is freed,
1193: * a call to vm_page_wait must really block.
1194: */
1195:
1196: simple_lock(&vm_page_queue_free_lock);
1197: if (vm_page_free_count < vm_page_free_target) {
1198: if (vm_page_free_wanted++ == 0)
1199: thread_wakeup((event_t)&vm_page_free_wanted);
1200: assert_wait((event_t)&vm_page_free_count, FALSE);
1201: simple_unlock(&vm_page_queue_free_lock);
1202: if (continuation != 0) {
1203: counter(c_vm_page_wait_block_user++);
1204: thread_block(continuation);
1205: } else {
1206: counter(c_vm_page_wait_block_kernel++);
1207: thread_block((void (*)(void)) 0);
1208: }
1209: } else
1210: simple_unlock(&vm_page_queue_free_lock);
1211: }
1212:
1213: /*
1214: * vm_page_alloc:
1215: *
1216: * Allocate and return a memory cell associated
1217: * with this VM object/offset pair.
1218: *
1219: * Object must be locked.
1220: */
1221:
1222: vm_page_t vm_page_alloc(
1223: vm_object_t object,
1224: vm_offset_t offset)
1225: {
1226: register vm_page_t mem;
1227:
1228: mem = vm_page_grab();
1229: if (mem == VM_PAGE_NULL)
1230: return VM_PAGE_NULL;
1231:
1232: vm_page_lock_queues();
1233: vm_page_insert(mem, object, offset);
1234: vm_page_unlock_queues();
1235:
1236: return mem;
1237: }
1238:
1239: /*
1240: * vm_page_free:
1241: *
1242: * Returns the given page to the free list,
1243: * disassociating it with any VM object.
1244: *
1245: * Object and page queues must be locked prior to entry.
1246: */
1247: void vm_page_free(
1248: register vm_page_t mem)
1249: {
1250: if (mem->free)
1251: panic("vm_page_free");
1252:
1253: if (mem->tabled)
1254: vm_page_remove(mem);
1255: VM_PAGE_QUEUES_REMOVE(mem);
1256:
1257: if (mem->wire_count != 0) {
1258: if (!mem->private && !mem->fictitious)
1259: vm_page_wire_count--;
1260: mem->wire_count = 0;
1261: }
1262:
1263: if (mem->laundry) {
1264: vm_page_laundry_count--;
1265: mem->laundry = FALSE;
1266: }
1267:
1268: PAGE_WAKEUP_DONE(mem);
1269:
1270: if (mem->absent)
1271: vm_object_absent_release(mem->object);
1272:
1273: /*
1274: * XXX The calls to vm_page_init here are
1275: * really overkill.
1276: */
1277:
1278: if (mem->private || mem->fictitious) {
1279: vm_page_init(mem, vm_page_fictitious_addr);
1280: mem->fictitious = TRUE;
1281: vm_page_release_fictitious(mem);
1282: } else {
1283: vm_page_init(mem, mem->phys_addr);
1284: vm_page_release(mem);
1285: }
1286: }
1287:
1288: /*
1289: * vm_page_wire:
1290: *
1291: * Mark this page as wired down by yet
1292: * another map, removing it from paging queues
1293: * as necessary.
1294: *
1295: * The page's object and the page queues must be locked.
1296: */
1297: void vm_page_wire(
1298: register vm_page_t mem)
1299: {
1300: VM_PAGE_CHECK(mem);
1301:
1302: if (mem->wire_count == 0) {
1303: VM_PAGE_QUEUES_REMOVE(mem);
1304: if (!mem->private && !mem->fictitious)
1305: vm_page_wire_count++;
1306: }
1307: mem->wire_count++;
1308: }
1309:
1310: /*
1311: * vm_page_unwire:
1312: *
1313: * Release one wiring of this page, potentially
1314: * enabling it to be paged again.
1315: *
1316: * The page's object and the page queues must be locked.
1317: */
1318: void vm_page_unwire(
1319: register vm_page_t mem)
1320: {
1321: VM_PAGE_CHECK(mem);
1322:
1323: if (--mem->wire_count == 0) {
1324: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1325: vm_page_active_count++;
1326: mem->active = TRUE;
1327: if (!mem->private && !mem->fictitious)
1328: vm_page_wire_count--;
1329: }
1330: }
1331:
1332: /*
1333: * vm_page_deactivate:
1334: *
1335: * Returns the given page to the inactive list,
1336: * indicating that no physical maps have access
1337: * to this page. [Used by the physical mapping system.]
1338: *
1339: * The page queues must be locked.
1340: */
1341: void vm_page_deactivate(
1342: register vm_page_t m)
1343: {
1344: VM_PAGE_CHECK(m);
1345:
1346: /*
1347: * This page is no longer very interesting. If it was
1348: * interesting (active or inactive/referenced), then we
1349: * clear the reference bit and (re)enter it in the
1350: * inactive queue. Note wired pages should not have
1351: * their reference bit cleared.
1352: */
1353:
1354: if (m->active || (m->inactive && m->reference)) {
1355: if (!m->fictitious && !m->absent)
1356: pmap_clear_reference(m->phys_addr);
1357: m->reference = FALSE;
1358: VM_PAGE_QUEUES_REMOVE(m);
1359: }
1360: if (m->wire_count == 0 && !m->inactive) {
1361: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1362: m->inactive = TRUE;
1363: vm_page_inactive_count++;
1364: }
1365: }
1366:
1367: /*
1368: * vm_page_activate:
1369: *
1370: * Put the specified page on the active list (if appropriate).
1371: *
1372: * The page queues must be locked.
1373: */
1374:
1375: void vm_page_activate(
1376: register vm_page_t m)
1377: {
1378: VM_PAGE_CHECK(m);
1379:
1380: if (m->inactive) {
1381: queue_remove(&vm_page_queue_inactive, m, vm_page_t,
1382: pageq);
1383: vm_page_inactive_count--;
1384: m->inactive = FALSE;
1385: }
1386: if (m->wire_count == 0) {
1387: if (m->active)
1388: panic("vm_page_activate: already active");
1389:
1390: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1391: m->active = TRUE;
1392: vm_page_active_count++;
1393: }
1394: }
1395:
1396: /*
1397: * vm_page_zero_fill:
1398: *
1399: * Zero-fill the specified page.
1400: */
1401: void vm_page_zero_fill(
1402: vm_page_t m)
1403: {
1404: VM_PAGE_CHECK(m);
1405:
1406: pmap_zero_page(m->phys_addr);
1407: }
1408:
1409: /*
1410: * vm_page_copy:
1411: *
1412: * Copy one page to another
1413: */
1414:
1415: void vm_page_copy(
1416: vm_page_t src_m,
1417: vm_page_t dest_m)
1418: {
1419: VM_PAGE_CHECK(src_m);
1420: VM_PAGE_CHECK(dest_m);
1421:
1422: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1423: }
1424:
1425: #if MACH_VM_DEBUG
1426: /*
1427: * Routine: vm_page_info
1428: * Purpose:
1429: * Return information about the global VP table.
1430: * Fills the buffer with as much information as possible
1431: * and returns the desired size of the buffer.
1432: * Conditions:
1433: * Nothing locked. The caller should provide
1434: * possibly-pageable memory.
1435: */
1436:
1437: unsigned int
1438: vm_page_info(
1439: hash_info_bucket_t *info,
1440: unsigned int count)
1441: {
1442: int i;
1443:
1444: if (vm_page_bucket_count < count)
1445: count = vm_page_bucket_count;
1446:
1447: for (i = 0; i < count; i++) {
1448: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1449: unsigned int bucket_count = 0;
1450: vm_page_t m;
1451:
1452: simple_lock(&bucket->lock);
1453: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
1454: bucket_count++;
1455: simple_unlock(&bucket->lock);
1456:
1457: /* don't touch pageable memory while holding locks */
1458: info[i].hib_count = bucket_count;
1459: }
1460:
1461: return vm_page_bucket_count;
1462: }
1463: #endif /* MACH_VM_DEBUG */
1464:
1465: #include <mach_kdb.h>
1466: #if MACH_KDB
1467: #define printf kdbprintf
1468:
1469: /*
1470: * Routine: vm_page_print [exported]
1471: */
1472: void vm_page_print(p)
1473: vm_page_t p;
1474: {
1475: iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
1476: printf(" offset 0x%X", (vm_offset_t) p->offset);
1477: printf("wire_count %d,", p->wire_count);
1478: printf(" %s",
1479: (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
1480: printf("%s",
1481: (p->free ? " free" : ""));
1482: printf("%s ",
1483: (p->laundry ? " laundry" : ""));
1484: printf("%s",
1485: (p->dirty ? "dirty" : "clean"));
1486: printf("%s",
1487: (p->busy ? " busy" : ""));
1488: printf("%s",
1489: (p->absent ? " absent" : ""));
1490: printf("%s",
1491: (p->error ? " error" : ""));
1492: printf("%s",
1493: (p->fictitious ? " fictitious" : ""));
1494: printf("%s",
1495: (p->private ? " private" : ""));
1496: printf("%s",
1497: (p->wanted ? " wanted" : ""));
1498: printf("%s,",
1499: (p->tabled ? "" : "not_tabled"));
1500: printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
1501: (vm_offset_t) p->phys_addr,
1502: (vm_offset_t) p->page_lock,
1503: (vm_offset_t) p->unlock_request);
1504: }
1505: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.