![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_resident.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_resident.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
1.1.1.4 root 30: * File: vm/vm_resident.c
1.1 root 31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Resident memory management module.
34: */
1.1.1.3 root 35:
36: #include <kern/printf.h>
37: #include <string.h>
1.1 root 38:
39: #include <mach/vm_prot.h>
40: #include <kern/counters.h>
1.1.1.3 root 41: #include <kern/debug.h>
1.1 root 42: #include <kern/sched_prim.h>
43: #include <kern/task.h>
44: #include <kern/thread.h>
45: #include <mach/vm_statistics.h>
1.1.1.3 root 46: #include <machine/vm_param.h>
1.1 root 47: #include <kern/xpr.h>
1.1.1.3 root 48: #include <kern/slab.h>
1.1 root 49: #include <vm/pmap.h>
50: #include <vm/vm_map.h>
51: #include <vm/vm_page.h>
52: #include <vm/vm_pageout.h>
53: #include <vm/vm_kern.h>
54:
55: #if MACH_VM_DEBUG
56: #include <mach/kern_return.h>
57: #include <mach_debug/hash_info.h>
58: #include <vm/vm_user.h>
59: #endif
60:
1.1.1.3 root 61: #if MACH_KDB
62: #include <ddb/db_output.h>
63: #include <vm/vm_print.h>
64: #endif /* MACH_KDB */
65:
1.1 root 66:
67: /*
1.1.1.4 root 68: * Associated with each page of user-allocatable memory is a
1.1 root 69: * page structure.
70: */
71:
72: /*
73: * These variables record the values returned by vm_page_bootstrap,
74: * for debugging purposes. The implementation of pmap_steal_memory
1.1.1.5 ! root 75: * here also uses them internally.
1.1 root 76: */
77:
78: vm_offset_t virtual_space_start;
79: vm_offset_t virtual_space_end;
80:
81: /*
82: * The vm_page_lookup() routine, which provides for fast
83: * (virtual memory object, offset) to page lookup, employs
84: * the following hash table. The vm_page_{insert,remove}
85: * routines install and remove associations in the table.
86: * [This table is often called the virtual-to-physical,
87: * or VP, table.]
88: */
89: typedef struct {
90: decl_simple_lock_data(,lock)
91: vm_page_t pages;
92: } vm_page_bucket_t;
93:
94: vm_page_bucket_t *vm_page_buckets; /* Array of buckets */
95: unsigned int vm_page_bucket_count = 0; /* How big is array? */
96: unsigned int vm_page_hash_mask; /* Mask for hash function */
97:
98: vm_page_t vm_page_queue_fictitious;
99: decl_simple_lock_data(,vm_page_queue_free_lock)
100: unsigned int vm_page_free_wanted;
101: int vm_page_fictitious_count;
1.1.1.2 root 102: int vm_page_external_count;
1.1 root 103:
1.1.1.5 ! root 104: /*
! 105: * This variable isn't directly used. It's merely a placeholder for the
! 106: * address used to synchronize threads waiting for pages to become
! 107: * available. The real value is returned by vm_page_free_mem().
! 108: */
! 109: unsigned int vm_page_free_avail;
1.1 root 110:
111: /*
112: * Occasionally, the virtual memory system uses
113: * resident page structures that do not refer to
114: * real pages, for example to leave a page with
115: * important state information in the VP table.
116: *
117: * These page structures are allocated the way
118: * most other kernel structures are.
119: */
1.1.1.3 root 120: struct kmem_cache vm_page_cache;
1.1 root 121:
122: /*
123: * Fictitious pages don't have a physical address,
124: * but we must initialize phys_addr to something.
125: * For debugging, this should be a strange value
126: * that the pmap module can recognize in assertions.
127: */
128: vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
129:
130: /*
131: * Resident page structures are also chained on
132: * queues that are used by the page replacement
133: * system (pageout daemon). These queues are
134: * defined here, but are shared by the pageout
135: * module.
136: */
137: queue_head_t vm_page_queue_active;
138: queue_head_t vm_page_queue_inactive;
139: decl_simple_lock_data(,vm_page_queue_lock)
140: int vm_page_active_count;
141: int vm_page_inactive_count;
142: int vm_page_wire_count;
143:
144: /*
145: * Several page replacement parameters are also
146: * shared with this module, so that page allocation
147: * (done here in vm_page_alloc) can trigger the
148: * pageout daemon.
149: */
150: int vm_page_free_target = 0;
151: int vm_page_free_min = 0;
152: int vm_page_inactive_target = 0;
153: int vm_page_free_reserved = 0;
154: int vm_page_laundry_count = 0;
1.1.1.2 root 155: int vm_page_external_limit = 0;
156:
1.1 root 157:
158: /*
159: * The VM system has a couple of heuristics for deciding
160: * that pages are "uninteresting" and should be placed
161: * on the inactive queue as likely candidates for replacement.
162: * These variables let the heuristics be controlled at run-time
163: * to make experimentation easier.
164: */
165:
166: boolean_t vm_page_deactivate_behind = TRUE;
167: boolean_t vm_page_deactivate_hint = TRUE;
168:
169: /*
170: * vm_page_bootstrap:
171: *
172: * Initializes the resident memory module.
173: *
174: * Allocates memory for the page cells, and
175: * for the object/offset-to-page hash table headers.
176: * Each page cell is initialized and placed on the free list.
177: * Returns the range of available kernel virtual memory.
178: */
179:
180: void vm_page_bootstrap(
181: vm_offset_t *startp,
182: vm_offset_t *endp)
183: {
184: int i;
185:
186: /*
187: * Initialize the page queues.
188: */
189:
190: simple_lock_init(&vm_page_queue_free_lock);
191: simple_lock_init(&vm_page_queue_lock);
192:
193: vm_page_queue_fictitious = VM_PAGE_NULL;
194: queue_init(&vm_page_queue_active);
195: queue_init(&vm_page_queue_inactive);
196:
197: vm_page_free_wanted = 0;
198:
199: /*
200: * Allocate (and initialize) the virtual-to-physical
201: * table hash buckets.
202: *
203: * The number of buckets should be a power of two to
204: * get a good hash function. The following computation
205: * chooses the first power of two that is greater
206: * than the number of physical pages in the system.
207: */
208:
209: if (vm_page_bucket_count == 0) {
210: unsigned int npages = pmap_free_pages();
211:
212: vm_page_bucket_count = 1;
213: while (vm_page_bucket_count < npages)
214: vm_page_bucket_count <<= 1;
215: }
216:
217: vm_page_hash_mask = vm_page_bucket_count - 1;
218:
219: if (vm_page_hash_mask & vm_page_bucket_count)
220: printf("vm_page_bootstrap: WARNING -- strange page hash\n");
221:
222: vm_page_buckets = (vm_page_bucket_t *)
223: pmap_steal_memory(vm_page_bucket_count *
224: sizeof(vm_page_bucket_t));
225:
226: for (i = 0; i < vm_page_bucket_count; i++) {
1.1.1.4 root 227: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1.1 root 228:
229: bucket->pages = VM_PAGE_NULL;
230: simple_lock_init(&bucket->lock);
231: }
232:
1.1.1.5 ! root 233: vm_page_setup();
1.1 root 234:
235: virtual_space_start = round_page(virtual_space_start);
236: virtual_space_end = trunc_page(virtual_space_end);
237:
238: *startp = virtual_space_start;
239: *endp = virtual_space_end;
240: }
241:
242: #ifndef MACHINE_PAGES
243: /*
1.1.1.5 ! root 244: * We implement pmap_steal_memory with the help
! 245: * of two simpler functions, pmap_virtual_space and vm_page_bootalloc.
1.1 root 246: */
247:
248: vm_offset_t pmap_steal_memory(
249: vm_size_t size)
250: {
251: vm_offset_t addr, vaddr, paddr;
252:
1.1.1.5 ! root 253: size = round_page(size);
1.1 root 254:
255: /*
256: * If this is the first call to pmap_steal_memory,
257: * we have to initialize ourself.
258: */
259:
260: if (virtual_space_start == virtual_space_end) {
261: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
262:
263: /*
264: * The initial values must be aligned properly, and
265: * we don't trust the pmap module to do it right.
266: */
267:
268: virtual_space_start = round_page(virtual_space_start);
269: virtual_space_end = trunc_page(virtual_space_end);
270: }
271:
272: /*
273: * Allocate virtual memory for this request.
274: */
275:
276: addr = virtual_space_start;
277: virtual_space_start += size;
278:
279: /*
280: * Allocate and map physical pages to back new virtual pages.
281: */
282:
283: for (vaddr = round_page(addr);
284: vaddr < addr + size;
285: vaddr += PAGE_SIZE) {
1.1.1.5 ! root 286: paddr = vm_page_bootalloc(PAGE_SIZE);
1.1 root 287:
288: /*
289: * XXX Logically, these mappings should be wired,
290: * but some pmap modules barf if they are.
291: */
292:
293: pmap_enter(kernel_pmap, vaddr, paddr,
294: VM_PROT_READ|VM_PROT_WRITE, FALSE);
295: }
296:
297: return addr;
298: }
299: #endif /* MACHINE_PAGES */
300:
301: /*
302: * Routine: vm_page_module_init
303: * Purpose:
304: * Second initialization pass, to be done after
305: * the basic VM system is ready.
306: */
307: void vm_page_module_init(void)
308: {
1.1.1.3 root 309: kmem_cache_init(&vm_page_cache, "vm_page", sizeof(struct vm_page), 0,
1.1.1.5 ! root 310: NULL, 0);
1.1 root 311: }
312:
313: /*
314: * vm_page_hash:
315: *
316: * Distributes the object/offset key pair among hash buckets.
317: *
318: * NOTE: To get a good hash function, the bucket count should
319: * be a power of two.
320: */
321: #define vm_page_hash(object, offset) \
322: (((unsigned int)(vm_offset_t)object + (unsigned int)atop(offset)) \
323: & vm_page_hash_mask)
324:
325: /*
326: * vm_page_insert: [ internal use only ]
327: *
328: * Inserts the given mem entry into the object/object-page
329: * table and object list.
330: *
331: * The object and page must be locked.
332: */
333:
334: void vm_page_insert(
1.1.1.4 root 335: vm_page_t mem,
336: vm_object_t object,
337: vm_offset_t offset)
1.1 root 338: {
1.1.1.4 root 339: vm_page_bucket_t *bucket;
1.1 root 340:
341: VM_PAGE_CHECK(mem);
342:
343: if (mem->tabled)
344: panic("vm_page_insert");
345:
346: /*
347: * Record the object/offset pair in this page
348: */
349:
350: mem->object = object;
351: mem->offset = offset;
352:
353: /*
354: * Insert it into the object_object/offset hash table
355: */
356:
357: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
358: simple_lock(&bucket->lock);
359: mem->next = bucket->pages;
360: bucket->pages = mem;
361: simple_unlock(&bucket->lock);
362:
363: /*
364: * Now link into the object's list of backed pages.
365: */
366:
367: queue_enter(&object->memq, mem, vm_page_t, listq);
368: mem->tabled = TRUE;
369:
370: /*
371: * Show that the object has one more resident page.
372: */
373:
374: object->resident_page_count++;
1.1.1.5 ! root 375: assert(object->resident_page_count != 0);
1.1.1.3 root 376:
377: if (object->can_persist && (object->ref_count == 0))
378: vm_object_cached_pages_update(1);
1.1 root 379:
380: /*
381: * Detect sequential access and inactivate previous page.
382: * We ignore busy pages.
383: */
384:
385: if (vm_page_deactivate_behind &&
386: (offset == object->last_alloc + PAGE_SIZE)) {
387: vm_page_t last_mem;
388:
389: last_mem = vm_page_lookup(object, object->last_alloc);
390: if ((last_mem != VM_PAGE_NULL) && !last_mem->busy)
391: vm_page_deactivate(last_mem);
392: }
393: object->last_alloc = offset;
394: }
395:
396: /*
397: * vm_page_replace:
398: *
399: * Exactly like vm_page_insert, except that we first
400: * remove any existing page at the given offset in object
401: * and we don't do deactivate-behind.
402: *
403: * The object and page must be locked.
404: */
405:
406: void vm_page_replace(
1.1.1.4 root 407: vm_page_t mem,
408: vm_object_t object,
409: vm_offset_t offset)
1.1 root 410: {
1.1.1.4 root 411: vm_page_bucket_t *bucket;
1.1 root 412:
413: VM_PAGE_CHECK(mem);
414:
415: if (mem->tabled)
416: panic("vm_page_replace");
417:
418: /*
419: * Record the object/offset pair in this page
420: */
421:
422: mem->object = object;
423: mem->offset = offset;
424:
425: /*
426: * Insert it into the object_object/offset hash table,
427: * replacing any page that might have been there.
428: */
429:
430: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
431: simple_lock(&bucket->lock);
432: if (bucket->pages) {
433: vm_page_t *mp = &bucket->pages;
1.1.1.4 root 434: vm_page_t m = *mp;
1.1 root 435: do {
436: if (m->object == object && m->offset == offset) {
437: /*
438: * Remove page from bucket and from object,
439: * and return it to the free list.
440: */
441: *mp = m->next;
442: queue_remove(&object->memq, m, vm_page_t,
443: listq);
444: m->tabled = FALSE;
445: object->resident_page_count--;
446:
1.1.1.3 root 447: if (object->can_persist
448: && (object->ref_count == 0))
449: vm_object_cached_pages_update(-1);
450:
1.1 root 451: /*
452: * Return page to the free list.
453: * Note the page is not tabled now, so this
454: * won't self-deadlock on the bucket lock.
455: */
456:
457: vm_page_free(m);
458: break;
459: }
460: mp = &m->next;
461: } while ((m = *mp) != 0);
462: mem->next = bucket->pages;
463: } else {
464: mem->next = VM_PAGE_NULL;
465: }
466: bucket->pages = mem;
467: simple_unlock(&bucket->lock);
468:
469: /*
470: * Now link into the object's list of backed pages.
471: */
472:
473: queue_enter(&object->memq, mem, vm_page_t, listq);
474: mem->tabled = TRUE;
475:
476: /*
477: * And show that the object has one more resident
478: * page.
479: */
480:
481: object->resident_page_count++;
1.1.1.5 ! root 482: assert(object->resident_page_count != 0);
1.1.1.3 root 483:
484: if (object->can_persist && (object->ref_count == 0))
485: vm_object_cached_pages_update(1);
1.1 root 486: }
487:
488: /*
489: * vm_page_remove: [ internal use only ]
490: *
491: * Removes the given mem entry from the object/offset-page
492: * table and the object page list.
493: *
494: * The object and page must be locked.
495: */
496:
497: void vm_page_remove(
1.1.1.4 root 498: vm_page_t mem)
1.1 root 499: {
1.1.1.4 root 500: vm_page_bucket_t *bucket;
501: vm_page_t this;
1.1 root 502:
503: assert(mem->tabled);
504: VM_PAGE_CHECK(mem);
505:
506: /*
507: * Remove from the object_object/offset hash table
508: */
509:
510: bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
511: simple_lock(&bucket->lock);
512: if ((this = bucket->pages) == mem) {
513: /* optimize for common case */
514:
515: bucket->pages = mem->next;
516: } else {
1.1.1.4 root 517: vm_page_t *prev;
1.1 root 518:
519: for (prev = &this->next;
520: (this = *prev) != mem;
521: prev = &this->next)
522: continue;
523: *prev = this->next;
524: }
525: simple_unlock(&bucket->lock);
526:
527: /*
528: * Now remove from the object's list of backed pages.
529: */
530:
531: queue_remove(&mem->object->memq, mem, vm_page_t, listq);
532:
533: /*
534: * And show that the object has one fewer resident
535: * page.
536: */
537:
538: mem->object->resident_page_count--;
539:
540: mem->tabled = FALSE;
1.1.1.3 root 541:
542: if (mem->object->can_persist && (mem->object->ref_count == 0))
543: vm_object_cached_pages_update(-1);
1.1 root 544: }
545:
546: /*
547: * vm_page_lookup:
548: *
549: * Returns the page associated with the object/offset
550: * pair specified; if none is found, VM_PAGE_NULL is returned.
551: *
552: * The object must be locked. No side effects.
553: */
554:
555: vm_page_t vm_page_lookup(
1.1.1.4 root 556: vm_object_t object,
557: vm_offset_t offset)
1.1 root 558: {
1.1.1.4 root 559: vm_page_t mem;
560: vm_page_bucket_t *bucket;
1.1 root 561:
562: /*
563: * Search the hash table for this object/offset pair
564: */
565:
566: bucket = &vm_page_buckets[vm_page_hash(object, offset)];
567:
568: simple_lock(&bucket->lock);
569: for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
570: VM_PAGE_CHECK(mem);
571: if ((mem->object == object) && (mem->offset == offset))
572: break;
573: }
574: simple_unlock(&bucket->lock);
575: return mem;
576: }
577:
578: /*
579: * vm_page_rename:
580: *
581: * Move the given memory entry from its
582: * current object to the specified target object/offset.
583: *
584: * The object must be locked.
585: */
586: void vm_page_rename(
1.1.1.4 root 587: vm_page_t mem,
588: vm_object_t new_object,
589: vm_offset_t new_offset)
1.1 root 590: {
591: /*
592: * Changes to mem->object require the page lock because
593: * the pageout daemon uses that lock to get the object.
594: */
595:
596: vm_page_lock_queues();
597: vm_page_remove(mem);
598: vm_page_insert(mem, new_object, new_offset);
599: vm_page_unlock_queues();
600: }
601:
1.1.1.5 ! root 602: static void vm_page_init_template(vm_page_t m)
! 603: {
! 604: m->object = VM_OBJECT_NULL; /* reset later */
! 605: m->offset = 0; /* reset later */
! 606: m->wire_count = 0;
! 607:
! 608: m->inactive = FALSE;
! 609: m->active = FALSE;
! 610: m->laundry = FALSE;
! 611: m->free = FALSE;
! 612: m->external = FALSE;
! 613:
! 614: m->busy = TRUE;
! 615: m->wanted = FALSE;
! 616: m->tabled = FALSE;
! 617: m->fictitious = FALSE;
! 618: m->private = FALSE;
! 619: m->absent = FALSE;
! 620: m->error = FALSE;
! 621: m->dirty = FALSE;
! 622: m->precious = FALSE;
! 623: m->reference = FALSE;
! 624:
! 625: m->page_lock = VM_PROT_NONE;
! 626: m->unlock_request = VM_PROT_NONE;
! 627: }
! 628:
1.1 root 629: /*
630: * vm_page_init:
631: *
632: * Initialize the fields in a new page.
633: * This takes a structure with random values and initializes it
634: * so that it can be given to vm_page_release or vm_page_insert.
635: */
636: void vm_page_init(
1.1.1.5 ! root 637: vm_page_t mem)
1.1 root 638: {
1.1.1.5 ! root 639: vm_page_init_template(mem);
1.1 root 640: }
641:
642: /*
643: * vm_page_grab_fictitious:
644: *
645: * Remove a fictitious page from the free list.
646: * Returns VM_PAGE_NULL if there are no free pages.
647: */
648:
649: vm_page_t vm_page_grab_fictitious(void)
650: {
1.1.1.4 root 651: vm_page_t m;
1.1 root 652:
653: simple_lock(&vm_page_queue_free_lock);
654: m = vm_page_queue_fictitious;
655: if (m != VM_PAGE_NULL) {
656: vm_page_fictitious_count--;
657: vm_page_queue_fictitious = (vm_page_t) m->pageq.next;
658: m->free = FALSE;
659: }
660: simple_unlock(&vm_page_queue_free_lock);
661:
662: return m;
663: }
664:
665: /*
666: * vm_page_release_fictitious:
667: *
668: * Release a fictitious page to the free list.
669: */
670:
1.1.1.5 ! root 671: static void vm_page_release_fictitious(
1.1.1.4 root 672: vm_page_t m)
1.1 root 673: {
674: simple_lock(&vm_page_queue_free_lock);
675: if (m->free)
676: panic("vm_page_release_fictitious");
677: m->free = TRUE;
678: m->pageq.next = (queue_entry_t) vm_page_queue_fictitious;
679: vm_page_queue_fictitious = m;
680: vm_page_fictitious_count++;
681: simple_unlock(&vm_page_queue_free_lock);
682: }
683:
684: /*
685: * vm_page_more_fictitious:
686: *
687: * Add more fictitious pages to the free list.
688: * Allowed to block.
689: */
690:
691: int vm_page_fictitious_quantum = 5;
692:
693: void vm_page_more_fictitious(void)
694: {
1.1.1.4 root 695: vm_page_t m;
1.1 root 696: int i;
697:
698: for (i = 0; i < vm_page_fictitious_quantum; i++) {
1.1.1.3 root 699: m = (vm_page_t) kmem_cache_alloc(&vm_page_cache);
1.1 root 700: if (m == VM_PAGE_NULL)
701: panic("vm_page_more_fictitious");
702:
1.1.1.5 ! root 703: vm_page_init(m);
! 704: m->phys_addr = vm_page_fictitious_addr;
1.1 root 705: m->fictitious = TRUE;
706: vm_page_release_fictitious(m);
707: }
708: }
709:
710: /*
711: * vm_page_convert:
712: *
713: * Attempt to convert a fictitious page into a real page.
1.1.1.5 ! root 714: *
! 715: * The object referenced by *MP must be locked.
1.1 root 716: */
717:
718: boolean_t vm_page_convert(
1.1.1.5 ! root 719: struct vm_page **mp,
1.1.1.2 root 720: boolean_t external)
1.1 root 721: {
1.1.1.5 ! root 722: struct vm_page *real_m, *fict_m;
! 723: vm_object_t object;
! 724: vm_offset_t offset;
! 725:
! 726: fict_m = *mp;
! 727:
! 728: assert(fict_m->fictitious);
! 729: assert(fict_m->phys_addr == vm_page_fictitious_addr);
! 730: assert(!fict_m->active);
! 731: assert(!fict_m->inactive);
1.1 root 732:
1.1.1.2 root 733: real_m = vm_page_grab(external);
1.1 root 734: if (real_m == VM_PAGE_NULL)
735: return FALSE;
736:
1.1.1.5 ! root 737: object = fict_m->object;
! 738: offset = fict_m->offset;
! 739: vm_page_remove(fict_m);
! 740:
! 741: memcpy(&real_m->vm_page_header,
! 742: &fict_m->vm_page_header,
! 743: sizeof(*fict_m) - VM_PAGE_HEADER_SIZE);
! 744: real_m->fictitious = FALSE;
! 745:
! 746: vm_page_insert(real_m, object, offset);
! 747:
! 748: assert(real_m->phys_addr != vm_page_fictitious_addr);
! 749: assert(fict_m->fictitious);
! 750: assert(fict_m->phys_addr == vm_page_fictitious_addr);
1.1 root 751:
1.1.1.5 ! root 752: vm_page_release_fictitious(fict_m);
! 753: *mp = real_m;
1.1 root 754: return TRUE;
755: }
756:
757: /*
758: * vm_page_grab:
759: *
760: * Remove a page from the free list.
761: * Returns VM_PAGE_NULL if the free list is too small.
762: */
763:
1.1.1.2 root 764: vm_page_t vm_page_grab(
765: boolean_t external)
1.1 root 766: {
1.1.1.4 root 767: vm_page_t mem;
1.1 root 768:
769: simple_lock(&vm_page_queue_free_lock);
770:
771: /*
772: * Only let privileged threads (involved in pageout)
1.1.1.2 root 773: * dip into the reserved pool or exceed the limit
774: * for externally-managed pages.
1.1 root 775: */
776:
1.1.1.5 ! root 777: if (((vm_page_mem_free() < vm_page_free_reserved)
1.1.1.2 root 778: || (external
779: && (vm_page_external_count > vm_page_external_limit)))
780: && !current_thread()->vm_privilege) {
1.1 root 781: simple_unlock(&vm_page_queue_free_lock);
782: return VM_PAGE_NULL;
783: }
784:
1.1.1.5 ! root 785: mem = vm_page_alloc_pa(0, VM_PAGE_SEL_DIRECTMAP, VM_PT_KERNEL);
! 786:
! 787: if (mem == NULL) {
! 788: simple_unlock(&vm_page_queue_free_lock);
! 789: return NULL;
! 790: }
1.1 root 791:
1.1.1.2 root 792: if (external)
793: vm_page_external_count++;
1.1.1.5 ! root 794:
1.1 root 795: mem->free = FALSE;
1.1.1.2 root 796: mem->extcounted = mem->external = external;
1.1 root 797: simple_unlock(&vm_page_queue_free_lock);
798:
799: /*
800: * Decide if we should poke the pageout daemon.
801: * We do this if the free count is less than the low
802: * water mark, or if the free count is less than the high
803: * water mark (but above the low water mark) and the inactive
804: * count is less than its target.
805: *
806: * We don't have the counts locked ... if they change a little,
807: * it doesn't really matter.
808: */
809:
1.1.1.5 ! root 810: if ((vm_page_mem_free() < vm_page_free_min) ||
! 811: ((vm_page_mem_free() < vm_page_free_target) &&
1.1 root 812: (vm_page_inactive_count < vm_page_inactive_target)))
813: thread_wakeup((event_t) &vm_page_free_wanted);
814:
815: return mem;
816: }
817:
1.1.1.4 root 818: vm_offset_t vm_page_grab_phys_addr(void)
1.1 root 819: {
1.1.1.2 root 820: vm_page_t p = vm_page_grab(FALSE);
1.1 root 821: if (p == VM_PAGE_NULL)
822: return -1;
823: else
824: return p->phys_addr;
825: }
826:
827: /*
1.1.1.5 ! root 828: * vm_page_release:
1.1 root 829: *
1.1.1.5 ! root 830: * Return a page to the free list.
1.1 root 831: */
832:
1.1.1.5 ! root 833: static void vm_page_release(
! 834: vm_page_t mem,
! 835: boolean_t external)
1.1 root 836: {
1.1.1.5 ! root 837: simple_lock(&vm_page_queue_free_lock);
! 838: if (mem->free)
! 839: panic("vm_page_release");
! 840: mem->free = TRUE;
! 841: vm_page_free_pa(mem, 0);
! 842: if (external)
! 843: vm_page_external_count--;
1.1 root 844:
1.1.1.5 ! root 845: /*
! 846: * Check if we should wake up someone waiting for page.
! 847: * But don't bother waking them unless they can allocate.
! 848: *
! 849: * We wakeup only one thread, to prevent starvation.
! 850: * Because the scheduling system handles wait queues FIFO,
! 851: * if we wakeup all waiting threads, one greedy thread
! 852: * can starve multiple niceguy threads. When the threads
! 853: * all wakeup, the greedy threads runs first, grabs the page,
! 854: * and waits for another page. It will be the first to run
! 855: * when the next page is freed.
! 856: *
! 857: * However, there is a slight danger here.
! 858: * The thread we wake might not use the free page.
! 859: * Then the other threads could wait indefinitely
! 860: * while the page goes unused. To forestall this,
! 861: * the pageout daemon will keep making free pages
! 862: * as long as vm_page_free_wanted is non-zero.
! 863: */
1.1 root 864:
1.1.1.5 ! root 865: if ((vm_page_free_wanted > 0) &&
! 866: (vm_page_mem_free() >= vm_page_free_reserved)) {
! 867: vm_page_free_wanted--;
! 868: thread_wakeup_one((event_t) &vm_page_free_avail);
! 869: }
1.1 root 870:
1.1.1.5 ! root 871: simple_unlock(&vm_page_queue_free_lock);
! 872: }
1.1 root 873:
1.1.1.5 ! root 874: /*
! 875: * vm_page_grab_contig:
! 876: *
! 877: * Remove a block of contiguous pages from the free list.
! 878: * Returns VM_PAGE_NULL if the request fails.
! 879: */
1.1 root 880:
1.1.1.5 ! root 881: vm_page_t vm_page_grab_contig(
! 882: vm_size_t size,
! 883: unsigned int selector)
! 884: {
! 885: unsigned int i, order, nr_pages;
! 886: vm_page_t mem;
1.1 root 887:
1.1.1.5 ! root 888: order = vm_page_order(size);
! 889: nr_pages = 1 << order;
1.1 root 890:
891: simple_lock(&vm_page_queue_free_lock);
892:
893: /*
1.1.1.5 ! root 894: * Only let privileged threads (involved in pageout)
! 895: * dip into the reserved pool or exceed the limit
! 896: * for externally-managed pages.
1.1 root 897: */
898:
1.1.1.5 ! root 899: if (((vm_page_mem_free() - nr_pages) <= vm_page_free_reserved)
! 900: && !current_thread()->vm_privilege) {
1.1 root 901: simple_unlock(&vm_page_queue_free_lock);
1.1.1.5 ! root 902: return VM_PAGE_NULL;
1.1 root 903: }
904:
1.1.1.5 ! root 905: /* TODO Allow caller to pass type */
! 906: mem = vm_page_alloc_pa(order, selector, VM_PT_KERNEL);
1.1 root 907:
1.1.1.5 ! root 908: if (mem == NULL) {
! 909: simple_unlock(&vm_page_queue_free_lock);
! 910: return NULL;
1.1 root 911: }
912:
1.1.1.5 ! root 913: for (i = 0; i < nr_pages; i++) {
! 914: mem[i].free = FALSE;
! 915: mem[i].extcounted = mem[i].external = 0;
1.1 root 916: }
917:
918: simple_unlock(&vm_page_queue_free_lock);
919:
920: /*
921: * Decide if we should poke the pageout daemon.
922: * We do this if the free count is less than the low
923: * water mark, or if the free count is less than the high
924: * water mark (but above the low water mark) and the inactive
925: * count is less than its target.
926: *
927: * We don't have the counts locked ... if they change a little,
928: * it doesn't really matter.
929: */
930:
1.1.1.5 ! root 931: if ((vm_page_mem_free() < vm_page_free_min) ||
! 932: ((vm_page_mem_free() < vm_page_free_target) &&
1.1 root 933: (vm_page_inactive_count < vm_page_inactive_target)))
1.1.1.5 ! root 934: thread_wakeup((event_t) &vm_page_free_wanted);
1.1 root 935:
1.1.1.5 ! root 936: return mem;
1.1 root 937: }
938:
939: /*
1.1.1.5 ! root 940: * vm_page_free_contig:
1.1 root 941: *
1.1.1.5 ! root 942: * Return a block of contiguous pages to the free list.
1.1 root 943: */
944:
1.1.1.5 ! root 945: void vm_page_free_contig(vm_page_t mem, vm_size_t size)
1.1 root 946: {
1.1.1.5 ! root 947: unsigned int i, order, nr_pages;
! 948:
! 949: order = vm_page_order(size);
! 950: nr_pages = 1 << order;
! 951:
1.1 root 952: simple_lock(&vm_page_queue_free_lock);
953:
1.1.1.5 ! root 954: for (i = 0; i < nr_pages; i++) {
! 955: if (mem[i].free)
! 956: panic("vm_page_free_contig");
! 957:
! 958: mem[i].free = TRUE;
! 959: }
! 960:
! 961: vm_page_free_pa(mem, order);
1.1 root 962:
963: if ((vm_page_free_wanted > 0) &&
1.1.1.5 ! root 964: (vm_page_mem_free() >= vm_page_free_reserved)) {
1.1 root 965: vm_page_free_wanted--;
1.1.1.5 ! root 966: thread_wakeup_one((event_t) &vm_page_free_avail);
1.1 root 967: }
968:
969: simple_unlock(&vm_page_queue_free_lock);
970: }
971:
972: /*
973: * vm_page_wait:
974: *
975: * Wait for a page to become available.
976: * If there are plenty of free pages, then we don't sleep.
977: */
978:
979: void vm_page_wait(
980: void (*continuation)(void))
981: {
982:
983: /*
984: * We can't use vm_page_free_reserved to make this
985: * determination. Consider: some thread might
986: * need to allocate two pages. The first allocation
987: * succeeds, the second fails. After the first page is freed,
988: * a call to vm_page_wait must really block.
989: */
990:
991: simple_lock(&vm_page_queue_free_lock);
1.1.1.5 ! root 992: if ((vm_page_mem_free() < vm_page_free_target)
1.1.1.2 root 993: || (vm_page_external_count > vm_page_external_limit)) {
1.1 root 994: if (vm_page_free_wanted++ == 0)
995: thread_wakeup((event_t)&vm_page_free_wanted);
1.1.1.5 ! root 996: assert_wait((event_t)&vm_page_free_avail, FALSE);
1.1 root 997: simple_unlock(&vm_page_queue_free_lock);
998: if (continuation != 0) {
999: counter(c_vm_page_wait_block_user++);
1000: thread_block(continuation);
1001: } else {
1002: counter(c_vm_page_wait_block_kernel++);
1003: thread_block((void (*)(void)) 0);
1004: }
1005: } else
1006: simple_unlock(&vm_page_queue_free_lock);
1007: }
1008:
1009: /*
1010: * vm_page_alloc:
1011: *
1012: * Allocate and return a memory cell associated
1013: * with this VM object/offset pair.
1014: *
1015: * Object must be locked.
1016: */
1017:
1018: vm_page_t vm_page_alloc(
1019: vm_object_t object,
1020: vm_offset_t offset)
1021: {
1.1.1.4 root 1022: vm_page_t mem;
1.1 root 1023:
1.1.1.2 root 1024: mem = vm_page_grab(!object->internal);
1.1 root 1025: if (mem == VM_PAGE_NULL)
1026: return VM_PAGE_NULL;
1027:
1028: vm_page_lock_queues();
1029: vm_page_insert(mem, object, offset);
1030: vm_page_unlock_queues();
1031:
1032: return mem;
1033: }
1034:
1035: /*
1036: * vm_page_free:
1037: *
1038: * Returns the given page to the free list,
1039: * disassociating it with any VM object.
1040: *
1041: * Object and page queues must be locked prior to entry.
1042: */
1043: void vm_page_free(
1.1.1.4 root 1044: vm_page_t mem)
1.1 root 1045: {
1046: if (mem->free)
1047: panic("vm_page_free");
1048:
1049: if (mem->tabled)
1050: vm_page_remove(mem);
1051: VM_PAGE_QUEUES_REMOVE(mem);
1052:
1053: if (mem->wire_count != 0) {
1054: if (!mem->private && !mem->fictitious)
1055: vm_page_wire_count--;
1056: mem->wire_count = 0;
1057: }
1058:
1059: if (mem->laundry) {
1060: vm_page_laundry_count--;
1061: mem->laundry = FALSE;
1062: }
1063:
1064: PAGE_WAKEUP_DONE(mem);
1065:
1066: if (mem->absent)
1067: vm_object_absent_release(mem->object);
1068:
1069: /*
1070: * XXX The calls to vm_page_init here are
1071: * really overkill.
1072: */
1073:
1074: if (mem->private || mem->fictitious) {
1.1.1.5 ! root 1075: vm_page_init(mem);
! 1076: mem->phys_addr = vm_page_fictitious_addr;
1.1 root 1077: mem->fictitious = TRUE;
1078: vm_page_release_fictitious(mem);
1079: } else {
1.1.1.2 root 1080: int external = mem->external && mem->extcounted;
1.1.1.5 ! root 1081: vm_page_init(mem);
1.1.1.2 root 1082: vm_page_release(mem, external);
1.1 root 1083: }
1084: }
1085:
1086: /*
1087: * vm_page_wire:
1088: *
1089: * Mark this page as wired down by yet
1090: * another map, removing it from paging queues
1091: * as necessary.
1092: *
1093: * The page's object and the page queues must be locked.
1094: */
1095: void vm_page_wire(
1.1.1.4 root 1096: vm_page_t mem)
1.1 root 1097: {
1098: VM_PAGE_CHECK(mem);
1099:
1100: if (mem->wire_count == 0) {
1101: VM_PAGE_QUEUES_REMOVE(mem);
1102: if (!mem->private && !mem->fictitious)
1103: vm_page_wire_count++;
1104: }
1105: mem->wire_count++;
1106: }
1107:
1108: /*
1109: * vm_page_unwire:
1110: *
1111: * Release one wiring of this page, potentially
1112: * enabling it to be paged again.
1113: *
1114: * The page's object and the page queues must be locked.
1115: */
1116: void vm_page_unwire(
1.1.1.4 root 1117: vm_page_t mem)
1.1 root 1118: {
1119: VM_PAGE_CHECK(mem);
1120:
1121: if (--mem->wire_count == 0) {
1122: queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
1123: vm_page_active_count++;
1124: mem->active = TRUE;
1125: if (!mem->private && !mem->fictitious)
1126: vm_page_wire_count--;
1127: }
1128: }
1129:
1130: /*
1131: * vm_page_deactivate:
1132: *
1133: * Returns the given page to the inactive list,
1134: * indicating that no physical maps have access
1135: * to this page. [Used by the physical mapping system.]
1136: *
1137: * The page queues must be locked.
1138: */
1139: void vm_page_deactivate(
1.1.1.4 root 1140: vm_page_t m)
1.1 root 1141: {
1142: VM_PAGE_CHECK(m);
1143:
1144: /*
1145: * This page is no longer very interesting. If it was
1146: * interesting (active or inactive/referenced), then we
1147: * clear the reference bit and (re)enter it in the
1148: * inactive queue. Note wired pages should not have
1149: * their reference bit cleared.
1150: */
1151:
1152: if (m->active || (m->inactive && m->reference)) {
1153: if (!m->fictitious && !m->absent)
1154: pmap_clear_reference(m->phys_addr);
1155: m->reference = FALSE;
1156: VM_PAGE_QUEUES_REMOVE(m);
1157: }
1158: if (m->wire_count == 0 && !m->inactive) {
1159: queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
1160: m->inactive = TRUE;
1161: vm_page_inactive_count++;
1162: }
1163: }
1164:
1165: /*
1166: * vm_page_activate:
1167: *
1168: * Put the specified page on the active list (if appropriate).
1169: *
1170: * The page queues must be locked.
1171: */
1172:
1173: void vm_page_activate(
1.1.1.4 root 1174: vm_page_t m)
1.1 root 1175: {
1176: VM_PAGE_CHECK(m);
1177:
1178: if (m->inactive) {
1179: queue_remove(&vm_page_queue_inactive, m, vm_page_t,
1180: pageq);
1181: vm_page_inactive_count--;
1182: m->inactive = FALSE;
1183: }
1184: if (m->wire_count == 0) {
1185: if (m->active)
1186: panic("vm_page_activate: already active");
1187:
1188: queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
1189: m->active = TRUE;
1190: vm_page_active_count++;
1191: }
1192: }
1193:
1194: /*
1195: * vm_page_zero_fill:
1196: *
1197: * Zero-fill the specified page.
1198: */
1199: void vm_page_zero_fill(
1200: vm_page_t m)
1201: {
1202: VM_PAGE_CHECK(m);
1203:
1204: pmap_zero_page(m->phys_addr);
1205: }
1206:
1207: /*
1208: * vm_page_copy:
1209: *
1210: * Copy one page to another
1211: */
1212:
1213: void vm_page_copy(
1214: vm_page_t src_m,
1215: vm_page_t dest_m)
1216: {
1217: VM_PAGE_CHECK(src_m);
1218: VM_PAGE_CHECK(dest_m);
1219:
1220: pmap_copy_page(src_m->phys_addr, dest_m->phys_addr);
1221: }
1222:
1223: #if MACH_VM_DEBUG
1224: /*
1225: * Routine: vm_page_info
1226: * Purpose:
1227: * Return information about the global VP table.
1228: * Fills the buffer with as much information as possible
1229: * and returns the desired size of the buffer.
1230: * Conditions:
1231: * Nothing locked. The caller should provide
1232: * possibly-pageable memory.
1233: */
1234:
1235: unsigned int
1236: vm_page_info(
1237: hash_info_bucket_t *info,
1238: unsigned int count)
1239: {
1240: int i;
1241:
1242: if (vm_page_bucket_count < count)
1243: count = vm_page_bucket_count;
1244:
1245: for (i = 0; i < count; i++) {
1246: vm_page_bucket_t *bucket = &vm_page_buckets[i];
1247: unsigned int bucket_count = 0;
1248: vm_page_t m;
1249:
1250: simple_lock(&bucket->lock);
1251: for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
1252: bucket_count++;
1253: simple_unlock(&bucket->lock);
1254:
1255: /* don't touch pageable memory while holding locks */
1256: info[i].hib_count = bucket_count;
1257: }
1258:
1259: return vm_page_bucket_count;
1260: }
1261: #endif /* MACH_VM_DEBUG */
1262:
1.1.1.3 root 1263:
1.1 root 1264: #if MACH_KDB
1265: #define printf kdbprintf
1266:
1267: /*
1268: * Routine: vm_page_print [exported]
1269: */
1270: void vm_page_print(p)
1.1.1.4 root 1271: const vm_page_t p;
1.1 root 1272: {
1273: iprintf("Page 0x%X: object 0x%X,", (vm_offset_t) p, (vm_offset_t) p->object);
1.1.1.4 root 1274: printf(" offset 0x%X", p->offset);
1.1 root 1275: printf("wire_count %d,", p->wire_count);
1276: printf(" %s",
1277: (p->active ? "active" : (p->inactive ? "inactive" : "loose")));
1278: printf("%s",
1279: (p->free ? " free" : ""));
1280: printf("%s ",
1281: (p->laundry ? " laundry" : ""));
1282: printf("%s",
1283: (p->dirty ? "dirty" : "clean"));
1284: printf("%s",
1285: (p->busy ? " busy" : ""));
1286: printf("%s",
1287: (p->absent ? " absent" : ""));
1288: printf("%s",
1289: (p->error ? " error" : ""));
1290: printf("%s",
1291: (p->fictitious ? " fictitious" : ""));
1292: printf("%s",
1293: (p->private ? " private" : ""));
1294: printf("%s",
1295: (p->wanted ? " wanted" : ""));
1296: printf("%s,",
1297: (p->tabled ? "" : "not_tabled"));
1298: printf("phys_addr = 0x%X, lock = 0x%X, unlock_request = 0x%X\n",
1.1.1.4 root 1299: p->phys_addr,
1.1 root 1300: (vm_offset_t) p->page_lock,
1301: (vm_offset_t) p->unlock_request);
1302: }
1303: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.