![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / i386|
Return to pmap.c CVS log | Up to [Apple XNU] / XNU / osfmk / i386 |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52:
53: /*
54: * File: pmap.c
55: * Author: Avadis Tevanian, Jr., Michael Wayne Young
56: * (These guys wrote the Vax version)
57: *
58: * Physical Map management code for Intel i386, i486, and i860.
59: *
60: * Manages physical address maps.
61: *
62: * In addition to hardware address maps, this
63: * module is called upon to provide software-use-only
64: * maps which may or may not be stored in the same
65: * form as hardware maps. These pseudo-maps are
66: * used to store intermediate results from copy
67: * operations to and from address spaces.
68: *
69: * Since the information managed by this module is
70: * also stored by the logical address mapping module,
71: * this module may throw away valid virtual-to-physical
72: * mappings at almost any time. However, invalidations
73: * of virtual-to-physical mappings must be done as
74: * requested.
75: *
76: * In order to cope with hardware architectures which
77: * make virtual-to-physical map invalidates expensive,
78: * this module may delay invalidate or reduced protection
79: * operations until such time as they are actually
80: * necessary. This module is given full information as
81: * to which processors are currently using which maps,
82: * and to when physical maps must be made correct.
83: */
84:
85: #include <cpus.h>
86:
87: #include <string.h>
88: #include <norma_vm.h>
89: #include <mach_kdb.h>
90: #include <mach_ldebug.h>
91:
92: #include <mach/machine/vm_types.h>
93:
94: #include <mach/boolean.h>
95: #include <kern/thread.h>
96: #include <kern/zalloc.h>
97:
98: #include <kern/lock.h>
99: #include <kern/spl.h>
100:
101: #include <vm/pmap.h>
102: #include <vm/vm_map.h>
103: #include <vm/vm_kern.h>
104: #include <mach/vm_param.h>
105: #include <mach/vm_prot.h>
106: #include <vm/vm_object.h>
107: #include <vm/vm_page.h>
108: #include <vm/vm_user.h>
109:
110: #include <mach/machine/vm_param.h>
111: #include <machine/thread.h>
112:
113: #include <kern/misc_protos.h> /* prototyping */
114: #include <i386/misc_protos.h>
115:
116: #include <i386/cpuid.h>
117:
118: #if MACH_KDB
119: #include <ddb/db_command.h>
120: #include <ddb/db_output.h>
121: #include <ddb/db_sym.h>
122: #include <ddb/db_print.h>
123: #endif /* MACH_KDB */
124:
125: #include <kern/xpr.h>
126:
127: #if NCPUS > 1
128: #include <i386/AT386/mp/mp_events.h>
129: #endif
130:
131: /*
132: * Forward declarations for internal functions.
133: */
134: void pmap_expand(
135: pmap_t map,
136: vm_offset_t v);
137:
138: extern void pmap_remove_range(
139: pmap_t pmap,
140: vm_offset_t va,
141: pt_entry_t *spte,
142: pt_entry_t *epte);
143:
144: void phys_attribute_clear(
145: vm_offset_t phys,
146: int bits);
147:
148: boolean_t phys_attribute_test(
149: vm_offset_t phys,
150: int bits);
151:
152: void pmap_set_modify(vm_offset_t phys);
153:
154: void phys_attribute_set(
155: vm_offset_t phys,
156: int bits);
157:
158:
159: #ifndef set_dirbase
160: void set_dirbase(vm_offset_t dirbase);
161: #endif /* set_dirbase */
162:
163: #define PA_TO_PTE(pa) (pa_to_pte((pa) - VM_MIN_KERNEL_ADDRESS))
164: #define iswired(pte) ((pte) & INTEL_PTE_WIRED)
165:
166: pmap_t real_pmap[NCPUS];
167:
168: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry);
169: #define WRITE_PTE_FAST(pte_p, pte_entry) *(pte_p) = (pte_entry);
170:
171: /*
172: * Private data structures.
173: */
174:
175: /*
176: * For each vm_page_t, there is a list of all currently
177: * valid virtual mappings of that page. An entry is
178: * a pv_entry_t; the list is the pv_table.
179: */
180:
181: typedef struct pv_entry {
182: struct pv_entry *next; /* next pv_entry */
183: pmap_t pmap; /* pmap where mapping lies */
184: vm_offset_t va; /* virtual address for mapping */
185: } *pv_entry_t;
186:
187: #define PV_ENTRY_NULL ((pv_entry_t) 0)
188:
189: pv_entry_t pv_head_table; /* array of entries, one per page */
190:
191: /*
192: * pv_list entries are kept on a list that can only be accessed
193: * with the pmap system locked (at SPLVM, not in the cpus_active set).
194: * The list is refilled from the pv_list_zone if it becomes empty.
195: */
196: pv_entry_t pv_free_list; /* free list at SPLVM */
197: decl_simple_lock_data(,pv_free_list_lock)
198:
199: #define PV_ALLOC(pv_e) { \
200: simple_lock(&pv_free_list_lock); \
201: if ((pv_e = pv_free_list) != 0) { \
202: pv_free_list = pv_e->next; \
203: } \
204: simple_unlock(&pv_free_list_lock); \
205: }
206:
207: #define PV_FREE(pv_e) { \
208: simple_lock(&pv_free_list_lock); \
209: pv_e->next = pv_free_list; \
210: pv_free_list = pv_e; \
211: simple_unlock(&pv_free_list_lock); \
212: }
213:
214: zone_t pv_list_zone; /* zone of pv_entry structures */
215:
216: /*
217: * Each entry in the pv_head_table is locked by a bit in the
218: * pv_lock_table. The lock bits are accessed by the physical
219: * address of the page they lock.
220: */
221:
222: char *pv_lock_table; /* pointer to array of bits */
223: #define pv_lock_table_size(n) (((n)+BYTE_SIZE-1)/BYTE_SIZE)
224:
225: /*
226: * First and last physical addresses that we maintain any information
227: * for. Initialized to zero so that pmap operations done before
228: * pmap_init won't touch any non-existent structures.
229: */
230: vm_offset_t vm_first_phys = (vm_offset_t) 0;
231: vm_offset_t vm_last_phys = (vm_offset_t) 0;
232: boolean_t pmap_initialized = FALSE;/* Has pmap_init completed? */
233:
234: /*
235: * Index into pv_head table, its lock bits, and the modify/reference
236: * bits starting at vm_first_phys.
237: */
238:
239: #define pa_index(pa) (atop(pa - vm_first_phys))
240:
241: #define pai_to_pvh(pai) (&pv_head_table[pai])
242: #define lock_pvh_pai(pai) bit_lock(pai, (void *)pv_lock_table)
243: #define unlock_pvh_pai(pai) bit_unlock(pai, (void *)pv_lock_table)
244:
245: /*
246: * Array of physical page attribites for managed pages.
247: * One byte per physical page.
248: */
249: char *pmap_phys_attributes;
250:
251: /*
252: * Physical page attributes. Copy bits from PTE definition.
253: */
254: #define PHYS_MODIFIED INTEL_PTE_MOD /* page modified */
255: #define PHYS_REFERENCED INTEL_PTE_REF /* page referenced */
256:
257: /*
258: * Amount of virtual memory mapped by one
259: * page-directory entry.
260: */
261: #define PDE_MAPPED_SIZE (pdetova(1))
262:
263: /*
264: * We allocate page table pages directly from the VM system
265: * through this object. It maps physical memory.
266: */
267: vm_object_t pmap_object = VM_OBJECT_NULL;
268:
269: /*
270: * Locking and TLB invalidation
271: */
272:
273: /*
274: * Locking Protocols:
275: *
276: * There are two structures in the pmap module that need locking:
277: * the pmaps themselves, and the per-page pv_lists (which are locked
278: * by locking the pv_lock_table entry that corresponds to the pv_head
279: * for the list in question.) Most routines want to lock a pmap and
280: * then do operations in it that require pv_list locking -- however
281: * pmap_remove_all and pmap_copy_on_write operate on a physical page
282: * basis and want to do the locking in the reverse order, i.e. lock
283: * a pv_list and then go through all the pmaps referenced by that list.
284: * To protect against deadlock between these two cases, the pmap_lock
285: * is used. There are three different locking protocols as a result:
286: *
287: * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
288: * the pmap.
289: *
290: * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
291: * lock on the pmap_lock (shared read), then lock the pmap
292: * and finally the pv_lists as needed [i.e. pmap lock before
293: * pv_list lock.]
294: *
295: * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
296: * Get a write lock on the pmap_lock (exclusive write); this
297: * also guaranteees exclusive access to the pv_lists. Lock the
298: * pmaps as needed.
299: *
300: * At no time may any routine hold more than one pmap lock or more than
301: * one pv_list lock. Because interrupt level routines can allocate
302: * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
303: * kernel_pmap can only be held at splhigh.
304: */
305:
306: #if NCPUS > 1
307: /*
308: * We raise the interrupt level to splhigh, to block interprocessor
309: * interrupts during pmap operations. We must take the CPU out of
310: * the cpus_active set while interrupts are blocked.
311: */
312: #define SPLVM(spl) { \
313: spl = splhigh(); \
314: mp_disable_preemption(); \
315: i_bit_clear(cpu_number(), &cpus_active); \
316: mp_enable_preemption(); \
317: }
318:
319: #define SPLX(spl) { \
320: mp_disable_preemption(); \
321: i_bit_set(cpu_number(), &cpus_active); \
322: mp_enable_preemption(); \
323: splx(spl); \
324: }
325:
326: /*
327: * Lock on pmap system
328: */
329: lock_t pmap_system_lock;
330:
331: #define PMAP_READ_LOCK(pmap, spl) { \
332: SPLVM(spl); \
333: lock_read(&pmap_system_lock); \
334: simple_lock(&(pmap)->lock); \
335: }
336:
337: #define PMAP_WRITE_LOCK(spl) { \
338: SPLVM(spl); \
339: lock_write(&pmap_system_lock); \
340: }
341:
342: #define PMAP_READ_UNLOCK(pmap, spl) { \
343: simple_unlock(&(pmap)->lock); \
344: lock_read_done(&pmap_system_lock); \
345: SPLX(spl); \
346: }
347:
348: #define PMAP_WRITE_UNLOCK(spl) { \
349: lock_write_done(&pmap_system_lock); \
350: SPLX(spl); \
351: }
352:
353: #define PMAP_WRITE_TO_READ_LOCK(pmap) { \
354: simple_lock(&(pmap)->lock); \
355: lock_write_to_read(&pmap_system_lock); \
356: }
357:
358: #define LOCK_PVH(index) lock_pvh_pai(index)
359:
360: #define UNLOCK_PVH(index) unlock_pvh_pai(index)
361:
362: #define PMAP_FLUSH_TLBS() \
363: { \
364: flush_tlb(); \
365: i386_signal_cpus(MP_TLB_FLUSH); \
366: }
367:
368: #define PMAP_RELOAD_TLBS() { \
369: i386_signal_cpus(MP_TLB_RELOAD); \
370: set_cr3(kernel_pmap->pdirbase); \
371: }
372:
373: #define PMAP_INVALIDATE_PAGE(map, addr) { \
374: if (map == kernel_pmap) \
375: invlpg((vm_offset_t) addr); \
376: else \
377: flush_tlb(); \
378: i386_signal_cpus(MP_TLB_FLUSH); \
379: }
380:
381: #else /* NCPUS > 1 */
382:
383: #if MACH_RT
384: #define SPLVM(spl) { (spl) = splhigh(); }
385: #define SPLX(spl) splx (spl)
386: #else /* MACH_RT */
387: #define SPLVM(spl)
388: #define SPLX(spl)
389: #endif /* MACH_RT */
390:
391: #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl)
392: #define PMAP_WRITE_LOCK(spl) SPLVM(spl)
393: #define PMAP_READ_UNLOCK(pmap, spl) SPLX(spl)
394: #define PMAP_WRITE_UNLOCK(spl) SPLX(spl)
395: #define PMAP_WRITE_TO_READ_LOCK(pmap)
396:
397: #if MACH_RT
398: #define LOCK_PVH(index) disable_preemption()
399: #define UNLOCK_PVH(index) enable_preemption()
400: #else /* MACH_RT */
401: #define LOCK_PVH(index)
402: #define UNLOCK_PVH(index)
403: #endif /* MACH_RT */
404:
405: #define PMAP_FLUSH_TLBS() flush_tlb()
406: #define PMAP_RELOAD_TLBS() set_cr3(kernel_pmap->pdirbase)
407: #define PMAP_INVALIDATE_PAGE(map, addr) { \
408: if (map == kernel_pmap) \
409: invlpg((vm_offset_t) addr); \
410: else \
411: flush_tlb(); \
412: }
413:
414: #endif /* NCPUS > 1 */
415:
416: #define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */
417:
418: #if NCPUS > 1
419: /*
420: * Structures to keep track of pending TLB invalidations
421: */
422: cpu_set cpus_active;
423: cpu_set cpus_idle;
424: volatile boolean_t cpu_update_needed[NCPUS];
425:
426:
427: #endif /* NCPUS > 1 */
428:
429: /*
430: * Other useful macros.
431: */
432: #define current_pmap() (vm_map_pmap(current_act()->map))
433: #define pmap_in_use(pmap, cpu) (((pmap)->cpus_using & (1 << (cpu))) != 0)
434:
435: struct pmap kernel_pmap_store;
436: pmap_t kernel_pmap;
437:
438: struct zone *pmap_zone; /* zone of pmap structures */
439:
440: int pmap_debug = 0; /* flag for debugging prints */
441: int ptes_per_vm_page; /* number of hardware ptes needed
442: to map one VM page. */
443: unsigned int inuse_ptepages_count = 0; /* debugging */
444:
445: /*
446: * Pmap cache. Cache is threaded through ref_count field of pmap.
447: * Max will eventually be constant -- variable for experimentation.
448: */
449: int pmap_cache_max = 32;
450: int pmap_alloc_chunk = 8;
451: pmap_t pmap_cache_list;
452: int pmap_cache_count;
453: decl_simple_lock_data(,pmap_cache_lock)
454:
455: extern vm_offset_t hole_start, hole_end;
456:
457: extern char end;
458:
459: /*
460: * Page directory for kernel.
461: */
462: pt_entry_t *kpde = 0; /* set by start.s - keep out of bss */
463:
464: #if DEBUG_ALIAS
465: #define PMAP_ALIAS_MAX 32
466: struct pmap_alias {
467: vm_offset_t rpc;
468: pmap_t pmap;
469: vm_offset_t va;
470: int cookie;
471: #define PMAP_ALIAS_COOKIE 0xdeadbeef
472: } pmap_aliasbuf[PMAP_ALIAS_MAX];
473: int pmap_alias_index = 0;
474: extern vm_offset_t get_rpc();
475:
476: #endif /* DEBUG_ALIAS */
477:
478: /*
479: * Given an offset and a map, compute the address of the
480: * pte. If the address is invalid with respect to the map
481: * then PT_ENTRY_NULL is returned (and the map may need to grow).
482: *
483: * This is only used in machine-dependent code.
484: */
485:
486: pt_entry_t *
487: pmap_pte(
488: register pmap_t pmap,
489: register vm_offset_t addr)
490: {
491: register pt_entry_t *ptp;
492: register pt_entry_t pte;
493:
494: pte = pmap->dirbase[pdenum(pmap, addr)];
495: if ((pte & INTEL_PTE_VALID) == 0)
496: return(PT_ENTRY_NULL);
497: ptp = (pt_entry_t *)ptetokv(pte);
498: return(&ptp[ptenum(addr)]);
499:
500: }
501:
502: #define pmap_pde(pmap, addr) (&(pmap)->dirbase[pdenum(pmap, addr)])
503:
504: #define DEBUG_PTE_PAGE 0
505:
506: #if DEBUG_PTE_PAGE
507: void
508: ptep_check(
509: ptep_t ptep)
510: {
511: register pt_entry_t *pte, *epte;
512: int ctu, ctw;
513:
514: /* check the use and wired counts */
515: if (ptep == PTE_PAGE_NULL)
516: return;
517: pte = pmap_pte(ptep->pmap, ptep->va);
518: epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
519: ctu = 0;
520: ctw = 0;
521: while (pte < epte) {
522: if (pte->pfn != 0) {
523: ctu++;
524: if (pte->wired)
525: ctw++;
526: }
527: pte += ptes_per_vm_page;
528: }
529:
530: if (ctu != ptep->use_count || ctw != ptep->wired_count) {
531: printf("use %d wired %d - actual use %d wired %d\n",
532: ptep->use_count, ptep->wired_count, ctu, ctw);
533: panic("pte count");
534: }
535: }
536: #endif /* DEBUG_PTE_PAGE */
537:
538: /*
539: * Map memory at initialization. The physical addresses being
540: * mapped are not managed and are never unmapped.
541: *
542: * For now, VM is already on, we only need to map the
543: * specified memory.
544: */
545: vm_offset_t
546: pmap_map(
547: register vm_offset_t virt,
548: register vm_offset_t start,
549: register vm_offset_t end,
550: register vm_prot_t prot)
551: {
552: register int ps;
553:
554: ps = PAGE_SIZE;
555: while (start < end) {
556: pmap_enter(kernel_pmap, virt, start, prot, FALSE);
557: virt += ps;
558: start += ps;
559: }
560: return(virt);
561: }
562:
563: /*
564: * Back-door routine for mapping kernel VM at initialization.
565: * Useful for mapping memory outside the range
566: * Sets no-cache, A, D.
567: * [vm_first_phys, vm_last_phys) (i.e., devices).
568: * Otherwise like pmap_map.
569: */
570: vm_offset_t
571: pmap_map_bd(
572: register vm_offset_t virt,
573: register vm_offset_t start,
574: register vm_offset_t end,
575: vm_prot_t prot)
576: {
577: register pt_entry_t template;
578: register pt_entry_t *pte;
579:
580: template = pa_to_pte(start)
581: | INTEL_PTE_NCACHE
582: | INTEL_PTE_REF
583: | INTEL_PTE_MOD
584: | INTEL_PTE_WIRED
585: | INTEL_PTE_VALID;
586: if (prot & VM_PROT_WRITE)
587: template |= INTEL_PTE_WRITE;
588:
589: while (start < end) {
590: pte = pmap_pte(kernel_pmap, virt);
591: if (pte == PT_ENTRY_NULL)
592: panic("pmap_map_bd: Invalid kernel address\n");
593: WRITE_PTE_FAST(pte, template)
594: pte_increment_pa(template);
595: virt += PAGE_SIZE;
596: start += PAGE_SIZE;
597: }
598:
599: PMAP_FLUSH_TLBS();
600:
601: return(virt);
602: }
603:
604: extern int cnvmem;
605: extern char *first_avail;
606: extern vm_offset_t virtual_avail, virtual_end;
607: extern vm_offset_t avail_start, avail_end, avail_next;
608:
609: /*
610: * Bootstrap the system enough to run with virtual memory.
611: * Map the kernel's code and data, and allocate the system page table.
612: * Called with mapping OFF. Page_size must already be set.
613: *
614: * Parameters:
615: * load_start: PA where kernel was loaded
616: * avail_start PA of first available physical page -
617: * after kernel page tables
618: * avail_end PA of last available physical page
619: * virtual_avail VA of first available page -
620: * after kernel page tables
621: * virtual_end VA of last available page -
622: * end of kernel address space
623: *
624: * &start_text start of kernel text
625: * &etext end of kernel text
626: */
627:
628: void
629: pmap_bootstrap(
630: vm_offset_t load_start)
631: {
632: vm_offset_t va, tva, paddr;
633: pt_entry_t template;
634: pt_entry_t *pde, *pte, *ptend;
635: vm_size_t morevm; /* VM space for kernel map */
636:
637: /*
638: * Set ptes_per_vm_page for general use.
639: */
640: ptes_per_vm_page = PAGE_SIZE / INTEL_PGBYTES;
641:
642: /*
643: * The kernel's pmap is statically allocated so we don't
644: * have to use pmap_create, which is unlikely to work
645: * correctly at this part of the boot sequence.
646: */
647:
648: kernel_pmap = &kernel_pmap_store;
649:
650: #if NCPUS > 1
651: lock_init(&pmap_system_lock,
652: FALSE, /* NOT a sleep lock */
653: ETAP_VM_PMAP_SYS,
654: ETAP_VM_PMAP_SYS_I);
655: #endif /* NCPUS > 1 */
656:
657: simple_lock_init(&kernel_pmap->lock, ETAP_VM_PMAP_KERNEL);
658: simple_lock_init(&pv_free_list_lock, ETAP_VM_PMAP_FREE);
659:
660: kernel_pmap->ref_count = 1;
661:
662: /*
663: * The kernel page directory has been allocated;
664: * its virtual address is in kpde.
665: *
666: * Enough kernel page table pages have been allocated
667: * to map low system memory, kernel text, kernel data/bss,
668: * kdb's symbols, and the page directory and page tables.
669: *
670: * No other physical memory has been allocated.
671: */
672:
673: /*
674: * Start mapping virtual memory to physical memory, 1-1,
675: * at end of mapped memory.
676: */
677:
678: virtual_avail = phystokv(avail_start);
679: virtual_end = phystokv(avail_end);
680:
681: pde = kpde;
682: pde += pdenum(kernel_pmap, virtual_avail);
683:
684: if (pte_to_pa(*pde) == 0) {
685: /* This pte has not been allocated */
686: pte = 0; ptend = 0;
687: }
688: else {
689: pte = (pt_entry_t *)ptetokv(*pde);
690: /* first pte of page */
691: ptend = pte+NPTES; /* last pte of page */
692: pte += ptenum(virtual_avail); /* point to pte that
693: maps first avail VA */
694: pde++; /* point pde to first empty slot */
695: }
696:
697: template = pa_to_pte(avail_start)
698: | INTEL_PTE_VALID
699: | INTEL_PTE_WRITE;
700:
701: for (va = virtual_avail; va < virtual_end; va += INTEL_PGBYTES) {
702: if (pte >= ptend) {
703: pte = (pt_entry_t *)phystokv(virtual_avail);
704: ptend = pte + NPTES;
705: virtual_avail = (vm_offset_t)ptend;
706: *pde = PA_TO_PTE((vm_offset_t) pte)
707: | INTEL_PTE_VALID
708: | INTEL_PTE_WRITE;
709: pde++;
710: }
711: WRITE_PTE_FAST(pte, template)
712: pte++;
713: pte_increment_pa(template);
714: }
715:
716: avail_start = virtual_avail - VM_MIN_KERNEL_ADDRESS;
717: avail_next = avail_start;
718:
719: /*
720: * Figure out maximum kernel address.
721: * Kernel virtual space is:
722: * - at least three times physical memory
723: * - at least VM_MIN_KERNEL_ADDRESS
724: * - limited by VM_MAX_KERNEL_ADDRESS
725: */
726:
727: morevm = 3*avail_end;
728: if (virtual_end + morevm > VM_MAX_KERNEL_ADDRESS)
729: morevm = virtual_end - VM_MAX_KERNEL_ADDRESS;
730:
731: /*
732: * startup requires additional virtual memory (for tables, buffers,
733: * etc.). The kd driver may also require some of that memory to
734: * access the graphics board.
735: *
736: */
737: *(int *)&template = 0;
738:
739: /*
740: * Leave room for kernel-loaded servers, which have been linked at
741: * addresses from VM_MIN_KERNEL_LOADED_ADDRESS to
742: * VM_MAX_KERNEL_LOADED_ADDRESS.
743: */
744: if (virtual_end + morevm < VM_MAX_KERNEL_LOADED_ADDRESS + 1)
745: morevm = VM_MAX_KERNEL_LOADED_ADDRESS + 1 - virtual_end;
746:
747:
748: virtual_end += morevm;
749: for (tva = va; tva < virtual_end; tva += INTEL_PGBYTES) {
750: if (pte >= ptend) {
751: pmap_next_page(&paddr);
752: pte = (pt_entry_t *)phystokv(paddr);
753: ptend = pte + NPTES;
754: *pde = PA_TO_PTE((vm_offset_t) pte)
755: | INTEL_PTE_VALID
756: | INTEL_PTE_WRITE;
757: pde++;
758: }
759: WRITE_PTE_FAST(pte, template)
760: pte++;
761: }
762:
763: virtual_avail = va;
764:
765: /* Push the virtual avail address above hole_end */
766: if (virtual_avail < hole_end)
767: virtual_avail = hole_end;
768:
769: /*
770: * c.f. comment above
771: *
772: */
773: virtual_end = va + morevm;
774: while (pte < ptend)
775: *pte++ = 0;
776:
777: /*
778: * invalidate user virtual addresses
779: */
780: memset((char *)kpde,
781: 0,
782: pdenum(kernel_pmap,VM_MIN_KERNEL_ADDRESS)*sizeof(pt_entry_t));
783: kernel_pmap->dirbase = kpde;
784: printf("Kernel virtual space from 0x%x to 0x%x.\n",
785: VM_MIN_KERNEL_ADDRESS, virtual_end);
786:
787: avail_start = avail_next;
788: printf("Available physical space from 0x%x to 0x%x\n",
789: avail_start, avail_end);
790:
791: kernel_pmap->pdirbase = kvtophys((vm_offset_t)kernel_pmap->dirbase);
792:
793: }
794:
795: void
796: pmap_virtual_space(
797: vm_offset_t *startp,
798: vm_offset_t *endp)
799: {
800: *startp = virtual_avail;
801: *endp = virtual_end;
802: }
803:
804: /*
805: * Initialize the pmap module.
806: * Called by vm_init, to initialize any structures that the pmap
807: * system needs to map virtual memory.
808: */
809: void
810: pmap_init(void)
811: {
812: register long npages;
813: vm_offset_t addr;
814: register vm_size_t s;
815: int i;
816:
817: /*
818: * Allocate memory for the pv_head_table and its lock bits,
819: * the modify bit array, and the pte_page table.
820: */
821:
822: npages = atop(avail_end - avail_start);
823: s = (vm_size_t) (sizeof(struct pv_entry) * npages
824: + pv_lock_table_size(npages)
825: + npages);
826:
827: s = round_page(s);
828: if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
829: panic("pmap_init");
830:
831: memset((char *)addr, 0, s);
832:
833: /*
834: * Allocate the structures first to preserve word-alignment.
835: */
836: pv_head_table = (pv_entry_t) addr;
837: addr = (vm_offset_t) (pv_head_table + npages);
838:
839: pv_lock_table = (char *) addr;
840: addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
841:
842: pmap_phys_attributes = (char *) addr;
843:
844: /*
845: * Create the zone of physical maps,
846: * and of the physical-to-virtual entries.
847: */
848: s = (vm_size_t) sizeof(struct pmap);
849: pmap_zone = zinit(s, 400*s, 4096, "pmap"); /* XXX */
850: s = (vm_size_t) sizeof(struct pv_entry);
851: pv_list_zone = zinit(s, 10000*s, 4096, "pv_list"); /* XXX */
852:
853: /*
854: * Only now, when all of the data structures are allocated,
855: * can we set vm_first_phys and vm_last_phys. If we set them
856: * too soon, the kmem_alloc_wired above will try to use these
857: * data structures and blow up.
858: */
859:
860: vm_first_phys = avail_start;
861: vm_last_phys = avail_end;
862: pmap_initialized = TRUE;
863:
864: /*
865: * Initializie pmap cache.
866: */
867: pmap_cache_list = PMAP_NULL;
868: pmap_cache_count = 0;
869: simple_lock_init(&pmap_cache_lock, ETAP_VM_PMAP_CACHE);
870: }
871:
872:
873: #define pmap_valid_page(x) ((avail_start <= x) && (x < avail_end))
874:
875:
876: #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
877:
878: boolean_t
879: pmap_verify_free(
880: vm_offset_t phys)
881: {
882: pv_entry_t pv_h;
883: int pai;
884: spl_t spl;
885: boolean_t result;
886:
887: assert(phys != vm_page_fictitious_addr);
888: if (!pmap_initialized)
889: return(TRUE);
890:
891: if (!pmap_valid_page(phys))
892: return(FALSE);
893:
894: PMAP_WRITE_LOCK(spl);
895:
896: pai = pa_index(phys);
897: pv_h = pai_to_pvh(pai);
898:
899: result = (pv_h->pmap == PMAP_NULL);
900: PMAP_WRITE_UNLOCK(spl);
901:
902: return(result);
903: }
904:
905: /*
906: * Create and return a physical map.
907: *
908: * If the size specified for the map
909: * is zero, the map is an actual physical
910: * map, and may be referenced by the
911: * hardware.
912: *
913: * If the size specified is non-zero,
914: * the map will be used in software only, and
915: * is bounded by that size.
916: */
917: pmap_t
918: pmap_create(
919: vm_size_t size)
920: {
921: register pmap_t p;
922: register pmap_statistics_t stats;
923:
924: /*
925: * A software use-only map doesn't even need a map.
926: */
927:
928: if (size != 0) {
929: return(PMAP_NULL);
930: }
931:
932: /*
933: * Try to get cached pmap, if this fails,
934: * allocate a pmap struct from the pmap_zone. Then allocate
935: * the page descriptor table from the pd_zone.
936: */
937:
938: simple_lock(&pmap_cache_lock);
939: while ((p = pmap_cache_list) == PMAP_NULL) {
940:
941: vm_offset_t dirbases;
942: register int i;
943:
944: simple_unlock(&pmap_cache_lock);
945:
946: #if NCPUS > 1
947: /*
948: * XXX NEEDS MP DOING ALLOC logic so that if multiple processors
949: * XXX get here, only one allocates a chunk of pmaps.
950: * (for now we'll just let it go - safe but wasteful)
951: */
952: #endif
953:
954: /*
955: * Allocate a chunck of pmaps. Single kmem_alloc_wired
956: * operation reduces kernel map fragmentation.
957: */
958:
959: if (kmem_alloc_wired(kernel_map, &dirbases,
960: pmap_alloc_chunk * INTEL_PGBYTES)
961: != KERN_SUCCESS)
962: panic("pmap_create.1");
963:
964: for (i = pmap_alloc_chunk; i > 0 ; i--) {
965: p = (pmap_t) zalloc(pmap_zone);
966: if (p == PMAP_NULL)
967: panic("pmap_create.2");
968:
969: /*
970: * Initialize pmap. Don't bother with
971: * ref count as cache list is threaded
972: * through it. It'll be set on cache removal.
973: */
974: p->dirbase = (pt_entry_t *) dirbases;
975: dirbases += INTEL_PGBYTES;
976: memcpy(p->dirbase, kpde, INTEL_PGBYTES);
977: p->pdirbase = kvtophys((vm_offset_t)p->dirbase);
978:
979: simple_lock_init(&p->lock, ETAP_VM_PMAP);
980: p->cpus_using = 0;
981:
982: /*
983: * Initialize statistics.
984: */
985: stats = &p->stats;
986: stats->resident_count = 0;
987: stats->wired_count = 0;
988:
989: /*
990: * Insert into cache
991: */
992: simple_lock(&pmap_cache_lock);
993: p->ref_count = (int) pmap_cache_list;
994: pmap_cache_list = p;
995: pmap_cache_count++;
996: simple_unlock(&pmap_cache_lock);
997: }
998: simple_lock(&pmap_cache_lock);
999: }
1000:
1001: assert(p->stats.resident_count == 0);
1002: assert(p->stats.wired_count == 0);
1003: p->stats.resident_count = 0;
1004: p->stats.wired_count = 0;
1005:
1006: pmap_cache_list = (pmap_t) p->ref_count;
1007: p->ref_count = 1;
1008: pmap_cache_count--;
1009: simple_unlock(&pmap_cache_lock);
1010:
1011: return(p);
1012: }
1013:
1014: /*
1015: * Retire the given physical map from service.
1016: * Should only be called if the map contains
1017: * no valid mappings.
1018: */
1019:
1020: void
1021: pmap_destroy(
1022: register pmap_t p)
1023: {
1024: register pt_entry_t *pdep;
1025: register vm_offset_t pa;
1026: register int c;
1027: spl_t s;
1028: register vm_page_t m;
1029:
1030: if (p == PMAP_NULL)
1031: return;
1032:
1033: SPLVM(s);
1034: simple_lock(&p->lock);
1035: c = --p->ref_count;
1036: if (c == 0) {
1037: register int my_cpu;
1038:
1039: mp_disable_preemption();
1040: my_cpu = cpu_number();
1041:
1042: /*
1043: * If some cpu is not using the physical pmap pointer that it
1044: * is supposed to be (see set_dirbase), we might be using the
1045: * pmap that is being destroyed! Make sure we are
1046: * physically on the right pmap:
1047: */
1048:
1049:
1050: if (real_pmap[my_cpu] == p) {
1051: PMAP_CPU_CLR(p, my_cpu);
1052: real_pmap[my_cpu] = kernel_pmap;
1053: PMAP_RELOAD_TLBS();
1054: }
1055: mp_enable_preemption();
1056: }
1057: simple_unlock(&p->lock);
1058: SPLX(s);
1059:
1060: if (c != 0) {
1061: return; /* still in use */
1062: }
1063:
1064: /*
1065: * Free the memory maps, then the
1066: * pmap structure.
1067: */
1068: pdep = p->dirbase;
1069: while (pdep < &p->dirbase[pdenum(p, LINEAR_KERNEL_ADDRESS)]) {
1070: if (*pdep & INTEL_PTE_VALID) {
1071: pa = pte_to_pa(*pdep);
1072: vm_object_lock(pmap_object);
1073: m = vm_page_lookup(pmap_object, pa);
1074: if (m == VM_PAGE_NULL)
1075: panic("pmap_destroy: pte page not in object");
1076: vm_page_lock_queues();
1077: vm_page_free(m);
1078: inuse_ptepages_count--;
1079: vm_object_unlock(pmap_object);
1080: vm_page_unlock_queues();
1081:
1082: /*
1083: * Clear pdes, this might be headed for the cache.
1084: */
1085: c = ptes_per_vm_page;
1086: do {
1087: *pdep = 0;
1088: pdep++;
1089: } while (--c > 0);
1090: }
1091: else {
1092: pdep += ptes_per_vm_page;
1093: }
1094:
1095: }
1096: assert(p->stats.resident_count == 0);
1097: assert(p->stats.wired_count == 0);
1098:
1099: /*
1100: * Add to cache if not already full
1101: */
1102: simple_lock(&pmap_cache_lock);
1103: if (pmap_cache_count <= pmap_cache_max) {
1104: p->ref_count = (int) pmap_cache_list;
1105: pmap_cache_list = p;
1106: pmap_cache_count++;
1107: simple_unlock(&pmap_cache_lock);
1108: }
1109: else {
1110: simple_unlock(&pmap_cache_lock);
1111: kmem_free(kernel_map, (vm_offset_t)p->dirbase, INTEL_PGBYTES);
1112: zfree(pmap_zone, (vm_offset_t) p);
1113: }
1114: }
1115:
1116: /*
1117: * Add a reference to the specified pmap.
1118: */
1119:
1120: void
1121: pmap_reference(
1122: register pmap_t p)
1123: {
1124: spl_t s;
1125:
1126: if (p != PMAP_NULL) {
1127: SPLVM(s);
1128: simple_lock(&p->lock);
1129: p->ref_count++;
1130: simple_unlock(&p->lock);
1131: SPLX(s);
1132: }
1133: }
1134:
1135: /*
1136: * Remove a range of hardware page-table entries.
1137: * The entries given are the first (inclusive)
1138: * and last (exclusive) entries for the VM pages.
1139: * The virtual address is the va for the first pte.
1140: *
1141: * The pmap must be locked.
1142: * If the pmap is not the kernel pmap, the range must lie
1143: * entirely within one pte-page. This is NOT checked.
1144: * Assumes that the pte-page exists.
1145: */
1146:
1147: /* static */
1148: void
1149: pmap_remove_range(
1150: pmap_t pmap,
1151: vm_offset_t va,
1152: pt_entry_t *spte,
1153: pt_entry_t *epte)
1154: {
1155: register pt_entry_t *cpte;
1156: int num_removed, num_unwired;
1157: int pai;
1158: vm_offset_t pa;
1159:
1160: #if DEBUG_PTE_PAGE
1161: if (pmap != kernel_pmap)
1162: ptep_check(get_pte_page(spte));
1163: #endif /* DEBUG_PTE_PAGE */
1164: num_removed = 0;
1165: num_unwired = 0;
1166:
1167: for (cpte = spte; cpte < epte;
1168: cpte += ptes_per_vm_page, va += PAGE_SIZE) {
1169:
1170: pa = pte_to_pa(*cpte);
1171: if (pa == 0)
1172: continue;
1173:
1174: num_removed++;
1175: if (iswired(*cpte))
1176: num_unwired++;
1177:
1178: if (!valid_page(pa)) {
1179:
1180: /*
1181: * Outside range of managed physical memory.
1182: * Just remove the mappings.
1183: */
1184: register int i = ptes_per_vm_page;
1185: register pt_entry_t *lpte = cpte;
1186: do {
1187: *lpte = 0;
1188: lpte++;
1189: } while (--i > 0);
1190: continue;
1191: }
1192:
1193: pai = pa_index(pa);
1194: LOCK_PVH(pai);
1195:
1196: /*
1197: * Get the modify and reference bits.
1198: */
1199: {
1200: register int i;
1201: register pt_entry_t *lpte;
1202:
1203: i = ptes_per_vm_page;
1204: lpte = cpte;
1205: do {
1206: pmap_phys_attributes[pai] |=
1207: *lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
1208: *lpte = 0;
1209: lpte++;
1210: } while (--i > 0);
1211: }
1212:
1213: /*
1214: * Remove the mapping from the pvlist for
1215: * this physical page.
1216: */
1217: {
1218: register pv_entry_t pv_h, prev, cur;
1219:
1220: pv_h = pai_to_pvh(pai);
1221: if (pv_h->pmap == PMAP_NULL) {
1222: panic("pmap_remove: null pv_list!");
1223: }
1224: if (pv_h->va == va && pv_h->pmap == pmap) {
1225: /*
1226: * Header is the pv_entry. Copy the next one
1227: * to header and free the next one (we cannot
1228: * free the header)
1229: */
1230: cur = pv_h->next;
1231: if (cur != PV_ENTRY_NULL) {
1232: *pv_h = *cur;
1233: PV_FREE(cur);
1234: }
1235: else {
1236: pv_h->pmap = PMAP_NULL;
1237: }
1238: }
1239: else {
1240: cur = pv_h;
1241: do {
1242: prev = cur;
1243: if ((cur = prev->next) == PV_ENTRY_NULL) {
1244: panic("pmap-remove: mapping not in pv_list!");
1245: }
1246: } while (cur->va != va || cur->pmap != pmap);
1247: prev->next = cur->next;
1248: PV_FREE(cur);
1249: }
1250: UNLOCK_PVH(pai);
1251: }
1252: }
1253:
1254: /*
1255: * Update the counts
1256: */
1257: assert(pmap->stats.resident_count >= num_removed);
1258: pmap->stats.resident_count -= num_removed;
1259: assert(pmap->stats.wired_count >= num_unwired);
1260: pmap->stats.wired_count -= num_unwired;
1261: }
1262:
1263: /*
1264: * Remove the given range of addresses
1265: * from the specified map.
1266: *
1267: * It is assumed that the start and end are properly
1268: * rounded to the hardware page size.
1269: */
1270:
1271: void
1272: pmap_remove(
1273: pmap_t map,
1274: vm_offset_t s,
1275: vm_offset_t e)
1276: {
1277: spl_t spl;
1278: register pt_entry_t *pde;
1279: register pt_entry_t *spte, *epte;
1280: vm_offset_t l;
1281:
1282: if (map == PMAP_NULL)
1283: return;
1284:
1285: PMAP_READ_LOCK(map, spl);
1286:
1287: pde = pmap_pde(map, s);
1288:
1289: while (s < e) {
1290: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1291: if (l > e)
1292: l = e;
1293: if (*pde & INTEL_PTE_VALID) {
1294: spte = (pt_entry_t *)ptetokv(*pde);
1295: spte = &spte[ptenum(s)];
1296: epte = &spte[intel_btop(l-s)];
1297: pmap_remove_range(map, s, spte, epte);
1298: }
1299: s = l;
1300: pde++;
1301: }
1302:
1303: PMAP_FLUSH_TLBS();
1304:
1305: PMAP_READ_UNLOCK(map, spl);
1306: }
1307:
1308: /*
1309: * Routine: pmap_page_protect
1310: *
1311: * Function:
1312: * Lower the permission for all mappings to a given
1313: * page.
1314: */
1315: void
1316: pmap_page_protect(
1317: vm_offset_t phys,
1318: vm_prot_t prot)
1319: {
1320: pv_entry_t pv_h, prev;
1321: register pv_entry_t pv_e;
1322: register pt_entry_t *pte;
1323: int pai;
1324: register pmap_t pmap;
1325: spl_t spl;
1326: boolean_t remove;
1327:
1328: assert(phys != vm_page_fictitious_addr);
1329: if (!valid_page(phys)) {
1330: /*
1331: * Not a managed page.
1332: */
1333: return;
1334: }
1335:
1336: /*
1337: * Determine the new protection.
1338: */
1339: switch (prot) {
1340: case VM_PROT_READ:
1341: case VM_PROT_READ|VM_PROT_EXECUTE:
1342: remove = FALSE;
1343: break;
1344: case VM_PROT_ALL:
1345: return; /* nothing to do */
1346: default:
1347: remove = TRUE;
1348: break;
1349: }
1350:
1351: /*
1352: * Lock the pmap system first, since we will be changing
1353: * several pmaps.
1354: */
1355:
1356: PMAP_WRITE_LOCK(spl);
1357:
1358: pai = pa_index(phys);
1359: pv_h = pai_to_pvh(pai);
1360:
1361: /*
1362: * Walk down PV list, changing or removing all mappings.
1363: * We do not have to lock the pv_list because we have
1364: * the entire pmap system locked.
1365: */
1366: if (pv_h->pmap != PMAP_NULL) {
1367:
1368: prev = pv_e = pv_h;
1369: do {
1370: pmap = pv_e->pmap;
1371: /*
1372: * Lock the pmap to block pmap_extract and similar routines.
1373: */
1374: simple_lock(&pmap->lock);
1375:
1376: {
1377: register vm_offset_t va;
1378:
1379: va = pv_e->va;
1380: pte = pmap_pte(pmap, va);
1381:
1382: /*
1383: * Consistency checks.
1384: */
1385: /* assert(*pte & INTEL_PTE_VALID); XXX */
1386: /* assert(pte_to_phys(*pte) == phys); */
1387:
1388: /*
1389: * Invalidate TLBs for all CPUs using this mapping.
1390: */
1391: PMAP_INVALIDATE_PAGE(pmap, va);
1392: }
1393:
1394: /*
1395: * Remove the mapping if new protection is NONE
1396: * or if write-protecting a kernel mapping.
1397: */
1398: if (remove || pmap == kernel_pmap) {
1399: /*
1400: * Remove the mapping, collecting any modify bits.
1401: */
1402: if (iswired(*pte))
1403: panic("pmap_remove_all removing a wired page");
1404:
1405: {
1406: register int i = ptes_per_vm_page;
1407:
1408: do {
1409: pmap_phys_attributes[pai] |=
1410: *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1411: *pte++ = 0;
1412: } while (--i > 0);
1413: }
1414:
1415: assert(pmap->stats.resident_count >= 1);
1416: pmap->stats.resident_count--;
1417:
1418: /*
1419: * Remove the pv_entry.
1420: */
1421: if (pv_e == pv_h) {
1422: /*
1423: * Fix up head later.
1424: */
1425: pv_h->pmap = PMAP_NULL;
1426: }
1427: else {
1428: /*
1429: * Delete this entry.
1430: */
1431: prev->next = pv_e->next;
1432: PV_FREE(pv_e);
1433: }
1434: }
1435: else {
1436: /*
1437: * Write-protect.
1438: */
1439: register int i = ptes_per_vm_page;
1440:
1441: do {
1442: *pte &= ~INTEL_PTE_WRITE;
1443: pte++;
1444: } while (--i > 0);
1445:
1446: /*
1447: * Advance prev.
1448: */
1449: prev = pv_e;
1450: }
1451:
1452: simple_unlock(&pmap->lock);
1453:
1454: } while ((pv_e = prev->next) != PV_ENTRY_NULL);
1455:
1456: /*
1457: * If pv_head mapping was removed, fix it up.
1458: */
1459: if (pv_h->pmap == PMAP_NULL) {
1460: pv_e = pv_h->next;
1461: if (pv_e != PV_ENTRY_NULL) {
1462: *pv_h = *pv_e;
1463: PV_FREE(pv_e);
1464: }
1465: }
1466: }
1467:
1468: PMAP_WRITE_UNLOCK(spl);
1469: }
1470:
1471: /*
1472: * Set the physical protection on the
1473: * specified range of this map as requested.
1474: * Will not increase permissions.
1475: */
1476: void
1477: pmap_protect(
1478: pmap_t map,
1479: vm_offset_t s,
1480: vm_offset_t e,
1481: vm_prot_t prot)
1482: {
1483: register pt_entry_t *pde;
1484: register pt_entry_t *spte, *epte;
1485: vm_offset_t l;
1486: spl_t spl;
1487:
1488:
1489: if (map == PMAP_NULL)
1490: return;
1491:
1492: /*
1493: * Determine the new protection.
1494: */
1495: switch (prot) {
1496: case VM_PROT_READ:
1497: case VM_PROT_READ|VM_PROT_EXECUTE:
1498: break;
1499: case VM_PROT_READ|VM_PROT_WRITE:
1500: case VM_PROT_ALL:
1501: return; /* nothing to do */
1502: default:
1503: pmap_remove(map, s, e);
1504: return;
1505: }
1506:
1507: /*
1508: * If write-protecting in the kernel pmap,
1509: * remove the mappings; the i386 ignores
1510: * the write-permission bit in kernel mode.
1511: *
1512: * XXX should be #if'd for i386
1513: */
1514:
1515: if (cpuid_family == CPUID_FAMILY_386)
1516: if (map == kernel_pmap) {
1517: pmap_remove(map, s, e);
1518: return;
1519: }
1520:
1521: SPLVM(spl);
1522: simple_lock(&map->lock);
1523:
1524:
1525: pde = pmap_pde(map, s);
1526: while (s < e) {
1527: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1528: if (l > e)
1529: l = e;
1530: if (*pde & INTEL_PTE_VALID) {
1531: spte = (pt_entry_t *)ptetokv(*pde);
1532: spte = &spte[ptenum(s)];
1533: epte = &spte[intel_btop(l-s)];
1534:
1535: while (spte < epte) {
1536: if (*spte & INTEL_PTE_VALID)
1537: *spte &= ~INTEL_PTE_WRITE;
1538: spte++;
1539: }
1540: }
1541: s = l;
1542: pde++;
1543: }
1544:
1545: PMAP_FLUSH_TLBS();
1546:
1547: simple_unlock(&map->lock);
1548: SPLX(spl);
1549: }
1550:
1551:
1552:
1553: /*
1554: * Insert the given physical page (p) at
1555: * the specified virtual address (v) in the
1556: * target physical map with the protection requested.
1557: *
1558: * If specified, the page will be wired down, meaning
1559: * that the related pte cannot be reclaimed.
1560: *
1561: * NB: This is the only routine which MAY NOT lazy-evaluate
1562: * or lose information. That is, this routine must actually
1563: * insert this page into the given map NOW.
1564: */
1565: void
1566: pmap_enter(
1567: register pmap_t pmap,
1568: vm_offset_t v,
1569: register vm_offset_t pa,
1570: vm_prot_t prot,
1571: boolean_t wired)
1572: {
1573: register pt_entry_t *pte;
1574: register pv_entry_t pv_h;
1575: register int i, pai;
1576: pv_entry_t pv_e;
1577: pt_entry_t template;
1578: spl_t spl;
1579: vm_offset_t old_pa;
1580:
1581: XPR(0x80000000, "%x/%x: pmap_enter %x/%x/%x\n",
1582: current_thread()->top_act,
1583: current_thread(),
1584: pmap, v, pa);
1585:
1586: assert(pa != vm_page_fictitious_addr);
1587: if (pmap_debug)
1588: printf("pmap(%x, %x)\n", v, pa);
1589: if (pmap == PMAP_NULL)
1590: return;
1591:
1592: if (cpuid_family == CPUID_FAMILY_386)
1593: if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0
1594: && !wired /* hack for io_wire */ ) {
1595: /*
1596: * Because the 386 ignores write protection in kernel mode,
1597: * we cannot enter a read-only kernel mapping, and must
1598: * remove an existing mapping if changing it.
1599: *
1600: * XXX should be #if'd for i386
1601: */
1602: PMAP_READ_LOCK(pmap, spl);
1603:
1604: pte = pmap_pte(pmap, v);
1605: if (pte != PT_ENTRY_NULL && pte_to_pa(*pte) != 0) {
1606: /*
1607: * Invalidate the translation buffer,
1608: * then remove the mapping.
1609: */
1610: PMAP_INVALIDATE_PAGE(pmap, v);
1611: pmap_remove_range(pmap, v, pte,
1612: pte + ptes_per_vm_page);
1613: }
1614: PMAP_READ_UNLOCK(pmap, spl);
1615: return;
1616: }
1617:
1618: /*
1619: * Must allocate a new pvlist entry while we're unlocked;
1620: * zalloc may cause pageout (which will lock the pmap system).
1621: * If we determine we need a pvlist entry, we will unlock
1622: * and allocate one. Then we will retry, throughing away
1623: * the allocated entry later (if we no longer need it).
1624: */
1625: pv_e = PV_ENTRY_NULL;
1626: Retry:
1627: PMAP_READ_LOCK(pmap, spl);
1628:
1629: /*
1630: * Expand pmap to include this pte. Assume that
1631: * pmap is always expanded to include enough hardware
1632: * pages to map one VM page.
1633: */
1634:
1635: while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
1636: /*
1637: * Must unlock to expand the pmap.
1638: */
1639: PMAP_READ_UNLOCK(pmap, spl);
1640:
1641: pmap_expand(pmap, v);
1642:
1643: PMAP_READ_LOCK(pmap, spl);
1644: }
1645: /*
1646: * Special case if the physical page is already mapped
1647: * at this address.
1648: */
1649: old_pa = pte_to_pa(*pte);
1650: if (old_pa == pa) {
1651: /*
1652: * May be changing its wired attribute or protection
1653: */
1654:
1655: template = pa_to_pte(pa) | INTEL_PTE_VALID;
1656: if (pmap != kernel_pmap)
1657: template |= INTEL_PTE_USER;
1658: if (prot & VM_PROT_WRITE)
1659: template |= INTEL_PTE_WRITE;
1660: if (wired) {
1661: template |= INTEL_PTE_WIRED;
1662: if (!iswired(*pte))
1663: pmap->stats.wired_count++;
1664: }
1665: else {
1666: if (iswired(*pte)) {
1667: assert(pmap->stats.wired_count >= 1);
1668: pmap->stats.wired_count--;
1669: }
1670: }
1671:
1672: PMAP_INVALIDATE_PAGE(pmap, v);
1673:
1674: i = ptes_per_vm_page;
1675: do {
1676: if (*pte & INTEL_PTE_MOD)
1677: template |= INTEL_PTE_MOD;
1678: WRITE_PTE(pte, template)
1679: pte++;
1680: pte_increment_pa(template);
1681: } while (--i > 0);
1682:
1683: goto Done;
1684: }
1685:
1686: /*
1687: * Outline of code from here:
1688: * 1) If va was mapped, update TLBs, remove the mapping
1689: * and remove old pvlist entry.
1690: * 2) Add pvlist entry for new mapping
1691: * 3) Enter new mapping.
1692: *
1693: * SHARING_FAULTS complicates this slightly in that it cannot
1694: * replace the mapping, but must remove it (because adding the
1695: * pvlist entry for the new mapping may remove others), and
1696: * hence always enters the new mapping at step 3)
1697: *
1698: * If the old physical page is not managed step 1) is skipped
1699: * (except for updating the TLBs), and the mapping is
1700: * overwritten at step 3). If the new physical page is not
1701: * managed, step 2) is skipped.
1702: */
1703:
1704: if (old_pa != (vm_offset_t) 0) {
1705:
1706: PMAP_INVALIDATE_PAGE(pmap, v);
1707:
1708: #if DEBUG_PTE_PAGE
1709: if (pmap != kernel_pmap)
1710: ptep_check(get_pte_page(pte));
1711: #endif /* DEBUG_PTE_PAGE */
1712:
1713: /*
1714: * Don't do anything to pages outside valid memory here.
1715: * Instead convince the code that enters a new mapping
1716: * to overwrite the old one.
1717: */
1718:
1719: if (valid_page(old_pa)) {
1720:
1721: pai = pa_index(old_pa);
1722: LOCK_PVH(pai);
1723:
1724: assert(pmap->stats.resident_count >= 1);
1725: pmap->stats.resident_count--;
1726: if (iswired(*pte)) {
1727: assert(pmap->stats.wired_count >= 1);
1728: pmap->stats.wired_count--;
1729: }
1730: i = ptes_per_vm_page;
1731: do {
1732: pmap_phys_attributes[pai] |=
1733: *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1734: WRITE_PTE(pte, 0)
1735: pte++;
1736: pte_increment_pa(template);
1737: } while (--i > 0);
1738:
1739: /*
1740: * Put pte back to beginning of page since it'll be
1741: * used later to enter the new page.
1742: */
1743: pte -= ptes_per_vm_page;
1744:
1745: /*
1746: * Remove the mapping from the pvlist for
1747: * this physical page.
1748: */
1749: {
1750: register pv_entry_t prev, cur;
1751:
1752: pv_h = pai_to_pvh(pai);
1753: if (pv_h->pmap == PMAP_NULL) {
1754: panic("pmap_enter: null pv_list!");
1755: }
1756: if (pv_h->va == v && pv_h->pmap == pmap) {
1757: /*
1758: * Header is the pv_entry. Copy the next one
1759: * to header and free the next one (we cannot
1760: * free the header)
1761: */
1762: cur = pv_h->next;
1763: if (cur != PV_ENTRY_NULL) {
1764: *pv_h = *cur;
1765: pv_e = cur;
1766: }
1767: else {
1768: pv_h->pmap = PMAP_NULL;
1769: }
1770: }
1771: else {
1772: cur = pv_h;
1773: do {
1774: prev = cur;
1775: if ((cur = prev->next) == PV_ENTRY_NULL) {
1776: panic("pmap_enter: mapping not in pv_list!");
1777: }
1778: } while (cur->va != v || cur->pmap != pmap);
1779: prev->next = cur->next;
1780: pv_e = cur;
1781: }
1782: }
1783: UNLOCK_PVH(pai);
1784: }
1785: else {
1786:
1787: /*
1788: * old_pa is not managed. Pretend it's zero so code
1789: * at Step 3) will enter new mapping (overwriting old
1790: * one). Do removal part of accounting.
1791: */
1792: old_pa = (vm_offset_t) 0;
1793: assert(pmap->stats.resident_count >= 1);
1794: pmap->stats.resident_count--;
1795: if (iswired(*pte)) {
1796: assert(pmap->stats.wired_count >= 1);
1797: pmap->stats.wired_count--;
1798: }
1799: }
1800: }
1801:
1802: if (valid_page(pa)) {
1803:
1804: /*
1805: * Step 2) Enter the mapping in the PV list for this
1806: * physical page.
1807: */
1808:
1809: pai = pa_index(pa);
1810:
1811:
1812: #if SHARING_FAULTS
1813: RetryPvList:
1814: /*
1815: * We can return here from the sharing fault code below
1816: * in case we removed the only entry on the pv list and thus
1817: * must enter the new one in the list header.
1818: */
1819: #endif /* SHARING_FAULTS */
1820: LOCK_PVH(pai);
1821: pv_h = pai_to_pvh(pai);
1822:
1823: if (pv_h->pmap == PMAP_NULL) {
1824: /*
1825: * No mappings yet
1826: */
1827: pv_h->va = v;
1828: pv_h->pmap = pmap;
1829: pv_h->next = PV_ENTRY_NULL;
1830: }
1831: else {
1832: #if DEBUG
1833: {
1834: /*
1835: * check that this mapping is not already there
1836: * or there is no alias for this mapping in the same map
1837: */
1838: pv_entry_t e = pv_h;
1839: while (e != PV_ENTRY_NULL) {
1840: if (e->pmap == pmap && e->va == v)
1841: panic("pmap_enter: already in pv_list");
1842: e = e->next;
1843: }
1844: }
1845: #endif /* DEBUG */
1846: #if SHARING_FAULTS
1847: {
1848: /*
1849: * do sharing faults.
1850: * if we find an entry on this pv list in the same address
1851: * space, remove it. we know there will not be more
1852: * than one.
1853: */
1854: pv_entry_t e = pv_h;
1855: pt_entry_t *opte;
1856:
1857: while (e != PV_ENTRY_NULL) {
1858: if (e->pmap == pmap) {
1859: /*
1860: * Remove it, drop pv list lock first.
1861: */
1862: UNLOCK_PVH(pai);
1863:
1864: opte = pmap_pte(pmap, e->va);
1865: assert(opte != PT_ENTRY_NULL);
1866: /*
1867: * Invalidate the translation buffer,
1868: * then remove the mapping.
1869: */
1870: PMAP_INVALIDATE_PAGE(pmap, e->va);
1871: pmap_remove_range(pmap, e->va, opte,
1872: opte + ptes_per_vm_page);
1873: /*
1874: * We could have remove the head entry,
1875: * so there could be no more entries
1876: * and so we have to use the pv head entry.
1877: * so, go back to the top and try the entry
1878: * again.
1879: */
1880: goto RetryPvList;
1881: }
1882: e = e->next;
1883: }
1884:
1885: /*
1886: * check that this mapping is not already there
1887: */
1888: e = pv_h;
1889: while (e != PV_ENTRY_NULL) {
1890: if (e->pmap == pmap)
1891: panic("pmap_enter: alias in pv_list");
1892: e = e->next;
1893: }
1894: }
1895: #endif /* SHARING_FAULTS */
1896: #if DEBUG_ALIAS
1897: {
1898: /*
1899: * check for aliases within the same address space.
1900: */
1901: pv_entry_t e = pv_h;
1902: vm_offset_t rpc = get_rpc();
1903:
1904: while (e != PV_ENTRY_NULL) {
1905: if (e->pmap == pmap) {
1906: /*
1907: * log this entry in the alias ring buffer
1908: * if it's not there already.
1909: */
1910: struct pmap_alias *pma;
1911: int ii, logit;
1912:
1913: logit = TRUE;
1914: for (ii = 0; ii < pmap_alias_index; ii++) {
1915: if (pmap_aliasbuf[ii].rpc == rpc) {
1916: /* found it in the log already */
1917: logit = FALSE;
1918: break;
1919: }
1920: }
1921: if (logit) {
1922: pma = &pmap_aliasbuf[pmap_alias_index];
1923: pma->pmap = pmap;
1924: pma->va = v;
1925: pma->rpc = rpc;
1926: pma->cookie = PMAP_ALIAS_COOKIE;
1927: if (++pmap_alias_index >= PMAP_ALIAS_MAX)
1928: panic("pmap_enter: exhausted alias log");
1929: }
1930: }
1931: e = e->next;
1932: }
1933: }
1934: #endif /* DEBUG_ALIAS */
1935: /*
1936: * Add new pv_entry after header.
1937: */
1938: if (pv_e == PV_ENTRY_NULL) {
1939: PV_ALLOC(pv_e);
1940: if (pv_e == PV_ENTRY_NULL) {
1941: UNLOCK_PVH(pai);
1942: PMAP_READ_UNLOCK(pmap, spl);
1943:
1944: /*
1945: * Refill from zone.
1946: */
1947: pv_e = (pv_entry_t) zalloc(pv_list_zone);
1948: goto Retry;
1949: }
1950: }
1951: pv_e->va = v;
1952: pv_e->pmap = pmap;
1953: pv_e->next = pv_h->next;
1954: pv_h->next = pv_e;
1955: /*
1956: * Remember that we used the pvlist entry.
1957: */
1958: pv_e = PV_ENTRY_NULL;
1959: }
1960: UNLOCK_PVH(pai);
1961: }
1962:
1963: /*
1964: * Step 3) Enter and count the mapping.
1965: */
1966:
1967: pmap->stats.resident_count++;
1968:
1969: /*
1970: * Build a template to speed up entering -
1971: * only the pfn changes.
1972: */
1973: template = pa_to_pte(pa) | INTEL_PTE_VALID;
1974: if (pmap != kernel_pmap)
1975: template |= INTEL_PTE_USER;
1976: if (prot & VM_PROT_WRITE)
1977: template |= INTEL_PTE_WRITE;
1978: if (wired) {
1979: template |= INTEL_PTE_WIRED;
1980: pmap->stats.wired_count++;
1981: }
1982: i = ptes_per_vm_page;
1983: do {
1984: WRITE_PTE(pte, template)
1985: pte++;
1986: pte_increment_pa(template);
1987: } while (--i > 0);
1988: Done:
1989: if (pv_e != PV_ENTRY_NULL) {
1990: PV_FREE(pv_e);
1991: }
1992:
1993: PMAP_READ_UNLOCK(pmap, spl);
1994: }
1995:
1996: /*
1997: * Routine: pmap_change_wiring
1998: * Function: Change the wiring attribute for a map/virtual-address
1999: * pair.
2000: * In/out conditions:
2001: * The mapping must already exist in the pmap.
2002: */
2003: void
2004: pmap_change_wiring(
2005: register pmap_t map,
2006: vm_offset_t v,
2007: boolean_t wired)
2008: {
2009: register pt_entry_t *pte;
2010: register int i;
2011: spl_t spl;
2012:
2013: /*
2014: * We must grab the pmap system lock because we may
2015: * change a pte_page queue.
2016: */
2017: PMAP_READ_LOCK(map, spl);
2018:
2019: if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
2020: panic("pmap_change_wiring: pte missing");
2021:
2022: if (wired && !iswired(*pte)) {
2023: /*
2024: * wiring down mapping
2025: */
2026: map->stats.wired_count++;
2027: i = ptes_per_vm_page;
2028: do {
2029: *pte++ |= INTEL_PTE_WIRED;
2030: } while (--i > 0);
2031: }
2032: else if (!wired && iswired(*pte)) {
2033: /*
2034: * unwiring mapping
2035: */
2036: assert(map->stats.wired_count >= 1);
2037: map->stats.wired_count--;
2038: i = ptes_per_vm_page;
2039: do {
2040: *pte++ &= ~INTEL_PTE_WIRED;
2041: } while (--i > 0);
2042: }
2043:
2044: PMAP_READ_UNLOCK(map, spl);
2045: }
2046:
2047: /*
2048: * Routine: pmap_extract
2049: * Function:
2050: * Extract the physical page address associated
2051: * with the given map/virtual_address pair.
2052: */
2053:
2054: vm_offset_t
2055: pmap_extract(
2056: register pmap_t pmap,
2057: vm_offset_t va)
2058: {
2059: register pt_entry_t *pte;
2060: register vm_offset_t pa;
2061: spl_t spl;
2062:
2063: SPLVM(spl);
2064: simple_lock(&pmap->lock);
2065: if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL)
2066: pa = (vm_offset_t) 0;
2067: else if (!(*pte & INTEL_PTE_VALID))
2068: pa = (vm_offset_t) 0;
2069: else
2070: pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK);
2071: simple_unlock(&pmap->lock);
2072: SPLX(spl);
2073: return(pa);
2074: }
2075:
2076: /*
2077: * Routine: pmap_expand
2078: *
2079: * Expands a pmap to be able to map the specified virtual address.
2080: *
2081: * Allocates new virtual memory for the P0 or P1 portion of the
2082: * pmap, then re-maps the physical pages that were in the old
2083: * pmap to be in the new pmap.
2084: *
2085: * Must be called with the pmap system and the pmap unlocked,
2086: * since these must be unlocked to use vm_allocate or vm_deallocate.
2087: * Thus it must be called in a loop that checks whether the map
2088: * has been expanded enough.
2089: * (We won't loop forever, since page tables aren't shrunk.)
2090: */
2091: void
2092: pmap_expand(
2093: register pmap_t map,
2094: register vm_offset_t v)
2095: {
2096: pt_entry_t *pdp;
2097: register vm_page_t m;
2098: register vm_offset_t pa;
2099: register int i;
2100: spl_t spl;
2101:
2102: if (map == kernel_pmap)
2103: panic("pmap_expand");
2104:
2105: /*
2106: * We cannot allocate the pmap_object in pmap_init,
2107: * because it is called before the zone package is up.
2108: * Allocate it now if it is missing.
2109: */
2110: if (pmap_object == VM_OBJECT_NULL)
2111: pmap_object = vm_object_allocate(avail_end);
2112:
2113: /*
2114: * Allocate a VM page for the level 2 page table entries.
2115: */
2116: while ((m = vm_page_grab()) == VM_PAGE_NULL)
2117: VM_PAGE_WAIT();
2118:
2119: /*
2120: * Map the page to its physical address so that it
2121: * can be found later.
2122: */
2123: pa = m->phys_addr;
2124: vm_object_lock(pmap_object);
2125: vm_page_insert(m, pmap_object, pa);
2126: vm_page_lock_queues();
2127: vm_page_wire(m);
2128: inuse_ptepages_count++;
2129: vm_object_unlock(pmap_object);
2130: vm_page_unlock_queues();
2131:
2132: /*
2133: * Zero the page.
2134: */
2135: memset((void *)phystokv(pa), 0, PAGE_SIZE);
2136:
2137: PMAP_READ_LOCK(map, spl);
2138: /*
2139: * See if someone else expanded us first
2140: */
2141: if (pmap_pte(map, v) != PT_ENTRY_NULL) {
2142: PMAP_READ_UNLOCK(map, spl);
2143: vm_object_lock(pmap_object);
2144: vm_page_lock_queues();
2145: vm_page_free(m);
2146: inuse_ptepages_count--;
2147: vm_page_unlock_queues();
2148: vm_object_unlock(pmap_object);
2149: return;
2150: }
2151:
2152: /*
2153: * Set the page directory entry for this page table.
2154: * If we have allocated more than one hardware page,
2155: * set several page directory entries.
2156: */
2157:
2158: i = ptes_per_vm_page;
2159: pdp = &map->dirbase[pdenum(map, v) & ~(i-1)];
2160: do {
2161: *pdp = pa_to_pte(pa)
2162: | INTEL_PTE_VALID
2163: | INTEL_PTE_USER
2164: | INTEL_PTE_WRITE;
2165: pdp++;
2166: pa += INTEL_PGBYTES;
2167: } while (--i > 0);
2168:
2169: PMAP_READ_UNLOCK(map, spl);
2170: return;
2171: }
2172:
2173: /*
2174: * Copy the range specified by src_addr/len
2175: * from the source map to the range dst_addr/len
2176: * in the destination map.
2177: *
2178: * This routine is only advisory and need not do anything.
2179: */
2180: #if 0
2181: void
2182: pmap_copy(
2183: pmap_t dst_pmap,
2184: pmap_t src_pmap,
2185: vm_offset_t dst_addr,
2186: vm_size_t len,
2187: vm_offset_t src_addr)
2188: {
2189: #ifdef lint
2190: dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
2191: #endif /* lint */
2192: }
2193: #endif/* 0 */
2194:
2195: int collect_ref;
2196: int collect_unref;
2197:
2198: /*
2199: * Routine: pmap_collect
2200: * Function:
2201: * Garbage collects the physical map system for
2202: * pages which are no longer used.
2203: * Success need not be guaranteed -- that is, there
2204: * may well be pages which are not referenced, but
2205: * others may be collected.
2206: * Usage:
2207: * Called by the pageout daemon when pages are scarce.
2208: */
2209: void
2210: pmap_collect(
2211: pmap_t p)
2212: {
2213: register pt_entry_t *pdp, *ptp;
2214: pt_entry_t *eptp;
2215: vm_offset_t pa;
2216: int wired;
2217: spl_t spl;
2218:
2219: if (p == PMAP_NULL)
2220: return;
2221:
2222: if (p == kernel_pmap)
2223: return;
2224:
2225: /*
2226: * Garbage collect map.
2227: */
2228: PMAP_READ_LOCK(p, spl);
2229: PMAP_FLUSH_TLBS();
2230:
2231: for (pdp = p->dirbase;
2232: pdp < &p->dirbase[pdenum(p, LINEAR_KERNEL_ADDRESS)];
2233: pdp += ptes_per_vm_page)
2234: {
2235: if (*pdp & INTEL_PTE_VALID)
2236: if(*pdp & INTEL_PTE_REF) {
2237: *pdp &= ~INTEL_PTE_REF;
2238: collect_ref++;
2239: } else {
2240: collect_unref++;
2241: pa = pte_to_pa(*pdp);
2242: ptp = (pt_entry_t *)phystokv(pa);
2243: eptp = ptp + NPTES*ptes_per_vm_page;
2244:
2245: /*
2246: * If the pte page has any wired mappings, we cannot
2247: * free it.
2248: */
2249: wired = 0;
2250: {
2251: register pt_entry_t *ptep;
2252: for (ptep = ptp; ptep < eptp; ptep++) {
2253: if (iswired(*ptep)) {
2254: wired = 1;
2255: break;
2256: }
2257: }
2258: }
2259: if (!wired) {
2260: /*
2261: * Remove the virtual addresses mapped by this pte page.
2262: */
2263: pmap_remove_range(p,
2264: pdetova(pdp - p->dirbase),
2265: ptp,
2266: eptp);
2267:
2268: /*
2269: * Invalidate the page directory pointer.
2270: */
2271: {
2272: register int i = ptes_per_vm_page;
2273: register pt_entry_t *pdep = pdp;
2274: do {
2275: *pdep++ = 0;
2276: } while (--i > 0);
2277: }
2278:
2279: PMAP_READ_UNLOCK(p, spl);
2280:
2281: /*
2282: * And free the pte page itself.
2283: */
2284: {
2285: register vm_page_t m;
2286:
2287: vm_object_lock(pmap_object);
2288: m = vm_page_lookup(pmap_object, pa);
2289: if (m == VM_PAGE_NULL)
2290: panic("pmap_collect: pte page not in object");
2291: vm_page_lock_queues();
2292: vm_page_free(m);
2293: inuse_ptepages_count--;
2294: vm_page_unlock_queues();
2295: vm_object_unlock(pmap_object);
2296: }
2297:
2298: PMAP_READ_LOCK(p, spl);
2299: }
2300: }
2301: }
2302: PMAP_READ_UNLOCK(p, spl);
2303: return;
2304:
2305: }
2306:
2307: /*
2308: * Routine: pmap_kernel
2309: * Function:
2310: * Returns the physical map handle for the kernel.
2311: */
2312: #if 0
2313: pmap_t
2314: pmap_kernel(void)
2315: {
2316: return (kernel_pmap);
2317: }
2318: #endif/* 0 */
2319:
2320: /*
2321: * pmap_zero_page zeros the specified (machine independent) page.
2322: * See machine/phys.c or machine/phys.s for implementation.
2323: */
2324: #if 0
2325: void
2326: pmap_zero_page(
2327: register vm_offset_t phys)
2328: {
2329: register int i;
2330:
2331: assert(phys != vm_page_fictitious_addr);
2332: i = PAGE_SIZE / INTEL_PGBYTES;
2333: phys = intel_pfn(phys);
2334:
2335: while (i--)
2336: zero_phys(phys++);
2337: }
2338: #endif/* 0 */
2339:
2340: /*
2341: * pmap_copy_page copies the specified (machine independent) page.
2342: * See machine/phys.c or machine/phys.s for implementation.
2343: */
2344: #if 0
2345: void
2346: pmap_copy_page(
2347: vm_offset_t src,
2348: vm_offset_t dst)
2349: {
2350: int i;
2351:
2352: assert(src != vm_page_fictitious_addr);
2353: assert(dst != vm_page_fictitious_addr);
2354: i = PAGE_SIZE / INTEL_PGBYTES;
2355:
2356: while (i--) {
2357: copy_phys(intel_pfn(src), intel_pfn(dst));
2358: src += INTEL_PGBYTES;
2359: dst += INTEL_PGBYTES;
2360: }
2361: }
2362: #endif/* 0 */
2363:
2364: /*
2365: * Routine: pmap_pageable
2366: * Function:
2367: * Make the specified pages (by pmap, offset)
2368: * pageable (or not) as requested.
2369: *
2370: * A page which is not pageable may not take
2371: * a fault; therefore, its page table entry
2372: * must remain valid for the duration.
2373: *
2374: * This routine is merely advisory; pmap_enter
2375: * will specify that these pages are to be wired
2376: * down (or not) as appropriate.
2377: */
2378: void
2379: pmap_pageable(
2380: pmap_t pmap,
2381: vm_offset_t start,
2382: vm_offset_t end,
2383: boolean_t pageable)
2384: {
2385: #ifdef lint
2386: pmap++; start++; end++; pageable++;
2387: #endif /* lint */
2388: }
2389:
2390: /*
2391: * Clear specified attribute bits.
2392: */
2393: void
2394: phys_attribute_clear(
2395: vm_offset_t phys,
2396: int bits)
2397: {
2398: pv_entry_t pv_h;
2399: register pv_entry_t pv_e;
2400: register pt_entry_t *pte;
2401: int pai;
2402: register pmap_t pmap;
2403: spl_t spl;
2404:
2405: assert(phys != vm_page_fictitious_addr);
2406: if (!valid_page(phys)) {
2407: /*
2408: * Not a managed page.
2409: */
2410: return;
2411: }
2412:
2413: /*
2414: * Lock the pmap system first, since we will be changing
2415: * several pmaps.
2416: */
2417:
2418: PMAP_WRITE_LOCK(spl);
2419:
2420: pai = pa_index(phys);
2421: pv_h = pai_to_pvh(pai);
2422:
2423: /*
2424: * Walk down PV list, clearing all modify or reference bits.
2425: * We do not have to lock the pv_list because we have
2426: * the entire pmap system locked.
2427: */
2428: if (pv_h->pmap != PMAP_NULL) {
2429: /*
2430: * There are some mappings.
2431: */
2432: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
2433:
2434: pmap = pv_e->pmap;
2435: /*
2436: * Lock the pmap to block pmap_extract and similar routines.
2437: */
2438: simple_lock(&pmap->lock);
2439:
2440: {
2441: register vm_offset_t va;
2442:
2443: va = pv_e->va;
2444: pte = pmap_pte(pmap, va);
2445:
2446: #if 0
2447: /*
2448: * Consistency checks.
2449: */
2450: assert(*pte & INTEL_PTE_VALID);
2451: /* assert(pte_to_phys(*pte) == phys); */
2452: #endif
2453:
2454: /*
2455: * Invalidate TLBs for all CPUs using this mapping.
2456: */
2457: PMAP_INVALIDATE_PAGE(pmap, va);
2458: }
2459:
2460: /*
2461: * Clear modify or reference bits.
2462: */
2463: {
2464: register int i = ptes_per_vm_page;
2465: do {
2466: *pte++ &= ~bits;
2467: } while (--i > 0);
2468: }
2469: simple_unlock(&pmap->lock);
2470: }
2471: }
2472:
2473: pmap_phys_attributes[pai] &= ~bits;
2474:
2475: PMAP_WRITE_UNLOCK(spl);
2476: }
2477:
2478: /*
2479: * Check specified attribute bits.
2480: */
2481: boolean_t
2482: phys_attribute_test(
2483: vm_offset_t phys,
2484: int bits)
2485: {
2486: pv_entry_t pv_h;
2487: register pv_entry_t pv_e;
2488: register pt_entry_t *pte;
2489: int pai;
2490: register pmap_t pmap;
2491: spl_t spl;
2492:
2493: assert(phys != vm_page_fictitious_addr);
2494: if (!valid_page(phys)) {
2495: /*
2496: * Not a managed page.
2497: */
2498: return (FALSE);
2499: }
2500:
2501: /*
2502: * Lock the pmap system first, since we will be checking
2503: * several pmaps.
2504: */
2505:
2506: PMAP_WRITE_LOCK(spl);
2507:
2508: pai = pa_index(phys);
2509: pv_h = pai_to_pvh(pai);
2510:
2511: if (pmap_phys_attributes[pai] & bits) {
2512: PMAP_WRITE_UNLOCK(spl);
2513: return (TRUE);
2514: }
2515:
2516: /*
2517: * Walk down PV list, checking all mappings.
2518: * We do not have to lock the pv_list because we have
2519: * the entire pmap system locked.
2520: */
2521: if (pv_h->pmap != PMAP_NULL) {
2522: /*
2523: * There are some mappings.
2524: */
2525: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
2526:
2527: pmap = pv_e->pmap;
2528: /*
2529: * Lock the pmap to block pmap_extract and similar routines.
2530: */
2531: simple_lock(&pmap->lock);
2532:
2533: {
2534: register vm_offset_t va;
2535:
2536: va = pv_e->va;
2537: pte = pmap_pte(pmap, va);
2538:
2539: #if 0
2540: /*
2541: * Consistency checks.
2542: */
2543: assert(*pte & INTEL_PTE_VALID);
2544: /* assert(pte_to_phys(*pte) == phys); */
2545: #endif
2546: }
2547:
2548: /*
2549: * Check modify or reference bits.
2550: */
2551: {
2552: register int i = ptes_per_vm_page;
2553:
2554: do {
2555: if (*pte++ & bits) {
2556: simple_unlock(&pmap->lock);
2557: PMAP_WRITE_UNLOCK(spl);
2558: return (TRUE);
2559: }
2560: } while (--i > 0);
2561: }
2562: simple_unlock(&pmap->lock);
2563: }
2564: }
2565: PMAP_WRITE_UNLOCK(spl);
2566: return (FALSE);
2567: }
2568:
2569: /*
2570: * Set specified attribute bits.
2571: */
2572: void
2573: phys_attribute_set(
2574: vm_offset_t phys,
2575: int bits)
2576: {
2577: int spl;
2578:
2579: assert(phys != vm_page_fictitious_addr);
2580: if (!valid_page(phys)) {
2581: /*
2582: * Not a managed page.
2583: */
2584: return;
2585: }
2586:
2587: /*
2588: * Lock the pmap system and set the requested bits in
2589: * the phys attributes array. Don't need to bother with
2590: * ptes because the test routine looks here first.
2591: */
2592:
2593: PMAP_WRITE_LOCK(spl);
2594: pmap_phys_attributes[pa_index(phys)] |= bits;
2595: PMAP_WRITE_UNLOCK(spl);
2596: }
2597:
2598: /*
2599: * Set the modify bit on the specified physical page.
2600: */
2601:
2602: void pmap_set_modify(
2603: register vm_offset_t phys)
2604: {
2605: phys_attribute_set(phys, PHYS_MODIFIED);
2606: }
2607:
2608: /*
2609: * Clear the modify bits on the specified physical page.
2610: */
2611:
2612: void
2613: pmap_clear_modify(
2614: register vm_offset_t phys)
2615: {
2616: phys_attribute_clear(phys, PHYS_MODIFIED);
2617: }
2618:
2619: /*
2620: * pmap_is_modified:
2621: *
2622: * Return whether or not the specified physical page is modified
2623: * by any physical maps.
2624: */
2625:
2626: boolean_t
2627: pmap_is_modified(
2628: register vm_offset_t phys)
2629: {
2630: return (phys_attribute_test(phys, PHYS_MODIFIED));
2631: }
2632:
2633: /*
2634: * pmap_clear_reference:
2635: *
2636: * Clear the reference bit on the specified physical page.
2637: */
2638:
2639: void
2640: pmap_clear_reference(
2641: vm_offset_t phys)
2642: {
2643: phys_attribute_clear(phys, PHYS_REFERENCED);
2644: }
2645:
2646: /*
2647: * pmap_is_referenced:
2648: *
2649: * Return whether or not the specified physical page is referenced
2650: * by any physical maps.
2651: */
2652:
2653: boolean_t
2654: pmap_is_referenced(
2655: vm_offset_t phys)
2656: {
2657: return (phys_attribute_test(phys, PHYS_REFERENCED));
2658: }
2659:
2660: /*
2661: * Set the modify bit on the specified range
2662: * of this map as requested.
2663: *
2664: * This optimization stands only if each time the dirty bit
2665: * in vm_page_t is tested, it is also tested in the pmap.
2666: */
2667: void
2668: pmap_modify_pages(
2669: pmap_t map,
2670: vm_offset_t s,
2671: vm_offset_t e)
2672: {
2673: spl_t spl;
2674: register pt_entry_t *pde;
2675: register pt_entry_t *spte, *epte;
2676: vm_offset_t l;
2677:
2678: if (map == PMAP_NULL)
2679: return;
2680:
2681: PMAP_READ_LOCK(map, spl);
2682:
2683: pde = pmap_pde(map, s);
2684: while (s && s < e) {
2685: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
2686: if (l > e)
2687: l = e;
2688: if (*pde & INTEL_PTE_VALID) {
2689: spte = (pt_entry_t *)ptetokv(*pde);
2690: if (l) {
2691: spte = &spte[ptenum(s)];
2692: epte = &spte[intel_btop(l-s)];
2693: } else {
2694: epte = &spte[intel_btop(PDE_MAPPED_SIZE)];
2695: spte = &spte[ptenum(s)];
2696: }
2697: while (spte < epte) {
2698: if (*spte & INTEL_PTE_VALID) {
2699: *spte |= (INTEL_PTE_MOD | INTEL_PTE_WRITE);
2700: }
2701: spte++;
2702: }
2703: }
2704: s = l;
2705: pde++;
2706: }
2707: PMAP_FLUSH_TLBS();
2708: PMAP_READ_UNLOCK(map, spl);
2709: }
2710:
2711:
2712: #if NCPUS > 1
2713:
2714: void inline
2715: pmap_wait_for_clear()
2716: {
2717: register int my_cpu;
2718: spl_t s;
2719: register pmap_t my_pmap;
2720:
2721: mp_disable_preemption();
2722: my_cpu = cpu_number();
2723:
2724:
2725: my_pmap = real_pmap[my_cpu];
2726:
2727: if (!(my_pmap && pmap_in_use(my_pmap, my_cpu)))
2728: my_pmap = kernel_pmap;
2729:
2730: /*
2731: * Raise spl to splhigh (above splip) to block out pmap_extract
2732: * from IO code (which would put this cpu back in the active
2733: * set).
2734: */
2735: s = splhigh();
2736:
2737: /*
2738: * Wait for any pmap updates in progress, on either user
2739: * or kernel pmap.
2740: */
2741: while (*(volatile hw_lock_t)&my_pmap->lock.interlock ||
2742: *(volatile hw_lock_t)&kernel_pmap->lock.interlock) {
2743: continue;
2744: }
2745:
2746: splx(s);
2747: mp_enable_preemption();
2748: }
2749:
2750: void
2751: pmap_flush_tlb_interrupt(void) {
2752: pmap_wait_for_clear();
2753:
2754: flush_tlb();
2755: }
2756:
2757: void
2758: pmap_reload_tlb_interrupt(void) {
2759: pmap_wait_for_clear();
2760:
2761: set_cr3(kernel_pmap->pdirbase);
2762: }
2763:
2764:
2765: #endif /* NCPUS > 1 */
2766:
2767: #if MACH_KDB
2768:
2769: /* show phys page mappings and attributes */
2770:
2771: extern void db_show_page(vm_offset_t pa);
2772:
2773: void
2774: db_show_page(vm_offset_t pa)
2775: {
2776: pv_entry_t pv_h;
2777: int pai;
2778: char attr;
2779:
2780: pai = pa_index(pa);
2781: pv_h = pai_to_pvh(pai);
2782:
2783: attr = pmap_phys_attributes[pai];
2784: printf("phys page %x ", pa);
2785: if (attr & PHYS_MODIFIED)
2786: printf("modified, ");
2787: if (attr & PHYS_REFERENCED)
2788: printf("referenced, ");
2789: if (pv_h->pmap || pv_h->next)
2790: printf(" mapped at\n");
2791: else
2792: printf(" not mapped\n");
2793: for (; pv_h; pv_h = pv_h->next)
2794: if (pv_h->pmap)
2795: printf("%x in pmap %x\n", pv_h->va, pv_h->pmap);
2796: }
2797:
2798: #endif /* MACH_KDB */
2799:
2800: #if MACH_KDB
2801: void db_kvtophys(vm_offset_t);
2802: void db_show_vaddrs(pt_entry_t *);
2803:
2804: /*
2805: * print out the results of kvtophys(arg)
2806: */
2807: void
2808: db_kvtophys(
2809: vm_offset_t vaddr)
2810: {
2811: db_printf("0x%x", kvtophys(vaddr));
2812: }
2813:
2814: /*
2815: * Walk the pages tables.
2816: */
2817: void
2818: db_show_vaddrs(
2819: pt_entry_t *dirbase)
2820: {
2821: pt_entry_t *ptep, *pdep, tmp;
2822: int x, y, pdecnt, ptecnt;
2823:
2824: if (dirbase == 0) {
2825: dirbase = kernel_pmap->dirbase;
2826: }
2827: if (dirbase == 0) {
2828: db_printf("need a dirbase...\n");
2829: return;
2830: }
2831: dirbase = (pt_entry_t *) ((unsigned long) dirbase & ~INTEL_OFFMASK);
2832:
2833: db_printf("dirbase: 0x%x\n", dirbase);
2834:
2835: pdecnt = ptecnt = 0;
2836: pdep = &dirbase[0];
2837: for (y = 0; y < NPDES; y++, pdep++) {
2838: if (((tmp = *pdep) & INTEL_PTE_VALID) == 0) {
2839: continue;
2840: }
2841: pdecnt++;
2842: ptep = (pt_entry_t *) ((*pdep) & ~INTEL_OFFMASK);
2843: db_printf("dir[%4d]: 0x%x\n", y, *pdep);
2844: for (x = 0; x < NPTES; x++, ptep++) {
2845: if (((tmp = *ptep) & INTEL_PTE_VALID) == 0) {
2846: continue;
2847: }
2848: ptecnt++;
2849: db_printf(" tab[%4d]: 0x%x, va=0x%x, pa=0x%x\n",
2850: x,
2851: *ptep,
2852: (y << 22) | (x << 12),
2853: *ptep & ~INTEL_OFFMASK);
2854: }
2855: }
2856:
2857: db_printf("total: %d tables, %d page table entries.\n", pdecnt, ptecnt);
2858:
2859: }
2860: #endif /* MACH_KDB */
2861:
2862: #include <mach_vm_debug.h>
2863: #if MACH_VM_DEBUG
2864: #include <vm/vm_debug.h>
2865:
2866: int
2867: pmap_list_resident_pages(
2868: register pmap_t pmap,
2869: register vm_offset_t *listp,
2870: register int space)
2871: {
2872: return 0;
2873: }
2874: #endif /* MACH_VM_DEBUG */
2875:
2876: #ifdef MACH_BSD
2877: /*
2878: * pmap_pagemove
2879: *
2880: * BSD support routine to reassign virtual addresses.
2881: */
2882:
2883: void
2884: pmap_movepage(unsigned long from, unsigned long to, vm_size_t size)
2885: {
2886: spl_t spl;
2887: pt_entry_t *pte, saved_pte;
2888: /* Lock the kernel map */
2889:
2890:
2891: while (size > 0) {
2892: PMAP_READ_LOCK(kernel_pmap, spl);
2893: pte = pmap_pte(kernel_pmap, from);
2894: if (pte == NULL)
2895: panic("pmap_pagemove from pte NULL");
2896: saved_pte = *pte;
2897: PMAP_READ_UNLOCK(kernel_pmap, spl);
2898:
2899: pmap_enter(kernel_pmap, to, i386_trunc_page(*pte),
2900: VM_PROT_READ|VM_PROT_WRITE, *pte & INTEL_PTE_WIRED);
2901:
2902: pmap_remove(kernel_pmap, from, from+PAGE_SIZE);
2903:
2904: PMAP_READ_LOCK(kernel_pmap, spl);
2905: pte = pmap_pte(kernel_pmap, to);
2906: if (pte == NULL)
2907: panic("pmap_pagemove 'to' pte NULL");
2908:
2909: *pte = saved_pte;
2910: PMAP_READ_UNLOCK(kernel_pmap, spl);
2911:
2912: from += PAGE_SIZE;
2913: to += PAGE_SIZE;
2914: size -= PAGE_SIZE;
2915: }
2916:
2917: /* Get the processors to update the TLBs */
2918: PMAP_FLUSH_TLBS();
2919:
2920: }
2921: #endif
2922:
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.