![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / i386 / intel|
Return to pmap.c CVS log | Up to [GnuMach] / Gnu-Mach / i386 / intel |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
4: * All Rights Reserved.
1.1.1.2 root 5: *
1.1 root 6: * Permission to use, copy, modify and distribute this software and its
7: * documentation is hereby granted, provided that both the copyright
8: * notice and this permission notice appear in all copies of the
9: * software, derivative works or modified versions, and any portions
10: * thereof, and that both notices appear in supporting documentation.
1.1.1.2 root 11: *
1.1 root 12: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
1.1.1.2 root 15: *
1.1 root 16: * Carnegie Mellon requests users of this software to return to
1.1.1.2 root 17: *
1.1 root 18: * Software Distribution Coordinator or [email protected]
19: * School of Computer Science
20: * Carnegie Mellon University
21: * Pittsburgh PA 15213-3890
1.1.1.2 root 22: *
1.1 root 23: * any improvements or extensions that they make and grant Carnegie Mellon
24: * the rights to redistribute these changes.
25: */
26: /*
27: * File: pmap.c
28: * Author: Avadis Tevanian, Jr., Michael Wayne Young
29: * (These guys wrote the Vax version)
30: *
1.1.1.3 root 31: * Physical Map management code for Intel i386, and i486.
1.1 root 32: *
33: * Manages physical address maps.
34: *
35: * In addition to hardware address maps, this
36: * module is called upon to provide software-use-only
37: * maps which may or may not be stored in the same
38: * form as hardware maps. These pseudo-maps are
39: * used to store intermediate results from copy
40: * operations to and from address spaces.
41: *
42: * Since the information managed by this module is
43: * also stored by the logical address mapping module,
44: * this module may throw away valid virtual-to-physical
45: * mappings at almost any time. However, invalidations
46: * of virtual-to-physical mappings must be done as
47: * requested.
48: *
49: * In order to cope with hardware architectures which
50: * make virtual-to-physical map invalidates expensive,
51: * this module may delay invalidate or reduced protection
52: * operations until such time as they are actually
53: * necessary. This module is given full information as
54: * to which processors are currently using which maps,
55: * and to when physical maps must be made correct.
56: */
57:
1.1.1.3 root 58: #include <string.h>
1.1 root 59:
60: #include <mach/machine/vm_types.h>
61:
62: #include <mach/boolean.h>
1.1.1.3 root 63: #include <kern/debug.h>
64: #include <kern/printf.h>
1.1 root 65: #include <kern/thread.h>
1.1.1.3 root 66: #include <kern/slab.h>
1.1 root 67:
68: #include <kern/lock.h>
69:
70: #include <vm/pmap.h>
71: #include <vm/vm_map.h>
72: #include <vm/vm_kern.h>
1.1.1.3 root 73: #include <i386/vm_param.h>
1.1 root 74: #include <mach/vm_prot.h>
75: #include <vm/vm_object.h>
76: #include <vm/vm_page.h>
77: #include <vm/vm_user.h>
78:
79: #include <mach/machine/vm_param.h>
1.1.1.3 root 80: #include <mach/xen.h>
1.1 root 81: #include <machine/thread.h>
1.1.1.3 root 82: #include <i386/cpu_number.h>
83: #include <i386/proc_reg.h>
84: #include <i386/locore.h>
85: #include <i386/model_dep.h>
1.1.1.7 ! root 86: #include <i386at/biosmem.h>
1.1.1.6 root 87: #include <i386at/model_dep.h>
1.1.1.3 root 88:
89: #ifdef MACH_PSEUDO_PHYS
90: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0;
91: #else /* MACH_PSEUDO_PHYS */
1.1 root 92: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry);
1.1.1.3 root 93: #endif /* MACH_PSEUDO_PHYS */
1.1 root 94:
95: /*
96: * Private data structures.
97: */
98:
99: /*
100: * For each vm_page_t, there is a list of all currently
101: * valid virtual mappings of that page. An entry is
102: * a pv_entry_t; the list is the pv_table.
103: */
104:
105: typedef struct pv_entry {
106: struct pv_entry *next; /* next pv_entry */
107: pmap_t pmap; /* pmap where mapping lies */
108: vm_offset_t va; /* virtual address for mapping */
109: } *pv_entry_t;
110:
111: #define PV_ENTRY_NULL ((pv_entry_t) 0)
112:
113: pv_entry_t pv_head_table; /* array of entries, one per page */
114:
115: /*
116: * pv_list entries are kept on a list that can only be accessed
117: * with the pmap system locked (at SPLVM, not in the cpus_active set).
1.1.1.3 root 118: * The list is refilled from the pv_list_cache if it becomes empty.
1.1 root 119: */
120: pv_entry_t pv_free_list; /* free list at SPLVM */
121: decl_simple_lock_data(, pv_free_list_lock)
122:
123: #define PV_ALLOC(pv_e) { \
124: simple_lock(&pv_free_list_lock); \
125: if ((pv_e = pv_free_list) != 0) { \
126: pv_free_list = pv_e->next; \
127: } \
128: simple_unlock(&pv_free_list_lock); \
129: }
130:
131: #define PV_FREE(pv_e) { \
132: simple_lock(&pv_free_list_lock); \
133: pv_e->next = pv_free_list; \
134: pv_free_list = pv_e; \
135: simple_unlock(&pv_free_list_lock); \
136: }
137:
1.1.1.3 root 138: struct kmem_cache pv_list_cache; /* cache of pv_entry structures */
1.1 root 139:
140: /*
141: * Each entry in the pv_head_table is locked by a bit in the
142: * pv_lock_table. The lock bits are accessed by the physical
143: * address of the page they lock.
144: */
145:
146: char *pv_lock_table; /* pointer to array of bits */
147: #define pv_lock_table_size(n) (((n)+BYTE_SIZE-1)/BYTE_SIZE)
148:
149: /* Has pmap_init completed? */
150: boolean_t pmap_initialized = FALSE;
151:
152: /*
153: * Range of kernel virtual addresses available for kernel memory mapping.
154: * Does not include the virtual addresses used to map physical memory 1-1.
155: * Initialized by pmap_bootstrap.
156: */
157: vm_offset_t kernel_virtual_start;
158: vm_offset_t kernel_virtual_end;
159:
160: /*
161: * Index into pv_head table, its lock bits, and the modify/reference
1.1.1.7 ! root 162: * bits.
1.1 root 163: */
1.1.1.7 ! root 164: #define pa_index(pa) vm_page_table_index(pa)
1.1 root 165:
166: #define pai_to_pvh(pai) (&pv_head_table[pai])
167: #define lock_pvh_pai(pai) (bit_lock(pai, pv_lock_table))
168: #define unlock_pvh_pai(pai) (bit_unlock(pai, pv_lock_table))
169:
170: /*
1.1.1.5 root 171: * Array of physical page attributes for managed pages.
1.1 root 172: * One byte per physical page.
173: */
174: char *pmap_phys_attributes;
175:
176: /*
177: * Physical page attributes. Copy bits from PTE definition.
178: */
179: #define PHYS_MODIFIED INTEL_PTE_MOD /* page modified */
180: #define PHYS_REFERENCED INTEL_PTE_REF /* page referenced */
181:
182: /*
183: * Amount of virtual memory mapped by one
184: * page-directory entry.
185: */
186: #define PDE_MAPPED_SIZE (pdenum2lin(1))
187:
188: /*
189: * We allocate page table pages directly from the VM system
190: * through this object. It maps physical memory.
191: */
192: vm_object_t pmap_object = VM_OBJECT_NULL;
193:
194: /*
195: * Locking and TLB invalidation
196: */
197:
198: /*
199: * Locking Protocols:
200: *
201: * There are two structures in the pmap module that need locking:
202: * the pmaps themselves, and the per-page pv_lists (which are locked
203: * by locking the pv_lock_table entry that corresponds to the pv_head
204: * for the list in question.) Most routines want to lock a pmap and
205: * then do operations in it that require pv_list locking -- however
206: * pmap_remove_all and pmap_copy_on_write operate on a physical page
207: * basis and want to do the locking in the reverse order, i.e. lock
208: * a pv_list and then go through all the pmaps referenced by that list.
209: * To protect against deadlock between these two cases, the pmap_lock
210: * is used. There are three different locking protocols as a result:
211: *
212: * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
213: * the pmap.
214: *
215: * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
216: * lock on the pmap_lock (shared read), then lock the pmap
217: * and finally the pv_lists as needed [i.e. pmap lock before
218: * pv_list lock.]
219: *
220: * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
221: * Get a write lock on the pmap_lock (exclusive write); this
222: * also guaranteees exclusive access to the pv_lists. Lock the
223: * pmaps as needed.
224: *
225: * At no time may any routine hold more than one pmap lock or more than
226: * one pv_list lock. Because interrupt level routines can allocate
227: * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
228: * kernel_pmap can only be held at splvm.
229: */
230:
231: #if NCPUS > 1
232: /*
233: * We raise the interrupt level to splvm, to block interprocessor
234: * interrupts during pmap operations. We must take the CPU out of
235: * the cpus_active set while interrupts are blocked.
236: */
237: #define SPLVM(spl) { \
238: spl = splvm(); \
239: i_bit_clear(cpu_number(), &cpus_active); \
240: }
241:
242: #define SPLX(spl) { \
243: i_bit_set(cpu_number(), &cpus_active); \
244: splx(spl); \
245: }
246:
247: /*
248: * Lock on pmap system
249: */
250: lock_data_t pmap_system_lock;
251:
252: #define PMAP_READ_LOCK(pmap, spl) { \
253: SPLVM(spl); \
254: lock_read(&pmap_system_lock); \
255: simple_lock(&(pmap)->lock); \
256: }
257:
258: #define PMAP_WRITE_LOCK(spl) { \
259: SPLVM(spl); \
260: lock_write(&pmap_system_lock); \
261: }
262:
263: #define PMAP_READ_UNLOCK(pmap, spl) { \
264: simple_unlock(&(pmap)->lock); \
265: lock_read_done(&pmap_system_lock); \
266: SPLX(spl); \
267: }
268:
269: #define PMAP_WRITE_UNLOCK(spl) { \
270: lock_write_done(&pmap_system_lock); \
271: SPLX(spl); \
272: }
273:
274: #define PMAP_WRITE_TO_READ_LOCK(pmap) { \
275: simple_lock(&(pmap)->lock); \
276: lock_write_to_read(&pmap_system_lock); \
277: }
278:
279: #define LOCK_PVH(index) (lock_pvh_pai(index))
280:
281: #define UNLOCK_PVH(index) (unlock_pvh_pai(index))
282:
283: #define PMAP_UPDATE_TLBS(pmap, s, e) \
284: { \
285: cpu_set cpu_mask = 1 << cpu_number(); \
286: cpu_set users; \
287: \
288: /* Since the pmap is locked, other updates are locked */ \
289: /* out, and any pmap_activate has finished. */ \
290: \
291: /* find other cpus using the pmap */ \
292: users = (pmap)->cpus_using & ~cpu_mask; \
293: if (users) { \
294: /* signal them, and wait for them to finish */ \
295: /* using the pmap */ \
296: signal_cpus(users, (pmap), (s), (e)); \
297: while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
298: continue; \
299: } \
300: \
301: /* invalidate our own TLB if pmap is in use */ \
302: if ((pmap)->cpus_using & cpu_mask) { \
1.1.1.3 root 303: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 304: } \
305: }
306:
1.1.1.2 root 307: #else /* NCPUS > 1 */
1.1 root 308:
1.1.1.3 root 309: #define SPLVM(spl) ((void)(spl))
310: #define SPLX(spl) ((void)(spl))
1.1 root 311:
312: #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl)
313: #define PMAP_WRITE_LOCK(spl) SPLVM(spl)
314: #define PMAP_READ_UNLOCK(pmap, spl) SPLX(spl)
315: #define PMAP_WRITE_UNLOCK(spl) SPLX(spl)
316: #define PMAP_WRITE_TO_READ_LOCK(pmap)
317:
318: #define LOCK_PVH(index)
319: #define UNLOCK_PVH(index)
320:
321: #define PMAP_UPDATE_TLBS(pmap, s, e) { \
322: /* invalidate our own TLB if pmap is in use */ \
323: if ((pmap)->cpus_using) { \
1.1.1.3 root 324: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 325: } \
326: }
327:
1.1.1.2 root 328: #endif /* NCPUS > 1 */
1.1 root 329:
1.1.1.3 root 330: #ifdef MACH_PV_PAGETABLES
331: #define INVALIDATE_TLB(pmap, s, e) do { \
332: if (__builtin_constant_p((e) - (s)) \
333: && (e) - (s) == PAGE_SIZE) \
334: hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s) : (s)); \
335: else \
336: hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \
337: } while(0)
338: #else /* MACH_PV_PAGETABLES */
339: /* It is hard to know when a TLB flush becomes less expensive than a bunch of
340: * invlpgs. But it surely is more expensive than just one invlpg. */
1.1.1.7 ! root 341: #define INVALIDATE_TLB(pmap, s, e) do { \
1.1.1.3 root 342: if (__builtin_constant_p((e) - (s)) \
343: && (e) - (s) == PAGE_SIZE) \
1.1.1.7 ! root 344: invlpg_linear((pmap) == kernel_pmap ? kvtolin(s) : (s)); \
1.1.1.3 root 345: else \
346: flush_tlb(); \
1.1.1.7 ! root 347: } while (0)
1.1.1.3 root 348: #endif /* MACH_PV_PAGETABLES */
1.1 root 349:
350:
351: #if NCPUS > 1
352: /*
353: * Structures to keep track of pending TLB invalidations
354: */
355:
356: #define UPDATE_LIST_SIZE 4
357:
358: struct pmap_update_item {
359: pmap_t pmap; /* pmap to invalidate */
360: vm_offset_t start; /* start address to invalidate */
361: vm_offset_t end; /* end address to invalidate */
362: } ;
363:
364: typedef struct pmap_update_item *pmap_update_item_t;
365:
366: /*
367: * List of pmap updates. If the list overflows,
368: * the last entry is changed to invalidate all.
369: */
370: struct pmap_update_list {
371: decl_simple_lock_data(, lock)
372: int count;
373: struct pmap_update_item item[UPDATE_LIST_SIZE];
374: } ;
375: typedef struct pmap_update_list *pmap_update_list_t;
376:
377: struct pmap_update_list cpu_update_list[NCPUS];
378:
1.1.1.2 root 379: #endif /* NCPUS > 1 */
1.1 root 380:
381: /*
382: * Other useful macros.
383: */
384: #define current_pmap() (vm_map_pmap(current_thread()->task->map))
385: #define pmap_in_use(pmap, cpu) (((pmap)->cpus_using & (1 << (cpu))) != 0)
386:
387: struct pmap kernel_pmap_store;
388: pmap_t kernel_pmap;
389:
1.1.1.3 root 390: struct kmem_cache pmap_cache; /* cache of pmap structures */
1.1.1.7 ! root 391: struct kmem_cache pd_cache; /* cache of page directories */
! 392: #if PAE
! 393: struct kmem_cache pdpt_cache; /* cache of page
! 394: directory pointer
! 395: tables */
! 396: #endif
1.1 root 397:
1.1.1.4 root 398: boolean_t pmap_debug = FALSE; /* flag for debugging prints */
1.1 root 399:
400: #if 0
401: int ptes_per_vm_page; /* number of hardware ptes needed
402: to map one VM page. */
403: #else
404: #define ptes_per_vm_page 1
405: #endif
406:
407: unsigned int inuse_ptepages_count = 0; /* debugging */
408:
409: /*
410: * Pointer to the basic page directory for the kernel.
411: * Initialized by pmap_bootstrap().
412: */
413: pt_entry_t *kernel_page_dir;
414:
1.1.1.4 root 415: /*
416: * Two slots for temporary physical page mapping, to allow for
417: * physical-to-physical transfers.
418: */
419: static pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS];
1.1 root 420:
421: static inline pt_entry_t *
1.1.1.4 root 422: pmap_pde(const pmap_t pmap, vm_offset_t addr)
1.1 root 423: {
1.1.1.7 ! root 424: pt_entry_t *page_dir;
1.1 root 425: if (pmap == kernel_pmap)
426: addr = kvtolin(addr);
1.1.1.7 ! root 427: #if PAE
! 428: page_dir = (pt_entry_t *) ptetokv(pmap->pdpbase[lin2pdpnum(addr)]);
! 429: #else
! 430: page_dir = pmap->dirbase;
! 431: #endif
! 432: return &page_dir[lin2pdenum(addr)];
1.1 root 433: }
434:
435: /*
436: * Given an offset and a map, compute the address of the
437: * pte. If the address is invalid with respect to the map
438: * then PT_ENTRY_NULL is returned (and the map may need to grow).
439: *
440: * This is only used internally.
441: */
442: pt_entry_t *
1.1.1.4 root 443: pmap_pte(const pmap_t pmap, vm_offset_t addr)
1.1 root 444: {
445: pt_entry_t *ptp;
446: pt_entry_t pte;
447:
1.1.1.7 ! root 448: #if PAE
! 449: if (pmap->pdpbase == 0)
! 450: return(PT_ENTRY_NULL);
! 451: #else
1.1 root 452: if (pmap->dirbase == 0)
453: return(PT_ENTRY_NULL);
1.1.1.7 ! root 454: #endif
1.1 root 455: pte = *pmap_pde(pmap, addr);
456: if ((pte & INTEL_PTE_VALID) == 0)
457: return(PT_ENTRY_NULL);
458: ptp = (pt_entry_t *)ptetokv(pte);
459: return(&ptp[ptenum(addr)]);
460: }
461:
462: #define DEBUG_PTE_PAGE 0
463:
464: #if DEBUG_PTE_PAGE
1.1.1.4 root 465: void ptep_check(ptep_t ptep)
1.1 root 466: {
1.1.1.4 root 467: pt_entry_t *pte, *epte;
1.1 root 468: int ctu, ctw;
469:
470: /* check the use and wired counts */
471: if (ptep == PTE_PAGE_NULL)
472: return;
473: pte = pmap_pte(ptep->pmap, ptep->va);
474: epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
475: ctu = 0;
476: ctw = 0;
477: while (pte < epte) {
478: if (pte->pfn != 0) {
479: ctu++;
480: if (pte->wired)
481: ctw++;
482: }
483: pte += ptes_per_vm_page;
484: }
485:
486: if (ctu != ptep->use_count || ctw != ptep->wired_count) {
487: printf("use %d wired %d - actual use %d wired %d\n",
488: ptep->use_count, ptep->wired_count, ctu, ctw);
489: panic("pte count");
490: }
491: }
1.1.1.2 root 492: #endif /* DEBUG_PTE_PAGE */
1.1 root 493:
494: /*
495: * Map memory at initialization. The physical addresses being
496: * mapped are not managed and are never unmapped.
497: *
498: * For now, VM is already on, we only need to map the
499: * specified memory.
500: */
1.1.1.4 root 501: vm_offset_t pmap_map(
502: vm_offset_t virt,
1.1.1.7 ! root 503: phys_addr_t start,
! 504: phys_addr_t end,
1.1.1.4 root 505: int prot)
1.1 root 506: {
1.1.1.4 root 507: int ps;
1.1 root 508:
509: ps = PAGE_SIZE;
510: while (start < end) {
511: pmap_enter(kernel_pmap, virt, start, prot, FALSE);
512: virt += ps;
513: start += ps;
514: }
515: return(virt);
516: }
517:
518: /*
1.1.1.2 root 519: * Back-door routine for mapping kernel VM at initialization.
1.1.1.7 ! root 520: * Useful for mapping memory outside the range of direct mapped
! 521: * physical memory (i.e., devices).
1.1 root 522: * Otherwise like pmap_map.
523: */
1.1.1.4 root 524: vm_offset_t pmap_map_bd(
525: vm_offset_t virt,
1.1.1.7 ! root 526: phys_addr_t start,
! 527: phys_addr_t end,
1.1.1.4 root 528: vm_prot_t prot)
1.1 root 529: {
1.1.1.4 root 530: pt_entry_t template;
531: pt_entry_t *pte;
532: int spl;
1.1.1.3 root 533: #ifdef MACH_PV_PAGETABLES
534: int n, i = 0;
535: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
536: #endif /* MACH_PV_PAGETABLES */
1.1 root 537:
538: template = pa_to_pte(start)
1.1.1.3 root 539: | INTEL_PTE_NCACHE|INTEL_PTE_WTHRU
1.1 root 540: | INTEL_PTE_VALID;
1.1.1.3 root 541: if (CPU_HAS_FEATURE(CPU_FEATURE_PGE))
542: template |= INTEL_PTE_GLOBAL;
1.1 root 543: if (prot & VM_PROT_WRITE)
544: template |= INTEL_PTE_WRITE;
545:
1.1.1.3 root 546: PMAP_READ_LOCK(pmap, spl);
1.1 root 547: while (start < end) {
548: pte = pmap_pte(kernel_pmap, virt);
549: if (pte == PT_ENTRY_NULL)
550: panic("pmap_map_bd: Invalid kernel address\n");
1.1.1.3 root 551: #ifdef MACH_PV_PAGETABLES
552: update[i].ptr = kv_to_ma(pte);
553: update[i].val = pa_to_ma(template);
554: i++;
555: if (i == HYP_BATCH_MMU_UPDATES) {
556: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
557: if (n != i)
558: panic("couldn't pmap_map_bd\n");
559: i = 0;
560: }
561: #else /* MACH_PV_PAGETABLES */
562: WRITE_PTE(pte, template)
563: #endif /* MACH_PV_PAGETABLES */
1.1 root 564: pte_increment_pa(template);
565: virt += PAGE_SIZE;
566: start += PAGE_SIZE;
567: }
1.1.1.3 root 568: #ifdef MACH_PV_PAGETABLES
569: if (i > HYP_BATCH_MMU_UPDATES)
570: panic("overflowed array in pmap_map_bd");
571: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
572: if (n != i)
573: panic("couldn't pmap_map_bd\n");
574: #endif /* MACH_PV_PAGETABLES */
575: PMAP_READ_UNLOCK(pmap, spl);
1.1 root 576: return(virt);
577: }
578:
579: /*
580: * Bootstrap the system enough to run with virtual memory.
581: * Allocate the kernel page directory and page tables,
582: * and direct-map all physical memory.
583: * Called with mapping off.
584: */
1.1.1.4 root 585: void pmap_bootstrap(void)
1.1 root 586: {
587: /*
588: * Mapping is turned off; we must reference only physical addresses.
589: * The load image of the system is to be mapped 1-1 physical = virtual.
590: */
591:
592: /*
593: * Set ptes_per_vm_page for general use.
594: */
595: #if 0
596: ptes_per_vm_page = PAGE_SIZE / INTEL_PGBYTES;
597: #endif
598:
599: /*
600: * The kernel's pmap is statically allocated so we don't
601: * have to use pmap_create, which is unlikely to work
602: * correctly at this part of the boot sequence.
603: */
604:
605: kernel_pmap = &kernel_pmap_store;
606:
607: #if NCPUS > 1
608: lock_init(&pmap_system_lock, FALSE); /* NOT a sleep lock */
1.1.1.2 root 609: #endif /* NCPUS > 1 */
1.1 root 610:
611: simple_lock_init(&kernel_pmap->lock);
612:
613: kernel_pmap->ref_count = 1;
614:
615: /*
616: * Determine the kernel virtual address range.
617: * It starts at the end of the physical memory
618: * mapped into the kernel address space,
619: * and extends to a stupid arbitrary limit beyond that.
620: */
1.1.1.7 ! root 621: kernel_virtual_start = phystokv(biosmem_directmap_end());
! 622: kernel_virtual_end = kernel_virtual_start + VM_KERNEL_MAP_SIZE;
1.1.1.3 root 623:
624: if (kernel_virtual_end < kernel_virtual_start
625: || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS)
626: kernel_virtual_end = VM_MAX_KERNEL_ADDRESS;
1.1 root 627:
628: /*
629: * Allocate and clear a kernel page directory.
630: */
1.1.1.3 root 631: /* Note: initial Xen mapping holds at least 512kB free mapped page.
632: * We use that for directly building our linear mapping. */
633: #if PAE
634: {
635: vm_offset_t addr;
636: init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr);
1.1.1.7 ! root 637: kernel_page_dir = (pt_entry_t*)phystokv(addr);
1.1.1.3 root 638: }
639: kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page());
640: {
641: int i;
642: for (i = 0; i < PDPNUM; i++)
1.1.1.7 ! root 643: WRITE_PTE(&kernel_pmap->pdpbase[i],
! 644: pa_to_pte(_kvtophys((void *) kernel_page_dir
! 645: + i * INTEL_PGBYTES))
! 646: | INTEL_PTE_VALID);
1.1.1.3 root 647: }
648: #else /* PAE */
649: kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page());
650: #endif /* PAE */
1.1 root 651: {
1.1.1.6 root 652: unsigned i;
1.1 root 653: for (i = 0; i < NPDES; i++)
1.1.1.7 ! root 654: kernel_page_dir[i] = 0;
1.1 root 655: }
656:
1.1.1.3 root 657: #ifdef MACH_PV_PAGETABLES
658: /* We don't actually deal with the CR3 register content at all */
659: hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3);
660: /*
661: * Xen may only provide as few as 512KB extra bootstrap linear memory,
662: * which is far from enough to map all available memory, so we need to
663: * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE)
664: * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is
665: * enough for a pagetable mapping 4GiB.
666: */
667: #ifdef PAE
668: #define NSUP_L1 4
669: #else
670: #define NSUP_L1 1
671: #endif
672: pt_entry_t *l1_map[NSUP_L1];
673: {
674: pt_entry_t *base = (pt_entry_t*) boot_info.pt_base;
675: vm_offset_t la;
676: int n_l1map;
677: for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS; la >= VM_MIN_KERNEL_ADDRESS; la += NPTES * PAGE_SIZE) {
678: #ifdef PAE
679: pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)]);
680: #else /* PAE */
681: pt_entry_t *l2_map = base;
682: #endif /* PAE */
683: /* Like lin2pdenum, but works with non-contiguous boot L3 */
684: l2_map += (la >> PDESHIFT) & PDEMASK;
685: if (!(*l2_map & INTEL_PTE_VALID)) {
686: struct mmu_update update;
1.1.1.6 root 687: unsigned j, n;
1.1.1.3 root 688:
689: l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page());
690: for (j = 0; j < NPTES; j++)
691: l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS) * NPTES + j)) << PAGE_SHIFT) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
692: pmap_set_page_readonly_init(l1_map[n_l1map]);
693: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map])))
694: panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map]));
695: update.ptr = kv_to_ma(l2_map);
696: update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
697: hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF);
698: if (n != 1)
699: panic("couldn't complete bootstrap map");
700: /* added the last L1 table, can stop */
701: if (++n_l1map >= NSUP_L1)
702: break;
703: }
704: }
705: }
706: #endif /* MACH_PV_PAGETABLES */
707:
1.1 root 708: /*
709: * Allocate and set up the kernel page tables.
710: */
711: {
712: vm_offset_t va;
1.1.1.3 root 713: pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0;
1.1 root 714:
715: /*
1.1.1.7 ! root 716: * Map virtual memory for all directly mappable physical memory, 1-1,
1.1 root 717: * Make any mappings completely in the kernel's text segment read-only.
718: *
719: * Also allocate some additional all-null page tables afterwards
720: * for kernel virtual memory allocation,
721: * because this PMAP module is too stupid
722: * to allocate new kernel page tables later.
723: * XX fix this
724: */
1.1.1.7 ! root 725: for (va = phystokv(0); va >= phystokv(0) && va < kernel_virtual_end; )
1.1 root 726: {
1.1.1.7 ! root 727: pt_entry_t *pde = kernel_page_dir + lin2pdenum_cont(kvtolin(va));
1.1.1.3 root 728: pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page());
1.1 root 729: pt_entry_t *pte;
730:
731: /* Initialize the page directory entry. */
1.1.1.3 root 732: WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable))
733: | INTEL_PTE_VALID | INTEL_PTE_WRITE);
1.1 root 734:
735: /* Initialize the page table. */
1.1.1.7 ! root 736: for (pte = ptable; (va < phystokv(biosmem_directmap_end())) && (pte < ptable+NPTES); pte++)
1.1 root 737: {
738: if ((pte - ptable) < ptenum(va))
739: {
1.1.1.3 root 740: WRITE_PTE(pte, 0);
1.1 root 741: }
1.1.1.2 root 742: else
1.1.1.3 root 743: #ifdef MACH_PV_PAGETABLES
744: if (va == (vm_offset_t) &hyp_shared_info)
745: {
746: *pte = boot_info.shared_info | INTEL_PTE_VALID | INTEL_PTE_WRITE;
747: va += INTEL_PGBYTES;
748: }
749: else
750: #endif /* MACH_PV_PAGETABLES */
1.1 root 751: {
1.1.1.2 root 752: extern char _start[], etext[];
1.1 root 753:
1.1.1.3 root 754: if (((va >= (vm_offset_t) _start)
1.1 root 755: && (va + INTEL_PGBYTES <= (vm_offset_t)etext))
1.1.1.3 root 756: #ifdef MACH_PV_PAGETABLES
757: || (va >= (vm_offset_t) boot_info.pt_base
758: && (va + INTEL_PGBYTES <=
759: (vm_offset_t) ptable + INTEL_PGBYTES))
760: #endif /* MACH_PV_PAGETABLES */
761: )
1.1 root 762: {
1.1.1.3 root 763: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
764: | INTEL_PTE_VALID | global);
1.1 root 765: }
766: else
767: {
1.1.1.3 root 768: #ifdef MACH_PV_PAGETABLES
769: /* Keep supplementary L1 pages read-only */
770: int i;
771: for (i = 0; i < NSUP_L1; i++)
772: if (va == (vm_offset_t) l1_map[i]) {
773: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
774: | INTEL_PTE_VALID | global);
775: break;
776: }
777: if (i == NSUP_L1)
778: #endif /* MACH_PV_PAGETABLES */
779: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
780: | INTEL_PTE_VALID | INTEL_PTE_WRITE | global)
781:
1.1 root 782: }
783: va += INTEL_PGBYTES;
784: }
785: }
786: for (; pte < ptable+NPTES; pte++)
787: {
1.1.1.4 root 788: if (va >= kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE && va < kernel_virtual_end)
789: {
790: pmap_mapwindow_t *win = &mapwindows[atop(va - (kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE))];
791: win->entry = pte;
792: win->vaddr = va;
793: }
1.1.1.3 root 794: WRITE_PTE(pte, 0);
1.1 root 795: va += INTEL_PGBYTES;
796: }
1.1.1.3 root 797: #ifdef MACH_PV_PAGETABLES
798: pmap_set_page_readonly_init(ptable);
799: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable)))
800: panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable));
801: #endif /* MACH_PV_PAGETABLES */
1.1 root 802: }
803: }
804:
1.1.1.3 root 805: /* Architecture-specific code will turn on paging
806: soon after we return from here. */
807: }
1.1 root 808:
1.1.1.3 root 809: #ifdef MACH_PV_PAGETABLES
810: /* These are only required because of Xen security policies */
1.1 root 811:
1.1.1.3 root 812: /* Set back a page read write */
813: void pmap_set_page_readwrite(void *_vaddr) {
814: vm_offset_t vaddr = (vm_offset_t) _vaddr;
815: vm_offset_t paddr = kvtophys(vaddr);
816: vm_offset_t canon_vaddr = phystokv(paddr);
817: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
818: panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
819: if (canon_vaddr != vaddr)
820: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
821: panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr));
822: }
823:
824: /* Set a page read only (so as to pin it for instance) */
825: void pmap_set_page_readonly(void *_vaddr) {
826: vm_offset_t vaddr = (vm_offset_t) _vaddr;
827: vm_offset_t paddr = kvtophys(vaddr);
828: vm_offset_t canon_vaddr = phystokv(paddr);
829: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
830: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
831: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
832: }
833: if (canon_vaddr != vaddr &&
834: *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID) {
835: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
836: panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr));
1.1 root 837: }
1.1.1.3 root 838: }
1.1 root 839:
1.1.1.3 root 840: /* This needs to be called instead of pmap_set_page_readonly as long as RC3
841: * still points to the bootstrap dirbase, to also fix the bootstrap table. */
842: void pmap_set_page_readonly_init(void *_vaddr) {
843: vm_offset_t vaddr = (vm_offset_t) _vaddr;
844: #if PAE
845: pt_entry_t *pdpbase = (void*) boot_info.pt_base;
846: /* The bootstrap table does not necessarily use contiguous pages for the pde tables */
847: pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)]);
848: #else
849: pt_entry_t *dirbase = (void*) boot_info.pt_base;
1.1 root 850: #endif
1.1.1.3 root 851: pt_entry_t *pte = &dirbase[lin2pdenum(vaddr) & PTEMASK];
852: /* Modify our future kernel map (can't use update_va_mapping for this)... */
853: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
854: if (!hyp_mmu_update_la (kvtolin(vaddr), pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID))
855: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
856: }
857: /* ... and the bootstrap map. */
858: if (*pte & INTEL_PTE_VALID) {
859: if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID, UVMF_NONE))
860: panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
861: }
862: }
1.1 root 863:
1.1.1.3 root 864: void pmap_clear_bootstrap_pagetable(pt_entry_t *base) {
1.1.1.6 root 865: unsigned i;
1.1.1.3 root 866: pt_entry_t *dir;
867: vm_offset_t va = 0;
868: #if PAE
1.1.1.6 root 869: unsigned j;
1.1.1.3 root 870: #endif /* PAE */
871: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base)))
872: panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base));
873: #if PAE
874: for (j = 0; j < PDPNUM; j++)
875: {
876: pt_entry_t pdpe = base[j];
877: if (pdpe & INTEL_PTE_VALID) {
878: dir = (pt_entry_t *) ptetokv(pdpe);
879: #else /* PAE */
880: dir = base;
881: #endif /* PAE */
882: for (i = 0; i < NPTES; i++) {
883: pt_entry_t pde = dir[i];
884: unsigned long pfn = atop(pte_to_pa(pde));
885: void *pgt = (void*) phystokv(ptoa(pfn));
886: if (pde & INTEL_PTE_VALID)
887: hyp_free_page(pfn, pgt);
888: va += NPTES * INTEL_PGBYTES;
889: if (va >= HYP_VIRT_START)
890: break;
891: }
892: #if PAE
893: hyp_free_page(atop(_kvtophys(dir)), dir);
894: } else
895: va += NPTES * NPTES * INTEL_PGBYTES;
896: if (va >= HYP_VIRT_START)
897: break;
898: }
899: #endif /* PAE */
900: hyp_free_page(atop(_kvtophys(base)), base);
1.1 root 901: }
1.1.1.3 root 902: #endif /* MACH_PV_PAGETABLES */
1.1 root 903:
1.1.1.4 root 904: /*
905: * Create a temporary mapping for a given physical entry
906: *
907: * This can be used to access physical pages which are not mapped 1:1 by
908: * phystokv().
909: */
910: pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry)
911: {
912: pmap_mapwindow_t *map;
913:
1.1.1.7 ! root 914: assert(entry != 0);
! 915:
1.1.1.4 root 916: /* Find an empty one. */
917: for (map = &mapwindows[0]; map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]; map++)
918: if (!(*map->entry))
919: break;
920: assert(map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]);
921:
1.1.1.7 ! root 922: #ifdef MACH_PV_PAGETABLES
! 923: if (!hyp_mmu_update_pte(kv_to_ma(map->entry), pa_to_ma(entry)))
! 924: panic("pmap_get_mapwindow");
! 925: #else /* MACH_PV_PAGETABLES */
1.1.1.4 root 926: WRITE_PTE(map->entry, entry);
1.1.1.7 ! root 927: #endif /* MACH_PV_PAGETABLES */
1.1.1.4 root 928: return map;
929: }
930:
931: /*
932: * Destroy a temporary mapping for a physical entry
933: */
934: void pmap_put_mapwindow(pmap_mapwindow_t *map)
935: {
1.1.1.7 ! root 936: #ifdef MACH_PV_PAGETABLES
! 937: if (!hyp_mmu_update_pte(kv_to_ma(map->entry), 0))
! 938: panic("pmap_put_mapwindow");
! 939: #else /* MACH_PV_PAGETABLES */
1.1.1.4 root 940: WRITE_PTE(map->entry, 0);
1.1.1.7 ! root 941: #endif /* MACH_PV_PAGETABLES */
1.1.1.4 root 942: PMAP_UPDATE_TLBS(kernel_pmap, map->vaddr, map->vaddr + PAGE_SIZE);
943: }
944:
945: void pmap_virtual_space(
946: vm_offset_t *startp,
947: vm_offset_t *endp)
1.1 root 948: {
949: *startp = kernel_virtual_start;
1.1.1.4 root 950: *endp = kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE;
1.1 root 951: }
952:
953: /*
954: * Initialize the pmap module.
955: * Called by vm_init, to initialize any structures that the pmap
956: * system needs to map virtual memory.
957: */
1.1.1.4 root 958: void pmap_init(void)
1.1 root 959: {
1.1.1.7 ! root 960: unsigned long npages;
1.1 root 961: vm_offset_t addr;
1.1.1.4 root 962: vm_size_t s;
1.1.1.3 root 963: #if NCPUS > 1
1.1 root 964: int i;
1.1.1.3 root 965: #endif /* NCPUS > 1 */
1.1 root 966:
967: /*
968: * Allocate memory for the pv_head_table and its lock bits,
969: * the modify bit array, and the pte_page table.
970: */
971:
1.1.1.7 ! root 972: npages = vm_page_table_size();
1.1 root 973: s = (vm_size_t) (sizeof(struct pv_entry) * npages
974: + pv_lock_table_size(npages)
975: + npages);
976:
977: s = round_page(s);
978: if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
979: panic("pmap_init");
1.1.1.4 root 980: memset((void *) addr, 0, s);
1.1 root 981:
982: /*
983: * Allocate the structures first to preserve word-alignment.
984: */
985: pv_head_table = (pv_entry_t) addr;
986: addr = (vm_offset_t) (pv_head_table + npages);
987:
988: pv_lock_table = (char *) addr;
989: addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
990:
991: pmap_phys_attributes = (char *) addr;
992:
993: /*
1.1.1.3 root 994: * Create the cache of physical maps,
1.1 root 995: * and of the physical-to-virtual entries.
996: */
997: s = (vm_size_t) sizeof(struct pmap);
1.1.1.6 root 998: kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, 0);
1.1.1.7 ! root 999: kmem_cache_init(&pd_cache, "pd",
! 1000: INTEL_PGBYTES, INTEL_PGBYTES, NULL,
! 1001: KMEM_CACHE_PHYSMEM);
! 1002: #if PAE
! 1003: kmem_cache_init(&pdpt_cache, "pdpt",
! 1004: PDPNUM * sizeof(pt_entry_t),
! 1005: PDPNUM * sizeof(pt_entry_t), NULL,
! 1006: KMEM_CACHE_PHYSMEM);
! 1007: #endif
1.1 root 1008: s = (vm_size_t) sizeof(struct pv_entry);
1.1.1.6 root 1009: kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, 0);
1.1 root 1010:
1011: #if NCPUS > 1
1012: /*
1013: * Set up the pmap request lists
1014: */
1015: for (i = 0; i < NCPUS; i++) {
1016: pmap_update_list_t up = &cpu_update_list[i];
1017:
1018: simple_lock_init(&up->lock);
1019: up->count = 0;
1020: }
1.1.1.2 root 1021: #endif /* NCPUS > 1 */
1.1 root 1022:
1023: /*
1024: * Indicate that the PMAP module is now fully initialized.
1025: */
1026: pmap_initialized = TRUE;
1027: }
1028:
1.1.1.7 ! root 1029: static inline boolean_t
! 1030: valid_page(phys_addr_t addr)
1.1 root 1031: {
1.1.1.7 ! root 1032: struct vm_page *p;
1.1 root 1033:
1034: if (!pmap_initialized)
1.1.1.7 ! root 1035: return FALSE;
1.1 root 1036:
1.1.1.7 ! root 1037: p = vm_page_lookup_pa(addr);
! 1038: return (p != NULL);
1.1 root 1039: }
1040:
1041: /*
1042: * Routine: pmap_page_table_page_alloc
1043: *
1044: * Allocates a new physical page to be used as a page-table page.
1045: *
1046: * Must be called with the pmap system and the pmap unlocked,
1047: * since these must be unlocked to use vm_page_grab.
1048: */
1049: vm_offset_t
1.1.1.4 root 1050: pmap_page_table_page_alloc(void)
1.1 root 1051: {
1.1.1.4 root 1052: vm_page_t m;
1053: vm_offset_t pa;
1.1 root 1054:
1055: check_simple_locks();
1056:
1057: /*
1058: * We cannot allocate the pmap_object in pmap_init,
1.1.1.3 root 1059: * because it is called before the cache package is up.
1.1 root 1060: * Allocate it now if it is missing.
1061: */
1062: if (pmap_object == VM_OBJECT_NULL)
1.1.1.7 ! root 1063: pmap_object = vm_object_allocate(vm_page_table_size() * PAGE_SIZE);
1.1 root 1064:
1065: /*
1066: * Allocate a VM page for the level 2 page table entries.
1067: */
1.1.1.7 ! root 1068: while ((m = vm_page_grab()) == VM_PAGE_NULL)
1.1 root 1069: VM_PAGE_WAIT((void (*)()) 0);
1070:
1071: /*
1072: * Map the page to its physical address so that it
1073: * can be found later.
1074: */
1075: pa = m->phys_addr;
1076: vm_object_lock(pmap_object);
1077: vm_page_insert(m, pmap_object, pa);
1078: vm_page_lock_queues();
1079: vm_page_wire(m);
1080: inuse_ptepages_count++;
1081: vm_page_unlock_queues();
1082: vm_object_unlock(pmap_object);
1083:
1084: /*
1085: * Zero the page.
1086: */
1.1.1.3 root 1087: memset((void *)phystokv(pa), 0, PAGE_SIZE);
1.1 root 1088:
1089: return pa;
1090: }
1091:
1.1.1.3 root 1092: #ifdef MACH_XEN
1093: void pmap_map_mfn(void *_addr, unsigned long mfn) {
1094: vm_offset_t addr = (vm_offset_t) _addr;
1095: pt_entry_t *pte, *pdp;
1096: vm_offset_t ptp;
1097: pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT;
1098:
1099: /* Add a ptp if none exist yet for this pte */
1100: if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) {
1101: ptp = phystokv(pmap_page_table_page_alloc());
1102: #ifdef MACH_PV_PAGETABLES
1103: pmap_set_page_readonly((void*) ptp);
1104: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp)))
1105: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1106: #endif /* MACH_PV_PAGETABLES */
1107: pdp = pmap_pde(kernel_pmap, addr);
1108:
1109: #ifdef MACH_PV_PAGETABLES
1110: if (!hyp_mmu_update_pte(kv_to_ma(pdp),
1111: pa_to_pte(kv_to_ma(ptp)) | INTEL_PTE_VALID
1112: | INTEL_PTE_USER
1113: | INTEL_PTE_WRITE))
1114: panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__,__LINE__,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp));
1115: #else /* MACH_PV_PAGETABLES */
1116: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
1117: | INTEL_PTE_USER
1118: | INTEL_PTE_WRITE;
1119: #endif /* MACH_PV_PAGETABLES */
1120: pte = pmap_pte(kernel_pmap, addr);
1121: }
1122:
1123: #ifdef MACH_PV_PAGETABLES
1124: if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID | INTEL_PTE_WRITE))
1125: panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__,__LINE__,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma));
1126: #else /* MACH_PV_PAGETABLES */
1127: /* Note: in this case, mfn is actually a pfn. */
1128: WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE);
1129: #endif /* MACH_PV_PAGETABLES */
1130: }
1131: #endif /* MACH_XEN */
1132:
1.1 root 1133: /*
1134: * Deallocate a page-table page.
1135: * The page-table page must have all mappings removed,
1136: * and be removed from its page directory.
1137: */
1138: void
1.1.1.4 root 1139: pmap_page_table_page_dealloc(vm_offset_t pa)
1.1 root 1140: {
1141: vm_page_t m;
1142:
1143: vm_object_lock(pmap_object);
1144: m = vm_page_lookup(pmap_object, pa);
1145: vm_page_lock_queues();
1146: vm_page_free(m);
1147: inuse_ptepages_count--;
1148: vm_page_unlock_queues();
1149: vm_object_unlock(pmap_object);
1150: }
1151:
1152: /*
1153: * Create and return a physical map.
1154: *
1155: * If the size specified for the map
1156: * is zero, the map is an actual physical
1157: * map, and may be referenced by the
1158: * hardware.
1159: *
1160: * If the size specified is non-zero,
1161: * the map will be used in software only, and
1162: * is bounded by that size.
1163: */
1.1.1.4 root 1164: pmap_t pmap_create(vm_size_t size)
1.1 root 1165: {
1.1.1.7 ! root 1166: pt_entry_t *page_dir[PDPNUM];
! 1167: int i;
1.1.1.4 root 1168: pmap_t p;
1169: pmap_statistics_t stats;
1.1 root 1170:
1171: /*
1172: * A software use-only map doesn't even need a map.
1173: */
1174:
1175: if (size != 0) {
1176: return(PMAP_NULL);
1177: }
1178:
1179: /*
1.1.1.3 root 1180: * Allocate a pmap struct from the pmap_cache. Then allocate
1181: * the page descriptor table.
1.1 root 1182: */
1183:
1.1.1.3 root 1184: p = (pmap_t) kmem_cache_alloc(&pmap_cache);
1.1 root 1185: if (p == PMAP_NULL)
1.1.1.7 ! root 1186: return PMAP_NULL;
1.1 root 1187:
1.1.1.7 ! root 1188: for (i = 0; i < PDPNUM; i++) {
! 1189: page_dir[i] = (pt_entry_t *) kmem_cache_alloc(&pd_cache);
! 1190: if (page_dir[i] == NULL) {
! 1191: i -= 1;
! 1192: while (i >= 0) {
! 1193: kmem_cache_free(&pd_cache,
! 1194: (vm_address_t) page_dir[i]);
! 1195: i -= 1;
! 1196: }
! 1197: kmem_cache_free(&pmap_cache, (vm_address_t) p);
! 1198: return PMAP_NULL;
! 1199: }
! 1200: memcpy(page_dir[i],
! 1201: (void *) kernel_page_dir + i * INTEL_PGBYTES,
! 1202: INTEL_PGBYTES);
! 1203: }
1.1 root 1204:
1.1.1.3 root 1205: #ifdef LINUX_DEV
1206: #if VM_MIN_KERNEL_ADDRESS != 0
1207: /* Do not map BIOS in user tasks */
1.1.1.7 ! root 1208: page_dir
! 1209: #if PAE
! 1210: [lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)]
! 1211: #else
! 1212: [0]
! 1213: #endif
! 1214: [lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)]
! 1215: = 0;
1.1.1.3 root 1216: #endif
1217: #endif
1218: #ifdef MACH_PV_PAGETABLES
1219: {
1220: for (i = 0; i < PDPNUM; i++)
1.1.1.7 ! root 1221: pmap_set_page_readonly((void *) page_dir[i]);
1.1.1.3 root 1222: }
1223: #endif /* MACH_PV_PAGETABLES */
1224:
1225: #if PAE
1.1.1.7 ! root 1226: p->pdpbase = (pt_entry_t *) kmem_cache_alloc(&pdpt_cache);
! 1227: if (p->pdpbase == NULL) {
! 1228: for (i = 0; i < PDPNUM; i++)
! 1229: kmem_cache_free(&pd_cache, (vm_address_t) page_dir[i]);
! 1230: kmem_cache_free(&pmap_cache, (vm_address_t) p);
! 1231: return PMAP_NULL;
! 1232: }
! 1233:
1.1.1.3 root 1234: {
1235: for (i = 0; i < PDPNUM; i++)
1.1.1.7 ! root 1236: WRITE_PTE(&p->pdpbase[i],
! 1237: pa_to_pte(kvtophys((vm_offset_t) page_dir[i]))
! 1238: | INTEL_PTE_VALID);
1.1.1.3 root 1239: }
1240: #ifdef MACH_PV_PAGETABLES
1241: pmap_set_page_readonly(p->pdpbase);
1242: #endif /* MACH_PV_PAGETABLES */
1.1.1.7 ! root 1243: #else /* PAE */
! 1244: p->dirbase = page_dir[0];
1.1.1.3 root 1245: #endif /* PAE */
1246:
1.1 root 1247: p->ref_count = 1;
1248:
1249: simple_lock_init(&p->lock);
1250: p->cpus_using = 0;
1251:
1252: /*
1253: * Initialize statistics.
1254: */
1255:
1256: stats = &p->stats;
1257: stats->resident_count = 0;
1258: stats->wired_count = 0;
1259:
1260: return(p);
1261: }
1262:
1263: /*
1264: * Retire the given physical map from service.
1265: * Should only be called if the map contains
1266: * no valid mappings.
1267: */
1268:
1.1.1.4 root 1269: void pmap_destroy(pmap_t p)
1.1 root 1270: {
1.1.1.7 ! root 1271: #if PAE
! 1272: int i;
! 1273: #endif
! 1274: boolean_t free_all;
! 1275: pt_entry_t *page_dir;
1.1.1.4 root 1276: pt_entry_t *pdep;
1.1.1.7 ! root 1277: phys_addr_t pa;
1.1.1.4 root 1278: int c, s;
1279: vm_page_t m;
1.1 root 1280:
1281: if (p == PMAP_NULL)
1282: return;
1283:
1284: SPLVM(s);
1285: simple_lock(&p->lock);
1286: c = --p->ref_count;
1287: simple_unlock(&p->lock);
1288: SPLX(s);
1289:
1290: if (c != 0) {
1291: return; /* still in use */
1292: }
1293:
1.1.1.7 ! root 1294: #if PAE
! 1295: for (i = 0; i <= lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); i++) {
! 1296: free_all = i < lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS);
! 1297: page_dir = (pt_entry_t *) ptetokv(p->pdpbase[i]);
! 1298: #else
! 1299: free_all = FALSE;
! 1300: page_dir = p->dirbase;
! 1301: #endif
! 1302:
! 1303: /*
! 1304: * Free the memory maps, then the
! 1305: * pmap structure.
! 1306: */
! 1307: for (pdep = page_dir;
! 1308: (free_all
! 1309: || pdep < &page_dir[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)])
! 1310: && pdep < &page_dir[NPTES];
! 1311: pdep += ptes_per_vm_page) {
! 1312: if (*pdep & INTEL_PTE_VALID) {
! 1313: pa = pte_to_pa(*pdep);
! 1314: vm_object_lock(pmap_object);
! 1315: m = vm_page_lookup(pmap_object, pa);
! 1316: if (m == VM_PAGE_NULL)
! 1317: panic("pmap_destroy: pte page not in object");
! 1318: vm_page_lock_queues();
! 1319: #ifdef MACH_PV_PAGETABLES
! 1320: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa)))
! 1321: panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa));
! 1322: pmap_set_page_readwrite((void*) phystokv(pa));
! 1323: #endif /* MACH_PV_PAGETABLES */
! 1324: vm_page_free(m);
! 1325: inuse_ptepages_count--;
! 1326: vm_page_unlock_queues();
! 1327: vm_object_unlock(pmap_object);
! 1328: }
1.1 root 1329: }
1.1.1.3 root 1330: #ifdef MACH_PV_PAGETABLES
1.1.1.7 ! root 1331: pmap_set_page_readwrite((void*) page_dir);
1.1.1.3 root 1332: #endif /* MACH_PV_PAGETABLES */
1.1.1.7 ! root 1333: kmem_cache_free(&pd_cache, (vm_offset_t) page_dir);
1.1.1.3 root 1334: #if PAE
1.1.1.7 ! root 1335: }
! 1336:
1.1.1.3 root 1337: #ifdef MACH_PV_PAGETABLES
1338: pmap_set_page_readwrite(p->pdpbase);
1339: #endif /* MACH_PV_PAGETABLES */
1.1.1.7 ! root 1340: kmem_cache_free(&pdpt_cache, (vm_offset_t) p->pdpbase);
1.1.1.3 root 1341: #endif /* PAE */
1342: kmem_cache_free(&pmap_cache, (vm_offset_t) p);
1.1 root 1343: }
1344:
1345: /*
1346: * Add a reference to the specified pmap.
1347: */
1348:
1.1.1.4 root 1349: void pmap_reference(pmap_t p)
1.1 root 1350: {
1351: int s;
1352: if (p != PMAP_NULL) {
1353: SPLVM(s);
1354: simple_lock(&p->lock);
1355: p->ref_count++;
1356: simple_unlock(&p->lock);
1357: SPLX(s);
1358: }
1359: }
1360:
1361: /*
1362: * Remove a range of hardware page-table entries.
1363: * The entries given are the first (inclusive)
1364: * and last (exclusive) entries for the VM pages.
1365: * The virtual address is the va for the first pte.
1366: *
1367: * The pmap must be locked.
1368: * If the pmap is not the kernel pmap, the range must lie
1369: * entirely within one pte-page. This is NOT checked.
1370: * Assumes that the pte-page exists.
1371: */
1372:
1373: /* static */
1.1.1.4 root 1374: void pmap_remove_range(
1375: pmap_t pmap,
1376: vm_offset_t va,
1377: pt_entry_t *spte,
1378: pt_entry_t *epte)
1.1 root 1379: {
1.1.1.4 root 1380: pt_entry_t *cpte;
1.1.1.7 ! root 1381: unsigned long num_removed, num_unwired;
! 1382: unsigned long pai;
! 1383: phys_addr_t pa;
1.1.1.3 root 1384: #ifdef MACH_PV_PAGETABLES
1385: int n, ii = 0;
1386: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1387: #endif /* MACH_PV_PAGETABLES */
1.1 root 1388:
1389: #if DEBUG_PTE_PAGE
1390: if (pmap != kernel_pmap)
1391: ptep_check(get_pte_page(spte));
1.1.1.2 root 1392: #endif /* DEBUG_PTE_PAGE */
1.1 root 1393: num_removed = 0;
1394: num_unwired = 0;
1395:
1396: for (cpte = spte; cpte < epte;
1397: cpte += ptes_per_vm_page, va += PAGE_SIZE) {
1398:
1399: if (*cpte == 0)
1400: continue;
1401: pa = pte_to_pa(*cpte);
1402:
1403: num_removed++;
1404: if (*cpte & INTEL_PTE_WIRED)
1405: num_unwired++;
1406:
1407: if (!valid_page(pa)) {
1408:
1409: /*
1410: * Outside range of managed physical memory.
1411: * Just remove the mappings.
1412: */
1.1.1.4 root 1413: int i = ptes_per_vm_page;
1414: pt_entry_t *lpte = cpte;
1.1 root 1415: do {
1.1.1.3 root 1416: #ifdef MACH_PV_PAGETABLES
1417: update[ii].ptr = kv_to_ma(lpte);
1418: update[ii].val = 0;
1419: ii++;
1420: if (ii == HYP_BATCH_MMU_UPDATES) {
1421: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1422: if (n != ii)
1423: panic("couldn't pmap_remove_range\n");
1424: ii = 0;
1425: }
1426: #else /* MACH_PV_PAGETABLES */
1.1 root 1427: *lpte = 0;
1.1.1.3 root 1428: #endif /* MACH_PV_PAGETABLES */
1.1 root 1429: lpte++;
1430: } while (--i > 0);
1431: continue;
1432: }
1433:
1434: pai = pa_index(pa);
1435: LOCK_PVH(pai);
1436:
1437: /*
1438: * Get the modify and reference bits.
1439: */
1440: {
1.1.1.4 root 1441: int i;
1442: pt_entry_t *lpte;
1.1 root 1443:
1444: i = ptes_per_vm_page;
1445: lpte = cpte;
1446: do {
1447: pmap_phys_attributes[pai] |=
1448: *lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 root 1449: #ifdef MACH_PV_PAGETABLES
1450: update[ii].ptr = kv_to_ma(lpte);
1451: update[ii].val = 0;
1452: ii++;
1453: if (ii == HYP_BATCH_MMU_UPDATES) {
1454: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1455: if (n != ii)
1456: panic("couldn't pmap_remove_range\n");
1457: ii = 0;
1458: }
1459: #else /* MACH_PV_PAGETABLES */
1.1 root 1460: *lpte = 0;
1.1.1.3 root 1461: #endif /* MACH_PV_PAGETABLES */
1.1 root 1462: lpte++;
1463: } while (--i > 0);
1464: }
1465:
1466: /*
1467: * Remove the mapping from the pvlist for
1468: * this physical page.
1469: */
1470: {
1.1.1.4 root 1471: pv_entry_t pv_h, prev, cur;
1.1 root 1472:
1473: pv_h = pai_to_pvh(pai);
1474: if (pv_h->pmap == PMAP_NULL) {
1475: panic("pmap_remove: null pv_list!");
1476: }
1477: if (pv_h->va == va && pv_h->pmap == pmap) {
1478: /*
1479: * Header is the pv_entry. Copy the next one
1480: * to header and free the next one (we cannot
1481: * free the header)
1482: */
1483: cur = pv_h->next;
1484: if (cur != PV_ENTRY_NULL) {
1485: *pv_h = *cur;
1486: PV_FREE(cur);
1487: }
1488: else {
1489: pv_h->pmap = PMAP_NULL;
1490: }
1491: }
1492: else {
1493: cur = pv_h;
1494: do {
1495: prev = cur;
1496: if ((cur = prev->next) == PV_ENTRY_NULL) {
1497: panic("pmap-remove: mapping not in pv_list!");
1498: }
1499: } while (cur->va != va || cur->pmap != pmap);
1500: prev->next = cur->next;
1501: PV_FREE(cur);
1502: }
1503: UNLOCK_PVH(pai);
1504: }
1505: }
1506:
1.1.1.3 root 1507: #ifdef MACH_PV_PAGETABLES
1508: if (ii > HYP_BATCH_MMU_UPDATES)
1509: panic("overflowed array in pmap_remove_range");
1510: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1511: if (n != ii)
1512: panic("couldn't pmap_remove_range\n");
1513: #endif /* MACH_PV_PAGETABLES */
1514:
1.1 root 1515: /*
1516: * Update the counts
1517: */
1518: pmap->stats.resident_count -= num_removed;
1519: pmap->stats.wired_count -= num_unwired;
1520: }
1521:
1522: /*
1523: * Remove the given range of addresses
1524: * from the specified map.
1525: *
1526: * It is assumed that the start and end are properly
1527: * rounded to the hardware page size.
1528: */
1529:
1.1.1.4 root 1530: void pmap_remove(
1531: pmap_t map,
1532: vm_offset_t s,
1533: vm_offset_t e)
1.1 root 1534: {
1535: int spl;
1.1.1.4 root 1536: pt_entry_t *pde;
1537: pt_entry_t *spte, *epte;
1.1 root 1538: vm_offset_t l;
1.1.1.3 root 1539: vm_offset_t _s = s;
1.1 root 1540:
1541: if (map == PMAP_NULL)
1542: return;
1543:
1544: PMAP_READ_LOCK(map, spl);
1545:
1546: pde = pmap_pde(map, s);
1547: while (s < e) {
1548: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1549: if (l > e)
1550: l = e;
1551: if (*pde & INTEL_PTE_VALID) {
1552: spte = (pt_entry_t *)ptetokv(*pde);
1553: spte = &spte[ptenum(s)];
1554: epte = &spte[intel_btop(l-s)];
1555: pmap_remove_range(map, s, spte, epte);
1556: }
1557: s = l;
1558: pde++;
1559: }
1.1.1.3 root 1560: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1561:
1562: PMAP_READ_UNLOCK(map, spl);
1563: }
1564:
1565: /*
1566: * Routine: pmap_page_protect
1567: *
1568: * Function:
1569: * Lower the permission for all mappings to a given
1570: * page.
1571: */
1.1.1.4 root 1572: void pmap_page_protect(
1.1.1.7 ! root 1573: phys_addr_t phys,
1.1.1.4 root 1574: vm_prot_t prot)
1.1 root 1575: {
1576: pv_entry_t pv_h, prev;
1.1.1.4 root 1577: pv_entry_t pv_e;
1578: pt_entry_t *pte;
1.1.1.7 ! root 1579: unsigned long pai;
1.1.1.4 root 1580: pmap_t pmap;
1.1 root 1581: int spl;
1582: boolean_t remove;
1583:
1584: assert(phys != vm_page_fictitious_addr);
1585: if (!valid_page(phys)) {
1586: /*
1587: * Not a managed page.
1588: */
1589: return;
1590: }
1591:
1592: /*
1593: * Determine the new protection.
1594: */
1595: switch (prot) {
1596: case VM_PROT_READ:
1597: case VM_PROT_READ|VM_PROT_EXECUTE:
1598: remove = FALSE;
1599: break;
1600: case VM_PROT_ALL:
1601: return; /* nothing to do */
1602: default:
1603: remove = TRUE;
1604: break;
1605: }
1606:
1607: /*
1608: * Lock the pmap system first, since we will be changing
1609: * several pmaps.
1610: */
1611:
1612: PMAP_WRITE_LOCK(spl);
1613:
1614: pai = pa_index(phys);
1615: pv_h = pai_to_pvh(pai);
1616:
1617: /*
1618: * Walk down PV list, changing or removing all mappings.
1619: * We do not have to lock the pv_list because we have
1620: * the entire pmap system locked.
1621: */
1622: if (pv_h->pmap != PMAP_NULL) {
1623:
1624: prev = pv_e = pv_h;
1625: do {
1.1.1.4 root 1626: vm_offset_t va;
1.1.1.3 root 1627:
1.1 root 1628: pmap = pv_e->pmap;
1629: /*
1630: * Lock the pmap to block pmap_extract and similar routines.
1631: */
1632: simple_lock(&pmap->lock);
1633:
1.1.1.3 root 1634: va = pv_e->va;
1635: pte = pmap_pte(pmap, va);
1.1 root 1636:
1.1.1.3 root 1637: /*
1638: * Consistency checks.
1639: */
1.1.1.5 root 1640: assert(*pte & INTEL_PTE_VALID);
1641: assert(pte_to_pa(*pte) == phys);
1.1 root 1642:
1643: /*
1644: * Remove the mapping if new protection is NONE
1645: * or if write-protecting a kernel mapping.
1646: */
1647: if (remove || pmap == kernel_pmap) {
1648: /*
1649: * Remove the mapping, collecting any modify bits.
1650: */
1651: if (*pte & INTEL_PTE_WIRED)
1.1.1.5 root 1652: panic("pmap_page_protect removing a wired page");
1.1 root 1653:
1654: {
1.1.1.4 root 1655: int i = ptes_per_vm_page;
1.1 root 1656:
1657: do {
1658: pmap_phys_attributes[pai] |=
1659: *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 root 1660: #ifdef MACH_PV_PAGETABLES
1661: if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0))
1662: panic("%s:%d could not clear pte %p\n",__FILE__,__LINE__,pte-1);
1663: #else /* MACH_PV_PAGETABLES */
1.1 root 1664: *pte++ = 0;
1.1.1.3 root 1665: #endif /* MACH_PV_PAGETABLES */
1.1 root 1666: } while (--i > 0);
1667: }
1668:
1669: pmap->stats.resident_count--;
1670:
1671: /*
1672: * Remove the pv_entry.
1673: */
1674: if (pv_e == pv_h) {
1675: /*
1676: * Fix up head later.
1677: */
1678: pv_h->pmap = PMAP_NULL;
1679: }
1680: else {
1681: /*
1682: * Delete this entry.
1683: */
1684: prev->next = pv_e->next;
1685: PV_FREE(pv_e);
1686: }
1687: }
1688: else {
1689: /*
1690: * Write-protect.
1691: */
1.1.1.4 root 1692: int i = ptes_per_vm_page;
1.1 root 1693:
1694: do {
1.1.1.3 root 1695: #ifdef MACH_PV_PAGETABLES
1696: if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE))
1697: panic("%s:%d could not disable write on pte %p\n",__FILE__,__LINE__,pte);
1698: #else /* MACH_PV_PAGETABLES */
1.1 root 1699: *pte &= ~INTEL_PTE_WRITE;
1.1.1.3 root 1700: #endif /* MACH_PV_PAGETABLES */
1.1 root 1701: pte++;
1702: } while (--i > 0);
1703:
1704: /*
1705: * Advance prev.
1706: */
1707: prev = pv_e;
1708: }
1.1.1.3 root 1709: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 1710:
1711: simple_unlock(&pmap->lock);
1712:
1713: } while ((pv_e = prev->next) != PV_ENTRY_NULL);
1714:
1715: /*
1716: * If pv_head mapping was removed, fix it up.
1717: */
1718: if (pv_h->pmap == PMAP_NULL) {
1719: pv_e = pv_h->next;
1720: if (pv_e != PV_ENTRY_NULL) {
1721: *pv_h = *pv_e;
1722: PV_FREE(pv_e);
1723: }
1724: }
1725: }
1726:
1727: PMAP_WRITE_UNLOCK(spl);
1728: }
1729:
1730: /*
1731: * Set the physical protection on the
1732: * specified range of this map as requested.
1733: * Will not increase permissions.
1734: */
1.1.1.4 root 1735: void pmap_protect(
1736: pmap_t map,
1737: vm_offset_t s,
1738: vm_offset_t e,
1739: vm_prot_t prot)
1740: {
1741: pt_entry_t *pde;
1742: pt_entry_t *spte, *epte;
1743: vm_offset_t l;
1.1 root 1744: int spl;
1.1.1.3 root 1745: vm_offset_t _s = s;
1.1 root 1746:
1747: if (map == PMAP_NULL)
1748: return;
1749:
1750: /*
1751: * Determine the new protection.
1752: */
1753: switch (prot) {
1754: case VM_PROT_READ:
1755: case VM_PROT_READ|VM_PROT_EXECUTE:
1756: break;
1757: case VM_PROT_READ|VM_PROT_WRITE:
1758: case VM_PROT_ALL:
1759: return; /* nothing to do */
1760: default:
1761: pmap_remove(map, s, e);
1762: return;
1763: }
1764:
1765: /*
1766: * If write-protecting in the kernel pmap,
1767: * remove the mappings; the i386 ignores
1768: * the write-permission bit in kernel mode.
1769: *
1770: * XXX should be #if'd for i386
1771: */
1772: if (map == kernel_pmap) {
1773: pmap_remove(map, s, e);
1774: return;
1775: }
1776:
1777: SPLVM(spl);
1778: simple_lock(&map->lock);
1779:
1780: pde = pmap_pde(map, s);
1781: while (s < e) {
1782: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1783: if (l > e)
1784: l = e;
1785: if (*pde & INTEL_PTE_VALID) {
1786: spte = (pt_entry_t *)ptetokv(*pde);
1787: spte = &spte[ptenum(s)];
1788: epte = &spte[intel_btop(l-s)];
1789:
1.1.1.3 root 1790: #ifdef MACH_PV_PAGETABLES
1791: int n, i = 0;
1792: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1793: #endif /* MACH_PV_PAGETABLES */
1794:
1.1 root 1795: while (spte < epte) {
1.1.1.3 root 1796: if (*spte & INTEL_PTE_VALID) {
1797: #ifdef MACH_PV_PAGETABLES
1798: update[i].ptr = kv_to_ma(spte);
1799: update[i].val = *spte & ~INTEL_PTE_WRITE;
1800: i++;
1801: if (i == HYP_BATCH_MMU_UPDATES) {
1802: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
1803: if (n != i)
1804: panic("couldn't pmap_protect\n");
1805: i = 0;
1806: }
1807: #else /* MACH_PV_PAGETABLES */
1.1 root 1808: *spte &= ~INTEL_PTE_WRITE;
1.1.1.3 root 1809: #endif /* MACH_PV_PAGETABLES */
1810: }
1.1 root 1811: spte++;
1812: }
1.1.1.3 root 1813: #ifdef MACH_PV_PAGETABLES
1814: if (i > HYP_BATCH_MMU_UPDATES)
1815: panic("overflowed array in pmap_protect");
1816: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
1817: if (n != i)
1818: panic("couldn't pmap_protect\n");
1819: #endif /* MACH_PV_PAGETABLES */
1.1 root 1820: }
1821: s = l;
1822: pde++;
1823: }
1.1.1.3 root 1824: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1825:
1826: simple_unlock(&map->lock);
1827: SPLX(spl);
1828: }
1829:
1830: /*
1831: * Insert the given physical page (p) at
1832: * the specified virtual address (v) in the
1833: * target physical map with the protection requested.
1834: *
1835: * If specified, the page will be wired down, meaning
1836: * that the related pte can not be reclaimed.
1837: *
1838: * NB: This is the only routine which MAY NOT lazy-evaluate
1839: * or lose information. That is, this routine must actually
1840: * insert this page into the given map NOW.
1841: */
1.1.1.4 root 1842: void pmap_enter(
1843: pmap_t pmap,
1844: vm_offset_t v,
1.1.1.7 ! root 1845: phys_addr_t pa,
1.1.1.4 root 1846: vm_prot_t prot,
1847: boolean_t wired)
1848: {
1.1.1.7 ! root 1849: boolean_t is_physmem;
1.1.1.4 root 1850: pt_entry_t *pte;
1851: pv_entry_t pv_h;
1.1.1.7 ! root 1852: unsigned long i, pai;
1.1 root 1853: pv_entry_t pv_e;
1854: pt_entry_t template;
1855: int spl;
1.1.1.7 ! root 1856: phys_addr_t old_pa;
1.1 root 1857:
1858: assert(pa != vm_page_fictitious_addr);
1.1.1.3 root 1859: if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa);
1.1 root 1860: if (pmap == PMAP_NULL)
1861: return;
1862:
1.1.1.3 root 1863: #if !MACH_KDB
1864: if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end))
1865: panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa);
1866: #endif
1.1 root 1867: if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0
1868: && !wired /* hack for io_wire */ ) {
1869: /*
1870: * Because the 386 ignores write protection in kernel mode,
1871: * we cannot enter a read-only kernel mapping, and must
1872: * remove an existing mapping if changing it.
1873: *
1874: * XXX should be #if'd for i386
1875: */
1876: PMAP_READ_LOCK(pmap, spl);
1877:
1878: pte = pmap_pte(pmap, v);
1879: if (pte != PT_ENTRY_NULL && *pte != 0) {
1880: /*
1881: * Invalidate the translation buffer,
1882: * then remove the mapping.
1883: */
1884: pmap_remove_range(pmap, v, pte,
1885: pte + ptes_per_vm_page);
1.1.1.3 root 1886: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1887: }
1888: PMAP_READ_UNLOCK(pmap, spl);
1889: return;
1890: }
1891:
1892: /*
1893: * Must allocate a new pvlist entry while we're unlocked;
1.1.1.3 root 1894: * Allocating may cause pageout (which will lock the pmap system).
1.1 root 1895: * If we determine we need a pvlist entry, we will unlock
1896: * and allocate one. Then we will retry, throughing away
1897: * the allocated entry later (if we no longer need it).
1898: */
1899: pv_e = PV_ENTRY_NULL;
1900: Retry:
1901: PMAP_READ_LOCK(pmap, spl);
1902:
1903: /*
1904: * Expand pmap to include this pte. Assume that
1905: * pmap is always expanded to include enough hardware
1906: * pages to map one VM page.
1907: */
1908:
1909: while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
1910: /*
1911: * Need to allocate a new page-table page.
1912: */
1913: vm_offset_t ptp;
1914: pt_entry_t *pdp;
1915: int i;
1916:
1917: if (pmap == kernel_pmap) {
1918: /*
1919: * Would have to enter the new page-table page in
1920: * EVERY pmap.
1921: */
1922: panic("pmap_expand kernel pmap to %#x", v);
1923: }
1924:
1925: /*
1926: * Unlock the pmap and allocate a new page-table page.
1927: */
1928: PMAP_READ_UNLOCK(pmap, spl);
1929:
1.1.1.3 root 1930: ptp = phystokv(pmap_page_table_page_alloc());
1.1 root 1931:
1932: /*
1933: * Re-lock the pmap and check that another thread has
1934: * not already allocated the page-table page. If it
1935: * has, discard the new page-table page (and try
1936: * again to make sure).
1937: */
1938: PMAP_READ_LOCK(pmap, spl);
1939:
1940: if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
1941: /*
1942: * Oops...
1943: */
1944: PMAP_READ_UNLOCK(pmap, spl);
1.1.1.3 root 1945: pmap_page_table_page_dealloc(kvtophys(ptp));
1.1 root 1946: PMAP_READ_LOCK(pmap, spl);
1947: continue;
1948: }
1949:
1950: /*
1951: * Enter the new page table page in the page directory.
1952: */
1953: i = ptes_per_vm_page;
1954: /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/
1955: pdp = pmap_pde(pmap, v);
1956: do {
1.1.1.3 root 1957: #ifdef MACH_PV_PAGETABLES
1958: pmap_set_page_readonly((void *) ptp);
1959: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp)))
1960: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1961: if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)),
1962: pa_to_pte(pa_to_ma(kvtophys(ptp))) | INTEL_PTE_VALID
1963: | INTEL_PTE_USER
1964: | INTEL_PTE_WRITE))
1965: panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp)));
1966: #else /* MACH_PV_PAGETABLES */
1967: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
1968: | INTEL_PTE_USER
1969: | INTEL_PTE_WRITE;
1970: #endif /* MACH_PV_PAGETABLES */
1.1 root 1971: pdp++;
1972: ptp += INTEL_PGBYTES;
1973: } while (--i > 0);
1974:
1975: /*
1976: * Now, get the address of the page-table entry.
1977: */
1978: continue;
1979: }
1980:
1.1.1.7 ! root 1981: if (vm_page_ready())
! 1982: is_physmem = (vm_page_lookup_pa(pa) != NULL);
! 1983: else
! 1984: is_physmem = (pa < biosmem_directmap_end());
! 1985:
1.1 root 1986: /*
1987: * Special case if the physical page is already mapped
1988: * at this address.
1989: */
1990: old_pa = pte_to_pa(*pte);
1991: if (*pte && old_pa == pa) {
1992: /*
1993: * May be changing its wired attribute or protection
1994: */
1.1.1.2 root 1995:
1.1 root 1996: if (wired && !(*pte & INTEL_PTE_WIRED))
1997: pmap->stats.wired_count++;
1998: else if (!wired && (*pte & INTEL_PTE_WIRED))
1999: pmap->stats.wired_count--;
2000:
2001: template = pa_to_pte(pa) | INTEL_PTE_VALID;
2002: if (pmap != kernel_pmap)
2003: template |= INTEL_PTE_USER;
2004: if (prot & VM_PROT_WRITE)
2005: template |= INTEL_PTE_WRITE;
1.1.1.3 root 2006: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
1.1.1.7 ! root 2007: && !is_physmem)
1.1.1.3 root 2008: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 2009: if (wired)
2010: template |= INTEL_PTE_WIRED;
2011: i = ptes_per_vm_page;
2012: do {
2013: if (*pte & INTEL_PTE_MOD)
2014: template |= INTEL_PTE_MOD;
1.1.1.3 root 2015: #ifdef MACH_PV_PAGETABLES
2016: if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))
2017: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
2018: #else /* MACH_PV_PAGETABLES */
1.1 root 2019: WRITE_PTE(pte, template)
1.1.1.3 root 2020: #endif /* MACH_PV_PAGETABLES */
1.1 root 2021: pte++;
2022: pte_increment_pa(template);
2023: } while (--i > 0);
1.1.1.3 root 2024: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 2025: }
2026: else {
2027:
2028: /*
2029: * Remove old mapping from the PV list if necessary.
2030: */
2031: if (*pte) {
2032: /*
2033: * Don't free the pte page if removing last
2034: * mapping - we will immediately replace it.
2035: */
2036: pmap_remove_range(pmap, v, pte,
2037: pte + ptes_per_vm_page);
1.1.1.3 root 2038: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 2039: }
2040:
2041: if (valid_page(pa)) {
2042:
2043: /*
2044: * Enter the mapping in the PV list for this
2045: * physical page.
2046: */
2047:
2048: pai = pa_index(pa);
2049: LOCK_PVH(pai);
2050: pv_h = pai_to_pvh(pai);
2051:
2052: if (pv_h->pmap == PMAP_NULL) {
2053: /*
2054: * No mappings yet
2055: */
2056: pv_h->va = v;
2057: pv_h->pmap = pmap;
2058: pv_h->next = PV_ENTRY_NULL;
2059: }
2060: else {
2061: #if DEBUG
2062: {
2063: /* check that this mapping is not already there */
2064: pv_entry_t e = pv_h;
2065: while (e != PV_ENTRY_NULL) {
2066: if (e->pmap == pmap && e->va == v)
2067: panic("pmap_enter: already in pv_list");
2068: e = e->next;
2069: }
2070: }
1.1.1.2 root 2071: #endif /* DEBUG */
2072:
1.1 root 2073: /*
2074: * Add new pv_entry after header.
2075: */
2076: if (pv_e == PV_ENTRY_NULL) {
2077: PV_ALLOC(pv_e);
2078: if (pv_e == PV_ENTRY_NULL) {
2079: UNLOCK_PVH(pai);
2080: PMAP_READ_UNLOCK(pmap, spl);
2081:
2082: /*
1.1.1.3 root 2083: * Refill from cache.
1.1 root 2084: */
1.1.1.3 root 2085: pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache);
1.1 root 2086: goto Retry;
2087: }
2088: }
2089: pv_e->va = v;
2090: pv_e->pmap = pmap;
2091: pv_e->next = pv_h->next;
2092: pv_h->next = pv_e;
2093: /*
2094: * Remember that we used the pvlist entry.
2095: */
2096: pv_e = PV_ENTRY_NULL;
2097: }
2098: UNLOCK_PVH(pai);
2099: }
2100:
2101: /*
2102: * And count the mapping.
2103: */
2104:
2105: pmap->stats.resident_count++;
2106: if (wired)
2107: pmap->stats.wired_count++;
2108:
2109: /*
2110: * Build a template to speed up entering -
2111: * only the pfn changes.
2112: */
2113: template = pa_to_pte(pa) | INTEL_PTE_VALID;
2114: if (pmap != kernel_pmap)
2115: template |= INTEL_PTE_USER;
2116: if (prot & VM_PROT_WRITE)
2117: template |= INTEL_PTE_WRITE;
1.1.1.3 root 2118: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
1.1.1.7 ! root 2119: && !is_physmem)
1.1.1.3 root 2120: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 2121: if (wired)
2122: template |= INTEL_PTE_WIRED;
2123: i = ptes_per_vm_page;
2124: do {
1.1.1.3 root 2125: #ifdef MACH_PV_PAGETABLES
2126: if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))))
2127: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
2128: #else /* MACH_PV_PAGETABLES */
1.1 root 2129: WRITE_PTE(pte, template)
1.1.1.3 root 2130: #endif /* MACH_PV_PAGETABLES */
1.1 root 2131: pte++;
2132: pte_increment_pa(template);
2133: } while (--i > 0);
2134: }
2135:
2136: if (pv_e != PV_ENTRY_NULL) {
2137: PV_FREE(pv_e);
2138: }
2139:
2140: PMAP_READ_UNLOCK(pmap, spl);
2141: }
2142:
2143: /*
2144: * Routine: pmap_change_wiring
2145: * Function: Change the wiring attribute for a map/virtual-address
2146: * pair.
2147: * In/out conditions:
2148: * The mapping must already exist in the pmap.
2149: */
1.1.1.4 root 2150: void pmap_change_wiring(
2151: pmap_t map,
2152: vm_offset_t v,
2153: boolean_t wired)
1.1 root 2154: {
1.1.1.4 root 2155: pt_entry_t *pte;
2156: int i;
2157: int spl;
1.1 root 2158:
2159: /*
2160: * We must grab the pmap system lock because we may
2161: * change a pte_page queue.
2162: */
2163: PMAP_READ_LOCK(map, spl);
2164:
2165: if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
2166: panic("pmap_change_wiring: pte missing");
2167:
2168: if (wired && !(*pte & INTEL_PTE_WIRED)) {
2169: /*
2170: * wiring down mapping
2171: */
2172: map->stats.wired_count++;
2173: i = ptes_per_vm_page;
2174: do {
2175: *pte++ |= INTEL_PTE_WIRED;
2176: } while (--i > 0);
2177: }
2178: else if (!wired && (*pte & INTEL_PTE_WIRED)) {
2179: /*
2180: * unwiring mapping
2181: */
2182: map->stats.wired_count--;
2183: i = ptes_per_vm_page;
2184: do {
1.1.1.3 root 2185: #ifdef MACH_PV_PAGETABLES
2186: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED)))
2187: panic("%s:%d could not wire down pte %p\n",__FILE__,__LINE__,pte);
2188: #else /* MACH_PV_PAGETABLES */
1.1 root 2189: *pte &= ~INTEL_PTE_WIRED;
1.1.1.3 root 2190: #endif /* MACH_PV_PAGETABLES */
2191: pte++;
1.1 root 2192: } while (--i > 0);
2193: }
2194:
2195: PMAP_READ_UNLOCK(map, spl);
2196: }
2197:
2198: /*
2199: * Routine: pmap_extract
2200: * Function:
2201: * Extract the physical page address associated
2202: * with the given map/virtual_address pair.
2203: */
2204:
1.1.1.7 ! root 2205: phys_addr_t pmap_extract(
1.1.1.4 root 2206: pmap_t pmap,
2207: vm_offset_t va)
1.1 root 2208: {
1.1.1.4 root 2209: pt_entry_t *pte;
1.1.1.7 ! root 2210: phys_addr_t pa;
1.1.1.4 root 2211: int spl;
1.1 root 2212:
2213: SPLVM(spl);
2214: simple_lock(&pmap->lock);
2215: if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL)
1.1.1.7 ! root 2216: pa = 0;
1.1 root 2217: else if (!(*pte & INTEL_PTE_VALID))
1.1.1.7 ! root 2218: pa = 0;
1.1 root 2219: else
2220: pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK);
2221: simple_unlock(&pmap->lock);
2222: SPLX(spl);
2223: return(pa);
2224: }
2225:
2226: /*
2227: * Copy the range specified by src_addr/len
2228: * from the source map to the range dst_addr/len
2229: * in the destination map.
2230: *
2231: * This routine is only advisory and need not do anything.
2232: */
2233: #if 0
2234: void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2235: pmap_t dst_pmap;
2236: pmap_t src_pmap;
2237: vm_offset_t dst_addr;
2238: vm_size_t len;
2239: vm_offset_t src_addr;
2240: {
2241: }
1.1.1.2 root 2242: #endif /* 0 */
1.1 root 2243:
2244: /*
2245: * Routine: pmap_collect
2246: * Function:
2247: * Garbage collects the physical map system for
2248: * pages which are no longer used.
2249: * Success need not be guaranteed -- that is, there
2250: * may well be pages which are not referenced, but
2251: * others may be collected.
2252: * Usage:
2253: * Called by the pageout daemon when pages are scarce.
2254: */
1.1.1.4 root 2255: void pmap_collect(pmap_t p)
1.1 root 2256: {
1.1.1.7 ! root 2257: int i;
! 2258: boolean_t free_all;
! 2259: pt_entry_t *page_dir;
1.1.1.4 root 2260: pt_entry_t *pdp, *ptp;
1.1 root 2261: pt_entry_t *eptp;
1.1.1.7 ! root 2262: phys_addr_t pa;
1.1 root 2263: int spl, wired;
2264:
2265: if (p == PMAP_NULL)
2266: return;
2267:
2268: if (p == kernel_pmap)
2269: return;
2270:
1.1.1.7 ! root 2271: #if PAE
! 2272: for (i = 0; i <= lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS); i++) {
! 2273: free_all = i < lin2pdpnum(LINEAR_MIN_KERNEL_ADDRESS);
! 2274: page_dir = (pt_entry_t *) ptetokv(p->pdpbase[i]);
! 2275: #else
! 2276: i = 0;
! 2277: free_all = FALSE;
! 2278: page_dir = p->dirbase;
! 2279: #endif
1.1 root 2280:
1.1.1.7 ! root 2281: /*
! 2282: * Garbage collect map.
! 2283: */
! 2284: PMAP_READ_LOCK(p, spl);
! 2285: for (pdp = page_dir;
! 2286: (free_all
! 2287: || pdp < &page_dir[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)])
! 2288: && pdp < &page_dir[NPTES];
! 2289: pdp += ptes_per_vm_page) {
! 2290: if (*pdp & INTEL_PTE_VALID) {
! 2291:
! 2292: pa = pte_to_pa(*pdp);
! 2293: ptp = (pt_entry_t *)phystokv(pa);
! 2294: eptp = ptp + NPTES*ptes_per_vm_page;
1.1 root 2295:
2296: /*
1.1.1.7 ! root 2297: * If the pte page has any wired mappings, we cannot
! 2298: * free it.
1.1 root 2299: */
1.1.1.7 ! root 2300: wired = 0;
! 2301: {
! 2302: pt_entry_t *ptep;
! 2303: for (ptep = ptp; ptep < eptp; ptep++) {
! 2304: if (*ptep & INTEL_PTE_WIRED) {
! 2305: wired = 1;
! 2306: break;
! 2307: }
! 2308: }
1.1 root 2309: }
1.1.1.7 ! root 2310: if (!wired) {
! 2311: /*
! 2312: * Remove the virtual addresses mapped by this pte page.
! 2313: */
! 2314: { /*XXX big hack*/
! 2315: vm_offset_t va = pdenum2lin(pdp - page_dir
! 2316: + i * NPTES);
! 2317: if (p == kernel_pmap)
! 2318: va = lintokv(va);
! 2319: pmap_remove_range(p,
! 2320: va,
! 2321: ptp,
! 2322: eptp);
! 2323: }
1.1 root 2324:
1.1.1.7 ! root 2325: /*
! 2326: * Invalidate the page directory pointer.
! 2327: */
! 2328: {
! 2329: int i = ptes_per_vm_page;
! 2330: pt_entry_t *pdep = pdp;
! 2331: do {
! 2332: #ifdef MACH_PV_PAGETABLES
! 2333: unsigned long pte = *pdep;
! 2334: void *ptable = (void*) ptetokv(pte);
! 2335: if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0)))
! 2336: panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1);
! 2337: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable)))
! 2338: panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable)));
! 2339: pmap_set_page_readwrite(ptable);
1.1.1.3 root 2340: #else /* MACH_PV_PAGETABLES */
1.1.1.7 ! root 2341: *pdep++ = 0;
1.1.1.3 root 2342: #endif /* MACH_PV_PAGETABLES */
1.1.1.7 ! root 2343: } while (--i > 0);
! 2344: }
1.1 root 2345:
1.1.1.7 ! root 2346: PMAP_READ_UNLOCK(p, spl);
1.1 root 2347:
1.1.1.7 ! root 2348: /*
! 2349: * And free the pte page itself.
! 2350: */
! 2351: {
! 2352: vm_page_t m;
1.1 root 2353:
1.1.1.7 ! root 2354: vm_object_lock(pmap_object);
! 2355: m = vm_page_lookup(pmap_object, pa);
! 2356: if (m == VM_PAGE_NULL)
! 2357: panic("pmap_collect: pte page not in object");
! 2358: vm_page_lock_queues();
! 2359: vm_page_free(m);
! 2360: inuse_ptepages_count--;
! 2361: vm_page_unlock_queues();
! 2362: vm_object_unlock(pmap_object);
! 2363: }
1.1 root 2364:
1.1.1.7 ! root 2365: PMAP_READ_LOCK(p, spl);
! 2366: }
1.1 root 2367: }
2368: }
1.1.1.7 ! root 2369: #if PAE
1.1 root 2370: }
1.1.1.7 ! root 2371: #endif
1.1.1.3 root 2372: PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
2373:
1.1 root 2374: PMAP_READ_UNLOCK(p, spl);
2375: return;
2376:
2377: }
2378:
2379: /*
2380: * Routine: pmap_activate
2381: * Function:
2382: * Binds the given physical map to the given
2383: * processor, and returns a hardware map description.
2384: */
2385: #if 0
2386: void pmap_activate(my_pmap, th, my_cpu)
2387: register pmap_t my_pmap;
2388: thread_t th;
2389: int my_cpu;
2390: {
2391: PMAP_ACTIVATE(my_pmap, th, my_cpu);
2392: }
1.1.1.2 root 2393: #endif /* 0 */
1.1 root 2394:
2395: /*
2396: * Routine: pmap_deactivate
2397: * Function:
2398: * Indicates that the given physical map is no longer
2399: * in use on the specified processor. (This is a macro
2400: * in pmap.h)
2401: */
2402: #if 0
2403: void pmap_deactivate(pmap, th, which_cpu)
2404: pmap_t pmap;
2405: thread_t th;
2406: int which_cpu;
2407: {
2408: PMAP_DEACTIVATE(pmap, th, which_cpu);
2409: }
1.1.1.2 root 2410: #endif /* 0 */
1.1 root 2411:
2412: /*
2413: * Routine: pmap_kernel
2414: * Function:
2415: * Returns the physical map handle for the kernel.
2416: */
2417: #if 0
2418: pmap_t pmap_kernel()
2419: {
2420: return (kernel_pmap);
2421: }
1.1.1.2 root 2422: #endif /* 0 */
1.1 root 2423:
2424: /*
2425: * pmap_zero_page zeros the specified (machine independent) page.
2426: * See machine/phys.c or machine/phys.s for implementation.
2427: */
2428: #if 0
2429: pmap_zero_page(phys)
2430: register vm_offset_t phys;
2431: {
2432: register int i;
2433:
2434: assert(phys != vm_page_fictitious_addr);
2435: i = PAGE_SIZE / INTEL_PGBYTES;
2436: phys = intel_pfn(phys);
2437:
2438: while (i--)
2439: zero_phys(phys++);
2440: }
1.1.1.2 root 2441: #endif /* 0 */
1.1 root 2442:
2443: /*
2444: * pmap_copy_page copies the specified (machine independent) page.
2445: * See machine/phys.c or machine/phys.s for implementation.
2446: */
2447: #if 0
2448: pmap_copy_page(src, dst)
2449: vm_offset_t src, dst;
2450: {
2451: int i;
2452:
2453: assert(src != vm_page_fictitious_addr);
2454: assert(dst != vm_page_fictitious_addr);
2455: i = PAGE_SIZE / INTEL_PGBYTES;
2456:
2457: while (i--) {
2458: copy_phys(intel_pfn(src), intel_pfn(dst));
2459: src += INTEL_PGBYTES;
2460: dst += INTEL_PGBYTES;
2461: }
2462: }
1.1.1.2 root 2463: #endif /* 0 */
1.1 root 2464:
2465: /*
2466: * Routine: pmap_pageable
2467: * Function:
2468: * Make the specified pages (by pmap, offset)
2469: * pageable (or not) as requested.
2470: *
2471: * A page which is not pageable may not take
2472: * a fault; therefore, its page table entry
2473: * must remain valid for the duration.
2474: *
2475: * This routine is merely advisory; pmap_enter
2476: * will specify that these pages are to be wired
2477: * down (or not) as appropriate.
2478: */
1.1.1.3 root 2479: void
1.1.1.4 root 2480: pmap_pageable(
2481: pmap_t pmap,
2482: vm_offset_t start,
2483: vm_offset_t end,
2484: boolean_t pageable)
2485: {
1.1 root 2486: }
2487:
2488: /*
2489: * Clear specified attribute bits.
2490: */
2491: void
1.1.1.4 root 2492: phys_attribute_clear(
1.1.1.7 ! root 2493: phys_addr_t phys,
1.1.1.4 root 2494: int bits)
1.1 root 2495: {
2496: pv_entry_t pv_h;
1.1.1.4 root 2497: pv_entry_t pv_e;
2498: pt_entry_t *pte;
1.1.1.7 ! root 2499: unsigned long pai;
1.1.1.4 root 2500: pmap_t pmap;
1.1 root 2501: int spl;
2502:
2503: assert(phys != vm_page_fictitious_addr);
2504: if (!valid_page(phys)) {
2505: /*
2506: * Not a managed page.
2507: */
2508: return;
2509: }
2510:
2511: /*
2512: * Lock the pmap system first, since we will be changing
2513: * several pmaps.
2514: */
2515:
2516: PMAP_WRITE_LOCK(spl);
2517:
2518: pai = pa_index(phys);
2519: pv_h = pai_to_pvh(pai);
2520:
2521: /*
2522: * Walk down PV list, clearing all modify or reference bits.
2523: * We do not have to lock the pv_list because we have
2524: * the entire pmap system locked.
2525: */
2526: if (pv_h->pmap != PMAP_NULL) {
2527: /*
2528: * There are some mappings.
2529: */
2530: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
1.1.1.4 root 2531: vm_offset_t va;
1.1 root 2532:
2533: pmap = pv_e->pmap;
2534: /*
2535: * Lock the pmap to block pmap_extract and similar routines.
2536: */
2537: simple_lock(&pmap->lock);
2538:
1.1.1.3 root 2539: va = pv_e->va;
2540: pte = pmap_pte(pmap, va);
1.1 root 2541:
1.1.1.3 root 2542: /*
2543: * Consistency checks.
2544: */
2545: assert(*pte & INTEL_PTE_VALID);
1.1.1.5 root 2546: assert(pte_to_pa(*pte) == phys);
1.1 root 2547:
2548: /*
2549: * Clear modify or reference bits.
2550: */
2551: {
1.1.1.4 root 2552: int i = ptes_per_vm_page;
1.1 root 2553: do {
1.1.1.3 root 2554: #ifdef MACH_PV_PAGETABLES
2555: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits)))
2556: panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__,__LINE__,bits,pte);
2557: #else /* MACH_PV_PAGETABLES */
1.1 root 2558: *pte &= ~bits;
1.1.1.3 root 2559: #endif /* MACH_PV_PAGETABLES */
1.1 root 2560: } while (--i > 0);
2561: }
1.1.1.3 root 2562: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 2563: simple_unlock(&pmap->lock);
2564: }
2565: }
2566:
2567: pmap_phys_attributes[pai] &= ~bits;
2568:
2569: PMAP_WRITE_UNLOCK(spl);
2570: }
2571:
2572: /*
2573: * Check specified attribute bits.
2574: */
2575: boolean_t
1.1.1.4 root 2576: phys_attribute_test(
1.1.1.7 ! root 2577: phys_addr_t phys,
1.1.1.4 root 2578: int bits)
1.1 root 2579: {
2580: pv_entry_t pv_h;
1.1.1.4 root 2581: pv_entry_t pv_e;
2582: pt_entry_t *pte;
1.1.1.7 ! root 2583: unsigned long pai;
1.1.1.4 root 2584: pmap_t pmap;
1.1 root 2585: int spl;
2586:
2587: assert(phys != vm_page_fictitious_addr);
2588: if (!valid_page(phys)) {
2589: /*
2590: * Not a managed page.
2591: */
2592: return (FALSE);
2593: }
2594:
2595: /*
2596: * Lock the pmap system first, since we will be checking
2597: * several pmaps.
2598: */
2599:
2600: PMAP_WRITE_LOCK(spl);
2601:
2602: pai = pa_index(phys);
2603: pv_h = pai_to_pvh(pai);
2604:
2605: if (pmap_phys_attributes[pai] & bits) {
2606: PMAP_WRITE_UNLOCK(spl);
2607: return (TRUE);
2608: }
2609:
2610: /*
2611: * Walk down PV list, checking all mappings.
2612: * We do not have to lock the pv_list because we have
2613: * the entire pmap system locked.
2614: */
2615: if (pv_h->pmap != PMAP_NULL) {
2616: /*
2617: * There are some mappings.
2618: */
2619: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
2620:
2621: pmap = pv_e->pmap;
2622: /*
2623: * Lock the pmap to block pmap_extract and similar routines.
2624: */
2625: simple_lock(&pmap->lock);
2626:
2627: {
1.1.1.4 root 2628: vm_offset_t va;
1.1 root 2629:
2630: va = pv_e->va;
2631: pte = pmap_pte(pmap, va);
2632:
2633: /*
2634: * Consistency checks.
2635: */
2636: assert(*pte & INTEL_PTE_VALID);
1.1.1.5 root 2637: assert(pte_to_pa(*pte) == phys);
1.1 root 2638: }
2639:
2640: /*
2641: * Check modify or reference bits.
2642: */
2643: {
1.1.1.4 root 2644: int i = ptes_per_vm_page;
1.1 root 2645:
2646: do {
2647: if (*pte & bits) {
2648: simple_unlock(&pmap->lock);
2649: PMAP_WRITE_UNLOCK(spl);
2650: return (TRUE);
2651: }
2652: } while (--i > 0);
2653: }
2654: simple_unlock(&pmap->lock);
2655: }
2656: }
2657: PMAP_WRITE_UNLOCK(spl);
2658: return (FALSE);
2659: }
2660:
2661: /*
2662: * Clear the modify bits on the specified physical page.
2663: */
2664:
1.1.1.7 ! root 2665: void pmap_clear_modify(phys_addr_t phys)
1.1 root 2666: {
2667: phys_attribute_clear(phys, PHYS_MODIFIED);
2668: }
2669:
2670: /*
2671: * pmap_is_modified:
2672: *
2673: * Return whether or not the specified physical page is modified
2674: * by any physical maps.
2675: */
2676:
1.1.1.7 ! root 2677: boolean_t pmap_is_modified(phys_addr_t phys)
1.1 root 2678: {
2679: return (phys_attribute_test(phys, PHYS_MODIFIED));
2680: }
2681:
2682: /*
2683: * pmap_clear_reference:
2684: *
2685: * Clear the reference bit on the specified physical page.
2686: */
2687:
1.1.1.7 ! root 2688: void pmap_clear_reference(phys_addr_t phys)
1.1 root 2689: {
2690: phys_attribute_clear(phys, PHYS_REFERENCED);
2691: }
2692:
2693: /*
2694: * pmap_is_referenced:
2695: *
2696: * Return whether or not the specified physical page is referenced
2697: * by any physical maps.
2698: */
2699:
1.1.1.7 ! root 2700: boolean_t pmap_is_referenced(phys_addr_t phys)
1.1 root 2701: {
2702: return (phys_attribute_test(phys, PHYS_REFERENCED));
2703: }
2704:
2705: #if NCPUS > 1
2706: /*
2707: * TLB Coherence Code (TLB "shootdown" code)
1.1.1.2 root 2708: *
1.1 root 2709: * Threads that belong to the same task share the same address space and
2710: * hence share a pmap. However, they may run on distinct cpus and thus
2711: * have distinct TLBs that cache page table entries. In order to guarantee
2712: * the TLBs are consistent, whenever a pmap is changed, all threads that
2713: * are active in that pmap must have their TLB updated. To keep track of
2714: * this information, the set of cpus that are currently using a pmap is
2715: * maintained within each pmap structure (cpus_using). Pmap_activate() and
2716: * pmap_deactivate add and remove, respectively, a cpu from this set.
2717: * Since the TLBs are not addressable over the bus, each processor must
2718: * flush its own TLB; a processor that needs to invalidate another TLB
2719: * needs to interrupt the processor that owns that TLB to signal the
2720: * update.
1.1.1.2 root 2721: *
1.1 root 2722: * Whenever a pmap is updated, the lock on that pmap is locked, and all
2723: * cpus using the pmap are signaled to invalidate. All threads that need
2724: * to activate a pmap must wait for the lock to clear to await any updates
2725: * in progress before using the pmap. They must ACQUIRE the lock to add
2726: * their cpu to the cpus_using set. An implicit assumption made
2727: * throughout the TLB code is that all kernel code that runs at or higher
2728: * than splvm blocks out update interrupts, and that such code does not
2729: * touch pageable pages.
1.1.1.2 root 2730: *
1.1 root 2731: * A shootdown interrupt serves another function besides signaling a
2732: * processor to invalidate. The interrupt routine (pmap_update_interrupt)
2733: * waits for the both the pmap lock (and the kernel pmap lock) to clear,
2734: * preventing user code from making implicit pmap updates while the
2735: * sending processor is performing its update. (This could happen via a
2736: * user data write reference that turns on the modify bit in the page
2737: * table). It must wait for any kernel updates that may have started
2738: * concurrently with a user pmap update because the IPC code
2739: * changes mappings.
2740: * Spinning on the VALUES of the locks is sufficient (rather than
2741: * having to acquire the locks) because any updates that occur subsequent
2742: * to finding the lock unlocked will be signaled via another interrupt.
1.1.1.2 root 2743: * (This assumes the interrupt is cleared before the low level interrupt code
2744: * calls pmap_update_interrupt()).
2745: *
1.1 root 2746: * The signaling processor must wait for any implicit updates in progress
2747: * to terminate before continuing with its update. Thus it must wait for an
2748: * acknowledgement of the interrupt from each processor for which such
2749: * references could be made. For maintaining this information, a set
1.1.1.2 root 2750: * cpus_active is used. A cpu is in this set if and only if it can
1.1 root 2751: * use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
2752: * this set; when all such cpus are removed, it is safe to update.
1.1.1.2 root 2753: *
1.1 root 2754: * Before attempting to acquire the update lock on a pmap, a cpu (A) must
2755: * be at least at the priority of the interprocessor interrupt
2756: * (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
2757: * kernel update; it would spin forever in pmap_update_interrupt() trying
2758: * to acquire the user pmap lock it had already acquired. Furthermore A
2759: * must remove itself from cpus_active. Otherwise, another cpu holding
2760: * the lock (B) could be in the process of sending an update signal to A,
2761: * and thus be waiting for A to remove itself from cpus_active. If A is
2762: * spinning on the lock at priority this will never happen and a deadlock
2763: * will result.
2764: */
2765:
2766: /*
2767: * Signal another CPU that it must flush its TLB
2768: */
1.1.1.4 root 2769: void signal_cpus(
2770: cpu_set use_list,
2771: pmap_t pmap,
2772: vm_offset_t start,
2773: vm_offset_t end)
1.1 root 2774: {
1.1.1.4 root 2775: int which_cpu, j;
2776: pmap_update_list_t update_list_p;
1.1 root 2777:
2778: while ((which_cpu = ffs(use_list)) != 0) {
2779: which_cpu -= 1; /* convert to 0 origin */
2780:
2781: update_list_p = &cpu_update_list[which_cpu];
2782: simple_lock(&update_list_p->lock);
2783:
2784: j = update_list_p->count;
2785: if (j >= UPDATE_LIST_SIZE) {
2786: /*
2787: * list overflowed. Change last item to
2788: * indicate overflow.
2789: */
2790: update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
2791: update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
2792: update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS;
2793: }
2794: else {
2795: update_list_p->item[j].pmap = pmap;
2796: update_list_p->item[j].start = start;
2797: update_list_p->item[j].end = end;
2798: update_list_p->count = j+1;
2799: }
2800: cpu_update_needed[which_cpu] = TRUE;
2801: simple_unlock(&update_list_p->lock);
2802:
2803: if ((cpus_idle & (1 << which_cpu)) == 0)
2804: interrupt_processor(which_cpu);
2805: use_list &= ~(1 << which_cpu);
2806: }
2807: }
2808:
1.1.1.4 root 2809: void process_pmap_updates(pmap_t my_pmap)
1.1 root 2810: {
1.1.1.4 root 2811: int my_cpu = cpu_number();
2812: pmap_update_list_t update_list_p;
2813: int j;
2814: pmap_t pmap;
1.1 root 2815:
2816: update_list_p = &cpu_update_list[my_cpu];
2817: simple_lock(&update_list_p->lock);
2818:
2819: for (j = 0; j < update_list_p->count; j++) {
2820: pmap = update_list_p->item[j].pmap;
2821: if (pmap == my_pmap ||
2822: pmap == kernel_pmap) {
2823:
1.1.1.3 root 2824: INVALIDATE_TLB(pmap,
2825: update_list_p->item[j].start,
1.1 root 2826: update_list_p->item[j].end);
2827: }
2828: }
2829: update_list_p->count = 0;
2830: cpu_update_needed[my_cpu] = FALSE;
2831: simple_unlock(&update_list_p->lock);
2832: }
2833:
2834: /*
2835: * Interrupt routine for TBIA requested from other processor.
2836: */
1.1.1.3 root 2837: void pmap_update_interrupt(void)
1.1 root 2838: {
1.1.1.4 root 2839: int my_cpu;
2840: pmap_t my_pmap;
2841: int s;
1.1 root 2842:
2843: my_cpu = cpu_number();
2844:
2845: /*
2846: * Exit now if we're idle. We'll pick up the update request
2847: * when we go active, and we must not put ourselves back in
2848: * the active set because we'll never process the interrupt
2849: * while we're idle (thus hanging the system).
2850: */
2851: if (cpus_idle & (1 << my_cpu))
2852: return;
2853:
2854: if (current_thread() == THREAD_NULL)
2855: my_pmap = kernel_pmap;
2856: else {
2857: my_pmap = current_pmap();
2858: if (!pmap_in_use(my_pmap, my_cpu))
2859: my_pmap = kernel_pmap;
2860: }
2861:
2862: /*
2863: * Raise spl to splvm (above splip) to block out pmap_extract
2864: * from IO code (which would put this cpu back in the active
2865: * set).
2866: */
2867: s = splvm();
2868:
2869: do {
2870:
2871: /*
2872: * Indicate that we're not using either user or kernel
2873: * pmap.
2874: */
2875: i_bit_clear(my_cpu, &cpus_active);
2876:
2877: /*
2878: * Wait for any pmap updates in progress, on either user
2879: * or kernel pmap.
2880: */
2881: while (*(volatile int *)&my_pmap->lock.lock_data ||
2882: *(volatile int *)&kernel_pmap->lock.lock_data)
2883: continue;
2884:
2885: process_pmap_updates(my_pmap);
2886:
2887: i_bit_set(my_cpu, &cpus_active);
2888:
2889: } while (cpu_update_needed[my_cpu]);
1.1.1.2 root 2890:
1.1 root 2891: splx(s);
2892: }
1.1.1.2 root 2893: #else /* NCPUS > 1 */
1.1 root 2894: /*
2895: * Dummy routine to satisfy external reference.
2896: */
1.1.1.4 root 2897: void pmap_update_interrupt(void)
1.1 root 2898: {
2899: /* should never be called. */
2900: }
1.1.1.2 root 2901: #endif /* NCPUS > 1 */
1.1 root 2902:
1.1.1.3 root 2903: #if defined(__i386__)
1.1 root 2904: /* Unmap page 0 to trap NULL references. */
2905: void
1.1.1.4 root 2906: pmap_unmap_page_zero (void)
1.1 root 2907: {
2908: int *pte;
2909:
1.1.1.4 root 2910: printf("Unmapping the zero page. Some BIOS functions may not be working any more.\n");
1.1 root 2911: pte = (int *) pmap_pte (kernel_pmap, 0);
1.1.1.3 root 2912: if (!pte)
2913: return;
1.1 root 2914: assert (pte);
1.1.1.3 root 2915: #ifdef MACH_PV_PAGETABLES
2916: if (!hyp_mmu_update_pte(kv_to_ma(pte), 0))
2917: printf("couldn't unmap page 0\n");
2918: #else /* MACH_PV_PAGETABLES */
1.1 root 2919: *pte = 0;
1.1.1.3 root 2920: INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE);
2921: #endif /* MACH_PV_PAGETABLES */
1.1 root 2922: }
1.1.1.3 root 2923: #endif /* __i386__ */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.