![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / i386 / intel|
Return to pmap.c CVS log | Up to [GnuMach] / Gnu-Mach / i386 / intel |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
4: * All Rights Reserved.
1.1.1.2 root 5: *
1.1 root 6: * Permission to use, copy, modify and distribute this software and its
7: * documentation is hereby granted, provided that both the copyright
8: * notice and this permission notice appear in all copies of the
9: * software, derivative works or modified versions, and any portions
10: * thereof, and that both notices appear in supporting documentation.
1.1.1.2 root 11: *
1.1 root 12: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
1.1.1.2 root 15: *
1.1 root 16: * Carnegie Mellon requests users of this software to return to
1.1.1.2 root 17: *
1.1 root 18: * Software Distribution Coordinator or [email protected]
19: * School of Computer Science
20: * Carnegie Mellon University
21: * Pittsburgh PA 15213-3890
1.1.1.2 root 22: *
1.1 root 23: * any improvements or extensions that they make and grant Carnegie Mellon
24: * the rights to redistribute these changes.
25: */
26: /*
27: * File: pmap.c
28: * Author: Avadis Tevanian, Jr., Michael Wayne Young
29: * (These guys wrote the Vax version)
30: *
1.1.1.3 ! root 31: * Physical Map management code for Intel i386, and i486.
1.1 root 32: *
33: * Manages physical address maps.
34: *
35: * In addition to hardware address maps, this
36: * module is called upon to provide software-use-only
37: * maps which may or may not be stored in the same
38: * form as hardware maps. These pseudo-maps are
39: * used to store intermediate results from copy
40: * operations to and from address spaces.
41: *
42: * Since the information managed by this module is
43: * also stored by the logical address mapping module,
44: * this module may throw away valid virtual-to-physical
45: * mappings at almost any time. However, invalidations
46: * of virtual-to-physical mappings must be done as
47: * requested.
48: *
49: * In order to cope with hardware architectures which
50: * make virtual-to-physical map invalidates expensive,
51: * this module may delay invalidate or reduced protection
52: * operations until such time as they are actually
53: * necessary. This module is given full information as
54: * to which processors are currently using which maps,
55: * and to when physical maps must be made correct.
56: */
57:
1.1.1.3 ! root 58: #include <string.h>
1.1 root 59:
60: #include <mach/machine/vm_types.h>
61:
62: #include <mach/boolean.h>
1.1.1.3 ! root 63: #include <kern/debug.h>
! 64: #include <kern/printf.h>
1.1 root 65: #include <kern/thread.h>
1.1.1.3 ! root 66: #include <kern/slab.h>
1.1 root 67:
68: #include <kern/lock.h>
69:
70: #include <vm/pmap.h>
71: #include <vm/vm_map.h>
72: #include <vm/vm_kern.h>
1.1.1.3 ! root 73: #include <i386/vm_param.h>
1.1 root 74: #include <mach/vm_prot.h>
75: #include <vm/vm_object.h>
76: #include <vm/vm_page.h>
77: #include <vm/vm_user.h>
78:
79: #include <mach/machine/vm_param.h>
1.1.1.3 ! root 80: #include <mach/xen.h>
1.1 root 81: #include <machine/thread.h>
1.1.1.3 ! root 82: #include <i386/cpu_number.h>
! 83: #include <i386/proc_reg.h>
! 84: #include <i386/locore.h>
! 85: #include <i386/model_dep.h>
! 86:
! 87: #ifdef MACH_PSEUDO_PHYS
! 88: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0;
! 89: #else /* MACH_PSEUDO_PHYS */
1.1 root 90: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry);
1.1.1.3 ! root 91: #endif /* MACH_PSEUDO_PHYS */
1.1 root 92:
93: /*
94: * Private data structures.
95: */
96:
97: /*
98: * For each vm_page_t, there is a list of all currently
99: * valid virtual mappings of that page. An entry is
100: * a pv_entry_t; the list is the pv_table.
101: */
102:
103: typedef struct pv_entry {
104: struct pv_entry *next; /* next pv_entry */
105: pmap_t pmap; /* pmap where mapping lies */
106: vm_offset_t va; /* virtual address for mapping */
107: } *pv_entry_t;
108:
109: #define PV_ENTRY_NULL ((pv_entry_t) 0)
110:
111: pv_entry_t pv_head_table; /* array of entries, one per page */
112:
113: /*
114: * pv_list entries are kept on a list that can only be accessed
115: * with the pmap system locked (at SPLVM, not in the cpus_active set).
1.1.1.3 ! root 116: * The list is refilled from the pv_list_cache if it becomes empty.
1.1 root 117: */
118: pv_entry_t pv_free_list; /* free list at SPLVM */
119: decl_simple_lock_data(, pv_free_list_lock)
120:
121: #define PV_ALLOC(pv_e) { \
122: simple_lock(&pv_free_list_lock); \
123: if ((pv_e = pv_free_list) != 0) { \
124: pv_free_list = pv_e->next; \
125: } \
126: simple_unlock(&pv_free_list_lock); \
127: }
128:
129: #define PV_FREE(pv_e) { \
130: simple_lock(&pv_free_list_lock); \
131: pv_e->next = pv_free_list; \
132: pv_free_list = pv_e; \
133: simple_unlock(&pv_free_list_lock); \
134: }
135:
1.1.1.3 ! root 136: struct kmem_cache pv_list_cache; /* cache of pv_entry structures */
1.1 root 137:
138: /*
139: * Each entry in the pv_head_table is locked by a bit in the
140: * pv_lock_table. The lock bits are accessed by the physical
141: * address of the page they lock.
142: */
143:
144: char *pv_lock_table; /* pointer to array of bits */
145: #define pv_lock_table_size(n) (((n)+BYTE_SIZE-1)/BYTE_SIZE)
146:
147: /* Has pmap_init completed? */
148: boolean_t pmap_initialized = FALSE;
149:
150: /*
151: * Range of kernel virtual addresses available for kernel memory mapping.
152: * Does not include the virtual addresses used to map physical memory 1-1.
153: * Initialized by pmap_bootstrap.
154: */
155: vm_offset_t kernel_virtual_start;
156: vm_offset_t kernel_virtual_end;
157:
158: /*
159: * Index into pv_head table, its lock bits, and the modify/reference
160: * bits starting at phys_first_addr.
161: */
162: #define pa_index(pa) (atop(pa - phys_first_addr))
163:
164: #define pai_to_pvh(pai) (&pv_head_table[pai])
165: #define lock_pvh_pai(pai) (bit_lock(pai, pv_lock_table))
166: #define unlock_pvh_pai(pai) (bit_unlock(pai, pv_lock_table))
167:
168: /*
169: * Array of physical page attribites for managed pages.
170: * One byte per physical page.
171: */
172: char *pmap_phys_attributes;
173:
174: /*
175: * Physical page attributes. Copy bits from PTE definition.
176: */
177: #define PHYS_MODIFIED INTEL_PTE_MOD /* page modified */
178: #define PHYS_REFERENCED INTEL_PTE_REF /* page referenced */
179:
180: /*
181: * Amount of virtual memory mapped by one
182: * page-directory entry.
183: */
184: #define PDE_MAPPED_SIZE (pdenum2lin(1))
185:
186: /*
187: * We allocate page table pages directly from the VM system
188: * through this object. It maps physical memory.
189: */
190: vm_object_t pmap_object = VM_OBJECT_NULL;
191:
192: /*
193: * Locking and TLB invalidation
194: */
195:
196: /*
197: * Locking Protocols:
198: *
199: * There are two structures in the pmap module that need locking:
200: * the pmaps themselves, and the per-page pv_lists (which are locked
201: * by locking the pv_lock_table entry that corresponds to the pv_head
202: * for the list in question.) Most routines want to lock a pmap and
203: * then do operations in it that require pv_list locking -- however
204: * pmap_remove_all and pmap_copy_on_write operate on a physical page
205: * basis and want to do the locking in the reverse order, i.e. lock
206: * a pv_list and then go through all the pmaps referenced by that list.
207: * To protect against deadlock between these two cases, the pmap_lock
208: * is used. There are three different locking protocols as a result:
209: *
210: * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
211: * the pmap.
212: *
213: * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
214: * lock on the pmap_lock (shared read), then lock the pmap
215: * and finally the pv_lists as needed [i.e. pmap lock before
216: * pv_list lock.]
217: *
218: * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
219: * Get a write lock on the pmap_lock (exclusive write); this
220: * also guaranteees exclusive access to the pv_lists. Lock the
221: * pmaps as needed.
222: *
223: * At no time may any routine hold more than one pmap lock or more than
224: * one pv_list lock. Because interrupt level routines can allocate
225: * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
226: * kernel_pmap can only be held at splvm.
227: */
228:
229: #if NCPUS > 1
230: /*
231: * We raise the interrupt level to splvm, to block interprocessor
232: * interrupts during pmap operations. We must take the CPU out of
233: * the cpus_active set while interrupts are blocked.
234: */
235: #define SPLVM(spl) { \
236: spl = splvm(); \
237: i_bit_clear(cpu_number(), &cpus_active); \
238: }
239:
240: #define SPLX(spl) { \
241: i_bit_set(cpu_number(), &cpus_active); \
242: splx(spl); \
243: }
244:
245: /*
246: * Lock on pmap system
247: */
248: lock_data_t pmap_system_lock;
249:
250: #define PMAP_READ_LOCK(pmap, spl) { \
251: SPLVM(spl); \
252: lock_read(&pmap_system_lock); \
253: simple_lock(&(pmap)->lock); \
254: }
255:
256: #define PMAP_WRITE_LOCK(spl) { \
257: SPLVM(spl); \
258: lock_write(&pmap_system_lock); \
259: }
260:
261: #define PMAP_READ_UNLOCK(pmap, spl) { \
262: simple_unlock(&(pmap)->lock); \
263: lock_read_done(&pmap_system_lock); \
264: SPLX(spl); \
265: }
266:
267: #define PMAP_WRITE_UNLOCK(spl) { \
268: lock_write_done(&pmap_system_lock); \
269: SPLX(spl); \
270: }
271:
272: #define PMAP_WRITE_TO_READ_LOCK(pmap) { \
273: simple_lock(&(pmap)->lock); \
274: lock_write_to_read(&pmap_system_lock); \
275: }
276:
277: #define LOCK_PVH(index) (lock_pvh_pai(index))
278:
279: #define UNLOCK_PVH(index) (unlock_pvh_pai(index))
280:
281: #define PMAP_UPDATE_TLBS(pmap, s, e) \
282: { \
283: cpu_set cpu_mask = 1 << cpu_number(); \
284: cpu_set users; \
285: \
286: /* Since the pmap is locked, other updates are locked */ \
287: /* out, and any pmap_activate has finished. */ \
288: \
289: /* find other cpus using the pmap */ \
290: users = (pmap)->cpus_using & ~cpu_mask; \
291: if (users) { \
292: /* signal them, and wait for them to finish */ \
293: /* using the pmap */ \
294: signal_cpus(users, (pmap), (s), (e)); \
295: while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
296: continue; \
297: } \
298: \
299: /* invalidate our own TLB if pmap is in use */ \
300: if ((pmap)->cpus_using & cpu_mask) { \
1.1.1.3 ! root 301: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 302: } \
303: }
304:
1.1.1.2 root 305: #else /* NCPUS > 1 */
1.1 root 306:
1.1.1.3 ! root 307: #define SPLVM(spl) ((void)(spl))
! 308: #define SPLX(spl) ((void)(spl))
1.1 root 309:
310: #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl)
311: #define PMAP_WRITE_LOCK(spl) SPLVM(spl)
312: #define PMAP_READ_UNLOCK(pmap, spl) SPLX(spl)
313: #define PMAP_WRITE_UNLOCK(spl) SPLX(spl)
314: #define PMAP_WRITE_TO_READ_LOCK(pmap)
315:
316: #define LOCK_PVH(index)
317: #define UNLOCK_PVH(index)
318:
319: #define PMAP_UPDATE_TLBS(pmap, s, e) { \
320: /* invalidate our own TLB if pmap is in use */ \
321: if ((pmap)->cpus_using) { \
1.1.1.3 ! root 322: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 323: } \
324: }
325:
1.1.1.2 root 326: #endif /* NCPUS > 1 */
1.1 root 327:
328: #define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */
329:
1.1.1.3 ! root 330: #ifdef MACH_PV_PAGETABLES
! 331: #if 1
! 332: #define INVALIDATE_TLB(pmap, s, e) hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL)
! 333: #else
! 334: #define INVALIDATE_TLB(pmap, s, e) do { \
! 335: if (__builtin_constant_p((e) - (s)) \
! 336: && (e) - (s) == PAGE_SIZE) \
! 337: hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s) : (s)); \
! 338: else \
! 339: hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \
! 340: } while(0)
! 341: #endif
! 342: #else /* MACH_PV_PAGETABLES */
! 343: #if 0
! 344: /* It is hard to know when a TLB flush becomes less expensive than a bunch of
! 345: * invlpgs. But it surely is more expensive than just one invlpg. */
! 346: #define INVALIDATE_TLB(pmap, s, e) { \
! 347: if (__builtin_constant_p((e) - (s)) \
! 348: && (e) - (s) == PAGE_SIZE) \
! 349: invlpg_linear(s); \
! 350: else \
! 351: flush_tlb(); \
1.1 root 352: }
1.1.1.3 ! root 353: #else
! 354: #define INVALIDATE_TLB(pmap, s, e) { \
! 355: (void) (pmap); \
! 356: (void) (s); \
! 357: (void) (e); \
1.1 root 358: flush_tlb(); \
359: }
1.1.1.3 ! root 360: #endif
! 361: #endif /* MACH_PV_PAGETABLES */
1.1 root 362:
363:
364: #if NCPUS > 1
365: /*
366: * Structures to keep track of pending TLB invalidations
367: */
368:
369: #define UPDATE_LIST_SIZE 4
370:
371: struct pmap_update_item {
372: pmap_t pmap; /* pmap to invalidate */
373: vm_offset_t start; /* start address to invalidate */
374: vm_offset_t end; /* end address to invalidate */
375: } ;
376:
377: typedef struct pmap_update_item *pmap_update_item_t;
378:
379: /*
380: * List of pmap updates. If the list overflows,
381: * the last entry is changed to invalidate all.
382: */
383: struct pmap_update_list {
384: decl_simple_lock_data(, lock)
385: int count;
386: struct pmap_update_item item[UPDATE_LIST_SIZE];
387: } ;
388: typedef struct pmap_update_list *pmap_update_list_t;
389:
390: struct pmap_update_list cpu_update_list[NCPUS];
391:
1.1.1.2 root 392: #endif /* NCPUS > 1 */
1.1 root 393:
394: /*
395: * Other useful macros.
396: */
397: #define current_pmap() (vm_map_pmap(current_thread()->task->map))
398: #define pmap_in_use(pmap, cpu) (((pmap)->cpus_using & (1 << (cpu))) != 0)
399:
400: struct pmap kernel_pmap_store;
401: pmap_t kernel_pmap;
402:
1.1.1.3 ! root 403: struct kmem_cache pmap_cache; /* cache of pmap structures */
1.1 root 404:
405: int pmap_debug = 0; /* flag for debugging prints */
406:
407: #if 0
408: int ptes_per_vm_page; /* number of hardware ptes needed
409: to map one VM page. */
410: #else
411: #define ptes_per_vm_page 1
412: #endif
413:
414: unsigned int inuse_ptepages_count = 0; /* debugging */
415:
416: extern char end;
417:
418: /*
419: * Pointer to the basic page directory for the kernel.
420: * Initialized by pmap_bootstrap().
421: */
422: pt_entry_t *kernel_page_dir;
423:
424: void pmap_remove_range(); /* forward */
425: #if NCPUS > 1
426: void signal_cpus(); /* forward */
1.1.1.2 root 427: #endif /* NCPUS > 1 */
1.1 root 428:
429: static inline pt_entry_t *
430: pmap_pde(pmap_t pmap, vm_offset_t addr)
431: {
432: if (pmap == kernel_pmap)
433: addr = kvtolin(addr);
434: return &pmap->dirbase[lin2pdenum(addr)];
435: }
436:
437: /*
438: * Given an offset and a map, compute the address of the
439: * pte. If the address is invalid with respect to the map
440: * then PT_ENTRY_NULL is returned (and the map may need to grow).
441: *
442: * This is only used internally.
443: */
444: pt_entry_t *
445: pmap_pte(pmap_t pmap, vm_offset_t addr)
446: {
447: pt_entry_t *ptp;
448: pt_entry_t pte;
449:
450: if (pmap->dirbase == 0)
451: return(PT_ENTRY_NULL);
452: pte = *pmap_pde(pmap, addr);
453: if ((pte & INTEL_PTE_VALID) == 0)
454: return(PT_ENTRY_NULL);
455: ptp = (pt_entry_t *)ptetokv(pte);
456: return(&ptp[ptenum(addr)]);
457: }
458:
459: #define DEBUG_PTE_PAGE 0
460:
461: #if DEBUG_PTE_PAGE
462: void ptep_check(ptep)
463: ptep_t ptep;
464: {
465: register pt_entry_t *pte, *epte;
466: int ctu, ctw;
467:
468: /* check the use and wired counts */
469: if (ptep == PTE_PAGE_NULL)
470: return;
471: pte = pmap_pte(ptep->pmap, ptep->va);
472: epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
473: ctu = 0;
474: ctw = 0;
475: while (pte < epte) {
476: if (pte->pfn != 0) {
477: ctu++;
478: if (pte->wired)
479: ctw++;
480: }
481: pte += ptes_per_vm_page;
482: }
483:
484: if (ctu != ptep->use_count || ctw != ptep->wired_count) {
485: printf("use %d wired %d - actual use %d wired %d\n",
486: ptep->use_count, ptep->wired_count, ctu, ctw);
487: panic("pte count");
488: }
489: }
1.1.1.2 root 490: #endif /* DEBUG_PTE_PAGE */
1.1 root 491:
492: /*
493: * Map memory at initialization. The physical addresses being
494: * mapped are not managed and are never unmapped.
495: *
496: * For now, VM is already on, we only need to map the
497: * specified memory.
498: */
499: vm_offset_t pmap_map(virt, start, end, prot)
500: register vm_offset_t virt;
501: register vm_offset_t start;
502: register vm_offset_t end;
503: register int prot;
504: {
505: register int ps;
506:
507: ps = PAGE_SIZE;
508: while (start < end) {
509: pmap_enter(kernel_pmap, virt, start, prot, FALSE);
510: virt += ps;
511: start += ps;
512: }
513: return(virt);
514: }
515:
516: /*
1.1.1.2 root 517: * Back-door routine for mapping kernel VM at initialization.
1.1 root 518: * Useful for mapping memory outside the range
519: * [phys_first_addr, phys_last_addr) (i.e., devices).
520: * Otherwise like pmap_map.
521: */
522: vm_offset_t pmap_map_bd(virt, start, end, prot)
523: register vm_offset_t virt;
524: register vm_offset_t start;
525: register vm_offset_t end;
526: vm_prot_t prot;
527: {
528: register pt_entry_t template;
529: register pt_entry_t *pte;
1.1.1.3 ! root 530: int spl;
! 531: #ifdef MACH_PV_PAGETABLES
! 532: int n, i = 0;
! 533: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
! 534: #endif /* MACH_PV_PAGETABLES */
1.1 root 535:
536: template = pa_to_pte(start)
1.1.1.3 ! root 537: | INTEL_PTE_NCACHE|INTEL_PTE_WTHRU
1.1 root 538: | INTEL_PTE_VALID;
1.1.1.3 ! root 539: if (CPU_HAS_FEATURE(CPU_FEATURE_PGE))
! 540: template |= INTEL_PTE_GLOBAL;
1.1 root 541: if (prot & VM_PROT_WRITE)
542: template |= INTEL_PTE_WRITE;
543:
1.1.1.3 ! root 544: PMAP_READ_LOCK(pmap, spl);
1.1 root 545: while (start < end) {
546: pte = pmap_pte(kernel_pmap, virt);
547: if (pte == PT_ENTRY_NULL)
548: panic("pmap_map_bd: Invalid kernel address\n");
1.1.1.3 ! root 549: #ifdef MACH_PV_PAGETABLES
! 550: update[i].ptr = kv_to_ma(pte);
! 551: update[i].val = pa_to_ma(template);
! 552: i++;
! 553: if (i == HYP_BATCH_MMU_UPDATES) {
! 554: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
! 555: if (n != i)
! 556: panic("couldn't pmap_map_bd\n");
! 557: i = 0;
! 558: }
! 559: #else /* MACH_PV_PAGETABLES */
! 560: WRITE_PTE(pte, template)
! 561: #endif /* MACH_PV_PAGETABLES */
1.1 root 562: pte_increment_pa(template);
563: virt += PAGE_SIZE;
564: start += PAGE_SIZE;
565: }
1.1.1.3 ! root 566: #ifdef MACH_PV_PAGETABLES
! 567: if (i > HYP_BATCH_MMU_UPDATES)
! 568: panic("overflowed array in pmap_map_bd");
! 569: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
! 570: if (n != i)
! 571: panic("couldn't pmap_map_bd\n");
! 572: #endif /* MACH_PV_PAGETABLES */
! 573: PMAP_READ_UNLOCK(pmap, spl);
1.1 root 574: return(virt);
575: }
576:
577: /*
578: * Bootstrap the system enough to run with virtual memory.
579: * Allocate the kernel page directory and page tables,
580: * and direct-map all physical memory.
581: * Called with mapping off.
582: */
583: void pmap_bootstrap()
584: {
585: /*
586: * Mapping is turned off; we must reference only physical addresses.
587: * The load image of the system is to be mapped 1-1 physical = virtual.
588: */
589:
590: /*
591: * Set ptes_per_vm_page for general use.
592: */
593: #if 0
594: ptes_per_vm_page = PAGE_SIZE / INTEL_PGBYTES;
595: #endif
596:
597: /*
598: * The kernel's pmap is statically allocated so we don't
599: * have to use pmap_create, which is unlikely to work
600: * correctly at this part of the boot sequence.
601: */
602:
603: kernel_pmap = &kernel_pmap_store;
604:
605: #if NCPUS > 1
606: lock_init(&pmap_system_lock, FALSE); /* NOT a sleep lock */
1.1.1.2 root 607: #endif /* NCPUS > 1 */
1.1 root 608:
609: simple_lock_init(&kernel_pmap->lock);
610:
611: kernel_pmap->ref_count = 1;
612:
613: /*
614: * Determine the kernel virtual address range.
615: * It starts at the end of the physical memory
616: * mapped into the kernel address space,
617: * and extends to a stupid arbitrary limit beyond that.
618: */
1.1.1.3 ! root 619: kernel_virtual_start = phystokv(phys_last_addr);
! 620: kernel_virtual_end = phystokv(phys_last_addr) + VM_KERNEL_MAP_SIZE;
! 621:
! 622: if (kernel_virtual_end < kernel_virtual_start
! 623: || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS)
! 624: kernel_virtual_end = VM_MAX_KERNEL_ADDRESS;
1.1 root 625:
626: /*
627: * Allocate and clear a kernel page directory.
628: */
1.1.1.3 ! root 629: /* Note: initial Xen mapping holds at least 512kB free mapped page.
! 630: * We use that for directly building our linear mapping. */
! 631: #if PAE
! 632: {
! 633: vm_offset_t addr;
! 634: init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr);
! 635: kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr);
! 636: }
! 637: kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page());
! 638: {
! 639: int i;
! 640: for (i = 0; i < PDPNUM; i++)
! 641: WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID);
! 642: }
! 643: #else /* PAE */
! 644: kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page());
! 645: #endif /* PAE */
1.1 root 646: {
647: int i;
648: for (i = 0; i < NPDES; i++)
649: kernel_pmap->dirbase[i] = 0;
650: }
651:
1.1.1.3 ! root 652: #ifdef MACH_PV_PAGETABLES
! 653: /* We don't actually deal with the CR3 register content at all */
! 654: hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3);
! 655: /*
! 656: * Xen may only provide as few as 512KB extra bootstrap linear memory,
! 657: * which is far from enough to map all available memory, so we need to
! 658: * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE)
! 659: * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is
! 660: * enough for a pagetable mapping 4GiB.
! 661: */
! 662: #ifdef PAE
! 663: #define NSUP_L1 4
! 664: #else
! 665: #define NSUP_L1 1
! 666: #endif
! 667: pt_entry_t *l1_map[NSUP_L1];
! 668: {
! 669: pt_entry_t *base = (pt_entry_t*) boot_info.pt_base;
! 670: vm_offset_t la;
! 671: int n_l1map;
! 672: for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS; la >= VM_MIN_KERNEL_ADDRESS; la += NPTES * PAGE_SIZE) {
! 673: #ifdef PAE
! 674: pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)]);
! 675: #else /* PAE */
! 676: pt_entry_t *l2_map = base;
! 677: #endif /* PAE */
! 678: /* Like lin2pdenum, but works with non-contiguous boot L3 */
! 679: l2_map += (la >> PDESHIFT) & PDEMASK;
! 680: if (!(*l2_map & INTEL_PTE_VALID)) {
! 681: struct mmu_update update;
! 682: int j, n;
! 683:
! 684: l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page());
! 685: for (j = 0; j < NPTES; j++)
! 686: l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS) * NPTES + j)) << PAGE_SHIFT) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
! 687: pmap_set_page_readonly_init(l1_map[n_l1map]);
! 688: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map])))
! 689: panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map]));
! 690: update.ptr = kv_to_ma(l2_map);
! 691: update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
! 692: hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF);
! 693: if (n != 1)
! 694: panic("couldn't complete bootstrap map");
! 695: /* added the last L1 table, can stop */
! 696: if (++n_l1map >= NSUP_L1)
! 697: break;
! 698: }
! 699: }
! 700: }
! 701: #endif /* MACH_PV_PAGETABLES */
! 702:
1.1 root 703: /*
704: * Allocate and set up the kernel page tables.
705: */
706: {
707: vm_offset_t va;
1.1.1.3 ! root 708: pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0;
1.1 root 709:
710: /*
711: * Map virtual memory for all known physical memory, 1-1,
712: * from phys_first_addr to phys_last_addr.
713: * Make any mappings completely in the kernel's text segment read-only.
714: *
715: * Also allocate some additional all-null page tables afterwards
716: * for kernel virtual memory allocation,
717: * because this PMAP module is too stupid
718: * to allocate new kernel page tables later.
719: * XX fix this
720: */
1.1.1.3 ! root 721: for (va = phystokv(phys_first_addr); va >= phystokv(phys_first_addr) && va < kernel_virtual_end; )
1.1 root 722: {
723: pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va));
1.1.1.3 ! root 724: pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page());
1.1 root 725: pt_entry_t *pte;
726:
727: /* Initialize the page directory entry. */
1.1.1.3 ! root 728: WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable))
! 729: | INTEL_PTE_VALID | INTEL_PTE_WRITE);
1.1 root 730:
731: /* Initialize the page table. */
1.1.1.3 ! root 732: for (pte = ptable; (va < phystokv(phys_last_addr)) && (pte < ptable+NPTES); pte++)
1.1 root 733: {
734: if ((pte - ptable) < ptenum(va))
735: {
1.1.1.3 ! root 736: WRITE_PTE(pte, 0);
1.1 root 737: }
1.1.1.2 root 738: else
1.1.1.3 ! root 739: #ifdef MACH_PV_PAGETABLES
! 740: if (va == (vm_offset_t) &hyp_shared_info)
! 741: {
! 742: *pte = boot_info.shared_info | INTEL_PTE_VALID | INTEL_PTE_WRITE;
! 743: va += INTEL_PGBYTES;
! 744: }
! 745: else
! 746: #endif /* MACH_PV_PAGETABLES */
1.1 root 747: {
1.1.1.2 root 748: extern char _start[], etext[];
1.1 root 749:
1.1.1.3 ! root 750: if (((va >= (vm_offset_t) _start)
1.1 root 751: && (va + INTEL_PGBYTES <= (vm_offset_t)etext))
1.1.1.3 ! root 752: #ifdef MACH_PV_PAGETABLES
! 753: || (va >= (vm_offset_t) boot_info.pt_base
! 754: && (va + INTEL_PGBYTES <=
! 755: (vm_offset_t) ptable + INTEL_PGBYTES))
! 756: #endif /* MACH_PV_PAGETABLES */
! 757: )
1.1 root 758: {
1.1.1.3 ! root 759: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
! 760: | INTEL_PTE_VALID | global);
1.1 root 761: }
762: else
763: {
1.1.1.3 ! root 764: #ifdef MACH_PV_PAGETABLES
! 765: /* Keep supplementary L1 pages read-only */
! 766: int i;
! 767: for (i = 0; i < NSUP_L1; i++)
! 768: if (va == (vm_offset_t) l1_map[i]) {
! 769: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
! 770: | INTEL_PTE_VALID | global);
! 771: break;
! 772: }
! 773: if (i == NSUP_L1)
! 774: #endif /* MACH_PV_PAGETABLES */
! 775: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
! 776: | INTEL_PTE_VALID | INTEL_PTE_WRITE | global)
! 777:
1.1 root 778: }
779: va += INTEL_PGBYTES;
780: }
781: }
782: for (; pte < ptable+NPTES; pte++)
783: {
1.1.1.3 ! root 784: WRITE_PTE(pte, 0);
1.1 root 785: va += INTEL_PGBYTES;
786: }
1.1.1.3 ! root 787: #ifdef MACH_PV_PAGETABLES
! 788: pmap_set_page_readonly_init(ptable);
! 789: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable)))
! 790: panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable));
! 791: #endif /* MACH_PV_PAGETABLES */
1.1 root 792: }
793: }
794:
1.1.1.3 ! root 795: /* Architecture-specific code will turn on paging
! 796: soon after we return from here. */
! 797: }
1.1 root 798:
1.1.1.3 ! root 799: #ifdef MACH_PV_PAGETABLES
! 800: /* These are only required because of Xen security policies */
1.1 root 801:
1.1.1.3 ! root 802: /* Set back a page read write */
! 803: void pmap_set_page_readwrite(void *_vaddr) {
! 804: vm_offset_t vaddr = (vm_offset_t) _vaddr;
! 805: vm_offset_t paddr = kvtophys(vaddr);
! 806: vm_offset_t canon_vaddr = phystokv(paddr);
! 807: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
! 808: panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
! 809: if (canon_vaddr != vaddr)
! 810: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
! 811: panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr));
! 812: }
! 813:
! 814: /* Set a page read only (so as to pin it for instance) */
! 815: void pmap_set_page_readonly(void *_vaddr) {
! 816: vm_offset_t vaddr = (vm_offset_t) _vaddr;
! 817: vm_offset_t paddr = kvtophys(vaddr);
! 818: vm_offset_t canon_vaddr = phystokv(paddr);
! 819: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
! 820: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
! 821: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
! 822: }
! 823: if (canon_vaddr != vaddr &&
! 824: *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID) {
! 825: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
! 826: panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr));
1.1 root 827: }
1.1.1.3 ! root 828: }
1.1 root 829:
1.1.1.3 ! root 830: /* This needs to be called instead of pmap_set_page_readonly as long as RC3
! 831: * still points to the bootstrap dirbase, to also fix the bootstrap table. */
! 832: void pmap_set_page_readonly_init(void *_vaddr) {
! 833: vm_offset_t vaddr = (vm_offset_t) _vaddr;
! 834: #if PAE
! 835: pt_entry_t *pdpbase = (void*) boot_info.pt_base;
! 836: /* The bootstrap table does not necessarily use contiguous pages for the pde tables */
! 837: pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)]);
! 838: #else
! 839: pt_entry_t *dirbase = (void*) boot_info.pt_base;
1.1 root 840: #endif
1.1.1.3 ! root 841: pt_entry_t *pte = &dirbase[lin2pdenum(vaddr) & PTEMASK];
! 842: /* Modify our future kernel map (can't use update_va_mapping for this)... */
! 843: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
! 844: if (!hyp_mmu_update_la (kvtolin(vaddr), pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID))
! 845: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
! 846: }
! 847: /* ... and the bootstrap map. */
! 848: if (*pte & INTEL_PTE_VALID) {
! 849: if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID, UVMF_NONE))
! 850: panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
! 851: }
! 852: }
1.1 root 853:
1.1.1.3 ! root 854: void pmap_clear_bootstrap_pagetable(pt_entry_t *base) {
! 855: int i;
! 856: pt_entry_t *dir;
! 857: vm_offset_t va = 0;
! 858: #if PAE
! 859: int j;
! 860: #endif /* PAE */
! 861: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base)))
! 862: panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base));
! 863: #if PAE
! 864: for (j = 0; j < PDPNUM; j++)
! 865: {
! 866: pt_entry_t pdpe = base[j];
! 867: if (pdpe & INTEL_PTE_VALID) {
! 868: dir = (pt_entry_t *) ptetokv(pdpe);
! 869: #else /* PAE */
! 870: dir = base;
! 871: #endif /* PAE */
! 872: for (i = 0; i < NPTES; i++) {
! 873: pt_entry_t pde = dir[i];
! 874: unsigned long pfn = atop(pte_to_pa(pde));
! 875: void *pgt = (void*) phystokv(ptoa(pfn));
! 876: if (pde & INTEL_PTE_VALID)
! 877: hyp_free_page(pfn, pgt);
! 878: va += NPTES * INTEL_PGBYTES;
! 879: if (va >= HYP_VIRT_START)
! 880: break;
! 881: }
! 882: #if PAE
! 883: hyp_free_page(atop(_kvtophys(dir)), dir);
! 884: } else
! 885: va += NPTES * NPTES * INTEL_PGBYTES;
! 886: if (va >= HYP_VIRT_START)
! 887: break;
! 888: }
! 889: #endif /* PAE */
! 890: hyp_free_page(atop(_kvtophys(base)), base);
1.1 root 891: }
1.1.1.3 ! root 892: #endif /* MACH_PV_PAGETABLES */
1.1 root 893:
894: void pmap_virtual_space(startp, endp)
895: vm_offset_t *startp;
896: vm_offset_t *endp;
897: {
898: *startp = kernel_virtual_start;
899: *endp = kernel_virtual_end;
900: }
901:
902: /*
903: * Initialize the pmap module.
904: * Called by vm_init, to initialize any structures that the pmap
905: * system needs to map virtual memory.
906: */
907: void pmap_init()
908: {
909: register long npages;
910: vm_offset_t addr;
911: register vm_size_t s;
1.1.1.3 ! root 912: #if NCPUS > 1
1.1 root 913: int i;
1.1.1.3 ! root 914: #endif /* NCPUS > 1 */
1.1 root 915:
916: /*
917: * Allocate memory for the pv_head_table and its lock bits,
918: * the modify bit array, and the pte_page table.
919: */
920:
921: npages = atop(phys_last_addr - phys_first_addr);
922: s = (vm_size_t) (sizeof(struct pv_entry) * npages
923: + pv_lock_table_size(npages)
924: + npages);
925:
926: s = round_page(s);
927: if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
928: panic("pmap_init");
1.1.1.3 ! root 929: memset((char *) addr, 0, s);
1.1 root 930:
931: /*
932: * Allocate the structures first to preserve word-alignment.
933: */
934: pv_head_table = (pv_entry_t) addr;
935: addr = (vm_offset_t) (pv_head_table + npages);
936:
937: pv_lock_table = (char *) addr;
938: addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
939:
940: pmap_phys_attributes = (char *) addr;
941:
942: /*
1.1.1.3 ! root 943: * Create the cache of physical maps,
1.1 root 944: * and of the physical-to-virtual entries.
945: */
946: s = (vm_size_t) sizeof(struct pmap);
1.1.1.3 ! root 947: kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0);
1.1 root 948: s = (vm_size_t) sizeof(struct pv_entry);
1.1.1.3 ! root 949: kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0);
1.1 root 950:
951: #if NCPUS > 1
952: /*
953: * Set up the pmap request lists
954: */
955: for (i = 0; i < NCPUS; i++) {
956: pmap_update_list_t up = &cpu_update_list[i];
957:
958: simple_lock_init(&up->lock);
959: up->count = 0;
960: }
1.1.1.2 root 961: #endif /* NCPUS > 1 */
1.1 root 962:
963: /*
964: * Indicate that the PMAP module is now fully initialized.
965: */
966: pmap_initialized = TRUE;
967: }
968:
969: #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
970:
971: boolean_t pmap_verify_free(phys)
972: vm_offset_t phys;
973: {
974: pv_entry_t pv_h;
975: int pai;
976: int spl;
977: boolean_t result;
978:
979: assert(phys != vm_page_fictitious_addr);
980: if (!pmap_initialized)
981: return(TRUE);
982:
983: if (!pmap_valid_page(phys))
984: return(FALSE);
985:
986: PMAP_WRITE_LOCK(spl);
987:
988: pai = pa_index(phys);
989: pv_h = pai_to_pvh(pai);
990:
991: result = (pv_h->pmap == PMAP_NULL);
992: PMAP_WRITE_UNLOCK(spl);
993:
994: return(result);
995: }
996:
997: /*
998: * Routine: pmap_page_table_page_alloc
999: *
1000: * Allocates a new physical page to be used as a page-table page.
1001: *
1002: * Must be called with the pmap system and the pmap unlocked,
1003: * since these must be unlocked to use vm_page_grab.
1004: */
1005: vm_offset_t
1006: pmap_page_table_page_alloc()
1007: {
1008: register vm_page_t m;
1009: register vm_offset_t pa;
1010:
1011: check_simple_locks();
1012:
1013: /*
1014: * We cannot allocate the pmap_object in pmap_init,
1.1.1.3 ! root 1015: * because it is called before the cache package is up.
1.1 root 1016: * Allocate it now if it is missing.
1017: */
1018: if (pmap_object == VM_OBJECT_NULL)
1019: pmap_object = vm_object_allocate(phys_last_addr - phys_first_addr);
1020:
1021: /*
1022: * Allocate a VM page for the level 2 page table entries.
1023: */
1.1.1.2 root 1024: while ((m = vm_page_grab(FALSE)) == VM_PAGE_NULL)
1.1 root 1025: VM_PAGE_WAIT((void (*)()) 0);
1026:
1027: /*
1028: * Map the page to its physical address so that it
1029: * can be found later.
1030: */
1031: pa = m->phys_addr;
1032: vm_object_lock(pmap_object);
1033: vm_page_insert(m, pmap_object, pa);
1034: vm_page_lock_queues();
1035: vm_page_wire(m);
1036: inuse_ptepages_count++;
1037: vm_page_unlock_queues();
1038: vm_object_unlock(pmap_object);
1039:
1040: /*
1041: * Zero the page.
1042: */
1.1.1.3 ! root 1043: memset((void *)phystokv(pa), 0, PAGE_SIZE);
1.1 root 1044:
1045: return pa;
1046: }
1047:
1.1.1.3 ! root 1048: #ifdef MACH_XEN
! 1049: void pmap_map_mfn(void *_addr, unsigned long mfn) {
! 1050: vm_offset_t addr = (vm_offset_t) _addr;
! 1051: pt_entry_t *pte, *pdp;
! 1052: vm_offset_t ptp;
! 1053: pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT;
! 1054:
! 1055: /* Add a ptp if none exist yet for this pte */
! 1056: if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) {
! 1057: ptp = phystokv(pmap_page_table_page_alloc());
! 1058: #ifdef MACH_PV_PAGETABLES
! 1059: pmap_set_page_readonly((void*) ptp);
! 1060: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp)))
! 1061: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
! 1062: #endif /* MACH_PV_PAGETABLES */
! 1063: pdp = pmap_pde(kernel_pmap, addr);
! 1064:
! 1065: #ifdef MACH_PV_PAGETABLES
! 1066: if (!hyp_mmu_update_pte(kv_to_ma(pdp),
! 1067: pa_to_pte(kv_to_ma(ptp)) | INTEL_PTE_VALID
! 1068: | INTEL_PTE_USER
! 1069: | INTEL_PTE_WRITE))
! 1070: panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__,__LINE__,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp));
! 1071: #else /* MACH_PV_PAGETABLES */
! 1072: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
! 1073: | INTEL_PTE_USER
! 1074: | INTEL_PTE_WRITE;
! 1075: #endif /* MACH_PV_PAGETABLES */
! 1076: pte = pmap_pte(kernel_pmap, addr);
! 1077: }
! 1078:
! 1079: #ifdef MACH_PV_PAGETABLES
! 1080: if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID | INTEL_PTE_WRITE))
! 1081: panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__,__LINE__,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma));
! 1082: #else /* MACH_PV_PAGETABLES */
! 1083: /* Note: in this case, mfn is actually a pfn. */
! 1084: WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE);
! 1085: #endif /* MACH_PV_PAGETABLES */
! 1086: }
! 1087: #endif /* MACH_XEN */
! 1088:
1.1 root 1089: /*
1090: * Deallocate a page-table page.
1091: * The page-table page must have all mappings removed,
1092: * and be removed from its page directory.
1093: */
1094: void
1095: pmap_page_table_page_dealloc(pa)
1096: vm_offset_t pa;
1097: {
1098: vm_page_t m;
1099:
1100: vm_object_lock(pmap_object);
1101: m = vm_page_lookup(pmap_object, pa);
1102: vm_page_lock_queues();
1103: vm_page_free(m);
1104: inuse_ptepages_count--;
1105: vm_page_unlock_queues();
1106: vm_object_unlock(pmap_object);
1107: }
1108:
1109: /*
1110: * Create and return a physical map.
1111: *
1112: * If the size specified for the map
1113: * is zero, the map is an actual physical
1114: * map, and may be referenced by the
1115: * hardware.
1116: *
1117: * If the size specified is non-zero,
1118: * the map will be used in software only, and
1119: * is bounded by that size.
1120: */
1121: pmap_t pmap_create(size)
1122: vm_size_t size;
1123: {
1124: register pmap_t p;
1125: register pmap_statistics_t stats;
1126:
1127: /*
1128: * A software use-only map doesn't even need a map.
1129: */
1130:
1131: if (size != 0) {
1132: return(PMAP_NULL);
1133: }
1134:
1135: /*
1.1.1.3 ! root 1136: * Allocate a pmap struct from the pmap_cache. Then allocate
! 1137: * the page descriptor table.
1.1 root 1138: */
1139:
1.1.1.3 ! root 1140: p = (pmap_t) kmem_cache_alloc(&pmap_cache);
1.1 root 1141: if (p == PMAP_NULL)
1142: panic("pmap_create");
1143:
1144: if (kmem_alloc_wired(kernel_map,
1.1.1.3 ! root 1145: (vm_offset_t *)&p->dirbase, PDPNUM * INTEL_PGBYTES)
1.1 root 1146: != KERN_SUCCESS)
1147: panic("pmap_create");
1148:
1.1.1.3 ! root 1149: memcpy(p->dirbase, kernel_page_dir, PDPNUM * INTEL_PGBYTES);
! 1150: #ifdef LINUX_DEV
! 1151: #if VM_MIN_KERNEL_ADDRESS != 0
! 1152: /* Do not map BIOS in user tasks */
! 1153: p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] = 0;
! 1154: #endif
! 1155: #endif
! 1156: #ifdef MACH_PV_PAGETABLES
! 1157: {
! 1158: int i;
! 1159: for (i = 0; i < PDPNUM; i++)
! 1160: pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES);
! 1161: }
! 1162: #endif /* MACH_PV_PAGETABLES */
! 1163:
! 1164: #if PAE
! 1165: if (kmem_alloc_wired(kernel_map,
! 1166: (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES)
! 1167: != KERN_SUCCESS)
! 1168: panic("pmap_create");
! 1169: {
! 1170: int i;
! 1171: for (i = 0; i < PDPNUM; i++)
! 1172: WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID);
! 1173: }
! 1174: #ifdef MACH_PV_PAGETABLES
! 1175: pmap_set_page_readonly(p->pdpbase);
! 1176: #endif /* MACH_PV_PAGETABLES */
! 1177: #endif /* PAE */
! 1178:
1.1 root 1179: p->ref_count = 1;
1180:
1181: simple_lock_init(&p->lock);
1182: p->cpus_using = 0;
1183:
1184: /*
1185: * Initialize statistics.
1186: */
1187:
1188: stats = &p->stats;
1189: stats->resident_count = 0;
1190: stats->wired_count = 0;
1191:
1192: return(p);
1193: }
1194:
1195: /*
1196: * Retire the given physical map from service.
1197: * Should only be called if the map contains
1198: * no valid mappings.
1199: */
1200:
1201: void pmap_destroy(p)
1202: register pmap_t p;
1203: {
1204: register pt_entry_t *pdep;
1205: register vm_offset_t pa;
1206: register int c, s;
1207: register vm_page_t m;
1208:
1209: if (p == PMAP_NULL)
1210: return;
1211:
1212: SPLVM(s);
1213: simple_lock(&p->lock);
1214: c = --p->ref_count;
1215: simple_unlock(&p->lock);
1216: SPLX(s);
1217:
1218: if (c != 0) {
1219: return; /* still in use */
1220: }
1221:
1222: /*
1223: * Free the memory maps, then the
1224: * pmap structure.
1225: */
1226: for (pdep = p->dirbase;
1227: pdep < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)];
1228: pdep += ptes_per_vm_page) {
1229: if (*pdep & INTEL_PTE_VALID) {
1230: pa = pte_to_pa(*pdep);
1231: vm_object_lock(pmap_object);
1232: m = vm_page_lookup(pmap_object, pa);
1233: if (m == VM_PAGE_NULL)
1234: panic("pmap_destroy: pte page not in object");
1235: vm_page_lock_queues();
1.1.1.3 ! root 1236: #ifdef MACH_PV_PAGETABLES
! 1237: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa)))
! 1238: panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa));
! 1239: pmap_set_page_readwrite((void*) phystokv(pa));
! 1240: #endif /* MACH_PV_PAGETABLES */
1.1 root 1241: vm_page_free(m);
1242: inuse_ptepages_count--;
1243: vm_page_unlock_queues();
1244: vm_object_unlock(pmap_object);
1245: }
1246: }
1.1.1.3 ! root 1247: #ifdef MACH_PV_PAGETABLES
! 1248: {
! 1249: int i;
! 1250: for (i = 0; i < PDPNUM; i++)
! 1251: pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES);
! 1252: }
! 1253: #endif /* MACH_PV_PAGETABLES */
! 1254: kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM * INTEL_PGBYTES);
! 1255: #if PAE
! 1256: #ifdef MACH_PV_PAGETABLES
! 1257: pmap_set_page_readwrite(p->pdpbase);
! 1258: #endif /* MACH_PV_PAGETABLES */
! 1259: kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES);
! 1260: #endif /* PAE */
! 1261: kmem_cache_free(&pmap_cache, (vm_offset_t) p);
1.1 root 1262: }
1263:
1264: /*
1265: * Add a reference to the specified pmap.
1266: */
1267:
1268: void pmap_reference(p)
1269: register pmap_t p;
1270: {
1271: int s;
1272: if (p != PMAP_NULL) {
1273: SPLVM(s);
1274: simple_lock(&p->lock);
1275: p->ref_count++;
1276: simple_unlock(&p->lock);
1277: SPLX(s);
1278: }
1279: }
1280:
1281: /*
1282: * Remove a range of hardware page-table entries.
1283: * The entries given are the first (inclusive)
1284: * and last (exclusive) entries for the VM pages.
1285: * The virtual address is the va for the first pte.
1286: *
1287: * The pmap must be locked.
1288: * If the pmap is not the kernel pmap, the range must lie
1289: * entirely within one pte-page. This is NOT checked.
1290: * Assumes that the pte-page exists.
1291: */
1292:
1293: /* static */
1294: void pmap_remove_range(pmap, va, spte, epte)
1295: pmap_t pmap;
1296: vm_offset_t va;
1297: pt_entry_t *spte;
1298: pt_entry_t *epte;
1299: {
1300: register pt_entry_t *cpte;
1301: int num_removed, num_unwired;
1302: int pai;
1303: vm_offset_t pa;
1.1.1.3 ! root 1304: #ifdef MACH_PV_PAGETABLES
! 1305: int n, ii = 0;
! 1306: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
! 1307: #endif /* MACH_PV_PAGETABLES */
1.1 root 1308:
1309: #if DEBUG_PTE_PAGE
1310: if (pmap != kernel_pmap)
1311: ptep_check(get_pte_page(spte));
1.1.1.2 root 1312: #endif /* DEBUG_PTE_PAGE */
1.1 root 1313: num_removed = 0;
1314: num_unwired = 0;
1315:
1316: for (cpte = spte; cpte < epte;
1317: cpte += ptes_per_vm_page, va += PAGE_SIZE) {
1318:
1319: if (*cpte == 0)
1320: continue;
1321: pa = pte_to_pa(*cpte);
1322:
1323: num_removed++;
1324: if (*cpte & INTEL_PTE_WIRED)
1325: num_unwired++;
1326:
1327: if (!valid_page(pa)) {
1328:
1329: /*
1330: * Outside range of managed physical memory.
1331: * Just remove the mappings.
1332: */
1333: register int i = ptes_per_vm_page;
1334: register pt_entry_t *lpte = cpte;
1335: do {
1.1.1.3 ! root 1336: #ifdef MACH_PV_PAGETABLES
! 1337: update[ii].ptr = kv_to_ma(lpte);
! 1338: update[ii].val = 0;
! 1339: ii++;
! 1340: if (ii == HYP_BATCH_MMU_UPDATES) {
! 1341: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
! 1342: if (n != ii)
! 1343: panic("couldn't pmap_remove_range\n");
! 1344: ii = 0;
! 1345: }
! 1346: #else /* MACH_PV_PAGETABLES */
1.1 root 1347: *lpte = 0;
1.1.1.3 ! root 1348: #endif /* MACH_PV_PAGETABLES */
1.1 root 1349: lpte++;
1350: } while (--i > 0);
1351: continue;
1352: }
1353:
1354: pai = pa_index(pa);
1355: LOCK_PVH(pai);
1356:
1357: /*
1358: * Get the modify and reference bits.
1359: */
1360: {
1361: register int i;
1362: register pt_entry_t *lpte;
1363:
1364: i = ptes_per_vm_page;
1365: lpte = cpte;
1366: do {
1367: pmap_phys_attributes[pai] |=
1368: *lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 ! root 1369: #ifdef MACH_PV_PAGETABLES
! 1370: update[ii].ptr = kv_to_ma(lpte);
! 1371: update[ii].val = 0;
! 1372: ii++;
! 1373: if (ii == HYP_BATCH_MMU_UPDATES) {
! 1374: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
! 1375: if (n != ii)
! 1376: panic("couldn't pmap_remove_range\n");
! 1377: ii = 0;
! 1378: }
! 1379: #else /* MACH_PV_PAGETABLES */
1.1 root 1380: *lpte = 0;
1.1.1.3 ! root 1381: #endif /* MACH_PV_PAGETABLES */
1.1 root 1382: lpte++;
1383: } while (--i > 0);
1384: }
1385:
1386: /*
1387: * Remove the mapping from the pvlist for
1388: * this physical page.
1389: */
1390: {
1391: register pv_entry_t pv_h, prev, cur;
1392:
1393: pv_h = pai_to_pvh(pai);
1394: if (pv_h->pmap == PMAP_NULL) {
1395: panic("pmap_remove: null pv_list!");
1396: }
1397: if (pv_h->va == va && pv_h->pmap == pmap) {
1398: /*
1399: * Header is the pv_entry. Copy the next one
1400: * to header and free the next one (we cannot
1401: * free the header)
1402: */
1403: cur = pv_h->next;
1404: if (cur != PV_ENTRY_NULL) {
1405: *pv_h = *cur;
1406: PV_FREE(cur);
1407: }
1408: else {
1409: pv_h->pmap = PMAP_NULL;
1410: }
1411: }
1412: else {
1413: cur = pv_h;
1414: do {
1415: prev = cur;
1416: if ((cur = prev->next) == PV_ENTRY_NULL) {
1417: panic("pmap-remove: mapping not in pv_list!");
1418: }
1419: } while (cur->va != va || cur->pmap != pmap);
1420: prev->next = cur->next;
1421: PV_FREE(cur);
1422: }
1423: UNLOCK_PVH(pai);
1424: }
1425: }
1426:
1.1.1.3 ! root 1427: #ifdef MACH_PV_PAGETABLES
! 1428: if (ii > HYP_BATCH_MMU_UPDATES)
! 1429: panic("overflowed array in pmap_remove_range");
! 1430: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
! 1431: if (n != ii)
! 1432: panic("couldn't pmap_remove_range\n");
! 1433: #endif /* MACH_PV_PAGETABLES */
! 1434:
1.1 root 1435: /*
1436: * Update the counts
1437: */
1438: pmap->stats.resident_count -= num_removed;
1439: pmap->stats.wired_count -= num_unwired;
1440: }
1441:
1442: /*
1443: * Remove the given range of addresses
1444: * from the specified map.
1445: *
1446: * It is assumed that the start and end are properly
1447: * rounded to the hardware page size.
1448: */
1449:
1450: void pmap_remove(map, s, e)
1451: pmap_t map;
1452: vm_offset_t s, e;
1453: {
1454: int spl;
1455: register pt_entry_t *pde;
1456: register pt_entry_t *spte, *epte;
1457: vm_offset_t l;
1.1.1.3 ! root 1458: vm_offset_t _s = s;
1.1 root 1459:
1460: if (map == PMAP_NULL)
1461: return;
1462:
1463: PMAP_READ_LOCK(map, spl);
1464:
1465: pde = pmap_pde(map, s);
1466: while (s < e) {
1467: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1468: if (l > e)
1469: l = e;
1470: if (*pde & INTEL_PTE_VALID) {
1471: spte = (pt_entry_t *)ptetokv(*pde);
1472: spte = &spte[ptenum(s)];
1473: epte = &spte[intel_btop(l-s)];
1474: pmap_remove_range(map, s, spte, epte);
1475: }
1476: s = l;
1477: pde++;
1478: }
1.1.1.3 ! root 1479: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1480:
1481: PMAP_READ_UNLOCK(map, spl);
1482: }
1483:
1484: /*
1485: * Routine: pmap_page_protect
1486: *
1487: * Function:
1488: * Lower the permission for all mappings to a given
1489: * page.
1490: */
1491: void pmap_page_protect(phys, prot)
1492: vm_offset_t phys;
1493: vm_prot_t prot;
1494: {
1495: pv_entry_t pv_h, prev;
1496: register pv_entry_t pv_e;
1497: register pt_entry_t *pte;
1498: int pai;
1499: register pmap_t pmap;
1500: int spl;
1501: boolean_t remove;
1502:
1503: assert(phys != vm_page_fictitious_addr);
1504: if (!valid_page(phys)) {
1505: /*
1506: * Not a managed page.
1507: */
1508: return;
1509: }
1510:
1511: /*
1512: * Determine the new protection.
1513: */
1514: switch (prot) {
1515: case VM_PROT_READ:
1516: case VM_PROT_READ|VM_PROT_EXECUTE:
1517: remove = FALSE;
1518: break;
1519: case VM_PROT_ALL:
1520: return; /* nothing to do */
1521: default:
1522: remove = TRUE;
1523: break;
1524: }
1525:
1526: /*
1527: * Lock the pmap system first, since we will be changing
1528: * several pmaps.
1529: */
1530:
1531: PMAP_WRITE_LOCK(spl);
1532:
1533: pai = pa_index(phys);
1534: pv_h = pai_to_pvh(pai);
1535:
1536: /*
1537: * Walk down PV list, changing or removing all mappings.
1538: * We do not have to lock the pv_list because we have
1539: * the entire pmap system locked.
1540: */
1541: if (pv_h->pmap != PMAP_NULL) {
1542:
1543: prev = pv_e = pv_h;
1544: do {
1.1.1.3 ! root 1545: register vm_offset_t va;
! 1546:
1.1 root 1547: pmap = pv_e->pmap;
1548: /*
1549: * Lock the pmap to block pmap_extract and similar routines.
1550: */
1551: simple_lock(&pmap->lock);
1552:
1.1.1.3 ! root 1553: va = pv_e->va;
! 1554: pte = pmap_pte(pmap, va);
1.1 root 1555:
1.1.1.3 ! root 1556: /*
! 1557: * Consistency checks.
! 1558: */
! 1559: /* assert(*pte & INTEL_PTE_VALID); XXX */
! 1560: /* assert(pte_to_phys(*pte) == phys); */
1.1 root 1561:
1562: /*
1563: * Remove the mapping if new protection is NONE
1564: * or if write-protecting a kernel mapping.
1565: */
1566: if (remove || pmap == kernel_pmap) {
1567: /*
1568: * Remove the mapping, collecting any modify bits.
1569: */
1570: if (*pte & INTEL_PTE_WIRED)
1571: panic("pmap_remove_all removing a wired page");
1572:
1573: {
1574: register int i = ptes_per_vm_page;
1575:
1576: do {
1577: pmap_phys_attributes[pai] |=
1578: *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 ! root 1579: #ifdef MACH_PV_PAGETABLES
! 1580: if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0))
! 1581: panic("%s:%d could not clear pte %p\n",__FILE__,__LINE__,pte-1);
! 1582: #else /* MACH_PV_PAGETABLES */
1.1 root 1583: *pte++ = 0;
1.1.1.3 ! root 1584: #endif /* MACH_PV_PAGETABLES */
1.1 root 1585: } while (--i > 0);
1586: }
1587:
1588: pmap->stats.resident_count--;
1589:
1590: /*
1591: * Remove the pv_entry.
1592: */
1593: if (pv_e == pv_h) {
1594: /*
1595: * Fix up head later.
1596: */
1597: pv_h->pmap = PMAP_NULL;
1598: }
1599: else {
1600: /*
1601: * Delete this entry.
1602: */
1603: prev->next = pv_e->next;
1604: PV_FREE(pv_e);
1605: }
1606: }
1607: else {
1608: /*
1609: * Write-protect.
1610: */
1611: register int i = ptes_per_vm_page;
1612:
1613: do {
1.1.1.3 ! root 1614: #ifdef MACH_PV_PAGETABLES
! 1615: if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE))
! 1616: panic("%s:%d could not disable write on pte %p\n",__FILE__,__LINE__,pte);
! 1617: #else /* MACH_PV_PAGETABLES */
1.1 root 1618: *pte &= ~INTEL_PTE_WRITE;
1.1.1.3 ! root 1619: #endif /* MACH_PV_PAGETABLES */
1.1 root 1620: pte++;
1621: } while (--i > 0);
1622:
1623: /*
1624: * Advance prev.
1625: */
1626: prev = pv_e;
1627: }
1.1.1.3 ! root 1628: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 1629:
1630: simple_unlock(&pmap->lock);
1631:
1632: } while ((pv_e = prev->next) != PV_ENTRY_NULL);
1633:
1634: /*
1635: * If pv_head mapping was removed, fix it up.
1636: */
1637: if (pv_h->pmap == PMAP_NULL) {
1638: pv_e = pv_h->next;
1639: if (pv_e != PV_ENTRY_NULL) {
1640: *pv_h = *pv_e;
1641: PV_FREE(pv_e);
1642: }
1643: }
1644: }
1645:
1646: PMAP_WRITE_UNLOCK(spl);
1647: }
1648:
1649: /*
1650: * Set the physical protection on the
1651: * specified range of this map as requested.
1652: * Will not increase permissions.
1653: */
1654: void pmap_protect(map, s, e, prot)
1655: pmap_t map;
1656: vm_offset_t s, e;
1657: vm_prot_t prot;
1658: {
1659: register pt_entry_t *pde;
1660: register pt_entry_t *spte, *epte;
1661: vm_offset_t l;
1662: int spl;
1.1.1.3 ! root 1663: vm_offset_t _s = s;
1.1 root 1664:
1665: if (map == PMAP_NULL)
1666: return;
1667:
1668: /*
1669: * Determine the new protection.
1670: */
1671: switch (prot) {
1672: case VM_PROT_READ:
1673: case VM_PROT_READ|VM_PROT_EXECUTE:
1674: break;
1675: case VM_PROT_READ|VM_PROT_WRITE:
1676: case VM_PROT_ALL:
1677: return; /* nothing to do */
1678: default:
1679: pmap_remove(map, s, e);
1680: return;
1681: }
1682:
1683: /*
1684: * If write-protecting in the kernel pmap,
1685: * remove the mappings; the i386 ignores
1686: * the write-permission bit in kernel mode.
1687: *
1688: * XXX should be #if'd for i386
1689: */
1690: if (map == kernel_pmap) {
1691: pmap_remove(map, s, e);
1692: return;
1693: }
1694:
1695: SPLVM(spl);
1696: simple_lock(&map->lock);
1697:
1698: pde = pmap_pde(map, s);
1699: while (s < e) {
1700: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1701: if (l > e)
1702: l = e;
1703: if (*pde & INTEL_PTE_VALID) {
1704: spte = (pt_entry_t *)ptetokv(*pde);
1705: spte = &spte[ptenum(s)];
1706: epte = &spte[intel_btop(l-s)];
1707:
1.1.1.3 ! root 1708: #ifdef MACH_PV_PAGETABLES
! 1709: int n, i = 0;
! 1710: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
! 1711: #endif /* MACH_PV_PAGETABLES */
! 1712:
1.1 root 1713: while (spte < epte) {
1.1.1.3 ! root 1714: if (*spte & INTEL_PTE_VALID) {
! 1715: #ifdef MACH_PV_PAGETABLES
! 1716: update[i].ptr = kv_to_ma(spte);
! 1717: update[i].val = *spte & ~INTEL_PTE_WRITE;
! 1718: i++;
! 1719: if (i == HYP_BATCH_MMU_UPDATES) {
! 1720: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
! 1721: if (n != i)
! 1722: panic("couldn't pmap_protect\n");
! 1723: i = 0;
! 1724: }
! 1725: #else /* MACH_PV_PAGETABLES */
1.1 root 1726: *spte &= ~INTEL_PTE_WRITE;
1.1.1.3 ! root 1727: #endif /* MACH_PV_PAGETABLES */
! 1728: }
1.1 root 1729: spte++;
1730: }
1.1.1.3 ! root 1731: #ifdef MACH_PV_PAGETABLES
! 1732: if (i > HYP_BATCH_MMU_UPDATES)
! 1733: panic("overflowed array in pmap_protect");
! 1734: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
! 1735: if (n != i)
! 1736: panic("couldn't pmap_protect\n");
! 1737: #endif /* MACH_PV_PAGETABLES */
1.1 root 1738: }
1739: s = l;
1740: pde++;
1741: }
1.1.1.3 ! root 1742: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1743:
1744: simple_unlock(&map->lock);
1745: SPLX(spl);
1746: }
1747:
1748: /*
1749: * Insert the given physical page (p) at
1750: * the specified virtual address (v) in the
1751: * target physical map with the protection requested.
1752: *
1753: * If specified, the page will be wired down, meaning
1754: * that the related pte can not be reclaimed.
1755: *
1756: * NB: This is the only routine which MAY NOT lazy-evaluate
1757: * or lose information. That is, this routine must actually
1758: * insert this page into the given map NOW.
1759: */
1760: void pmap_enter(pmap, v, pa, prot, wired)
1761: register pmap_t pmap;
1762: vm_offset_t v;
1763: register vm_offset_t pa;
1764: vm_prot_t prot;
1765: boolean_t wired;
1766: {
1767: register pt_entry_t *pte;
1768: register pv_entry_t pv_h;
1769: register int i, pai;
1770: pv_entry_t pv_e;
1771: pt_entry_t template;
1772: int spl;
1773: vm_offset_t old_pa;
1774:
1775: assert(pa != vm_page_fictitious_addr);
1.1.1.3 ! root 1776: if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa);
1.1 root 1777: if (pmap == PMAP_NULL)
1778: return;
1779:
1.1.1.3 ! root 1780: #if !MACH_KDB
! 1781: if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end))
! 1782: panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa);
! 1783: #endif
1.1 root 1784: if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0
1785: && !wired /* hack for io_wire */ ) {
1786: /*
1787: * Because the 386 ignores write protection in kernel mode,
1788: * we cannot enter a read-only kernel mapping, and must
1789: * remove an existing mapping if changing it.
1790: *
1791: * XXX should be #if'd for i386
1792: */
1793: PMAP_READ_LOCK(pmap, spl);
1794:
1795: pte = pmap_pte(pmap, v);
1796: if (pte != PT_ENTRY_NULL && *pte != 0) {
1797: /*
1798: * Invalidate the translation buffer,
1799: * then remove the mapping.
1800: */
1801: pmap_remove_range(pmap, v, pte,
1802: pte + ptes_per_vm_page);
1.1.1.3 ! root 1803: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1804: }
1805: PMAP_READ_UNLOCK(pmap, spl);
1806: return;
1807: }
1808:
1809: /*
1810: * Must allocate a new pvlist entry while we're unlocked;
1.1.1.3 ! root 1811: * Allocating may cause pageout (which will lock the pmap system).
1.1 root 1812: * If we determine we need a pvlist entry, we will unlock
1813: * and allocate one. Then we will retry, throughing away
1814: * the allocated entry later (if we no longer need it).
1815: */
1816: pv_e = PV_ENTRY_NULL;
1817: Retry:
1818: PMAP_READ_LOCK(pmap, spl);
1819:
1820: /*
1821: * Expand pmap to include this pte. Assume that
1822: * pmap is always expanded to include enough hardware
1823: * pages to map one VM page.
1824: */
1825:
1826: while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
1827: /*
1828: * Need to allocate a new page-table page.
1829: */
1830: vm_offset_t ptp;
1831: pt_entry_t *pdp;
1832: int i;
1833:
1834: if (pmap == kernel_pmap) {
1835: /*
1836: * Would have to enter the new page-table page in
1837: * EVERY pmap.
1838: */
1839: panic("pmap_expand kernel pmap to %#x", v);
1840: }
1841:
1842: /*
1843: * Unlock the pmap and allocate a new page-table page.
1844: */
1845: PMAP_READ_UNLOCK(pmap, spl);
1846:
1.1.1.3 ! root 1847: ptp = phystokv(pmap_page_table_page_alloc());
1.1 root 1848:
1849: /*
1850: * Re-lock the pmap and check that another thread has
1851: * not already allocated the page-table page. If it
1852: * has, discard the new page-table page (and try
1853: * again to make sure).
1854: */
1855: PMAP_READ_LOCK(pmap, spl);
1856:
1857: if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
1858: /*
1859: * Oops...
1860: */
1861: PMAP_READ_UNLOCK(pmap, spl);
1.1.1.3 ! root 1862: pmap_page_table_page_dealloc(kvtophys(ptp));
1.1 root 1863: PMAP_READ_LOCK(pmap, spl);
1864: continue;
1865: }
1866:
1867: /*
1868: * Enter the new page table page in the page directory.
1869: */
1870: i = ptes_per_vm_page;
1871: /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/
1872: pdp = pmap_pde(pmap, v);
1873: do {
1.1.1.3 ! root 1874: #ifdef MACH_PV_PAGETABLES
! 1875: pmap_set_page_readonly((void *) ptp);
! 1876: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp)))
! 1877: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
! 1878: if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)),
! 1879: pa_to_pte(pa_to_ma(kvtophys(ptp))) | INTEL_PTE_VALID
! 1880: | INTEL_PTE_USER
! 1881: | INTEL_PTE_WRITE))
! 1882: panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp)));
! 1883: #else /* MACH_PV_PAGETABLES */
! 1884: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
! 1885: | INTEL_PTE_USER
! 1886: | INTEL_PTE_WRITE;
! 1887: #endif /* MACH_PV_PAGETABLES */
1.1 root 1888: pdp++;
1889: ptp += INTEL_PGBYTES;
1890: } while (--i > 0);
1891:
1892: /*
1893: * Now, get the address of the page-table entry.
1894: */
1895: continue;
1896: }
1897:
1898: /*
1899: * Special case if the physical page is already mapped
1900: * at this address.
1901: */
1902: old_pa = pte_to_pa(*pte);
1903: if (*pte && old_pa == pa) {
1904: /*
1905: * May be changing its wired attribute or protection
1906: */
1.1.1.2 root 1907:
1.1 root 1908: if (wired && !(*pte & INTEL_PTE_WIRED))
1909: pmap->stats.wired_count++;
1910: else if (!wired && (*pte & INTEL_PTE_WIRED))
1911: pmap->stats.wired_count--;
1912:
1913: template = pa_to_pte(pa) | INTEL_PTE_VALID;
1914: if (pmap != kernel_pmap)
1915: template |= INTEL_PTE_USER;
1916: if (prot & VM_PROT_WRITE)
1917: template |= INTEL_PTE_WRITE;
1.1.1.3 ! root 1918: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
! 1919: && pa >= phys_last_addr)
! 1920: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 1921: if (wired)
1922: template |= INTEL_PTE_WIRED;
1923: i = ptes_per_vm_page;
1924: do {
1925: if (*pte & INTEL_PTE_MOD)
1926: template |= INTEL_PTE_MOD;
1.1.1.3 ! root 1927: #ifdef MACH_PV_PAGETABLES
! 1928: if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))
! 1929: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
! 1930: #else /* MACH_PV_PAGETABLES */
1.1 root 1931: WRITE_PTE(pte, template)
1.1.1.3 ! root 1932: #endif /* MACH_PV_PAGETABLES */
1.1 root 1933: pte++;
1934: pte_increment_pa(template);
1935: } while (--i > 0);
1.1.1.3 ! root 1936: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1937: }
1938: else {
1939:
1940: /*
1941: * Remove old mapping from the PV list if necessary.
1942: */
1943: if (*pte) {
1944: /*
1945: * Don't free the pte page if removing last
1946: * mapping - we will immediately replace it.
1947: */
1948: pmap_remove_range(pmap, v, pte,
1949: pte + ptes_per_vm_page);
1.1.1.3 ! root 1950: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1951: }
1952:
1953: if (valid_page(pa)) {
1954:
1955: /*
1956: * Enter the mapping in the PV list for this
1957: * physical page.
1958: */
1959:
1960: pai = pa_index(pa);
1961: LOCK_PVH(pai);
1962: pv_h = pai_to_pvh(pai);
1963:
1964: if (pv_h->pmap == PMAP_NULL) {
1965: /*
1966: * No mappings yet
1967: */
1968: pv_h->va = v;
1969: pv_h->pmap = pmap;
1970: pv_h->next = PV_ENTRY_NULL;
1971: }
1972: else {
1973: #if DEBUG
1974: {
1975: /* check that this mapping is not already there */
1976: pv_entry_t e = pv_h;
1977: while (e != PV_ENTRY_NULL) {
1978: if (e->pmap == pmap && e->va == v)
1979: panic("pmap_enter: already in pv_list");
1980: e = e->next;
1981: }
1982: }
1.1.1.2 root 1983: #endif /* DEBUG */
1984:
1.1 root 1985: /*
1986: * Add new pv_entry after header.
1987: */
1988: if (pv_e == PV_ENTRY_NULL) {
1989: PV_ALLOC(pv_e);
1990: if (pv_e == PV_ENTRY_NULL) {
1991: UNLOCK_PVH(pai);
1992: PMAP_READ_UNLOCK(pmap, spl);
1993:
1994: /*
1.1.1.3 ! root 1995: * Refill from cache.
1.1 root 1996: */
1.1.1.3 ! root 1997: pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache);
1.1 root 1998: goto Retry;
1999: }
2000: }
2001: pv_e->va = v;
2002: pv_e->pmap = pmap;
2003: pv_e->next = pv_h->next;
2004: pv_h->next = pv_e;
2005: /*
2006: * Remember that we used the pvlist entry.
2007: */
2008: pv_e = PV_ENTRY_NULL;
2009: }
2010: UNLOCK_PVH(pai);
2011: }
2012:
2013: /*
2014: * And count the mapping.
2015: */
2016:
2017: pmap->stats.resident_count++;
2018: if (wired)
2019: pmap->stats.wired_count++;
2020:
2021: /*
2022: * Build a template to speed up entering -
2023: * only the pfn changes.
2024: */
2025: template = pa_to_pte(pa) | INTEL_PTE_VALID;
2026: if (pmap != kernel_pmap)
2027: template |= INTEL_PTE_USER;
2028: if (prot & VM_PROT_WRITE)
2029: template |= INTEL_PTE_WRITE;
1.1.1.3 ! root 2030: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
! 2031: && pa >= phys_last_addr)
! 2032: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 2033: if (wired)
2034: template |= INTEL_PTE_WIRED;
2035: i = ptes_per_vm_page;
2036: do {
1.1.1.3 ! root 2037: #ifdef MACH_PV_PAGETABLES
! 2038: if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))))
! 2039: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
! 2040: #else /* MACH_PV_PAGETABLES */
1.1 root 2041: WRITE_PTE(pte, template)
1.1.1.3 ! root 2042: #endif /* MACH_PV_PAGETABLES */
1.1 root 2043: pte++;
2044: pte_increment_pa(template);
2045: } while (--i > 0);
2046: }
2047:
2048: if (pv_e != PV_ENTRY_NULL) {
2049: PV_FREE(pv_e);
2050: }
2051:
2052: PMAP_READ_UNLOCK(pmap, spl);
2053: }
2054:
2055: /*
2056: * Routine: pmap_change_wiring
2057: * Function: Change the wiring attribute for a map/virtual-address
2058: * pair.
2059: * In/out conditions:
2060: * The mapping must already exist in the pmap.
2061: */
2062: void pmap_change_wiring(map, v, wired)
2063: register pmap_t map;
2064: vm_offset_t v;
2065: boolean_t wired;
2066: {
2067: register pt_entry_t *pte;
2068: register int i;
2069: int spl;
2070:
2071: /*
2072: * We must grab the pmap system lock because we may
2073: * change a pte_page queue.
2074: */
2075: PMAP_READ_LOCK(map, spl);
2076:
2077: if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
2078: panic("pmap_change_wiring: pte missing");
2079:
2080: if (wired && !(*pte & INTEL_PTE_WIRED)) {
2081: /*
2082: * wiring down mapping
2083: */
2084: map->stats.wired_count++;
2085: i = ptes_per_vm_page;
2086: do {
2087: *pte++ |= INTEL_PTE_WIRED;
2088: } while (--i > 0);
2089: }
2090: else if (!wired && (*pte & INTEL_PTE_WIRED)) {
2091: /*
2092: * unwiring mapping
2093: */
2094: map->stats.wired_count--;
2095: i = ptes_per_vm_page;
2096: do {
1.1.1.3 ! root 2097: #ifdef MACH_PV_PAGETABLES
! 2098: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED)))
! 2099: panic("%s:%d could not wire down pte %p\n",__FILE__,__LINE__,pte);
! 2100: #else /* MACH_PV_PAGETABLES */
1.1 root 2101: *pte &= ~INTEL_PTE_WIRED;
1.1.1.3 ! root 2102: #endif /* MACH_PV_PAGETABLES */
! 2103: pte++;
1.1 root 2104: } while (--i > 0);
2105: }
2106:
2107: PMAP_READ_UNLOCK(map, spl);
2108: }
2109:
2110: /*
2111: * Routine: pmap_extract
2112: * Function:
2113: * Extract the physical page address associated
2114: * with the given map/virtual_address pair.
2115: */
2116:
2117: vm_offset_t pmap_extract(pmap, va)
2118: register pmap_t pmap;
2119: vm_offset_t va;
2120: {
2121: register pt_entry_t *pte;
2122: register vm_offset_t pa;
2123: int spl;
2124:
2125: SPLVM(spl);
2126: simple_lock(&pmap->lock);
2127: if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL)
2128: pa = (vm_offset_t) 0;
2129: else if (!(*pte & INTEL_PTE_VALID))
2130: pa = (vm_offset_t) 0;
2131: else
2132: pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK);
2133: simple_unlock(&pmap->lock);
2134: SPLX(spl);
2135: return(pa);
2136: }
2137:
2138: /*
2139: * Copy the range specified by src_addr/len
2140: * from the source map to the range dst_addr/len
2141: * in the destination map.
2142: *
2143: * This routine is only advisory and need not do anything.
2144: */
2145: #if 0
2146: void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2147: pmap_t dst_pmap;
2148: pmap_t src_pmap;
2149: vm_offset_t dst_addr;
2150: vm_size_t len;
2151: vm_offset_t src_addr;
2152: {
2153: #ifdef lint
2154: dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
1.1.1.2 root 2155: #endif /* lint */
1.1 root 2156: }
1.1.1.2 root 2157: #endif /* 0 */
1.1 root 2158:
2159: /*
2160: * Routine: pmap_collect
2161: * Function:
2162: * Garbage collects the physical map system for
2163: * pages which are no longer used.
2164: * Success need not be guaranteed -- that is, there
2165: * may well be pages which are not referenced, but
2166: * others may be collected.
2167: * Usage:
2168: * Called by the pageout daemon when pages are scarce.
2169: */
2170: void pmap_collect(p)
2171: pmap_t p;
2172: {
2173: register pt_entry_t *pdp, *ptp;
2174: pt_entry_t *eptp;
2175: vm_offset_t pa;
2176: int spl, wired;
2177:
2178: if (p == PMAP_NULL)
2179: return;
2180:
2181: if (p == kernel_pmap)
2182: return;
2183:
2184: /*
2185: * Garbage collect map.
2186: */
2187: PMAP_READ_LOCK(p, spl);
2188: for (pdp = p->dirbase;
2189: pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)];
2190: pdp += ptes_per_vm_page)
2191: {
2192: if (*pdp & INTEL_PTE_VALID) {
2193:
2194: pa = pte_to_pa(*pdp);
2195: ptp = (pt_entry_t *)phystokv(pa);
2196: eptp = ptp + NPTES*ptes_per_vm_page;
2197:
2198: /*
2199: * If the pte page has any wired mappings, we cannot
2200: * free it.
2201: */
2202: wired = 0;
2203: {
2204: register pt_entry_t *ptep;
2205: for (ptep = ptp; ptep < eptp; ptep++) {
2206: if (*ptep & INTEL_PTE_WIRED) {
2207: wired = 1;
2208: break;
2209: }
2210: }
2211: }
2212: if (!wired) {
2213: /*
2214: * Remove the virtual addresses mapped by this pte page.
2215: */
2216: { /*XXX big hack*/
2217: vm_offset_t va = pdenum2lin(pdp - p->dirbase);
2218: if (p == kernel_pmap)
2219: va = lintokv(va);
2220: pmap_remove_range(p,
2221: va,
2222: ptp,
2223: eptp);
2224: }
2225:
2226: /*
2227: * Invalidate the page directory pointer.
2228: */
2229: {
2230: register int i = ptes_per_vm_page;
2231: register pt_entry_t *pdep = pdp;
2232: do {
1.1.1.3 ! root 2233: #ifdef MACH_PV_PAGETABLES
! 2234: unsigned long pte = *pdep;
! 2235: void *ptable = (void*) ptetokv(pte);
! 2236: if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0)))
! 2237: panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1);
! 2238: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable)))
! 2239: panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable)));
! 2240: pmap_set_page_readwrite(ptable);
! 2241: #else /* MACH_PV_PAGETABLES */
1.1 root 2242: *pdep++ = 0;
1.1.1.3 ! root 2243: #endif /* MACH_PV_PAGETABLES */
1.1 root 2244: } while (--i > 0);
2245: }
2246:
2247: PMAP_READ_UNLOCK(p, spl);
2248:
2249: /*
2250: * And free the pte page itself.
2251: */
2252: {
2253: register vm_page_t m;
2254:
2255: vm_object_lock(pmap_object);
2256: m = vm_page_lookup(pmap_object, pa);
2257: if (m == VM_PAGE_NULL)
2258: panic("pmap_collect: pte page not in object");
2259: vm_page_lock_queues();
2260: vm_page_free(m);
2261: inuse_ptepages_count--;
2262: vm_page_unlock_queues();
2263: vm_object_unlock(pmap_object);
2264: }
2265:
2266: PMAP_READ_LOCK(p, spl);
2267: }
2268: }
2269: }
1.1.1.3 ! root 2270: PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
! 2271:
1.1 root 2272: PMAP_READ_UNLOCK(p, spl);
2273: return;
2274:
2275: }
2276:
2277: /*
2278: * Routine: pmap_activate
2279: * Function:
2280: * Binds the given physical map to the given
2281: * processor, and returns a hardware map description.
2282: */
2283: #if 0
2284: void pmap_activate(my_pmap, th, my_cpu)
2285: register pmap_t my_pmap;
2286: thread_t th;
2287: int my_cpu;
2288: {
2289: PMAP_ACTIVATE(my_pmap, th, my_cpu);
2290: }
1.1.1.2 root 2291: #endif /* 0 */
1.1 root 2292:
2293: /*
2294: * Routine: pmap_deactivate
2295: * Function:
2296: * Indicates that the given physical map is no longer
2297: * in use on the specified processor. (This is a macro
2298: * in pmap.h)
2299: */
2300: #if 0
2301: void pmap_deactivate(pmap, th, which_cpu)
2302: pmap_t pmap;
2303: thread_t th;
2304: int which_cpu;
2305: {
2306: #ifdef lint
2307: pmap++; th++; which_cpu++;
1.1.1.2 root 2308: #endif /* lint */
1.1 root 2309: PMAP_DEACTIVATE(pmap, th, which_cpu);
2310: }
1.1.1.2 root 2311: #endif /* 0 */
1.1 root 2312:
2313: /*
2314: * Routine: pmap_kernel
2315: * Function:
2316: * Returns the physical map handle for the kernel.
2317: */
2318: #if 0
2319: pmap_t pmap_kernel()
2320: {
2321: return (kernel_pmap);
2322: }
1.1.1.2 root 2323: #endif /* 0 */
1.1 root 2324:
2325: /*
2326: * pmap_zero_page zeros the specified (machine independent) page.
2327: * See machine/phys.c or machine/phys.s for implementation.
2328: */
2329: #if 0
2330: pmap_zero_page(phys)
2331: register vm_offset_t phys;
2332: {
2333: register int i;
2334:
2335: assert(phys != vm_page_fictitious_addr);
2336: i = PAGE_SIZE / INTEL_PGBYTES;
2337: phys = intel_pfn(phys);
2338:
2339: while (i--)
2340: zero_phys(phys++);
2341: }
1.1.1.2 root 2342: #endif /* 0 */
1.1 root 2343:
2344: /*
2345: * pmap_copy_page copies the specified (machine independent) page.
2346: * See machine/phys.c or machine/phys.s for implementation.
2347: */
2348: #if 0
2349: pmap_copy_page(src, dst)
2350: vm_offset_t src, dst;
2351: {
2352: int i;
2353:
2354: assert(src != vm_page_fictitious_addr);
2355: assert(dst != vm_page_fictitious_addr);
2356: i = PAGE_SIZE / INTEL_PGBYTES;
2357:
2358: while (i--) {
2359: copy_phys(intel_pfn(src), intel_pfn(dst));
2360: src += INTEL_PGBYTES;
2361: dst += INTEL_PGBYTES;
2362: }
2363: }
1.1.1.2 root 2364: #endif /* 0 */
1.1 root 2365:
2366: /*
2367: * Routine: pmap_pageable
2368: * Function:
2369: * Make the specified pages (by pmap, offset)
2370: * pageable (or not) as requested.
2371: *
2372: * A page which is not pageable may not take
2373: * a fault; therefore, its page table entry
2374: * must remain valid for the duration.
2375: *
2376: * This routine is merely advisory; pmap_enter
2377: * will specify that these pages are to be wired
2378: * down (or not) as appropriate.
2379: */
1.1.1.3 ! root 2380: void
1.1 root 2381: pmap_pageable(pmap, start, end, pageable)
2382: pmap_t pmap;
2383: vm_offset_t start;
2384: vm_offset_t end;
2385: boolean_t pageable;
2386: {
2387: #ifdef lint
2388: pmap++; start++; end++; pageable++;
1.1.1.2 root 2389: #endif /* lint */
1.1 root 2390: }
2391:
2392: /*
2393: * Clear specified attribute bits.
2394: */
2395: void
2396: phys_attribute_clear(phys, bits)
2397: vm_offset_t phys;
2398: int bits;
2399: {
2400: pv_entry_t pv_h;
2401: register pv_entry_t pv_e;
2402: register pt_entry_t *pte;
2403: int pai;
2404: register pmap_t pmap;
2405: int spl;
2406:
2407: assert(phys != vm_page_fictitious_addr);
2408: if (!valid_page(phys)) {
2409: /*
2410: * Not a managed page.
2411: */
2412: return;
2413: }
2414:
2415: /*
2416: * Lock the pmap system first, since we will be changing
2417: * several pmaps.
2418: */
2419:
2420: PMAP_WRITE_LOCK(spl);
2421:
2422: pai = pa_index(phys);
2423: pv_h = pai_to_pvh(pai);
2424:
2425: /*
2426: * Walk down PV list, clearing all modify or reference bits.
2427: * We do not have to lock the pv_list because we have
2428: * the entire pmap system locked.
2429: */
2430: if (pv_h->pmap != PMAP_NULL) {
2431: /*
2432: * There are some mappings.
2433: */
2434: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
1.1.1.3 ! root 2435: register vm_offset_t va;
1.1 root 2436:
2437: pmap = pv_e->pmap;
2438: /*
2439: * Lock the pmap to block pmap_extract and similar routines.
2440: */
2441: simple_lock(&pmap->lock);
2442:
1.1.1.3 ! root 2443: va = pv_e->va;
! 2444: pte = pmap_pte(pmap, va);
1.1 root 2445:
2446: #if 0
1.1.1.3 ! root 2447: /*
! 2448: * Consistency checks.
! 2449: */
! 2450: assert(*pte & INTEL_PTE_VALID);
! 2451: /* assert(pte_to_phys(*pte) == phys); */
1.1 root 2452: #endif
2453:
2454: /*
2455: * Clear modify or reference bits.
2456: */
2457: {
2458: register int i = ptes_per_vm_page;
2459: do {
1.1.1.3 ! root 2460: #ifdef MACH_PV_PAGETABLES
! 2461: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits)))
! 2462: panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__,__LINE__,bits,pte);
! 2463: #else /* MACH_PV_PAGETABLES */
1.1 root 2464: *pte &= ~bits;
1.1.1.3 ! root 2465: #endif /* MACH_PV_PAGETABLES */
1.1 root 2466: } while (--i > 0);
2467: }
1.1.1.3 ! root 2468: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 2469: simple_unlock(&pmap->lock);
2470: }
2471: }
2472:
2473: pmap_phys_attributes[pai] &= ~bits;
2474:
2475: PMAP_WRITE_UNLOCK(spl);
2476: }
2477:
2478: /*
2479: * Check specified attribute bits.
2480: */
2481: boolean_t
2482: phys_attribute_test(phys, bits)
2483: vm_offset_t phys;
2484: int bits;
2485: {
2486: pv_entry_t pv_h;
2487: register pv_entry_t pv_e;
2488: register pt_entry_t *pte;
2489: int pai;
2490: register pmap_t pmap;
2491: int spl;
2492:
2493: assert(phys != vm_page_fictitious_addr);
2494: if (!valid_page(phys)) {
2495: /*
2496: * Not a managed page.
2497: */
2498: return (FALSE);
2499: }
2500:
2501: /*
2502: * Lock the pmap system first, since we will be checking
2503: * several pmaps.
2504: */
2505:
2506: PMAP_WRITE_LOCK(spl);
2507:
2508: pai = pa_index(phys);
2509: pv_h = pai_to_pvh(pai);
2510:
2511: if (pmap_phys_attributes[pai] & bits) {
2512: PMAP_WRITE_UNLOCK(spl);
2513: return (TRUE);
2514: }
2515:
2516: /*
2517: * Walk down PV list, checking all mappings.
2518: * We do not have to lock the pv_list because we have
2519: * the entire pmap system locked.
2520: */
2521: if (pv_h->pmap != PMAP_NULL) {
2522: /*
2523: * There are some mappings.
2524: */
2525: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
2526:
2527: pmap = pv_e->pmap;
2528: /*
2529: * Lock the pmap to block pmap_extract and similar routines.
2530: */
2531: simple_lock(&pmap->lock);
2532:
2533: {
2534: register vm_offset_t va;
2535:
2536: va = pv_e->va;
2537: pte = pmap_pte(pmap, va);
2538:
2539: #if 0
2540: /*
2541: * Consistency checks.
2542: */
2543: assert(*pte & INTEL_PTE_VALID);
2544: /* assert(pte_to_phys(*pte) == phys); */
2545: #endif
2546: }
2547:
2548: /*
2549: * Check modify or reference bits.
2550: */
2551: {
2552: register int i = ptes_per_vm_page;
2553:
2554: do {
2555: if (*pte & bits) {
2556: simple_unlock(&pmap->lock);
2557: PMAP_WRITE_UNLOCK(spl);
2558: return (TRUE);
2559: }
2560: } while (--i > 0);
2561: }
2562: simple_unlock(&pmap->lock);
2563: }
2564: }
2565: PMAP_WRITE_UNLOCK(spl);
2566: return (FALSE);
2567: }
2568:
2569: /*
2570: * Clear the modify bits on the specified physical page.
2571: */
2572:
2573: void pmap_clear_modify(phys)
2574: register vm_offset_t phys;
2575: {
2576: phys_attribute_clear(phys, PHYS_MODIFIED);
2577: }
2578:
2579: /*
2580: * pmap_is_modified:
2581: *
2582: * Return whether or not the specified physical page is modified
2583: * by any physical maps.
2584: */
2585:
2586: boolean_t pmap_is_modified(phys)
2587: register vm_offset_t phys;
2588: {
2589: return (phys_attribute_test(phys, PHYS_MODIFIED));
2590: }
2591:
2592: /*
2593: * pmap_clear_reference:
2594: *
2595: * Clear the reference bit on the specified physical page.
2596: */
2597:
2598: void pmap_clear_reference(phys)
2599: vm_offset_t phys;
2600: {
2601: phys_attribute_clear(phys, PHYS_REFERENCED);
2602: }
2603:
2604: /*
2605: * pmap_is_referenced:
2606: *
2607: * Return whether or not the specified physical page is referenced
2608: * by any physical maps.
2609: */
2610:
2611: boolean_t pmap_is_referenced(phys)
2612: vm_offset_t phys;
2613: {
2614: return (phys_attribute_test(phys, PHYS_REFERENCED));
2615: }
2616:
2617: #if NCPUS > 1
2618: /*
2619: * TLB Coherence Code (TLB "shootdown" code)
1.1.1.2 root 2620: *
1.1 root 2621: * Threads that belong to the same task share the same address space and
2622: * hence share a pmap. However, they may run on distinct cpus and thus
2623: * have distinct TLBs that cache page table entries. In order to guarantee
2624: * the TLBs are consistent, whenever a pmap is changed, all threads that
2625: * are active in that pmap must have their TLB updated. To keep track of
2626: * this information, the set of cpus that are currently using a pmap is
2627: * maintained within each pmap structure (cpus_using). Pmap_activate() and
2628: * pmap_deactivate add and remove, respectively, a cpu from this set.
2629: * Since the TLBs are not addressable over the bus, each processor must
2630: * flush its own TLB; a processor that needs to invalidate another TLB
2631: * needs to interrupt the processor that owns that TLB to signal the
2632: * update.
1.1.1.2 root 2633: *
1.1 root 2634: * Whenever a pmap is updated, the lock on that pmap is locked, and all
2635: * cpus using the pmap are signaled to invalidate. All threads that need
2636: * to activate a pmap must wait for the lock to clear to await any updates
2637: * in progress before using the pmap. They must ACQUIRE the lock to add
2638: * their cpu to the cpus_using set. An implicit assumption made
2639: * throughout the TLB code is that all kernel code that runs at or higher
2640: * than splvm blocks out update interrupts, and that such code does not
2641: * touch pageable pages.
1.1.1.2 root 2642: *
1.1 root 2643: * A shootdown interrupt serves another function besides signaling a
2644: * processor to invalidate. The interrupt routine (pmap_update_interrupt)
2645: * waits for the both the pmap lock (and the kernel pmap lock) to clear,
2646: * preventing user code from making implicit pmap updates while the
2647: * sending processor is performing its update. (This could happen via a
2648: * user data write reference that turns on the modify bit in the page
2649: * table). It must wait for any kernel updates that may have started
2650: * concurrently with a user pmap update because the IPC code
2651: * changes mappings.
2652: * Spinning on the VALUES of the locks is sufficient (rather than
2653: * having to acquire the locks) because any updates that occur subsequent
2654: * to finding the lock unlocked will be signaled via another interrupt.
1.1.1.2 root 2655: * (This assumes the interrupt is cleared before the low level interrupt code
2656: * calls pmap_update_interrupt()).
2657: *
1.1 root 2658: * The signaling processor must wait for any implicit updates in progress
2659: * to terminate before continuing with its update. Thus it must wait for an
2660: * acknowledgement of the interrupt from each processor for which such
2661: * references could be made. For maintaining this information, a set
1.1.1.2 root 2662: * cpus_active is used. A cpu is in this set if and only if it can
1.1 root 2663: * use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
2664: * this set; when all such cpus are removed, it is safe to update.
1.1.1.2 root 2665: *
1.1 root 2666: * Before attempting to acquire the update lock on a pmap, a cpu (A) must
2667: * be at least at the priority of the interprocessor interrupt
2668: * (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
2669: * kernel update; it would spin forever in pmap_update_interrupt() trying
2670: * to acquire the user pmap lock it had already acquired. Furthermore A
2671: * must remove itself from cpus_active. Otherwise, another cpu holding
2672: * the lock (B) could be in the process of sending an update signal to A,
2673: * and thus be waiting for A to remove itself from cpus_active. If A is
2674: * spinning on the lock at priority this will never happen and a deadlock
2675: * will result.
2676: */
2677:
2678: /*
2679: * Signal another CPU that it must flush its TLB
2680: */
2681: void signal_cpus(use_list, pmap, start, end)
2682: cpu_set use_list;
2683: pmap_t pmap;
2684: vm_offset_t start, end;
2685: {
2686: register int which_cpu, j;
2687: register pmap_update_list_t update_list_p;
2688:
2689: while ((which_cpu = ffs(use_list)) != 0) {
2690: which_cpu -= 1; /* convert to 0 origin */
2691:
2692: update_list_p = &cpu_update_list[which_cpu];
2693: simple_lock(&update_list_p->lock);
2694:
2695: j = update_list_p->count;
2696: if (j >= UPDATE_LIST_SIZE) {
2697: /*
2698: * list overflowed. Change last item to
2699: * indicate overflow.
2700: */
2701: update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
2702: update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
2703: update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS;
2704: }
2705: else {
2706: update_list_p->item[j].pmap = pmap;
2707: update_list_p->item[j].start = start;
2708: update_list_p->item[j].end = end;
2709: update_list_p->count = j+1;
2710: }
2711: cpu_update_needed[which_cpu] = TRUE;
2712: simple_unlock(&update_list_p->lock);
2713:
2714: if ((cpus_idle & (1 << which_cpu)) == 0)
2715: interrupt_processor(which_cpu);
2716: use_list &= ~(1 << which_cpu);
2717: }
2718: }
2719:
2720: void process_pmap_updates(my_pmap)
2721: register pmap_t my_pmap;
2722: {
2723: register int my_cpu = cpu_number();
2724: register pmap_update_list_t update_list_p;
2725: register int j;
2726: register pmap_t pmap;
2727:
2728: update_list_p = &cpu_update_list[my_cpu];
2729: simple_lock(&update_list_p->lock);
2730:
2731: for (j = 0; j < update_list_p->count; j++) {
2732: pmap = update_list_p->item[j].pmap;
2733: if (pmap == my_pmap ||
2734: pmap == kernel_pmap) {
2735:
1.1.1.3 ! root 2736: INVALIDATE_TLB(pmap,
! 2737: update_list_p->item[j].start,
1.1 root 2738: update_list_p->item[j].end);
2739: }
2740: }
2741: update_list_p->count = 0;
2742: cpu_update_needed[my_cpu] = FALSE;
2743: simple_unlock(&update_list_p->lock);
2744: }
2745:
2746: /*
2747: * Interrupt routine for TBIA requested from other processor.
2748: */
1.1.1.3 ! root 2749: void pmap_update_interrupt(void)
1.1 root 2750: {
2751: register int my_cpu;
2752: register pmap_t my_pmap;
2753: int s;
2754:
2755: my_cpu = cpu_number();
2756:
2757: /*
2758: * Exit now if we're idle. We'll pick up the update request
2759: * when we go active, and we must not put ourselves back in
2760: * the active set because we'll never process the interrupt
2761: * while we're idle (thus hanging the system).
2762: */
2763: if (cpus_idle & (1 << my_cpu))
2764: return;
2765:
2766: if (current_thread() == THREAD_NULL)
2767: my_pmap = kernel_pmap;
2768: else {
2769: my_pmap = current_pmap();
2770: if (!pmap_in_use(my_pmap, my_cpu))
2771: my_pmap = kernel_pmap;
2772: }
2773:
2774: /*
2775: * Raise spl to splvm (above splip) to block out pmap_extract
2776: * from IO code (which would put this cpu back in the active
2777: * set).
2778: */
2779: s = splvm();
2780:
2781: do {
2782:
2783: /*
2784: * Indicate that we're not using either user or kernel
2785: * pmap.
2786: */
2787: i_bit_clear(my_cpu, &cpus_active);
2788:
2789: /*
2790: * Wait for any pmap updates in progress, on either user
2791: * or kernel pmap.
2792: */
2793: while (*(volatile int *)&my_pmap->lock.lock_data ||
2794: *(volatile int *)&kernel_pmap->lock.lock_data)
2795: continue;
2796:
2797: process_pmap_updates(my_pmap);
2798:
2799: i_bit_set(my_cpu, &cpus_active);
2800:
2801: } while (cpu_update_needed[my_cpu]);
1.1.1.2 root 2802:
1.1 root 2803: splx(s);
2804: }
1.1.1.2 root 2805: #else /* NCPUS > 1 */
1.1 root 2806: /*
2807: * Dummy routine to satisfy external reference.
2808: */
2809: void pmap_update_interrupt()
2810: {
2811: /* should never be called. */
2812: }
1.1.1.2 root 2813: #endif /* NCPUS > 1 */
1.1 root 2814:
1.1.1.3 ! root 2815: #if defined(__i386__)
1.1 root 2816: /* Unmap page 0 to trap NULL references. */
2817: void
2818: pmap_unmap_page_zero ()
2819: {
2820: int *pte;
2821:
2822: pte = (int *) pmap_pte (kernel_pmap, 0);
1.1.1.3 ! root 2823: if (!pte)
! 2824: return;
1.1 root 2825: assert (pte);
1.1.1.3 ! root 2826: #ifdef MACH_PV_PAGETABLES
! 2827: if (!hyp_mmu_update_pte(kv_to_ma(pte), 0))
! 2828: printf("couldn't unmap page 0\n");
! 2829: #else /* MACH_PV_PAGETABLES */
1.1 root 2830: *pte = 0;
1.1.1.3 ! root 2831: INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE);
! 2832: #endif /* MACH_PV_PAGETABLES */
1.1 root 2833: }
1.1.1.3 ! root 2834: #endif /* __i386__ */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.