![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / i386 / intel|
Return to pmap.c CVS log | Up to [GnuMach] / Gnu-Mach / i386 / intel |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
4: * All Rights Reserved.
1.1.1.2 root 5: *
1.1 root 6: * Permission to use, copy, modify and distribute this software and its
7: * documentation is hereby granted, provided that both the copyright
8: * notice and this permission notice appear in all copies of the
9: * software, derivative works or modified versions, and any portions
10: * thereof, and that both notices appear in supporting documentation.
1.1.1.2 root 11: *
1.1 root 12: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
1.1.1.2 root 15: *
1.1 root 16: * Carnegie Mellon requests users of this software to return to
1.1.1.2 root 17: *
1.1 root 18: * Software Distribution Coordinator or [email protected]
19: * School of Computer Science
20: * Carnegie Mellon University
21: * Pittsburgh PA 15213-3890
1.1.1.2 root 22: *
1.1 root 23: * any improvements or extensions that they make and grant Carnegie Mellon
24: * the rights to redistribute these changes.
25: */
26: /*
27: * File: pmap.c
28: * Author: Avadis Tevanian, Jr., Michael Wayne Young
29: * (These guys wrote the Vax version)
30: *
1.1.1.3 root 31: * Physical Map management code for Intel i386, and i486.
1.1 root 32: *
33: * Manages physical address maps.
34: *
35: * In addition to hardware address maps, this
36: * module is called upon to provide software-use-only
37: * maps which may or may not be stored in the same
38: * form as hardware maps. These pseudo-maps are
39: * used to store intermediate results from copy
40: * operations to and from address spaces.
41: *
42: * Since the information managed by this module is
43: * also stored by the logical address mapping module,
44: * this module may throw away valid virtual-to-physical
45: * mappings at almost any time. However, invalidations
46: * of virtual-to-physical mappings must be done as
47: * requested.
48: *
49: * In order to cope with hardware architectures which
50: * make virtual-to-physical map invalidates expensive,
51: * this module may delay invalidate or reduced protection
52: * operations until such time as they are actually
53: * necessary. This module is given full information as
54: * to which processors are currently using which maps,
55: * and to when physical maps must be made correct.
56: */
57:
1.1.1.3 root 58: #include <string.h>
1.1 root 59:
60: #include <mach/machine/vm_types.h>
61:
62: #include <mach/boolean.h>
1.1.1.3 root 63: #include <kern/debug.h>
64: #include <kern/printf.h>
1.1 root 65: #include <kern/thread.h>
1.1.1.3 root 66: #include <kern/slab.h>
1.1 root 67:
68: #include <kern/lock.h>
69:
70: #include <vm/pmap.h>
71: #include <vm/vm_map.h>
72: #include <vm/vm_kern.h>
1.1.1.3 root 73: #include <i386/vm_param.h>
1.1 root 74: #include <mach/vm_prot.h>
75: #include <vm/vm_object.h>
76: #include <vm/vm_page.h>
77: #include <vm/vm_user.h>
78:
79: #include <mach/machine/vm_param.h>
1.1.1.3 root 80: #include <mach/xen.h>
1.1 root 81: #include <machine/thread.h>
1.1.1.3 root 82: #include <i386/cpu_number.h>
83: #include <i386/proc_reg.h>
84: #include <i386/locore.h>
85: #include <i386/model_dep.h>
1.1.1.6 ! root 86: #include <i386at/model_dep.h>
1.1.1.3 root 87:
88: #ifdef MACH_PSEUDO_PHYS
89: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = pte_entry?pa_to_ma(pte_entry):0;
90: #else /* MACH_PSEUDO_PHYS */
1.1 root 91: #define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry);
1.1.1.3 root 92: #endif /* MACH_PSEUDO_PHYS */
1.1 root 93:
94: /*
95: * Private data structures.
96: */
97:
98: /*
99: * For each vm_page_t, there is a list of all currently
100: * valid virtual mappings of that page. An entry is
101: * a pv_entry_t; the list is the pv_table.
102: */
103:
104: typedef struct pv_entry {
105: struct pv_entry *next; /* next pv_entry */
106: pmap_t pmap; /* pmap where mapping lies */
107: vm_offset_t va; /* virtual address for mapping */
108: } *pv_entry_t;
109:
110: #define PV_ENTRY_NULL ((pv_entry_t) 0)
111:
112: pv_entry_t pv_head_table; /* array of entries, one per page */
113:
114: /*
115: * pv_list entries are kept on a list that can only be accessed
116: * with the pmap system locked (at SPLVM, not in the cpus_active set).
1.1.1.3 root 117: * The list is refilled from the pv_list_cache if it becomes empty.
1.1 root 118: */
119: pv_entry_t pv_free_list; /* free list at SPLVM */
120: decl_simple_lock_data(, pv_free_list_lock)
121:
122: #define PV_ALLOC(pv_e) { \
123: simple_lock(&pv_free_list_lock); \
124: if ((pv_e = pv_free_list) != 0) { \
125: pv_free_list = pv_e->next; \
126: } \
127: simple_unlock(&pv_free_list_lock); \
128: }
129:
130: #define PV_FREE(pv_e) { \
131: simple_lock(&pv_free_list_lock); \
132: pv_e->next = pv_free_list; \
133: pv_free_list = pv_e; \
134: simple_unlock(&pv_free_list_lock); \
135: }
136:
1.1.1.3 root 137: struct kmem_cache pv_list_cache; /* cache of pv_entry structures */
1.1 root 138:
139: /*
140: * Each entry in the pv_head_table is locked by a bit in the
141: * pv_lock_table. The lock bits are accessed by the physical
142: * address of the page they lock.
143: */
144:
145: char *pv_lock_table; /* pointer to array of bits */
146: #define pv_lock_table_size(n) (((n)+BYTE_SIZE-1)/BYTE_SIZE)
147:
148: /* Has pmap_init completed? */
149: boolean_t pmap_initialized = FALSE;
150:
151: /*
152: * Range of kernel virtual addresses available for kernel memory mapping.
153: * Does not include the virtual addresses used to map physical memory 1-1.
154: * Initialized by pmap_bootstrap.
155: */
156: vm_offset_t kernel_virtual_start;
157: vm_offset_t kernel_virtual_end;
158:
159: /*
160: * Index into pv_head table, its lock bits, and the modify/reference
161: * bits starting at phys_first_addr.
162: */
163: #define pa_index(pa) (atop(pa - phys_first_addr))
164:
165: #define pai_to_pvh(pai) (&pv_head_table[pai])
166: #define lock_pvh_pai(pai) (bit_lock(pai, pv_lock_table))
167: #define unlock_pvh_pai(pai) (bit_unlock(pai, pv_lock_table))
168:
169: /*
1.1.1.5 root 170: * Array of physical page attributes for managed pages.
1.1 root 171: * One byte per physical page.
172: */
173: char *pmap_phys_attributes;
174:
175: /*
176: * Physical page attributes. Copy bits from PTE definition.
177: */
178: #define PHYS_MODIFIED INTEL_PTE_MOD /* page modified */
179: #define PHYS_REFERENCED INTEL_PTE_REF /* page referenced */
180:
181: /*
182: * Amount of virtual memory mapped by one
183: * page-directory entry.
184: */
185: #define PDE_MAPPED_SIZE (pdenum2lin(1))
186:
187: /*
188: * We allocate page table pages directly from the VM system
189: * through this object. It maps physical memory.
190: */
191: vm_object_t pmap_object = VM_OBJECT_NULL;
192:
193: /*
194: * Locking and TLB invalidation
195: */
196:
197: /*
198: * Locking Protocols:
199: *
200: * There are two structures in the pmap module that need locking:
201: * the pmaps themselves, and the per-page pv_lists (which are locked
202: * by locking the pv_lock_table entry that corresponds to the pv_head
203: * for the list in question.) Most routines want to lock a pmap and
204: * then do operations in it that require pv_list locking -- however
205: * pmap_remove_all and pmap_copy_on_write operate on a physical page
206: * basis and want to do the locking in the reverse order, i.e. lock
207: * a pv_list and then go through all the pmaps referenced by that list.
208: * To protect against deadlock between these two cases, the pmap_lock
209: * is used. There are three different locking protocols as a result:
210: *
211: * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
212: * the pmap.
213: *
214: * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
215: * lock on the pmap_lock (shared read), then lock the pmap
216: * and finally the pv_lists as needed [i.e. pmap lock before
217: * pv_list lock.]
218: *
219: * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
220: * Get a write lock on the pmap_lock (exclusive write); this
221: * also guaranteees exclusive access to the pv_lists. Lock the
222: * pmaps as needed.
223: *
224: * At no time may any routine hold more than one pmap lock or more than
225: * one pv_list lock. Because interrupt level routines can allocate
226: * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
227: * kernel_pmap can only be held at splvm.
228: */
229:
230: #if NCPUS > 1
231: /*
232: * We raise the interrupt level to splvm, to block interprocessor
233: * interrupts during pmap operations. We must take the CPU out of
234: * the cpus_active set while interrupts are blocked.
235: */
236: #define SPLVM(spl) { \
237: spl = splvm(); \
238: i_bit_clear(cpu_number(), &cpus_active); \
239: }
240:
241: #define SPLX(spl) { \
242: i_bit_set(cpu_number(), &cpus_active); \
243: splx(spl); \
244: }
245:
246: /*
247: * Lock on pmap system
248: */
249: lock_data_t pmap_system_lock;
250:
251: #define PMAP_READ_LOCK(pmap, spl) { \
252: SPLVM(spl); \
253: lock_read(&pmap_system_lock); \
254: simple_lock(&(pmap)->lock); \
255: }
256:
257: #define PMAP_WRITE_LOCK(spl) { \
258: SPLVM(spl); \
259: lock_write(&pmap_system_lock); \
260: }
261:
262: #define PMAP_READ_UNLOCK(pmap, spl) { \
263: simple_unlock(&(pmap)->lock); \
264: lock_read_done(&pmap_system_lock); \
265: SPLX(spl); \
266: }
267:
268: #define PMAP_WRITE_UNLOCK(spl) { \
269: lock_write_done(&pmap_system_lock); \
270: SPLX(spl); \
271: }
272:
273: #define PMAP_WRITE_TO_READ_LOCK(pmap) { \
274: simple_lock(&(pmap)->lock); \
275: lock_write_to_read(&pmap_system_lock); \
276: }
277:
278: #define LOCK_PVH(index) (lock_pvh_pai(index))
279:
280: #define UNLOCK_PVH(index) (unlock_pvh_pai(index))
281:
282: #define PMAP_UPDATE_TLBS(pmap, s, e) \
283: { \
284: cpu_set cpu_mask = 1 << cpu_number(); \
285: cpu_set users; \
286: \
287: /* Since the pmap is locked, other updates are locked */ \
288: /* out, and any pmap_activate has finished. */ \
289: \
290: /* find other cpus using the pmap */ \
291: users = (pmap)->cpus_using & ~cpu_mask; \
292: if (users) { \
293: /* signal them, and wait for them to finish */ \
294: /* using the pmap */ \
295: signal_cpus(users, (pmap), (s), (e)); \
296: while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
297: continue; \
298: } \
299: \
300: /* invalidate our own TLB if pmap is in use */ \
301: if ((pmap)->cpus_using & cpu_mask) { \
1.1.1.3 root 302: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 303: } \
304: }
305:
1.1.1.2 root 306: #else /* NCPUS > 1 */
1.1 root 307:
1.1.1.3 root 308: #define SPLVM(spl) ((void)(spl))
309: #define SPLX(spl) ((void)(spl))
1.1 root 310:
311: #define PMAP_READ_LOCK(pmap, spl) SPLVM(spl)
312: #define PMAP_WRITE_LOCK(spl) SPLVM(spl)
313: #define PMAP_READ_UNLOCK(pmap, spl) SPLX(spl)
314: #define PMAP_WRITE_UNLOCK(spl) SPLX(spl)
315: #define PMAP_WRITE_TO_READ_LOCK(pmap)
316:
317: #define LOCK_PVH(index)
318: #define UNLOCK_PVH(index)
319:
320: #define PMAP_UPDATE_TLBS(pmap, s, e) { \
321: /* invalidate our own TLB if pmap is in use */ \
322: if ((pmap)->cpus_using) { \
1.1.1.3 root 323: INVALIDATE_TLB((pmap), (s), (e)); \
1.1 root 324: } \
325: }
326:
1.1.1.2 root 327: #endif /* NCPUS > 1 */
1.1 root 328:
329: #define MAX_TBIS_SIZE 32 /* > this -> TBIA */ /* XXX */
330:
1.1.1.3 root 331: #ifdef MACH_PV_PAGETABLES
332: #if 1
333: #define INVALIDATE_TLB(pmap, s, e) hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL)
334: #else
335: #define INVALIDATE_TLB(pmap, s, e) do { \
336: if (__builtin_constant_p((e) - (s)) \
337: && (e) - (s) == PAGE_SIZE) \
338: hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s) : (s)); \
339: else \
340: hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \
341: } while(0)
342: #endif
343: #else /* MACH_PV_PAGETABLES */
344: #if 0
345: /* It is hard to know when a TLB flush becomes less expensive than a bunch of
346: * invlpgs. But it surely is more expensive than just one invlpg. */
347: #define INVALIDATE_TLB(pmap, s, e) { \
348: if (__builtin_constant_p((e) - (s)) \
349: && (e) - (s) == PAGE_SIZE) \
350: invlpg_linear(s); \
351: else \
352: flush_tlb(); \
1.1 root 353: }
1.1.1.3 root 354: #else
355: #define INVALIDATE_TLB(pmap, s, e) { \
356: (void) (pmap); \
357: (void) (s); \
358: (void) (e); \
1.1 root 359: flush_tlb(); \
360: }
1.1.1.3 root 361: #endif
362: #endif /* MACH_PV_PAGETABLES */
1.1 root 363:
364:
365: #if NCPUS > 1
366: /*
367: * Structures to keep track of pending TLB invalidations
368: */
369:
370: #define UPDATE_LIST_SIZE 4
371:
372: struct pmap_update_item {
373: pmap_t pmap; /* pmap to invalidate */
374: vm_offset_t start; /* start address to invalidate */
375: vm_offset_t end; /* end address to invalidate */
376: } ;
377:
378: typedef struct pmap_update_item *pmap_update_item_t;
379:
380: /*
381: * List of pmap updates. If the list overflows,
382: * the last entry is changed to invalidate all.
383: */
384: struct pmap_update_list {
385: decl_simple_lock_data(, lock)
386: int count;
387: struct pmap_update_item item[UPDATE_LIST_SIZE];
388: } ;
389: typedef struct pmap_update_list *pmap_update_list_t;
390:
391: struct pmap_update_list cpu_update_list[NCPUS];
392:
1.1.1.2 root 393: #endif /* NCPUS > 1 */
1.1 root 394:
395: /*
396: * Other useful macros.
397: */
398: #define current_pmap() (vm_map_pmap(current_thread()->task->map))
399: #define pmap_in_use(pmap, cpu) (((pmap)->cpus_using & (1 << (cpu))) != 0)
400:
401: struct pmap kernel_pmap_store;
402: pmap_t kernel_pmap;
403:
1.1.1.3 root 404: struct kmem_cache pmap_cache; /* cache of pmap structures */
1.1 root 405:
1.1.1.4 root 406: boolean_t pmap_debug = FALSE; /* flag for debugging prints */
1.1 root 407:
408: #if 0
409: int ptes_per_vm_page; /* number of hardware ptes needed
410: to map one VM page. */
411: #else
412: #define ptes_per_vm_page 1
413: #endif
414:
415: unsigned int inuse_ptepages_count = 0; /* debugging */
416:
417: /*
418: * Pointer to the basic page directory for the kernel.
419: * Initialized by pmap_bootstrap().
420: */
421: pt_entry_t *kernel_page_dir;
422:
1.1.1.4 root 423: /*
424: * Two slots for temporary physical page mapping, to allow for
425: * physical-to-physical transfers.
426: */
427: static pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS];
1.1 root 428:
429: static inline pt_entry_t *
1.1.1.4 root 430: pmap_pde(const pmap_t pmap, vm_offset_t addr)
1.1 root 431: {
432: if (pmap == kernel_pmap)
433: addr = kvtolin(addr);
434: return &pmap->dirbase[lin2pdenum(addr)];
435: }
436:
437: /*
438: * Given an offset and a map, compute the address of the
439: * pte. If the address is invalid with respect to the map
440: * then PT_ENTRY_NULL is returned (and the map may need to grow).
441: *
442: * This is only used internally.
443: */
444: pt_entry_t *
1.1.1.4 root 445: pmap_pte(const pmap_t pmap, vm_offset_t addr)
1.1 root 446: {
447: pt_entry_t *ptp;
448: pt_entry_t pte;
449:
450: if (pmap->dirbase == 0)
451: return(PT_ENTRY_NULL);
452: pte = *pmap_pde(pmap, addr);
453: if ((pte & INTEL_PTE_VALID) == 0)
454: return(PT_ENTRY_NULL);
455: ptp = (pt_entry_t *)ptetokv(pte);
456: return(&ptp[ptenum(addr)]);
457: }
458:
459: #define DEBUG_PTE_PAGE 0
460:
461: #if DEBUG_PTE_PAGE
1.1.1.4 root 462: void ptep_check(ptep_t ptep)
1.1 root 463: {
1.1.1.4 root 464: pt_entry_t *pte, *epte;
1.1 root 465: int ctu, ctw;
466:
467: /* check the use and wired counts */
468: if (ptep == PTE_PAGE_NULL)
469: return;
470: pte = pmap_pte(ptep->pmap, ptep->va);
471: epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
472: ctu = 0;
473: ctw = 0;
474: while (pte < epte) {
475: if (pte->pfn != 0) {
476: ctu++;
477: if (pte->wired)
478: ctw++;
479: }
480: pte += ptes_per_vm_page;
481: }
482:
483: if (ctu != ptep->use_count || ctw != ptep->wired_count) {
484: printf("use %d wired %d - actual use %d wired %d\n",
485: ptep->use_count, ptep->wired_count, ctu, ctw);
486: panic("pte count");
487: }
488: }
1.1.1.2 root 489: #endif /* DEBUG_PTE_PAGE */
1.1 root 490:
491: /*
492: * Map memory at initialization. The physical addresses being
493: * mapped are not managed and are never unmapped.
494: *
495: * For now, VM is already on, we only need to map the
496: * specified memory.
497: */
1.1.1.4 root 498: vm_offset_t pmap_map(
499: vm_offset_t virt,
500: vm_offset_t start,
501: vm_offset_t end,
502: int prot)
1.1 root 503: {
1.1.1.4 root 504: int ps;
1.1 root 505:
506: ps = PAGE_SIZE;
507: while (start < end) {
508: pmap_enter(kernel_pmap, virt, start, prot, FALSE);
509: virt += ps;
510: start += ps;
511: }
512: return(virt);
513: }
514:
515: /*
1.1.1.2 root 516: * Back-door routine for mapping kernel VM at initialization.
1.1 root 517: * Useful for mapping memory outside the range
518: * [phys_first_addr, phys_last_addr) (i.e., devices).
519: * Otherwise like pmap_map.
520: */
1.1.1.4 root 521: vm_offset_t pmap_map_bd(
522: vm_offset_t virt,
523: vm_offset_t start,
524: vm_offset_t end,
525: vm_prot_t prot)
1.1 root 526: {
1.1.1.4 root 527: pt_entry_t template;
528: pt_entry_t *pte;
529: int spl;
1.1.1.3 root 530: #ifdef MACH_PV_PAGETABLES
531: int n, i = 0;
532: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
533: #endif /* MACH_PV_PAGETABLES */
1.1 root 534:
535: template = pa_to_pte(start)
1.1.1.3 root 536: | INTEL_PTE_NCACHE|INTEL_PTE_WTHRU
1.1 root 537: | INTEL_PTE_VALID;
1.1.1.3 root 538: if (CPU_HAS_FEATURE(CPU_FEATURE_PGE))
539: template |= INTEL_PTE_GLOBAL;
1.1 root 540: if (prot & VM_PROT_WRITE)
541: template |= INTEL_PTE_WRITE;
542:
1.1.1.3 root 543: PMAP_READ_LOCK(pmap, spl);
1.1 root 544: while (start < end) {
545: pte = pmap_pte(kernel_pmap, virt);
546: if (pte == PT_ENTRY_NULL)
547: panic("pmap_map_bd: Invalid kernel address\n");
1.1.1.3 root 548: #ifdef MACH_PV_PAGETABLES
549: update[i].ptr = kv_to_ma(pte);
550: update[i].val = pa_to_ma(template);
551: i++;
552: if (i == HYP_BATCH_MMU_UPDATES) {
553: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
554: if (n != i)
555: panic("couldn't pmap_map_bd\n");
556: i = 0;
557: }
558: #else /* MACH_PV_PAGETABLES */
559: WRITE_PTE(pte, template)
560: #endif /* MACH_PV_PAGETABLES */
1.1 root 561: pte_increment_pa(template);
562: virt += PAGE_SIZE;
563: start += PAGE_SIZE;
564: }
1.1.1.3 root 565: #ifdef MACH_PV_PAGETABLES
566: if (i > HYP_BATCH_MMU_UPDATES)
567: panic("overflowed array in pmap_map_bd");
568: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
569: if (n != i)
570: panic("couldn't pmap_map_bd\n");
571: #endif /* MACH_PV_PAGETABLES */
572: PMAP_READ_UNLOCK(pmap, spl);
1.1 root 573: return(virt);
574: }
575:
576: /*
577: * Bootstrap the system enough to run with virtual memory.
578: * Allocate the kernel page directory and page tables,
579: * and direct-map all physical memory.
580: * Called with mapping off.
581: */
1.1.1.4 root 582: void pmap_bootstrap(void)
1.1 root 583: {
584: /*
585: * Mapping is turned off; we must reference only physical addresses.
586: * The load image of the system is to be mapped 1-1 physical = virtual.
587: */
588:
589: /*
590: * Set ptes_per_vm_page for general use.
591: */
592: #if 0
593: ptes_per_vm_page = PAGE_SIZE / INTEL_PGBYTES;
594: #endif
595:
596: /*
597: * The kernel's pmap is statically allocated so we don't
598: * have to use pmap_create, which is unlikely to work
599: * correctly at this part of the boot sequence.
600: */
601:
602: kernel_pmap = &kernel_pmap_store;
603:
604: #if NCPUS > 1
605: lock_init(&pmap_system_lock, FALSE); /* NOT a sleep lock */
1.1.1.2 root 606: #endif /* NCPUS > 1 */
1.1 root 607:
608: simple_lock_init(&kernel_pmap->lock);
609:
610: kernel_pmap->ref_count = 1;
611:
612: /*
613: * Determine the kernel virtual address range.
614: * It starts at the end of the physical memory
615: * mapped into the kernel address space,
616: * and extends to a stupid arbitrary limit beyond that.
617: */
1.1.1.3 root 618: kernel_virtual_start = phystokv(phys_last_addr);
619: kernel_virtual_end = phystokv(phys_last_addr) + VM_KERNEL_MAP_SIZE;
620:
621: if (kernel_virtual_end < kernel_virtual_start
622: || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS)
623: kernel_virtual_end = VM_MAX_KERNEL_ADDRESS;
1.1 root 624:
625: /*
626: * Allocate and clear a kernel page directory.
627: */
1.1.1.3 root 628: /* Note: initial Xen mapping holds at least 512kB free mapped page.
629: * We use that for directly building our linear mapping. */
630: #if PAE
631: {
632: vm_offset_t addr;
633: init_alloc_aligned(PDPNUM * INTEL_PGBYTES, &addr);
634: kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr);
635: }
636: kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page());
637: {
638: int i;
639: for (i = 0; i < PDPNUM; i++)
640: WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID);
641: }
642: #else /* PAE */
643: kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page());
644: #endif /* PAE */
1.1 root 645: {
1.1.1.6 ! root 646: unsigned i;
1.1 root 647: for (i = 0; i < NPDES; i++)
648: kernel_pmap->dirbase[i] = 0;
649: }
650:
1.1.1.3 root 651: #ifdef MACH_PV_PAGETABLES
652: /* We don't actually deal with the CR3 register content at all */
653: hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3);
654: /*
655: * Xen may only provide as few as 512KB extra bootstrap linear memory,
656: * which is far from enough to map all available memory, so we need to
657: * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE)
658: * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is
659: * enough for a pagetable mapping 4GiB.
660: */
661: #ifdef PAE
662: #define NSUP_L1 4
663: #else
664: #define NSUP_L1 1
665: #endif
666: pt_entry_t *l1_map[NSUP_L1];
667: {
668: pt_entry_t *base = (pt_entry_t*) boot_info.pt_base;
669: vm_offset_t la;
670: int n_l1map;
671: for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS; la >= VM_MIN_KERNEL_ADDRESS; la += NPTES * PAGE_SIZE) {
672: #ifdef PAE
673: pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)]);
674: #else /* PAE */
675: pt_entry_t *l2_map = base;
676: #endif /* PAE */
677: /* Like lin2pdenum, but works with non-contiguous boot L3 */
678: l2_map += (la >> PDESHIFT) & PDEMASK;
679: if (!(*l2_map & INTEL_PTE_VALID)) {
680: struct mmu_update update;
1.1.1.6 ! root 681: unsigned j, n;
1.1.1.3 root 682:
683: l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page());
684: for (j = 0; j < NPTES; j++)
685: l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS) * NPTES + j)) << PAGE_SHIFT) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
686: pmap_set_page_readonly_init(l1_map[n_l1map]);
687: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map])))
688: panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map]));
689: update.ptr = kv_to_ma(l2_map);
690: update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID | INTEL_PTE_WRITE;
691: hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF);
692: if (n != 1)
693: panic("couldn't complete bootstrap map");
694: /* added the last L1 table, can stop */
695: if (++n_l1map >= NSUP_L1)
696: break;
697: }
698: }
699: }
700: #endif /* MACH_PV_PAGETABLES */
701:
1.1 root 702: /*
703: * Allocate and set up the kernel page tables.
704: */
705: {
706: vm_offset_t va;
1.1.1.3 root 707: pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE) ? INTEL_PTE_GLOBAL : 0;
1.1 root 708:
709: /*
710: * Map virtual memory for all known physical memory, 1-1,
711: * from phys_first_addr to phys_last_addr.
712: * Make any mappings completely in the kernel's text segment read-only.
713: *
714: * Also allocate some additional all-null page tables afterwards
715: * for kernel virtual memory allocation,
716: * because this PMAP module is too stupid
717: * to allocate new kernel page tables later.
718: * XX fix this
719: */
1.1.1.3 root 720: for (va = phystokv(phys_first_addr); va >= phystokv(phys_first_addr) && va < kernel_virtual_end; )
1.1 root 721: {
722: pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va));
1.1.1.3 root 723: pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page());
1.1 root 724: pt_entry_t *pte;
725:
726: /* Initialize the page directory entry. */
1.1.1.3 root 727: WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable))
728: | INTEL_PTE_VALID | INTEL_PTE_WRITE);
1.1 root 729:
730: /* Initialize the page table. */
1.1.1.3 root 731: for (pte = ptable; (va < phystokv(phys_last_addr)) && (pte < ptable+NPTES); pte++)
1.1 root 732: {
733: if ((pte - ptable) < ptenum(va))
734: {
1.1.1.3 root 735: WRITE_PTE(pte, 0);
1.1 root 736: }
1.1.1.2 root 737: else
1.1.1.3 root 738: #ifdef MACH_PV_PAGETABLES
739: if (va == (vm_offset_t) &hyp_shared_info)
740: {
741: *pte = boot_info.shared_info | INTEL_PTE_VALID | INTEL_PTE_WRITE;
742: va += INTEL_PGBYTES;
743: }
744: else
745: #endif /* MACH_PV_PAGETABLES */
1.1 root 746: {
1.1.1.2 root 747: extern char _start[], etext[];
1.1 root 748:
1.1.1.3 root 749: if (((va >= (vm_offset_t) _start)
1.1 root 750: && (va + INTEL_PGBYTES <= (vm_offset_t)etext))
1.1.1.3 root 751: #ifdef MACH_PV_PAGETABLES
752: || (va >= (vm_offset_t) boot_info.pt_base
753: && (va + INTEL_PGBYTES <=
754: (vm_offset_t) ptable + INTEL_PGBYTES))
755: #endif /* MACH_PV_PAGETABLES */
756: )
1.1 root 757: {
1.1.1.3 root 758: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
759: | INTEL_PTE_VALID | global);
1.1 root 760: }
761: else
762: {
1.1.1.3 root 763: #ifdef MACH_PV_PAGETABLES
764: /* Keep supplementary L1 pages read-only */
765: int i;
766: for (i = 0; i < NSUP_L1; i++)
767: if (va == (vm_offset_t) l1_map[i]) {
768: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
769: | INTEL_PTE_VALID | global);
770: break;
771: }
772: if (i == NSUP_L1)
773: #endif /* MACH_PV_PAGETABLES */
774: WRITE_PTE(pte, pa_to_pte(_kvtophys(va))
775: | INTEL_PTE_VALID | INTEL_PTE_WRITE | global)
776:
1.1 root 777: }
778: va += INTEL_PGBYTES;
779: }
780: }
781: for (; pte < ptable+NPTES; pte++)
782: {
1.1.1.4 root 783: if (va >= kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE && va < kernel_virtual_end)
784: {
785: pmap_mapwindow_t *win = &mapwindows[atop(va - (kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE))];
786: win->entry = pte;
787: win->vaddr = va;
788: }
1.1.1.3 root 789: WRITE_PTE(pte, 0);
1.1 root 790: va += INTEL_PGBYTES;
791: }
1.1.1.3 root 792: #ifdef MACH_PV_PAGETABLES
793: pmap_set_page_readonly_init(ptable);
794: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable)))
795: panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable));
796: #endif /* MACH_PV_PAGETABLES */
1.1 root 797: }
798: }
799:
1.1.1.3 root 800: /* Architecture-specific code will turn on paging
801: soon after we return from here. */
802: }
1.1 root 803:
1.1.1.3 root 804: #ifdef MACH_PV_PAGETABLES
805: /* These are only required because of Xen security policies */
1.1 root 806:
1.1.1.3 root 807: /* Set back a page read write */
808: void pmap_set_page_readwrite(void *_vaddr) {
809: vm_offset_t vaddr = (vm_offset_t) _vaddr;
810: vm_offset_t paddr = kvtophys(vaddr);
811: vm_offset_t canon_vaddr = phystokv(paddr);
812: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
813: panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
814: if (canon_vaddr != vaddr)
815: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID | INTEL_PTE_WRITE, UVMF_NONE))
816: panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr));
817: }
818:
819: /* Set a page read only (so as to pin it for instance) */
820: void pmap_set_page_readonly(void *_vaddr) {
821: vm_offset_t vaddr = (vm_offset_t) _vaddr;
822: vm_offset_t paddr = kvtophys(vaddr);
823: vm_offset_t canon_vaddr = phystokv(paddr);
824: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
825: if (hyp_do_update_va_mapping (kvtolin(vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
826: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
827: }
828: if (canon_vaddr != vaddr &&
829: *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID) {
830: if (hyp_do_update_va_mapping (kvtolin(canon_vaddr), pa_to_pte (pa_to_ma(paddr)) | INTEL_PTE_VALID, UVMF_NONE))
831: panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr));
1.1 root 832: }
1.1.1.3 root 833: }
1.1 root 834:
1.1.1.3 root 835: /* This needs to be called instead of pmap_set_page_readonly as long as RC3
836: * still points to the bootstrap dirbase, to also fix the bootstrap table. */
837: void pmap_set_page_readonly_init(void *_vaddr) {
838: vm_offset_t vaddr = (vm_offset_t) _vaddr;
839: #if PAE
840: pt_entry_t *pdpbase = (void*) boot_info.pt_base;
841: /* The bootstrap table does not necessarily use contiguous pages for the pde tables */
842: pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)]);
843: #else
844: pt_entry_t *dirbase = (void*) boot_info.pt_base;
1.1 root 845: #endif
1.1.1.3 root 846: pt_entry_t *pte = &dirbase[lin2pdenum(vaddr) & PTEMASK];
847: /* Modify our future kernel map (can't use update_va_mapping for this)... */
848: if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID) {
849: if (!hyp_mmu_update_la (kvtolin(vaddr), pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID))
850: panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
851: }
852: /* ... and the bootstrap map. */
853: if (*pte & INTEL_PTE_VALID) {
854: if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr)) | INTEL_PTE_VALID, UVMF_NONE))
855: panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
856: }
857: }
1.1 root 858:
1.1.1.3 root 859: void pmap_clear_bootstrap_pagetable(pt_entry_t *base) {
1.1.1.6 ! root 860: unsigned i;
1.1.1.3 root 861: pt_entry_t *dir;
862: vm_offset_t va = 0;
863: #if PAE
1.1.1.6 ! root 864: unsigned j;
1.1.1.3 root 865: #endif /* PAE */
866: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base)))
867: panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base));
868: #if PAE
869: for (j = 0; j < PDPNUM; j++)
870: {
871: pt_entry_t pdpe = base[j];
872: if (pdpe & INTEL_PTE_VALID) {
873: dir = (pt_entry_t *) ptetokv(pdpe);
874: #else /* PAE */
875: dir = base;
876: #endif /* PAE */
877: for (i = 0; i < NPTES; i++) {
878: pt_entry_t pde = dir[i];
879: unsigned long pfn = atop(pte_to_pa(pde));
880: void *pgt = (void*) phystokv(ptoa(pfn));
881: if (pde & INTEL_PTE_VALID)
882: hyp_free_page(pfn, pgt);
883: va += NPTES * INTEL_PGBYTES;
884: if (va >= HYP_VIRT_START)
885: break;
886: }
887: #if PAE
888: hyp_free_page(atop(_kvtophys(dir)), dir);
889: } else
890: va += NPTES * NPTES * INTEL_PGBYTES;
891: if (va >= HYP_VIRT_START)
892: break;
893: }
894: #endif /* PAE */
895: hyp_free_page(atop(_kvtophys(base)), base);
1.1 root 896: }
1.1.1.3 root 897: #endif /* MACH_PV_PAGETABLES */
1.1 root 898:
1.1.1.4 root 899: /*
900: * Create a temporary mapping for a given physical entry
901: *
902: * This can be used to access physical pages which are not mapped 1:1 by
903: * phystokv().
904: */
905: pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry)
906: {
907: pmap_mapwindow_t *map;
908:
909: /* Find an empty one. */
910: for (map = &mapwindows[0]; map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]; map++)
911: if (!(*map->entry))
912: break;
913: assert(map < &mapwindows[sizeof (mapwindows) / sizeof (*mapwindows)]);
914:
915: WRITE_PTE(map->entry, entry);
916: return map;
917: }
918:
919: /*
920: * Destroy a temporary mapping for a physical entry
921: */
922: void pmap_put_mapwindow(pmap_mapwindow_t *map)
923: {
924: WRITE_PTE(map->entry, 0);
925: PMAP_UPDATE_TLBS(kernel_pmap, map->vaddr, map->vaddr + PAGE_SIZE);
926: }
927:
928: void pmap_virtual_space(
929: vm_offset_t *startp,
930: vm_offset_t *endp)
1.1 root 931: {
932: *startp = kernel_virtual_start;
1.1.1.4 root 933: *endp = kernel_virtual_end - PMAP_NMAPWINDOWS * PAGE_SIZE;
1.1 root 934: }
935:
936: /*
937: * Initialize the pmap module.
938: * Called by vm_init, to initialize any structures that the pmap
939: * system needs to map virtual memory.
940: */
1.1.1.4 root 941: void pmap_init(void)
1.1 root 942: {
1.1.1.4 root 943: long npages;
1.1 root 944: vm_offset_t addr;
1.1.1.4 root 945: vm_size_t s;
1.1.1.3 root 946: #if NCPUS > 1
1.1 root 947: int i;
1.1.1.3 root 948: #endif /* NCPUS > 1 */
1.1 root 949:
950: /*
951: * Allocate memory for the pv_head_table and its lock bits,
952: * the modify bit array, and the pte_page table.
953: */
954:
955: npages = atop(phys_last_addr - phys_first_addr);
956: s = (vm_size_t) (sizeof(struct pv_entry) * npages
957: + pv_lock_table_size(npages)
958: + npages);
959:
960: s = round_page(s);
961: if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
962: panic("pmap_init");
1.1.1.4 root 963: memset((void *) addr, 0, s);
1.1 root 964:
965: /*
966: * Allocate the structures first to preserve word-alignment.
967: */
968: pv_head_table = (pv_entry_t) addr;
969: addr = (vm_offset_t) (pv_head_table + npages);
970:
971: pv_lock_table = (char *) addr;
972: addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
973:
974: pmap_phys_attributes = (char *) addr;
975:
976: /*
1.1.1.3 root 977: * Create the cache of physical maps,
1.1 root 978: * and of the physical-to-virtual entries.
979: */
980: s = (vm_size_t) sizeof(struct pmap);
1.1.1.6 ! root 981: kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, 0);
1.1 root 982: s = (vm_size_t) sizeof(struct pv_entry);
1.1.1.6 ! root 983: kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, 0);
1.1 root 984:
985: #if NCPUS > 1
986: /*
987: * Set up the pmap request lists
988: */
989: for (i = 0; i < NCPUS; i++) {
990: pmap_update_list_t up = &cpu_update_list[i];
991:
992: simple_lock_init(&up->lock);
993: up->count = 0;
994: }
1.1.1.2 root 995: #endif /* NCPUS > 1 */
1.1 root 996:
997: /*
998: * Indicate that the PMAP module is now fully initialized.
999: */
1000: pmap_initialized = TRUE;
1001: }
1002:
1003: #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
1004:
1.1.1.4 root 1005: boolean_t pmap_verify_free(vm_offset_t phys)
1.1 root 1006: {
1007: pv_entry_t pv_h;
1008: int pai;
1009: int spl;
1010: boolean_t result;
1011:
1012: assert(phys != vm_page_fictitious_addr);
1013: if (!pmap_initialized)
1014: return(TRUE);
1015:
1016: if (!pmap_valid_page(phys))
1017: return(FALSE);
1018:
1019: PMAP_WRITE_LOCK(spl);
1020:
1021: pai = pa_index(phys);
1022: pv_h = pai_to_pvh(pai);
1023:
1024: result = (pv_h->pmap == PMAP_NULL);
1025: PMAP_WRITE_UNLOCK(spl);
1026:
1027: return(result);
1028: }
1029:
1030: /*
1031: * Routine: pmap_page_table_page_alloc
1032: *
1033: * Allocates a new physical page to be used as a page-table page.
1034: *
1035: * Must be called with the pmap system and the pmap unlocked,
1036: * since these must be unlocked to use vm_page_grab.
1037: */
1038: vm_offset_t
1.1.1.4 root 1039: pmap_page_table_page_alloc(void)
1.1 root 1040: {
1.1.1.4 root 1041: vm_page_t m;
1042: vm_offset_t pa;
1.1 root 1043:
1044: check_simple_locks();
1045:
1046: /*
1047: * We cannot allocate the pmap_object in pmap_init,
1.1.1.3 root 1048: * because it is called before the cache package is up.
1.1 root 1049: * Allocate it now if it is missing.
1050: */
1051: if (pmap_object == VM_OBJECT_NULL)
1052: pmap_object = vm_object_allocate(phys_last_addr - phys_first_addr);
1053:
1054: /*
1055: * Allocate a VM page for the level 2 page table entries.
1056: */
1.1.1.2 root 1057: while ((m = vm_page_grab(FALSE)) == VM_PAGE_NULL)
1.1 root 1058: VM_PAGE_WAIT((void (*)()) 0);
1059:
1060: /*
1061: * Map the page to its physical address so that it
1062: * can be found later.
1063: */
1064: pa = m->phys_addr;
1065: vm_object_lock(pmap_object);
1066: vm_page_insert(m, pmap_object, pa);
1067: vm_page_lock_queues();
1068: vm_page_wire(m);
1069: inuse_ptepages_count++;
1070: vm_page_unlock_queues();
1071: vm_object_unlock(pmap_object);
1072:
1073: /*
1074: * Zero the page.
1075: */
1.1.1.3 root 1076: memset((void *)phystokv(pa), 0, PAGE_SIZE);
1.1 root 1077:
1078: return pa;
1079: }
1080:
1.1.1.3 root 1081: #ifdef MACH_XEN
1082: void pmap_map_mfn(void *_addr, unsigned long mfn) {
1083: vm_offset_t addr = (vm_offset_t) _addr;
1084: pt_entry_t *pte, *pdp;
1085: vm_offset_t ptp;
1086: pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT;
1087:
1088: /* Add a ptp if none exist yet for this pte */
1089: if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) {
1090: ptp = phystokv(pmap_page_table_page_alloc());
1091: #ifdef MACH_PV_PAGETABLES
1092: pmap_set_page_readonly((void*) ptp);
1093: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp)))
1094: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1095: #endif /* MACH_PV_PAGETABLES */
1096: pdp = pmap_pde(kernel_pmap, addr);
1097:
1098: #ifdef MACH_PV_PAGETABLES
1099: if (!hyp_mmu_update_pte(kv_to_ma(pdp),
1100: pa_to_pte(kv_to_ma(ptp)) | INTEL_PTE_VALID
1101: | INTEL_PTE_USER
1102: | INTEL_PTE_WRITE))
1103: panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__,__LINE__,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp));
1104: #else /* MACH_PV_PAGETABLES */
1105: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
1106: | INTEL_PTE_USER
1107: | INTEL_PTE_WRITE;
1108: #endif /* MACH_PV_PAGETABLES */
1109: pte = pmap_pte(kernel_pmap, addr);
1110: }
1111:
1112: #ifdef MACH_PV_PAGETABLES
1113: if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID | INTEL_PTE_WRITE))
1114: panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__,__LINE__,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma));
1115: #else /* MACH_PV_PAGETABLES */
1116: /* Note: in this case, mfn is actually a pfn. */
1117: WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE);
1118: #endif /* MACH_PV_PAGETABLES */
1119: }
1120: #endif /* MACH_XEN */
1121:
1.1 root 1122: /*
1123: * Deallocate a page-table page.
1124: * The page-table page must have all mappings removed,
1125: * and be removed from its page directory.
1126: */
1127: void
1.1.1.4 root 1128: pmap_page_table_page_dealloc(vm_offset_t pa)
1.1 root 1129: {
1130: vm_page_t m;
1131:
1132: vm_object_lock(pmap_object);
1133: m = vm_page_lookup(pmap_object, pa);
1134: vm_page_lock_queues();
1135: vm_page_free(m);
1136: inuse_ptepages_count--;
1137: vm_page_unlock_queues();
1138: vm_object_unlock(pmap_object);
1139: }
1140:
1141: /*
1142: * Create and return a physical map.
1143: *
1144: * If the size specified for the map
1145: * is zero, the map is an actual physical
1146: * map, and may be referenced by the
1147: * hardware.
1148: *
1149: * If the size specified is non-zero,
1150: * the map will be used in software only, and
1151: * is bounded by that size.
1152: */
1.1.1.4 root 1153: pmap_t pmap_create(vm_size_t size)
1.1 root 1154: {
1.1.1.4 root 1155: pmap_t p;
1156: pmap_statistics_t stats;
1.1 root 1157:
1158: /*
1159: * A software use-only map doesn't even need a map.
1160: */
1161:
1162: if (size != 0) {
1163: return(PMAP_NULL);
1164: }
1165:
1166: /*
1.1.1.3 root 1167: * Allocate a pmap struct from the pmap_cache. Then allocate
1168: * the page descriptor table.
1.1 root 1169: */
1170:
1.1.1.3 root 1171: p = (pmap_t) kmem_cache_alloc(&pmap_cache);
1.1 root 1172: if (p == PMAP_NULL)
1173: panic("pmap_create");
1174:
1175: if (kmem_alloc_wired(kernel_map,
1.1.1.3 root 1176: (vm_offset_t *)&p->dirbase, PDPNUM * INTEL_PGBYTES)
1.1 root 1177: != KERN_SUCCESS)
1178: panic("pmap_create");
1179:
1.1.1.3 root 1180: memcpy(p->dirbase, kernel_page_dir, PDPNUM * INTEL_PGBYTES);
1181: #ifdef LINUX_DEV
1182: #if VM_MIN_KERNEL_ADDRESS != 0
1183: /* Do not map BIOS in user tasks */
1184: p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)] = 0;
1185: #endif
1186: #endif
1187: #ifdef MACH_PV_PAGETABLES
1188: {
1189: int i;
1190: for (i = 0; i < PDPNUM; i++)
1191: pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES);
1192: }
1193: #endif /* MACH_PV_PAGETABLES */
1194:
1195: #if PAE
1196: if (kmem_alloc_wired(kernel_map,
1197: (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES)
1198: != KERN_SUCCESS)
1199: panic("pmap_create");
1200: {
1201: int i;
1202: for (i = 0; i < PDPNUM; i++)
1203: WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID);
1204: }
1205: #ifdef MACH_PV_PAGETABLES
1206: pmap_set_page_readonly(p->pdpbase);
1207: #endif /* MACH_PV_PAGETABLES */
1208: #endif /* PAE */
1209:
1.1 root 1210: p->ref_count = 1;
1211:
1212: simple_lock_init(&p->lock);
1213: p->cpus_using = 0;
1214:
1215: /*
1216: * Initialize statistics.
1217: */
1218:
1219: stats = &p->stats;
1220: stats->resident_count = 0;
1221: stats->wired_count = 0;
1222:
1223: return(p);
1224: }
1225:
1226: /*
1227: * Retire the given physical map from service.
1228: * Should only be called if the map contains
1229: * no valid mappings.
1230: */
1231:
1.1.1.4 root 1232: void pmap_destroy(pmap_t p)
1.1 root 1233: {
1.1.1.4 root 1234: pt_entry_t *pdep;
1235: vm_offset_t pa;
1236: int c, s;
1237: vm_page_t m;
1.1 root 1238:
1239: if (p == PMAP_NULL)
1240: return;
1241:
1242: SPLVM(s);
1243: simple_lock(&p->lock);
1244: c = --p->ref_count;
1245: simple_unlock(&p->lock);
1246: SPLX(s);
1247:
1248: if (c != 0) {
1249: return; /* still in use */
1250: }
1251:
1252: /*
1253: * Free the memory maps, then the
1254: * pmap structure.
1255: */
1256: for (pdep = p->dirbase;
1257: pdep < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)];
1258: pdep += ptes_per_vm_page) {
1259: if (*pdep & INTEL_PTE_VALID) {
1260: pa = pte_to_pa(*pdep);
1261: vm_object_lock(pmap_object);
1262: m = vm_page_lookup(pmap_object, pa);
1263: if (m == VM_PAGE_NULL)
1264: panic("pmap_destroy: pte page not in object");
1265: vm_page_lock_queues();
1.1.1.3 root 1266: #ifdef MACH_PV_PAGETABLES
1267: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa)))
1268: panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa));
1269: pmap_set_page_readwrite((void*) phystokv(pa));
1270: #endif /* MACH_PV_PAGETABLES */
1.1 root 1271: vm_page_free(m);
1272: inuse_ptepages_count--;
1273: vm_page_unlock_queues();
1274: vm_object_unlock(pmap_object);
1275: }
1276: }
1.1.1.3 root 1277: #ifdef MACH_PV_PAGETABLES
1278: {
1279: int i;
1280: for (i = 0; i < PDPNUM; i++)
1281: pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES);
1282: }
1283: #endif /* MACH_PV_PAGETABLES */
1284: kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM * INTEL_PGBYTES);
1285: #if PAE
1286: #ifdef MACH_PV_PAGETABLES
1287: pmap_set_page_readwrite(p->pdpbase);
1288: #endif /* MACH_PV_PAGETABLES */
1289: kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES);
1290: #endif /* PAE */
1291: kmem_cache_free(&pmap_cache, (vm_offset_t) p);
1.1 root 1292: }
1293:
1294: /*
1295: * Add a reference to the specified pmap.
1296: */
1297:
1.1.1.4 root 1298: void pmap_reference(pmap_t p)
1.1 root 1299: {
1300: int s;
1301: if (p != PMAP_NULL) {
1302: SPLVM(s);
1303: simple_lock(&p->lock);
1304: p->ref_count++;
1305: simple_unlock(&p->lock);
1306: SPLX(s);
1307: }
1308: }
1309:
1310: /*
1311: * Remove a range of hardware page-table entries.
1312: * The entries given are the first (inclusive)
1313: * and last (exclusive) entries for the VM pages.
1314: * The virtual address is the va for the first pte.
1315: *
1316: * The pmap must be locked.
1317: * If the pmap is not the kernel pmap, the range must lie
1318: * entirely within one pte-page. This is NOT checked.
1319: * Assumes that the pte-page exists.
1320: */
1321:
1322: /* static */
1.1.1.4 root 1323: void pmap_remove_range(
1324: pmap_t pmap,
1325: vm_offset_t va,
1326: pt_entry_t *spte,
1327: pt_entry_t *epte)
1.1 root 1328: {
1.1.1.4 root 1329: pt_entry_t *cpte;
1.1 root 1330: int num_removed, num_unwired;
1331: int pai;
1332: vm_offset_t pa;
1.1.1.3 root 1333: #ifdef MACH_PV_PAGETABLES
1334: int n, ii = 0;
1335: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1336: #endif /* MACH_PV_PAGETABLES */
1.1 root 1337:
1338: #if DEBUG_PTE_PAGE
1339: if (pmap != kernel_pmap)
1340: ptep_check(get_pte_page(spte));
1.1.1.2 root 1341: #endif /* DEBUG_PTE_PAGE */
1.1 root 1342: num_removed = 0;
1343: num_unwired = 0;
1344:
1345: for (cpte = spte; cpte < epte;
1346: cpte += ptes_per_vm_page, va += PAGE_SIZE) {
1347:
1348: if (*cpte == 0)
1349: continue;
1350: pa = pte_to_pa(*cpte);
1351:
1352: num_removed++;
1353: if (*cpte & INTEL_PTE_WIRED)
1354: num_unwired++;
1355:
1356: if (!valid_page(pa)) {
1357:
1358: /*
1359: * Outside range of managed physical memory.
1360: * Just remove the mappings.
1361: */
1.1.1.4 root 1362: int i = ptes_per_vm_page;
1363: pt_entry_t *lpte = cpte;
1.1 root 1364: do {
1.1.1.3 root 1365: #ifdef MACH_PV_PAGETABLES
1366: update[ii].ptr = kv_to_ma(lpte);
1367: update[ii].val = 0;
1368: ii++;
1369: if (ii == HYP_BATCH_MMU_UPDATES) {
1370: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1371: if (n != ii)
1372: panic("couldn't pmap_remove_range\n");
1373: ii = 0;
1374: }
1375: #else /* MACH_PV_PAGETABLES */
1.1 root 1376: *lpte = 0;
1.1.1.3 root 1377: #endif /* MACH_PV_PAGETABLES */
1.1 root 1378: lpte++;
1379: } while (--i > 0);
1380: continue;
1381: }
1382:
1383: pai = pa_index(pa);
1384: LOCK_PVH(pai);
1385:
1386: /*
1387: * Get the modify and reference bits.
1388: */
1389: {
1.1.1.4 root 1390: int i;
1391: pt_entry_t *lpte;
1.1 root 1392:
1393: i = ptes_per_vm_page;
1394: lpte = cpte;
1395: do {
1396: pmap_phys_attributes[pai] |=
1397: *lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 root 1398: #ifdef MACH_PV_PAGETABLES
1399: update[ii].ptr = kv_to_ma(lpte);
1400: update[ii].val = 0;
1401: ii++;
1402: if (ii == HYP_BATCH_MMU_UPDATES) {
1403: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1404: if (n != ii)
1405: panic("couldn't pmap_remove_range\n");
1406: ii = 0;
1407: }
1408: #else /* MACH_PV_PAGETABLES */
1.1 root 1409: *lpte = 0;
1.1.1.3 root 1410: #endif /* MACH_PV_PAGETABLES */
1.1 root 1411: lpte++;
1412: } while (--i > 0);
1413: }
1414:
1415: /*
1416: * Remove the mapping from the pvlist for
1417: * this physical page.
1418: */
1419: {
1.1.1.4 root 1420: pv_entry_t pv_h, prev, cur;
1.1 root 1421:
1422: pv_h = pai_to_pvh(pai);
1423: if (pv_h->pmap == PMAP_NULL) {
1424: panic("pmap_remove: null pv_list!");
1425: }
1426: if (pv_h->va == va && pv_h->pmap == pmap) {
1427: /*
1428: * Header is the pv_entry. Copy the next one
1429: * to header and free the next one (we cannot
1430: * free the header)
1431: */
1432: cur = pv_h->next;
1433: if (cur != PV_ENTRY_NULL) {
1434: *pv_h = *cur;
1435: PV_FREE(cur);
1436: }
1437: else {
1438: pv_h->pmap = PMAP_NULL;
1439: }
1440: }
1441: else {
1442: cur = pv_h;
1443: do {
1444: prev = cur;
1445: if ((cur = prev->next) == PV_ENTRY_NULL) {
1446: panic("pmap-remove: mapping not in pv_list!");
1447: }
1448: } while (cur->va != va || cur->pmap != pmap);
1449: prev->next = cur->next;
1450: PV_FREE(cur);
1451: }
1452: UNLOCK_PVH(pai);
1453: }
1454: }
1455:
1.1.1.3 root 1456: #ifdef MACH_PV_PAGETABLES
1457: if (ii > HYP_BATCH_MMU_UPDATES)
1458: panic("overflowed array in pmap_remove_range");
1459: hyp_mmu_update(kvtolin(&update), ii, kvtolin(&n), DOMID_SELF);
1460: if (n != ii)
1461: panic("couldn't pmap_remove_range\n");
1462: #endif /* MACH_PV_PAGETABLES */
1463:
1.1 root 1464: /*
1465: * Update the counts
1466: */
1467: pmap->stats.resident_count -= num_removed;
1468: pmap->stats.wired_count -= num_unwired;
1469: }
1470:
1471: /*
1472: * Remove the given range of addresses
1473: * from the specified map.
1474: *
1475: * It is assumed that the start and end are properly
1476: * rounded to the hardware page size.
1477: */
1478:
1.1.1.4 root 1479: void pmap_remove(
1480: pmap_t map,
1481: vm_offset_t s,
1482: vm_offset_t e)
1.1 root 1483: {
1484: int spl;
1.1.1.4 root 1485: pt_entry_t *pde;
1486: pt_entry_t *spte, *epte;
1.1 root 1487: vm_offset_t l;
1.1.1.3 root 1488: vm_offset_t _s = s;
1.1 root 1489:
1490: if (map == PMAP_NULL)
1491: return;
1492:
1493: PMAP_READ_LOCK(map, spl);
1494:
1495: pde = pmap_pde(map, s);
1496: while (s < e) {
1497: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1498: if (l > e)
1499: l = e;
1500: if (*pde & INTEL_PTE_VALID) {
1501: spte = (pt_entry_t *)ptetokv(*pde);
1502: spte = &spte[ptenum(s)];
1503: epte = &spte[intel_btop(l-s)];
1504: pmap_remove_range(map, s, spte, epte);
1505: }
1506: s = l;
1507: pde++;
1508: }
1.1.1.3 root 1509: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1510:
1511: PMAP_READ_UNLOCK(map, spl);
1512: }
1513:
1514: /*
1515: * Routine: pmap_page_protect
1516: *
1517: * Function:
1518: * Lower the permission for all mappings to a given
1519: * page.
1520: */
1.1.1.4 root 1521: void pmap_page_protect(
1522: vm_offset_t phys,
1523: vm_prot_t prot)
1.1 root 1524: {
1525: pv_entry_t pv_h, prev;
1.1.1.4 root 1526: pv_entry_t pv_e;
1527: pt_entry_t *pte;
1.1 root 1528: int pai;
1.1.1.4 root 1529: pmap_t pmap;
1.1 root 1530: int spl;
1531: boolean_t remove;
1532:
1533: assert(phys != vm_page_fictitious_addr);
1534: if (!valid_page(phys)) {
1535: /*
1536: * Not a managed page.
1537: */
1538: return;
1539: }
1540:
1541: /*
1542: * Determine the new protection.
1543: */
1544: switch (prot) {
1545: case VM_PROT_READ:
1546: case VM_PROT_READ|VM_PROT_EXECUTE:
1547: remove = FALSE;
1548: break;
1549: case VM_PROT_ALL:
1550: return; /* nothing to do */
1551: default:
1552: remove = TRUE;
1553: break;
1554: }
1555:
1556: /*
1557: * Lock the pmap system first, since we will be changing
1558: * several pmaps.
1559: */
1560:
1561: PMAP_WRITE_LOCK(spl);
1562:
1563: pai = pa_index(phys);
1564: pv_h = pai_to_pvh(pai);
1565:
1566: /*
1567: * Walk down PV list, changing or removing all mappings.
1568: * We do not have to lock the pv_list because we have
1569: * the entire pmap system locked.
1570: */
1571: if (pv_h->pmap != PMAP_NULL) {
1572:
1573: prev = pv_e = pv_h;
1574: do {
1.1.1.4 root 1575: vm_offset_t va;
1.1.1.3 root 1576:
1.1 root 1577: pmap = pv_e->pmap;
1578: /*
1579: * Lock the pmap to block pmap_extract and similar routines.
1580: */
1581: simple_lock(&pmap->lock);
1582:
1.1.1.3 root 1583: va = pv_e->va;
1584: pte = pmap_pte(pmap, va);
1.1 root 1585:
1.1.1.3 root 1586: /*
1587: * Consistency checks.
1588: */
1.1.1.5 root 1589: assert(*pte & INTEL_PTE_VALID);
1590: assert(pte_to_pa(*pte) == phys);
1.1 root 1591:
1592: /*
1593: * Remove the mapping if new protection is NONE
1594: * or if write-protecting a kernel mapping.
1595: */
1596: if (remove || pmap == kernel_pmap) {
1597: /*
1598: * Remove the mapping, collecting any modify bits.
1599: */
1600: if (*pte & INTEL_PTE_WIRED)
1.1.1.5 root 1601: panic("pmap_page_protect removing a wired page");
1.1 root 1602:
1603: {
1.1.1.4 root 1604: int i = ptes_per_vm_page;
1.1 root 1605:
1606: do {
1607: pmap_phys_attributes[pai] |=
1608: *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1.1.1.3 root 1609: #ifdef MACH_PV_PAGETABLES
1610: if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0))
1611: panic("%s:%d could not clear pte %p\n",__FILE__,__LINE__,pte-1);
1612: #else /* MACH_PV_PAGETABLES */
1.1 root 1613: *pte++ = 0;
1.1.1.3 root 1614: #endif /* MACH_PV_PAGETABLES */
1.1 root 1615: } while (--i > 0);
1616: }
1617:
1618: pmap->stats.resident_count--;
1619:
1620: /*
1621: * Remove the pv_entry.
1622: */
1623: if (pv_e == pv_h) {
1624: /*
1625: * Fix up head later.
1626: */
1627: pv_h->pmap = PMAP_NULL;
1628: }
1629: else {
1630: /*
1631: * Delete this entry.
1632: */
1633: prev->next = pv_e->next;
1634: PV_FREE(pv_e);
1635: }
1636: }
1637: else {
1638: /*
1639: * Write-protect.
1640: */
1.1.1.4 root 1641: int i = ptes_per_vm_page;
1.1 root 1642:
1643: do {
1.1.1.3 root 1644: #ifdef MACH_PV_PAGETABLES
1645: if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE))
1646: panic("%s:%d could not disable write on pte %p\n",__FILE__,__LINE__,pte);
1647: #else /* MACH_PV_PAGETABLES */
1.1 root 1648: *pte &= ~INTEL_PTE_WRITE;
1.1.1.3 root 1649: #endif /* MACH_PV_PAGETABLES */
1.1 root 1650: pte++;
1651: } while (--i > 0);
1652:
1653: /*
1654: * Advance prev.
1655: */
1656: prev = pv_e;
1657: }
1.1.1.3 root 1658: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 1659:
1660: simple_unlock(&pmap->lock);
1661:
1662: } while ((pv_e = prev->next) != PV_ENTRY_NULL);
1663:
1664: /*
1665: * If pv_head mapping was removed, fix it up.
1666: */
1667: if (pv_h->pmap == PMAP_NULL) {
1668: pv_e = pv_h->next;
1669: if (pv_e != PV_ENTRY_NULL) {
1670: *pv_h = *pv_e;
1671: PV_FREE(pv_e);
1672: }
1673: }
1674: }
1675:
1676: PMAP_WRITE_UNLOCK(spl);
1677: }
1678:
1679: /*
1680: * Set the physical protection on the
1681: * specified range of this map as requested.
1682: * Will not increase permissions.
1683: */
1.1.1.4 root 1684: void pmap_protect(
1685: pmap_t map,
1686: vm_offset_t s,
1687: vm_offset_t e,
1688: vm_prot_t prot)
1689: {
1690: pt_entry_t *pde;
1691: pt_entry_t *spte, *epte;
1692: vm_offset_t l;
1.1 root 1693: int spl;
1.1.1.3 root 1694: vm_offset_t _s = s;
1.1 root 1695:
1696: if (map == PMAP_NULL)
1697: return;
1698:
1699: /*
1700: * Determine the new protection.
1701: */
1702: switch (prot) {
1703: case VM_PROT_READ:
1704: case VM_PROT_READ|VM_PROT_EXECUTE:
1705: break;
1706: case VM_PROT_READ|VM_PROT_WRITE:
1707: case VM_PROT_ALL:
1708: return; /* nothing to do */
1709: default:
1710: pmap_remove(map, s, e);
1711: return;
1712: }
1713:
1714: /*
1715: * If write-protecting in the kernel pmap,
1716: * remove the mappings; the i386 ignores
1717: * the write-permission bit in kernel mode.
1718: *
1719: * XXX should be #if'd for i386
1720: */
1721: if (map == kernel_pmap) {
1722: pmap_remove(map, s, e);
1723: return;
1724: }
1725:
1726: SPLVM(spl);
1727: simple_lock(&map->lock);
1728:
1729: pde = pmap_pde(map, s);
1730: while (s < e) {
1731: l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
1732: if (l > e)
1733: l = e;
1734: if (*pde & INTEL_PTE_VALID) {
1735: spte = (pt_entry_t *)ptetokv(*pde);
1736: spte = &spte[ptenum(s)];
1737: epte = &spte[intel_btop(l-s)];
1738:
1.1.1.3 root 1739: #ifdef MACH_PV_PAGETABLES
1740: int n, i = 0;
1741: struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1742: #endif /* MACH_PV_PAGETABLES */
1743:
1.1 root 1744: while (spte < epte) {
1.1.1.3 root 1745: if (*spte & INTEL_PTE_VALID) {
1746: #ifdef MACH_PV_PAGETABLES
1747: update[i].ptr = kv_to_ma(spte);
1748: update[i].val = *spte & ~INTEL_PTE_WRITE;
1749: i++;
1750: if (i == HYP_BATCH_MMU_UPDATES) {
1751: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
1752: if (n != i)
1753: panic("couldn't pmap_protect\n");
1754: i = 0;
1755: }
1756: #else /* MACH_PV_PAGETABLES */
1.1 root 1757: *spte &= ~INTEL_PTE_WRITE;
1.1.1.3 root 1758: #endif /* MACH_PV_PAGETABLES */
1759: }
1.1 root 1760: spte++;
1761: }
1.1.1.3 root 1762: #ifdef MACH_PV_PAGETABLES
1763: if (i > HYP_BATCH_MMU_UPDATES)
1764: panic("overflowed array in pmap_protect");
1765: hyp_mmu_update(kvtolin(&update), i, kvtolin(&n), DOMID_SELF);
1766: if (n != i)
1767: panic("couldn't pmap_protect\n");
1768: #endif /* MACH_PV_PAGETABLES */
1.1 root 1769: }
1770: s = l;
1771: pde++;
1772: }
1.1.1.3 root 1773: PMAP_UPDATE_TLBS(map, _s, e);
1.1 root 1774:
1775: simple_unlock(&map->lock);
1776: SPLX(spl);
1777: }
1778:
1779: /*
1780: * Insert the given physical page (p) at
1781: * the specified virtual address (v) in the
1782: * target physical map with the protection requested.
1783: *
1784: * If specified, the page will be wired down, meaning
1785: * that the related pte can not be reclaimed.
1786: *
1787: * NB: This is the only routine which MAY NOT lazy-evaluate
1788: * or lose information. That is, this routine must actually
1789: * insert this page into the given map NOW.
1790: */
1.1.1.4 root 1791: void pmap_enter(
1792: pmap_t pmap,
1793: vm_offset_t v,
1794: vm_offset_t pa,
1795: vm_prot_t prot,
1796: boolean_t wired)
1797: {
1798: pt_entry_t *pte;
1799: pv_entry_t pv_h;
1800: int i, pai;
1.1 root 1801: pv_entry_t pv_e;
1802: pt_entry_t template;
1803: int spl;
1804: vm_offset_t old_pa;
1805:
1806: assert(pa != vm_page_fictitious_addr);
1.1.1.3 root 1807: if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa);
1.1 root 1808: if (pmap == PMAP_NULL)
1809: return;
1810:
1.1.1.3 root 1811: #if !MACH_KDB
1812: if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end))
1813: panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa);
1814: #endif
1.1 root 1815: if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0
1816: && !wired /* hack for io_wire */ ) {
1817: /*
1818: * Because the 386 ignores write protection in kernel mode,
1819: * we cannot enter a read-only kernel mapping, and must
1820: * remove an existing mapping if changing it.
1821: *
1822: * XXX should be #if'd for i386
1823: */
1824: PMAP_READ_LOCK(pmap, spl);
1825:
1826: pte = pmap_pte(pmap, v);
1827: if (pte != PT_ENTRY_NULL && *pte != 0) {
1828: /*
1829: * Invalidate the translation buffer,
1830: * then remove the mapping.
1831: */
1832: pmap_remove_range(pmap, v, pte,
1833: pte + ptes_per_vm_page);
1.1.1.3 root 1834: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1835: }
1836: PMAP_READ_UNLOCK(pmap, spl);
1837: return;
1838: }
1839:
1840: /*
1841: * Must allocate a new pvlist entry while we're unlocked;
1.1.1.3 root 1842: * Allocating may cause pageout (which will lock the pmap system).
1.1 root 1843: * If we determine we need a pvlist entry, we will unlock
1844: * and allocate one. Then we will retry, throughing away
1845: * the allocated entry later (if we no longer need it).
1846: */
1847: pv_e = PV_ENTRY_NULL;
1848: Retry:
1849: PMAP_READ_LOCK(pmap, spl);
1850:
1851: /*
1852: * Expand pmap to include this pte. Assume that
1853: * pmap is always expanded to include enough hardware
1854: * pages to map one VM page.
1855: */
1856:
1857: while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
1858: /*
1859: * Need to allocate a new page-table page.
1860: */
1861: vm_offset_t ptp;
1862: pt_entry_t *pdp;
1863: int i;
1864:
1865: if (pmap == kernel_pmap) {
1866: /*
1867: * Would have to enter the new page-table page in
1868: * EVERY pmap.
1869: */
1870: panic("pmap_expand kernel pmap to %#x", v);
1871: }
1872:
1873: /*
1874: * Unlock the pmap and allocate a new page-table page.
1875: */
1876: PMAP_READ_UNLOCK(pmap, spl);
1877:
1.1.1.3 root 1878: ptp = phystokv(pmap_page_table_page_alloc());
1.1 root 1879:
1880: /*
1881: * Re-lock the pmap and check that another thread has
1882: * not already allocated the page-table page. If it
1883: * has, discard the new page-table page (and try
1884: * again to make sure).
1885: */
1886: PMAP_READ_LOCK(pmap, spl);
1887:
1888: if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
1889: /*
1890: * Oops...
1891: */
1892: PMAP_READ_UNLOCK(pmap, spl);
1.1.1.3 root 1893: pmap_page_table_page_dealloc(kvtophys(ptp));
1.1 root 1894: PMAP_READ_LOCK(pmap, spl);
1895: continue;
1896: }
1897:
1898: /*
1899: * Enter the new page table page in the page directory.
1900: */
1901: i = ptes_per_vm_page;
1902: /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/
1903: pdp = pmap_pde(pmap, v);
1904: do {
1.1.1.3 root 1905: #ifdef MACH_PV_PAGETABLES
1906: pmap_set_page_readonly((void *) ptp);
1907: if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp)))
1908: panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1909: if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)),
1910: pa_to_pte(pa_to_ma(kvtophys(ptp))) | INTEL_PTE_VALID
1911: | INTEL_PTE_USER
1912: | INTEL_PTE_WRITE))
1913: panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__,__LINE__, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp)));
1914: #else /* MACH_PV_PAGETABLES */
1915: *pdp = pa_to_pte(kvtophys(ptp)) | INTEL_PTE_VALID
1916: | INTEL_PTE_USER
1917: | INTEL_PTE_WRITE;
1918: #endif /* MACH_PV_PAGETABLES */
1.1 root 1919: pdp++;
1920: ptp += INTEL_PGBYTES;
1921: } while (--i > 0);
1922:
1923: /*
1924: * Now, get the address of the page-table entry.
1925: */
1926: continue;
1927: }
1928:
1929: /*
1930: * Special case if the physical page is already mapped
1931: * at this address.
1932: */
1933: old_pa = pte_to_pa(*pte);
1934: if (*pte && old_pa == pa) {
1935: /*
1936: * May be changing its wired attribute or protection
1937: */
1.1.1.2 root 1938:
1.1 root 1939: if (wired && !(*pte & INTEL_PTE_WIRED))
1940: pmap->stats.wired_count++;
1941: else if (!wired && (*pte & INTEL_PTE_WIRED))
1942: pmap->stats.wired_count--;
1943:
1944: template = pa_to_pte(pa) | INTEL_PTE_VALID;
1945: if (pmap != kernel_pmap)
1946: template |= INTEL_PTE_USER;
1947: if (prot & VM_PROT_WRITE)
1948: template |= INTEL_PTE_WRITE;
1.1.1.3 root 1949: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
1950: && pa >= phys_last_addr)
1951: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 1952: if (wired)
1953: template |= INTEL_PTE_WIRED;
1954: i = ptes_per_vm_page;
1955: do {
1956: if (*pte & INTEL_PTE_MOD)
1957: template |= INTEL_PTE_MOD;
1.1.1.3 root 1958: #ifdef MACH_PV_PAGETABLES
1959: if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))
1960: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
1961: #else /* MACH_PV_PAGETABLES */
1.1 root 1962: WRITE_PTE(pte, template)
1.1.1.3 root 1963: #endif /* MACH_PV_PAGETABLES */
1.1 root 1964: pte++;
1965: pte_increment_pa(template);
1966: } while (--i > 0);
1.1.1.3 root 1967: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1968: }
1969: else {
1970:
1971: /*
1972: * Remove old mapping from the PV list if necessary.
1973: */
1974: if (*pte) {
1975: /*
1976: * Don't free the pte page if removing last
1977: * mapping - we will immediately replace it.
1978: */
1979: pmap_remove_range(pmap, v, pte,
1980: pte + ptes_per_vm_page);
1.1.1.3 root 1981: PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
1.1 root 1982: }
1983:
1984: if (valid_page(pa)) {
1985:
1986: /*
1987: * Enter the mapping in the PV list for this
1988: * physical page.
1989: */
1990:
1991: pai = pa_index(pa);
1992: LOCK_PVH(pai);
1993: pv_h = pai_to_pvh(pai);
1994:
1995: if (pv_h->pmap == PMAP_NULL) {
1996: /*
1997: * No mappings yet
1998: */
1999: pv_h->va = v;
2000: pv_h->pmap = pmap;
2001: pv_h->next = PV_ENTRY_NULL;
2002: }
2003: else {
2004: #if DEBUG
2005: {
2006: /* check that this mapping is not already there */
2007: pv_entry_t e = pv_h;
2008: while (e != PV_ENTRY_NULL) {
2009: if (e->pmap == pmap && e->va == v)
2010: panic("pmap_enter: already in pv_list");
2011: e = e->next;
2012: }
2013: }
1.1.1.2 root 2014: #endif /* DEBUG */
2015:
1.1 root 2016: /*
2017: * Add new pv_entry after header.
2018: */
2019: if (pv_e == PV_ENTRY_NULL) {
2020: PV_ALLOC(pv_e);
2021: if (pv_e == PV_ENTRY_NULL) {
2022: UNLOCK_PVH(pai);
2023: PMAP_READ_UNLOCK(pmap, spl);
2024:
2025: /*
1.1.1.3 root 2026: * Refill from cache.
1.1 root 2027: */
1.1.1.3 root 2028: pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache);
1.1 root 2029: goto Retry;
2030: }
2031: }
2032: pv_e->va = v;
2033: pv_e->pmap = pmap;
2034: pv_e->next = pv_h->next;
2035: pv_h->next = pv_e;
2036: /*
2037: * Remember that we used the pvlist entry.
2038: */
2039: pv_e = PV_ENTRY_NULL;
2040: }
2041: UNLOCK_PVH(pai);
2042: }
2043:
2044: /*
2045: * And count the mapping.
2046: */
2047:
2048: pmap->stats.resident_count++;
2049: if (wired)
2050: pmap->stats.wired_count++;
2051:
2052: /*
2053: * Build a template to speed up entering -
2054: * only the pfn changes.
2055: */
2056: template = pa_to_pte(pa) | INTEL_PTE_VALID;
2057: if (pmap != kernel_pmap)
2058: template |= INTEL_PTE_USER;
2059: if (prot & VM_PROT_WRITE)
2060: template |= INTEL_PTE_WRITE;
1.1.1.3 root 2061: if (machine_slot[cpu_number()].cpu_type >= CPU_TYPE_I486
2062: && pa >= phys_last_addr)
2063: template |= INTEL_PTE_NCACHE|INTEL_PTE_WTHRU;
1.1 root 2064: if (wired)
2065: template |= INTEL_PTE_WIRED;
2066: i = ptes_per_vm_page;
2067: do {
1.1.1.3 root 2068: #ifdef MACH_PV_PAGETABLES
2069: if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))))
2070: panic("%s:%d could not set pte %p to %p\n",__FILE__,__LINE__,pte,template);
2071: #else /* MACH_PV_PAGETABLES */
1.1 root 2072: WRITE_PTE(pte, template)
1.1.1.3 root 2073: #endif /* MACH_PV_PAGETABLES */
1.1 root 2074: pte++;
2075: pte_increment_pa(template);
2076: } while (--i > 0);
2077: }
2078:
2079: if (pv_e != PV_ENTRY_NULL) {
2080: PV_FREE(pv_e);
2081: }
2082:
2083: PMAP_READ_UNLOCK(pmap, spl);
2084: }
2085:
2086: /*
2087: * Routine: pmap_change_wiring
2088: * Function: Change the wiring attribute for a map/virtual-address
2089: * pair.
2090: * In/out conditions:
2091: * The mapping must already exist in the pmap.
2092: */
1.1.1.4 root 2093: void pmap_change_wiring(
2094: pmap_t map,
2095: vm_offset_t v,
2096: boolean_t wired)
1.1 root 2097: {
1.1.1.4 root 2098: pt_entry_t *pte;
2099: int i;
2100: int spl;
1.1 root 2101:
2102: /*
2103: * We must grab the pmap system lock because we may
2104: * change a pte_page queue.
2105: */
2106: PMAP_READ_LOCK(map, spl);
2107:
2108: if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
2109: panic("pmap_change_wiring: pte missing");
2110:
2111: if (wired && !(*pte & INTEL_PTE_WIRED)) {
2112: /*
2113: * wiring down mapping
2114: */
2115: map->stats.wired_count++;
2116: i = ptes_per_vm_page;
2117: do {
2118: *pte++ |= INTEL_PTE_WIRED;
2119: } while (--i > 0);
2120: }
2121: else if (!wired && (*pte & INTEL_PTE_WIRED)) {
2122: /*
2123: * unwiring mapping
2124: */
2125: map->stats.wired_count--;
2126: i = ptes_per_vm_page;
2127: do {
1.1.1.3 root 2128: #ifdef MACH_PV_PAGETABLES
2129: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED)))
2130: panic("%s:%d could not wire down pte %p\n",__FILE__,__LINE__,pte);
2131: #else /* MACH_PV_PAGETABLES */
1.1 root 2132: *pte &= ~INTEL_PTE_WIRED;
1.1.1.3 root 2133: #endif /* MACH_PV_PAGETABLES */
2134: pte++;
1.1 root 2135: } while (--i > 0);
2136: }
2137:
2138: PMAP_READ_UNLOCK(map, spl);
2139: }
2140:
2141: /*
2142: * Routine: pmap_extract
2143: * Function:
2144: * Extract the physical page address associated
2145: * with the given map/virtual_address pair.
2146: */
2147:
1.1.1.4 root 2148: vm_offset_t pmap_extract(
2149: pmap_t pmap,
2150: vm_offset_t va)
1.1 root 2151: {
1.1.1.4 root 2152: pt_entry_t *pte;
2153: vm_offset_t pa;
2154: int spl;
1.1 root 2155:
2156: SPLVM(spl);
2157: simple_lock(&pmap->lock);
2158: if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL)
2159: pa = (vm_offset_t) 0;
2160: else if (!(*pte & INTEL_PTE_VALID))
2161: pa = (vm_offset_t) 0;
2162: else
2163: pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK);
2164: simple_unlock(&pmap->lock);
2165: SPLX(spl);
2166: return(pa);
2167: }
2168:
2169: /*
2170: * Copy the range specified by src_addr/len
2171: * from the source map to the range dst_addr/len
2172: * in the destination map.
2173: *
2174: * This routine is only advisory and need not do anything.
2175: */
2176: #if 0
2177: void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2178: pmap_t dst_pmap;
2179: pmap_t src_pmap;
2180: vm_offset_t dst_addr;
2181: vm_size_t len;
2182: vm_offset_t src_addr;
2183: {
2184: }
1.1.1.2 root 2185: #endif /* 0 */
1.1 root 2186:
2187: /*
2188: * Routine: pmap_collect
2189: * Function:
2190: * Garbage collects the physical map system for
2191: * pages which are no longer used.
2192: * Success need not be guaranteed -- that is, there
2193: * may well be pages which are not referenced, but
2194: * others may be collected.
2195: * Usage:
2196: * Called by the pageout daemon when pages are scarce.
2197: */
1.1.1.4 root 2198: void pmap_collect(pmap_t p)
1.1 root 2199: {
1.1.1.4 root 2200: pt_entry_t *pdp, *ptp;
1.1 root 2201: pt_entry_t *eptp;
2202: vm_offset_t pa;
2203: int spl, wired;
2204:
2205: if (p == PMAP_NULL)
2206: return;
2207:
2208: if (p == kernel_pmap)
2209: return;
2210:
2211: /*
2212: * Garbage collect map.
2213: */
2214: PMAP_READ_LOCK(p, spl);
2215: for (pdp = p->dirbase;
2216: pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)];
2217: pdp += ptes_per_vm_page)
2218: {
2219: if (*pdp & INTEL_PTE_VALID) {
2220:
2221: pa = pte_to_pa(*pdp);
2222: ptp = (pt_entry_t *)phystokv(pa);
2223: eptp = ptp + NPTES*ptes_per_vm_page;
2224:
2225: /*
2226: * If the pte page has any wired mappings, we cannot
2227: * free it.
2228: */
2229: wired = 0;
2230: {
1.1.1.4 root 2231: pt_entry_t *ptep;
1.1 root 2232: for (ptep = ptp; ptep < eptp; ptep++) {
2233: if (*ptep & INTEL_PTE_WIRED) {
2234: wired = 1;
2235: break;
2236: }
2237: }
2238: }
2239: if (!wired) {
2240: /*
2241: * Remove the virtual addresses mapped by this pte page.
2242: */
2243: { /*XXX big hack*/
2244: vm_offset_t va = pdenum2lin(pdp - p->dirbase);
2245: if (p == kernel_pmap)
2246: va = lintokv(va);
2247: pmap_remove_range(p,
2248: va,
2249: ptp,
2250: eptp);
2251: }
2252:
2253: /*
2254: * Invalidate the page directory pointer.
2255: */
2256: {
1.1.1.4 root 2257: int i = ptes_per_vm_page;
2258: pt_entry_t *pdep = pdp;
1.1 root 2259: do {
1.1.1.3 root 2260: #ifdef MACH_PV_PAGETABLES
2261: unsigned long pte = *pdep;
2262: void *ptable = (void*) ptetokv(pte);
2263: if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0)))
2264: panic("%s:%d could not clear pde %p\n",__FILE__,__LINE__,pdep-1);
2265: if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable)))
2266: panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable)));
2267: pmap_set_page_readwrite(ptable);
2268: #else /* MACH_PV_PAGETABLES */
1.1 root 2269: *pdep++ = 0;
1.1.1.3 root 2270: #endif /* MACH_PV_PAGETABLES */
1.1 root 2271: } while (--i > 0);
2272: }
2273:
2274: PMAP_READ_UNLOCK(p, spl);
2275:
2276: /*
2277: * And free the pte page itself.
2278: */
2279: {
1.1.1.4 root 2280: vm_page_t m;
1.1 root 2281:
2282: vm_object_lock(pmap_object);
2283: m = vm_page_lookup(pmap_object, pa);
2284: if (m == VM_PAGE_NULL)
2285: panic("pmap_collect: pte page not in object");
2286: vm_page_lock_queues();
2287: vm_page_free(m);
2288: inuse_ptepages_count--;
2289: vm_page_unlock_queues();
2290: vm_object_unlock(pmap_object);
2291: }
2292:
2293: PMAP_READ_LOCK(p, spl);
2294: }
2295: }
2296: }
1.1.1.3 root 2297: PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
2298:
1.1 root 2299: PMAP_READ_UNLOCK(p, spl);
2300: return;
2301:
2302: }
2303:
2304: /*
2305: * Routine: pmap_activate
2306: * Function:
2307: * Binds the given physical map to the given
2308: * processor, and returns a hardware map description.
2309: */
2310: #if 0
2311: void pmap_activate(my_pmap, th, my_cpu)
2312: register pmap_t my_pmap;
2313: thread_t th;
2314: int my_cpu;
2315: {
2316: PMAP_ACTIVATE(my_pmap, th, my_cpu);
2317: }
1.1.1.2 root 2318: #endif /* 0 */
1.1 root 2319:
2320: /*
2321: * Routine: pmap_deactivate
2322: * Function:
2323: * Indicates that the given physical map is no longer
2324: * in use on the specified processor. (This is a macro
2325: * in pmap.h)
2326: */
2327: #if 0
2328: void pmap_deactivate(pmap, th, which_cpu)
2329: pmap_t pmap;
2330: thread_t th;
2331: int which_cpu;
2332: {
2333: PMAP_DEACTIVATE(pmap, th, which_cpu);
2334: }
1.1.1.2 root 2335: #endif /* 0 */
1.1 root 2336:
2337: /*
2338: * Routine: pmap_kernel
2339: * Function:
2340: * Returns the physical map handle for the kernel.
2341: */
2342: #if 0
2343: pmap_t pmap_kernel()
2344: {
2345: return (kernel_pmap);
2346: }
1.1.1.2 root 2347: #endif /* 0 */
1.1 root 2348:
2349: /*
2350: * pmap_zero_page zeros the specified (machine independent) page.
2351: * See machine/phys.c or machine/phys.s for implementation.
2352: */
2353: #if 0
2354: pmap_zero_page(phys)
2355: register vm_offset_t phys;
2356: {
2357: register int i;
2358:
2359: assert(phys != vm_page_fictitious_addr);
2360: i = PAGE_SIZE / INTEL_PGBYTES;
2361: phys = intel_pfn(phys);
2362:
2363: while (i--)
2364: zero_phys(phys++);
2365: }
1.1.1.2 root 2366: #endif /* 0 */
1.1 root 2367:
2368: /*
2369: * pmap_copy_page copies the specified (machine independent) page.
2370: * See machine/phys.c or machine/phys.s for implementation.
2371: */
2372: #if 0
2373: pmap_copy_page(src, dst)
2374: vm_offset_t src, dst;
2375: {
2376: int i;
2377:
2378: assert(src != vm_page_fictitious_addr);
2379: assert(dst != vm_page_fictitious_addr);
2380: i = PAGE_SIZE / INTEL_PGBYTES;
2381:
2382: while (i--) {
2383: copy_phys(intel_pfn(src), intel_pfn(dst));
2384: src += INTEL_PGBYTES;
2385: dst += INTEL_PGBYTES;
2386: }
2387: }
1.1.1.2 root 2388: #endif /* 0 */
1.1 root 2389:
2390: /*
2391: * Routine: pmap_pageable
2392: * Function:
2393: * Make the specified pages (by pmap, offset)
2394: * pageable (or not) as requested.
2395: *
2396: * A page which is not pageable may not take
2397: * a fault; therefore, its page table entry
2398: * must remain valid for the duration.
2399: *
2400: * This routine is merely advisory; pmap_enter
2401: * will specify that these pages are to be wired
2402: * down (or not) as appropriate.
2403: */
1.1.1.3 root 2404: void
1.1.1.4 root 2405: pmap_pageable(
2406: pmap_t pmap,
2407: vm_offset_t start,
2408: vm_offset_t end,
2409: boolean_t pageable)
2410: {
1.1 root 2411: }
2412:
2413: /*
2414: * Clear specified attribute bits.
2415: */
2416: void
1.1.1.4 root 2417: phys_attribute_clear(
2418: vm_offset_t phys,
2419: int bits)
1.1 root 2420: {
2421: pv_entry_t pv_h;
1.1.1.4 root 2422: pv_entry_t pv_e;
2423: pt_entry_t *pte;
1.1 root 2424: int pai;
1.1.1.4 root 2425: pmap_t pmap;
1.1 root 2426: int spl;
2427:
2428: assert(phys != vm_page_fictitious_addr);
2429: if (!valid_page(phys)) {
2430: /*
2431: * Not a managed page.
2432: */
2433: return;
2434: }
2435:
2436: /*
2437: * Lock the pmap system first, since we will be changing
2438: * several pmaps.
2439: */
2440:
2441: PMAP_WRITE_LOCK(spl);
2442:
2443: pai = pa_index(phys);
2444: pv_h = pai_to_pvh(pai);
2445:
2446: /*
2447: * Walk down PV list, clearing all modify or reference bits.
2448: * We do not have to lock the pv_list because we have
2449: * the entire pmap system locked.
2450: */
2451: if (pv_h->pmap != PMAP_NULL) {
2452: /*
2453: * There are some mappings.
2454: */
2455: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
1.1.1.4 root 2456: vm_offset_t va;
1.1 root 2457:
2458: pmap = pv_e->pmap;
2459: /*
2460: * Lock the pmap to block pmap_extract and similar routines.
2461: */
2462: simple_lock(&pmap->lock);
2463:
1.1.1.3 root 2464: va = pv_e->va;
2465: pte = pmap_pte(pmap, va);
1.1 root 2466:
1.1.1.3 root 2467: /*
2468: * Consistency checks.
2469: */
2470: assert(*pte & INTEL_PTE_VALID);
1.1.1.5 root 2471: assert(pte_to_pa(*pte) == phys);
1.1 root 2472:
2473: /*
2474: * Clear modify or reference bits.
2475: */
2476: {
1.1.1.4 root 2477: int i = ptes_per_vm_page;
1.1 root 2478: do {
1.1.1.3 root 2479: #ifdef MACH_PV_PAGETABLES
2480: if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits)))
2481: panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__,__LINE__,bits,pte);
2482: #else /* MACH_PV_PAGETABLES */
1.1 root 2483: *pte &= ~bits;
1.1.1.3 root 2484: #endif /* MACH_PV_PAGETABLES */
1.1 root 2485: } while (--i > 0);
2486: }
1.1.1.3 root 2487: PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1.1 root 2488: simple_unlock(&pmap->lock);
2489: }
2490: }
2491:
2492: pmap_phys_attributes[pai] &= ~bits;
2493:
2494: PMAP_WRITE_UNLOCK(spl);
2495: }
2496:
2497: /*
2498: * Check specified attribute bits.
2499: */
2500: boolean_t
1.1.1.4 root 2501: phys_attribute_test(
2502: vm_offset_t phys,
2503: int bits)
1.1 root 2504: {
2505: pv_entry_t pv_h;
1.1.1.4 root 2506: pv_entry_t pv_e;
2507: pt_entry_t *pte;
1.1 root 2508: int pai;
1.1.1.4 root 2509: pmap_t pmap;
1.1 root 2510: int spl;
2511:
2512: assert(phys != vm_page_fictitious_addr);
2513: if (!valid_page(phys)) {
2514: /*
2515: * Not a managed page.
2516: */
2517: return (FALSE);
2518: }
2519:
2520: /*
2521: * Lock the pmap system first, since we will be checking
2522: * several pmaps.
2523: */
2524:
2525: PMAP_WRITE_LOCK(spl);
2526:
2527: pai = pa_index(phys);
2528: pv_h = pai_to_pvh(pai);
2529:
2530: if (pmap_phys_attributes[pai] & bits) {
2531: PMAP_WRITE_UNLOCK(spl);
2532: return (TRUE);
2533: }
2534:
2535: /*
2536: * Walk down PV list, checking all mappings.
2537: * We do not have to lock the pv_list because we have
2538: * the entire pmap system locked.
2539: */
2540: if (pv_h->pmap != PMAP_NULL) {
2541: /*
2542: * There are some mappings.
2543: */
2544: for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
2545:
2546: pmap = pv_e->pmap;
2547: /*
2548: * Lock the pmap to block pmap_extract and similar routines.
2549: */
2550: simple_lock(&pmap->lock);
2551:
2552: {
1.1.1.4 root 2553: vm_offset_t va;
1.1 root 2554:
2555: va = pv_e->va;
2556: pte = pmap_pte(pmap, va);
2557:
2558: /*
2559: * Consistency checks.
2560: */
2561: assert(*pte & INTEL_PTE_VALID);
1.1.1.5 root 2562: assert(pte_to_pa(*pte) == phys);
1.1 root 2563: }
2564:
2565: /*
2566: * Check modify or reference bits.
2567: */
2568: {
1.1.1.4 root 2569: int i = ptes_per_vm_page;
1.1 root 2570:
2571: do {
2572: if (*pte & bits) {
2573: simple_unlock(&pmap->lock);
2574: PMAP_WRITE_UNLOCK(spl);
2575: return (TRUE);
2576: }
2577: } while (--i > 0);
2578: }
2579: simple_unlock(&pmap->lock);
2580: }
2581: }
2582: PMAP_WRITE_UNLOCK(spl);
2583: return (FALSE);
2584: }
2585:
2586: /*
2587: * Clear the modify bits on the specified physical page.
2588: */
2589:
1.1.1.4 root 2590: void pmap_clear_modify(vm_offset_t phys)
1.1 root 2591: {
2592: phys_attribute_clear(phys, PHYS_MODIFIED);
2593: }
2594:
2595: /*
2596: * pmap_is_modified:
2597: *
2598: * Return whether or not the specified physical page is modified
2599: * by any physical maps.
2600: */
2601:
1.1.1.4 root 2602: boolean_t pmap_is_modified(vm_offset_t phys)
1.1 root 2603: {
2604: return (phys_attribute_test(phys, PHYS_MODIFIED));
2605: }
2606:
2607: /*
2608: * pmap_clear_reference:
2609: *
2610: * Clear the reference bit on the specified physical page.
2611: */
2612:
1.1.1.4 root 2613: void pmap_clear_reference(vm_offset_t phys)
1.1 root 2614: {
2615: phys_attribute_clear(phys, PHYS_REFERENCED);
2616: }
2617:
2618: /*
2619: * pmap_is_referenced:
2620: *
2621: * Return whether or not the specified physical page is referenced
2622: * by any physical maps.
2623: */
2624:
1.1.1.4 root 2625: boolean_t pmap_is_referenced(vm_offset_t phys)
1.1 root 2626: {
2627: return (phys_attribute_test(phys, PHYS_REFERENCED));
2628: }
2629:
2630: #if NCPUS > 1
2631: /*
2632: * TLB Coherence Code (TLB "shootdown" code)
1.1.1.2 root 2633: *
1.1 root 2634: * Threads that belong to the same task share the same address space and
2635: * hence share a pmap. However, they may run on distinct cpus and thus
2636: * have distinct TLBs that cache page table entries. In order to guarantee
2637: * the TLBs are consistent, whenever a pmap is changed, all threads that
2638: * are active in that pmap must have their TLB updated. To keep track of
2639: * this information, the set of cpus that are currently using a pmap is
2640: * maintained within each pmap structure (cpus_using). Pmap_activate() and
2641: * pmap_deactivate add and remove, respectively, a cpu from this set.
2642: * Since the TLBs are not addressable over the bus, each processor must
2643: * flush its own TLB; a processor that needs to invalidate another TLB
2644: * needs to interrupt the processor that owns that TLB to signal the
2645: * update.
1.1.1.2 root 2646: *
1.1 root 2647: * Whenever a pmap is updated, the lock on that pmap is locked, and all
2648: * cpus using the pmap are signaled to invalidate. All threads that need
2649: * to activate a pmap must wait for the lock to clear to await any updates
2650: * in progress before using the pmap. They must ACQUIRE the lock to add
2651: * their cpu to the cpus_using set. An implicit assumption made
2652: * throughout the TLB code is that all kernel code that runs at or higher
2653: * than splvm blocks out update interrupts, and that such code does not
2654: * touch pageable pages.
1.1.1.2 root 2655: *
1.1 root 2656: * A shootdown interrupt serves another function besides signaling a
2657: * processor to invalidate. The interrupt routine (pmap_update_interrupt)
2658: * waits for the both the pmap lock (and the kernel pmap lock) to clear,
2659: * preventing user code from making implicit pmap updates while the
2660: * sending processor is performing its update. (This could happen via a
2661: * user data write reference that turns on the modify bit in the page
2662: * table). It must wait for any kernel updates that may have started
2663: * concurrently with a user pmap update because the IPC code
2664: * changes mappings.
2665: * Spinning on the VALUES of the locks is sufficient (rather than
2666: * having to acquire the locks) because any updates that occur subsequent
2667: * to finding the lock unlocked will be signaled via another interrupt.
1.1.1.2 root 2668: * (This assumes the interrupt is cleared before the low level interrupt code
2669: * calls pmap_update_interrupt()).
2670: *
1.1 root 2671: * The signaling processor must wait for any implicit updates in progress
2672: * to terminate before continuing with its update. Thus it must wait for an
2673: * acknowledgement of the interrupt from each processor for which such
2674: * references could be made. For maintaining this information, a set
1.1.1.2 root 2675: * cpus_active is used. A cpu is in this set if and only if it can
1.1 root 2676: * use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
2677: * this set; when all such cpus are removed, it is safe to update.
1.1.1.2 root 2678: *
1.1 root 2679: * Before attempting to acquire the update lock on a pmap, a cpu (A) must
2680: * be at least at the priority of the interprocessor interrupt
2681: * (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
2682: * kernel update; it would spin forever in pmap_update_interrupt() trying
2683: * to acquire the user pmap lock it had already acquired. Furthermore A
2684: * must remove itself from cpus_active. Otherwise, another cpu holding
2685: * the lock (B) could be in the process of sending an update signal to A,
2686: * and thus be waiting for A to remove itself from cpus_active. If A is
2687: * spinning on the lock at priority this will never happen and a deadlock
2688: * will result.
2689: */
2690:
2691: /*
2692: * Signal another CPU that it must flush its TLB
2693: */
1.1.1.4 root 2694: void signal_cpus(
2695: cpu_set use_list,
2696: pmap_t pmap,
2697: vm_offset_t start,
2698: vm_offset_t end)
1.1 root 2699: {
1.1.1.4 root 2700: int which_cpu, j;
2701: pmap_update_list_t update_list_p;
1.1 root 2702:
2703: while ((which_cpu = ffs(use_list)) != 0) {
2704: which_cpu -= 1; /* convert to 0 origin */
2705:
2706: update_list_p = &cpu_update_list[which_cpu];
2707: simple_lock(&update_list_p->lock);
2708:
2709: j = update_list_p->count;
2710: if (j >= UPDATE_LIST_SIZE) {
2711: /*
2712: * list overflowed. Change last item to
2713: * indicate overflow.
2714: */
2715: update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
2716: update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
2717: update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS;
2718: }
2719: else {
2720: update_list_p->item[j].pmap = pmap;
2721: update_list_p->item[j].start = start;
2722: update_list_p->item[j].end = end;
2723: update_list_p->count = j+1;
2724: }
2725: cpu_update_needed[which_cpu] = TRUE;
2726: simple_unlock(&update_list_p->lock);
2727:
2728: if ((cpus_idle & (1 << which_cpu)) == 0)
2729: interrupt_processor(which_cpu);
2730: use_list &= ~(1 << which_cpu);
2731: }
2732: }
2733:
1.1.1.4 root 2734: void process_pmap_updates(pmap_t my_pmap)
1.1 root 2735: {
1.1.1.4 root 2736: int my_cpu = cpu_number();
2737: pmap_update_list_t update_list_p;
2738: int j;
2739: pmap_t pmap;
1.1 root 2740:
2741: update_list_p = &cpu_update_list[my_cpu];
2742: simple_lock(&update_list_p->lock);
2743:
2744: for (j = 0; j < update_list_p->count; j++) {
2745: pmap = update_list_p->item[j].pmap;
2746: if (pmap == my_pmap ||
2747: pmap == kernel_pmap) {
2748:
1.1.1.3 root 2749: INVALIDATE_TLB(pmap,
2750: update_list_p->item[j].start,
1.1 root 2751: update_list_p->item[j].end);
2752: }
2753: }
2754: update_list_p->count = 0;
2755: cpu_update_needed[my_cpu] = FALSE;
2756: simple_unlock(&update_list_p->lock);
2757: }
2758:
2759: /*
2760: * Interrupt routine for TBIA requested from other processor.
2761: */
1.1.1.3 root 2762: void pmap_update_interrupt(void)
1.1 root 2763: {
1.1.1.4 root 2764: int my_cpu;
2765: pmap_t my_pmap;
2766: int s;
1.1 root 2767:
2768: my_cpu = cpu_number();
2769:
2770: /*
2771: * Exit now if we're idle. We'll pick up the update request
2772: * when we go active, and we must not put ourselves back in
2773: * the active set because we'll never process the interrupt
2774: * while we're idle (thus hanging the system).
2775: */
2776: if (cpus_idle & (1 << my_cpu))
2777: return;
2778:
2779: if (current_thread() == THREAD_NULL)
2780: my_pmap = kernel_pmap;
2781: else {
2782: my_pmap = current_pmap();
2783: if (!pmap_in_use(my_pmap, my_cpu))
2784: my_pmap = kernel_pmap;
2785: }
2786:
2787: /*
2788: * Raise spl to splvm (above splip) to block out pmap_extract
2789: * from IO code (which would put this cpu back in the active
2790: * set).
2791: */
2792: s = splvm();
2793:
2794: do {
2795:
2796: /*
2797: * Indicate that we're not using either user or kernel
2798: * pmap.
2799: */
2800: i_bit_clear(my_cpu, &cpus_active);
2801:
2802: /*
2803: * Wait for any pmap updates in progress, on either user
2804: * or kernel pmap.
2805: */
2806: while (*(volatile int *)&my_pmap->lock.lock_data ||
2807: *(volatile int *)&kernel_pmap->lock.lock_data)
2808: continue;
2809:
2810: process_pmap_updates(my_pmap);
2811:
2812: i_bit_set(my_cpu, &cpus_active);
2813:
2814: } while (cpu_update_needed[my_cpu]);
1.1.1.2 root 2815:
1.1 root 2816: splx(s);
2817: }
1.1.1.2 root 2818: #else /* NCPUS > 1 */
1.1 root 2819: /*
2820: * Dummy routine to satisfy external reference.
2821: */
1.1.1.4 root 2822: void pmap_update_interrupt(void)
1.1 root 2823: {
2824: /* should never be called. */
2825: }
1.1.1.2 root 2826: #endif /* NCPUS > 1 */
1.1 root 2827:
1.1.1.3 root 2828: #if defined(__i386__)
1.1 root 2829: /* Unmap page 0 to trap NULL references. */
2830: void
1.1.1.4 root 2831: pmap_unmap_page_zero (void)
1.1 root 2832: {
2833: int *pte;
2834:
1.1.1.4 root 2835: printf("Unmapping the zero page. Some BIOS functions may not be working any more.\n");
1.1 root 2836: pte = (int *) pmap_pte (kernel_pmap, 0);
1.1.1.3 root 2837: if (!pte)
2838: return;
1.1 root 2839: assert (pte);
1.1.1.3 root 2840: #ifdef MACH_PV_PAGETABLES
2841: if (!hyp_mmu_update_pte(kv_to_ma(pte), 0))
2842: printf("couldn't unmap page 0\n");
2843: #else /* MACH_PV_PAGETABLES */
1.1 root 2844: *pte = 0;
1.1.1.3 root 2845: INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE);
2846: #endif /* MACH_PV_PAGETABLES */
1.1 root 2847: }
1.1.1.3 root 2848: #endif /* __i386__ */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.