Annotation of qemu/xen-mapcache.c, revision 1.1.1.3
1.1 root 1: /*
2: * Copyright (C) 2011 Citrix Ltd.
3: *
4: * This work is licensed under the terms of the GNU GPL, version 2. See
5: * the COPYING file in the top-level directory.
6: *
1.1.1.3 ! root 7: * Contributions after 2012-01-13 are licensed under the terms of the
! 8: * GNU GPL, version 2 or (at your option) any later version.
1.1 root 9: */
10:
11: #include "config.h"
12:
13: #include <sys/resource.h>
14:
15: #include "hw/xen_backend.h"
16: #include "blockdev.h"
17: #include "bitmap.h"
18:
19: #include <xen/hvm/params.h>
20: #include <sys/mman.h>
21:
22: #include "xen-mapcache.h"
23: #include "trace.h"
24:
25:
26: //#define MAPCACHE_DEBUG
27:
28: #ifdef MAPCACHE_DEBUG
29: # define DPRINTF(fmt, ...) do { \
30: fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
31: } while (0)
32: #else
33: # define DPRINTF(fmt, ...) do { } while (0)
34: #endif
35:
36: #if defined(__i386__)
37: # define MCACHE_BUCKET_SHIFT 16
38: # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
39: #elif defined(__x86_64__)
40: # define MCACHE_BUCKET_SHIFT 20
41: # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
42: #endif
43: #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
44:
1.1.1.2 root 45: /* This is the size of the virtual address space reserve to QEMU that will not
46: * be use by MapCache.
47: * From empirical tests I observed that qemu use 75MB more than the
48: * max_mcache_size.
49: */
50: #define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
51:
1.1 root 52: #define mapcache_lock() ((void)0)
53: #define mapcache_unlock() ((void)0)
54:
55: typedef struct MapCacheEntry {
56: target_phys_addr_t paddr_index;
57: uint8_t *vaddr_base;
58: unsigned long *valid_mapping;
59: uint8_t lock;
60: target_phys_addr_t size;
61: struct MapCacheEntry *next;
62: } MapCacheEntry;
63:
64: typedef struct MapCacheRev {
65: uint8_t *vaddr_req;
66: target_phys_addr_t paddr_index;
67: target_phys_addr_t size;
68: QTAILQ_ENTRY(MapCacheRev) next;
69: } MapCacheRev;
70:
71: typedef struct MapCache {
72: MapCacheEntry *entry;
73: unsigned long nr_buckets;
74: QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
75:
76: /* For most cases (>99.9%), the page address is the same. */
77: target_phys_addr_t last_address_index;
78: uint8_t *last_address_vaddr;
79: unsigned long max_mcache_size;
80: unsigned int mcache_bucket_shift;
1.1.1.3 ! root 81:
! 82: phys_offset_to_gaddr_t phys_offset_to_gaddr;
! 83: void *opaque;
1.1 root 84: } MapCache;
85:
86: static MapCache *mapcache;
87:
88: static inline int test_bits(int nr, int size, const unsigned long *addr)
89: {
90: unsigned long res = find_next_zero_bit(addr, size + nr, nr);
91: if (res >= nr + size)
92: return 1;
93: else
94: return 0;
95: }
96:
1.1.1.3 ! root 97: void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque)
1.1 root 98: {
99: unsigned long size;
100: struct rlimit rlimit_as;
101:
1.1.1.2 root 102: mapcache = g_malloc0(sizeof (MapCache));
1.1 root 103:
1.1.1.3 ! root 104: mapcache->phys_offset_to_gaddr = f;
! 105: mapcache->opaque = opaque;
! 106:
1.1 root 107: QTAILQ_INIT(&mapcache->locked_entries);
108: mapcache->last_address_index = -1;
109:
1.1.1.2 root 110: if (geteuid() == 0) {
111: rlimit_as.rlim_cur = RLIM_INFINITY;
112: rlimit_as.rlim_max = RLIM_INFINITY;
113: mapcache->max_mcache_size = MCACHE_MAX_SIZE;
1.1 root 114: } else {
1.1.1.2 root 115: getrlimit(RLIMIT_AS, &rlimit_as);
116: rlimit_as.rlim_cur = rlimit_as.rlim_max;
117:
118: if (rlimit_as.rlim_max != RLIM_INFINITY) {
119: fprintf(stderr, "Warning: QEMU's maximum size of virtual"
120: " memory is not infinity.\n");
121: }
122: if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
123: mapcache->max_mcache_size = rlimit_as.rlim_max -
124: NON_MCACHE_MEMORY_SIZE;
125: } else {
126: mapcache->max_mcache_size = MCACHE_MAX_SIZE;
127: }
1.1 root 128: }
129:
130: setrlimit(RLIMIT_AS, &rlimit_as);
131:
132: mapcache->nr_buckets =
133: (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
134: (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
135: (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
136:
137: size = mapcache->nr_buckets * sizeof (MapCacheEntry);
138: size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
139: DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
140: mapcache->nr_buckets, size);
1.1.1.2 root 141: mapcache->entry = g_malloc0(size);
1.1 root 142: }
143:
144: static void xen_remap_bucket(MapCacheEntry *entry,
145: target_phys_addr_t size,
146: target_phys_addr_t address_index)
147: {
148: uint8_t *vaddr_base;
149: xen_pfn_t *pfns;
150: int *err;
151: unsigned int i;
152: target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
153:
154: trace_xen_remap_bucket(address_index);
155:
1.1.1.2 root 156: pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
157: err = g_malloc0(nb_pfn * sizeof (int));
1.1 root 158:
159: if (entry->vaddr_base != NULL) {
160: if (munmap(entry->vaddr_base, entry->size) != 0) {
161: perror("unmap fails");
162: exit(-1);
163: }
164: }
165: if (entry->valid_mapping != NULL) {
1.1.1.2 root 166: g_free(entry->valid_mapping);
1.1 root 167: entry->valid_mapping = NULL;
168: }
169:
170: for (i = 0; i < nb_pfn; i++) {
171: pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
172: }
173:
174: vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
175: pfns, err, nb_pfn);
176: if (vaddr_base == NULL) {
177: perror("xc_map_foreign_bulk");
178: exit(-1);
179: }
180:
181: entry->vaddr_base = vaddr_base;
182: entry->paddr_index = address_index;
183: entry->size = size;
1.1.1.2 root 184: entry->valid_mapping = (unsigned long *) g_malloc0(sizeof(unsigned long) *
1.1 root 185: BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
186:
187: bitmap_zero(entry->valid_mapping, nb_pfn);
188: for (i = 0; i < nb_pfn; i++) {
189: if (!err[i]) {
190: bitmap_set(entry->valid_mapping, i, 1);
191: }
192: }
193:
1.1.1.2 root 194: g_free(pfns);
195: g_free(err);
1.1 root 196: }
197:
198: uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
199: uint8_t lock)
200: {
201: MapCacheEntry *entry, *pentry = NULL;
1.1.1.3 ! root 202: target_phys_addr_t address_index;
! 203: target_phys_addr_t address_offset;
1.1 root 204: target_phys_addr_t __size = size;
1.1.1.3 ! root 205: bool translated = false;
! 206:
! 207: tryagain:
! 208: address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
! 209: address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
1.1 root 210:
211: trace_xen_map_cache(phys_addr);
212:
213: if (address_index == mapcache->last_address_index && !lock && !__size) {
214: trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
215: return mapcache->last_address_vaddr + address_offset;
216: }
217:
218: /* size is always a multiple of MCACHE_BUCKET_SIZE */
1.1.1.3 ! root 219: if (size) {
! 220: __size = size + address_offset;
! 221: if (__size % MCACHE_BUCKET_SIZE) {
! 222: __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
! 223: }
! 224: } else {
1.1 root 225: __size = MCACHE_BUCKET_SIZE;
1.1.1.3 ! root 226: }
1.1 root 227:
228: entry = &mapcache->entry[address_index % mapcache->nr_buckets];
229:
230: while (entry && entry->lock && entry->vaddr_base &&
231: (entry->paddr_index != address_index || entry->size != __size ||
232: !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
233: entry->valid_mapping))) {
234: pentry = entry;
235: entry = entry->next;
236: }
237: if (!entry) {
1.1.1.2 root 238: entry = g_malloc0(sizeof (MapCacheEntry));
1.1 root 239: pentry->next = entry;
240: xen_remap_bucket(entry, __size, address_index);
241: } else if (!entry->lock) {
242: if (!entry->vaddr_base || entry->paddr_index != address_index ||
243: entry->size != __size ||
244: !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
245: entry->valid_mapping)) {
246: xen_remap_bucket(entry, __size, address_index);
247: }
248: }
249:
250: if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
251: entry->valid_mapping)) {
252: mapcache->last_address_index = -1;
1.1.1.3 ! root 253: if (!translated && mapcache->phys_offset_to_gaddr) {
! 254: phys_addr = mapcache->phys_offset_to_gaddr(phys_addr, size, mapcache->opaque);
! 255: translated = true;
! 256: goto tryagain;
! 257: }
1.1 root 258: trace_xen_map_cache_return(NULL);
259: return NULL;
260: }
261:
262: mapcache->last_address_index = address_index;
263: mapcache->last_address_vaddr = entry->vaddr_base;
264: if (lock) {
1.1.1.2 root 265: MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
1.1 root 266: entry->lock++;
267: reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
268: reventry->paddr_index = mapcache->last_address_index;
269: reventry->size = entry->size;
270: QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
271: }
272:
273: trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
274: return mapcache->last_address_vaddr + address_offset;
275: }
276:
277: ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
278: {
1.1.1.2 root 279: MapCacheEntry *entry = NULL;
1.1 root 280: MapCacheRev *reventry;
281: target_phys_addr_t paddr_index;
282: target_phys_addr_t size;
283: int found = 0;
284:
285: QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
286: if (reventry->vaddr_req == ptr) {
287: paddr_index = reventry->paddr_index;
288: size = reventry->size;
289: found = 1;
290: break;
291: }
292: }
293: if (!found) {
294: fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
295: QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
296: DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
297: reventry->vaddr_req);
298: }
299: abort();
300: return 0;
301: }
302:
303: entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
304: while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
305: entry = entry->next;
306: }
307: if (!entry) {
308: DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
309: return 0;
310: }
311: return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
312: ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
313: }
314:
315: void xen_invalidate_map_cache_entry(uint8_t *buffer)
316: {
317: MapCacheEntry *entry = NULL, *pentry = NULL;
318: MapCacheRev *reventry;
319: target_phys_addr_t paddr_index;
320: target_phys_addr_t size;
321: int found = 0;
322:
323: if (mapcache->last_address_vaddr == buffer) {
324: mapcache->last_address_index = -1;
325: }
326:
327: QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
328: if (reventry->vaddr_req == buffer) {
329: paddr_index = reventry->paddr_index;
330: size = reventry->size;
331: found = 1;
332: break;
333: }
334: }
335: if (!found) {
336: DPRINTF("%s, could not find %p\n", __func__, buffer);
337: QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
338: DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
339: }
340: return;
341: }
342: QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
1.1.1.2 root 343: g_free(reventry);
1.1 root 344:
345: entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
346: while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
347: pentry = entry;
348: entry = entry->next;
349: }
350: if (!entry) {
351: DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
352: return;
353: }
354: entry->lock--;
355: if (entry->lock > 0 || pentry == NULL) {
356: return;
357: }
358:
359: pentry->next = entry->next;
360: if (munmap(entry->vaddr_base, entry->size) != 0) {
361: perror("unmap fails");
362: exit(-1);
363: }
1.1.1.2 root 364: g_free(entry->valid_mapping);
365: g_free(entry);
1.1 root 366: }
367:
368: void xen_invalidate_map_cache(void)
369: {
370: unsigned long i;
371: MapCacheRev *reventry;
372:
373: /* Flush pending AIO before destroying the mapcache */
1.1.1.3 ! root 374: bdrv_drain_all();
1.1 root 375:
376: QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
377: DPRINTF("There should be no locked mappings at this time, "
378: "but "TARGET_FMT_plx" -> %p is present\n",
379: reventry->paddr_index, reventry->vaddr_req);
380: }
381:
382: mapcache_lock();
383:
384: for (i = 0; i < mapcache->nr_buckets; i++) {
385: MapCacheEntry *entry = &mapcache->entry[i];
386:
387: if (entry->vaddr_base == NULL) {
388: continue;
389: }
1.1.1.3 ! root 390: if (entry->lock > 0) {
! 391: continue;
! 392: }
1.1 root 393:
394: if (munmap(entry->vaddr_base, entry->size) != 0) {
395: perror("unmap fails");
396: exit(-1);
397: }
398:
399: entry->paddr_index = 0;
400: entry->vaddr_base = NULL;
401: entry->size = 0;
1.1.1.2 root 402: g_free(entry->valid_mapping);
1.1 root 403: entry->valid_mapping = NULL;
404: }
405:
406: mapcache->last_address_index = -1;
407: mapcache->last_address_vaddr = NULL;
408:
409: mapcache_unlock();
410: }
unix.superglobalmegacorp.com
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to xen-mapcache.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu