![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to zalloc.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple Darwin 0.x] / kernel / kern|
Return to zalloc.c CVS log | Up to [Apple Darwin 0.x] / kernel / kern |
1.1 root 1: /*
2: * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * Portions Copyright (c) 1999 Apple Computer, Inc. All Rights
7: * Reserved. This file contains Original Code and/or Modifications of
8: * Original Code as defined in and that are subject to the Apple Public
9: * Source License Version 1.1 (the "License"). You may not use this file
10: * except in compliance with the License. Please obtain a copy of the
11: * License at http://www.apple.com/publicsource and read it before using
12: * this file.
13: *
14: * The Original Code and all software distributed under the License are
15: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
19: * License for the specific language governing rights and limitations
20: * under the License.
21: *
22: * @APPLE_LICENSE_HEADER_END@
23: */
24:
25: /*
26: * Mach Operating System
27: * Copyright (c) 1993,1991,1990,1989,1988,1987 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: * Copyright (c) 1995, 1997 Apple Computer, Inc.
52: *
53: * HISTORY
54: *
55: * 23 June 1995 ? at NeXT
56: * Pulled over from CMU (MK83), added local
57: * mods: freespace suballocator and sorted
58: * zone free lists to reduce fragmentation.
59: */
60: /*
61: * File: kern/zalloc.c
62: * Author: Avadis Tevanian, Jr.
63: *
64: * Zone-based memory allocator. A zone is a collection of fixed size
65: * data blocks for which quick allocation/deallocation is possible.
66: */
67:
68: #import <mach/features.h>
69:
70: #include <kern/macro_help.h>
71: #include <kern/sched.h>
72: #include <kern/time_out.h>
73: #include <kern/zalloc.h>
74: #include <mach/vm_param.h>
75: #include <vm/vm_kern.h>
76:
77: #if MACH_DEBUG
78: #include <mach/kern_return.h>
79: #include <mach/machine/vm_types.h>
80: #include <mach_debug/zone_info.h>
81: #include <kern/host.h>
82: #include <vm/vm_map.h>
83: #include <vm/vm_user.h>
84: #include <vm/vm_kern.h>
85: #endif
86:
1.1.1.2 ! root 87: #if DIAGNOSTIC
! 88: vm_offset_t zlowest = (vm_offset_t)(-1);
! 89: vm_offset_t zhighest = (vm_offset_t)0;
! 90: #define WATERMARK_ZONE(zone, element) \
! 91: MACRO_BEGIN \
! 92: if ((element) < (zone)->lowest) \
! 93: (zone)->lowest = (element); \
! 94: if ((element) < zlowest) \
! 95: zlowest = (element); \
! 96: if ((element) > (zone)->highest) \
! 97: (zone)->highest = (element); \
! 98: if ((element) > zhighest) \
! 99: zhighest = (element); \
! 100: MACRO_END
! 101: #else
! 102: #define WATERMARK_ZONE(zone, element)
! 103: #endif
! 104:
! 105: /*
! 106: * 1 - fill freed memory with 0xdeadface
! 107: * 2 - check_zone at zfree
! 108: * 4 - check_zone at zget
! 109: * 8 - check_zone at zalloc
! 110: * 10 - duplicate the free list pointer chain
! 111: */
! 112: unsigned zone_check = 0;
! 113:
! 114: void *memset __P((void *, int , size_t));
! 115:
1.1 root 116: #define ADD_TO_ZONE(zone, element) \
117: MACRO_BEGIN \
118: vm_offset_t cur, *last; \
1.1.1.2 ! root 119: if (zone_check & 1) \
! 120: (void) memset((void *)(element), 0xdeadface, \
! 121: (size_t) (zone)->elem_size); \
! 122: WATERMARK_ZONE((zone), (element)); \
1.1 root 123: if ( (zone)->last_insert != 0 && \
124: (element) > (zone)->last_insert ) \
125: (vm_offset_t)last = (zone)->last_insert; \
126: else \
127: last = &(zone)->free_elements; \
128: while ( (cur = *last) != 0 && \
129: (element) > cur ) \
130: (vm_offset_t)last = cur; \
131: *(vm_offset_t *)(element) = cur; \
1.1.1.2 ! root 132: if (zone_check & 0x10) \
! 133: *((vm_offset_t *)(element) + 1) = cur; \
1.1 root 134: *last = (element); \
135: (zone)->last_insert = (element); \
136: (zone)->count--; \
137: MACRO_END
138:
139: #define REMOVE_FROM_ZONE(zone, ret, type) \
140: MACRO_BEGIN \
141: (ret) = (type) (zone)->free_elements; \
142: if ((ret) != (type) 0) { \
143: (zone)->count++; \
144: (zone)->free_elements = *(vm_offset_t *)(ret); \
145: if ((zone)->last_insert == (vm_offset_t)(ret)) \
146: (zone)->last_insert = 0; \
147: } \
148: MACRO_END
149:
150: zone_t zone_zone; /* this is the zone containing other zones */
151:
152: boolean_t zone_ignore_overflow = TRUE;
153:
154: vm_map_t zone_map = VM_MAP_NULL;
155: vm_size_t zone_map_size;
156: vm_size_t zone_map_size_min = 12 * 1024 * 1024;
157: vm_size_t zone_map_size_max = 128 * 1024 * 1024;
158: vm_offset_t zone_min, zone_max;
159:
160: /*
161: * The VM system gives us an initial chunk of memory.
162: * It has to be big enough to allocate the zone_zone
163: * and some initial kernel data structures, like kernel maps.
164: * It is advantageous to make it bigger than really necessary,
165: * because this memory is more efficient than normal kernel
166: * virtual memory. (It doesn't have vm_page structures backing it
167: * and it may have other machine-dependent advantages.)
168: * So for best performance, zdata_size should approximate
169: * the amount of memory you expect the zone system to consume.
170: */
171:
172: vm_offset_t zdata;
173: vm_size_t zdata_size = 512 * 1024;
174:
175: #define lock_zone(zone) \
176: MACRO_BEGIN \
177: if ((zone)->pageable) { \
178: lock_write(&(zone)->complex_lock); \
179: } else { \
180: spl_t s = splhigh(); \
181: simple_lock(&(zone)->lock); \
182: (zone)->lock_ipl = s; \
183: } \
184: MACRO_END
185:
186: #define unlock_zone(zone) \
187: MACRO_BEGIN \
188: if ((zone)->pageable) { \
189: lock_done(&(zone)->complex_lock); \
190: } else { \
191: spl_t s = (zone)->lock_ipl; \
192: simple_unlock(&(zone)->lock); \
193: splx(s); \
194: } \
195: MACRO_END
196:
197: #define lock_zone_init(zone) \
198: MACRO_BEGIN \
199: if ((zone)->pageable) { \
200: lock_init(&(zone)->complex_lock, TRUE); \
201: } else { \
202: simple_lock_init(&(zone)->lock); \
203: } \
204: MACRO_END
205:
206: vm_offset_t zget_space(
207: struct zone_free_space *free_space,
208: vm_size_t size,
209: boolean_t canblock);
210:
211: decl_simple_lock_data(,zget_space_lock)
212:
213: /*
214: * A free list entry, which is created
215: * at the front of an available region
216: * of memory. The 'pred' field points
217: * at the 'next' pointer of the previous
218: * entry (or the head pointer).
219: */
220: struct zone_free_space_entry {
221: struct zone_free_space_entry *next;
222: vm_size_t length;
223: struct zone_free_space_entry **pred;
224: int _pad[1];
225: };
226: #define ZONE_MIN_ALLOC 16 /*
227: * *Must* be:
228: * a power of two
229: * and
230: * at least sizeof (struct
231: * zone_free_space_entry)
232: */
233:
234: /*
235: * An entry in the free list hint table,
236: * which allows for quickly locating a
237: * suitably sized region needed to satisfy
238: * a request.
239: */
240: struct zone_free_space_hint {
241: struct zone_free_space_entry *entry;
242: int _pad[3];
243: };
244:
245: /*
246: * Structure used to manage memory which
247: * has been obtained from the system, but
248: * which is not currently owned by any zone.
249: * The main two parameters are 'alloc_unit'
250: * and 'alloc_max'. The former specifies the
251: * allocation granularity. Allocation requests
252: * will be rounded up to this size, which must
253: * be a power of two. The size of the largest
254: * (suggested) allocation request is specified
255: * by 'alloc_max'. The free list is headed by
256: * 'entries', and is a doubly linked list of
257: * free regions, sorted by ascending address.
258: * There also is a table of hints, indexed by
259: * region size, which is used to speed up the
260: * allocations when the free list becomes
261: * especially large. It contains one entry
262: * per size value between alloc_unit and alloc_max.
263: */
264: struct zone_free_space {
265: vm_size_t alloc_unit;
266: vm_size_t alloc_max;
267: struct zone_free_space_entry *entries;
268: integer_t num_entries;
269: integer_t hash_shift;
270: struct zone_free_space_hint *hints;
271: integer_t num_hints;
272: };
273:
274: struct zone_free_space *zone_free_space[8];
275: integer_t zone_free_space_count;
276:
277: /*
278: * There is a default allocation space (zone_free_space[0]),
279: * which is special. It is statically allocated, contains
280: * a single hint, and space is not reclaimed from it. It
281: * is used to allocate space for non collectable zones,
282: * as well as resources needed for bootstrapping purposes.
283: */
284: struct zone_free_space_hint _zone_default_space_hint;
285: struct zone_free_space _zone_default_space;
286: #define zone_default_space (&_zone_default_space)
287:
288: static void zone_free_space_select(zone_t zone);
289:
290: #define zone_collectable(z) ((z)->free_space != 0 && \
291: (z)->free_space != zone_default_space)
292:
293: /*
294: * Compute the hash address for an element
295: * of a particular size. Oversized requests
296: * all land in the last bucket.
297: */
298: static __inline__
299: integer_t
300: zone_free_space_hash(
301: struct zone_free_space *freespace,
302: vm_size_t size
303: )
304: {
305: integer_t hash;
306:
307: if ((hash = size >> freespace->hash_shift) >
308: freespace->num_hints)
309: return (freespace->num_hints);
310: else
311: return (hash);
312: }
313: #define zone_free_space_hint_from_hash(freespace, hash) \
314: ((freespace)->hints + (hash) - 1)
315:
316: /*
317: * Return the hint entry for an element of
318: * a particular size.
319: */
320: static __inline__
321: struct zone_free_space_hint *
322: zone_free_space_hint(
323: struct zone_free_space *freespace,
324: vm_size_t size
325: )
326: {
327: return zone_free_space_hint_from_hash(freespace,
328: zone_free_space_hash(freespace, size));
329: }
330:
331: /*
332: * Conditionally insert the entry into the
333: * hint table such that the hint indicates
334: * the lowest addressed entry of the particular
335: * size.
336: */
337: static __inline__
338: void
339: zone_free_space_hint_insert(
340: struct zone_free_space *freespace,
341: struct zone_free_space_entry *entry
342: )
343: {
344: struct zone_free_space_hint *hint;
345:
346: hint = zone_free_space_hint(freespace, entry->length);
347: if (hint->entry == 0 || entry < hint->entry)
348: hint->entry = entry;
349: }
350:
351: /*
352: * Delete the entry from the hint table (if
353: * present), and locate the next entry of the
354: * particular size by searching forwards in
355: * the free list. For the 'alloc_max' hint,
356: * we choose the next entry of size >= alloc_max.
357: */
358: static
359: void
360: zone_free_space_hint_delete(
361: struct zone_free_space *freespace,
362: struct zone_free_space_entry *entry
363: )
364: {
365: integer_t hash;
366: struct zone_free_space_hint *hint;
367:
368: hash = zone_free_space_hash(freespace, entry->length);
369: hint = zone_free_space_hint_from_hash(freespace, hash);
370:
371: if (entry == hint->entry) {
372: if (hash < freespace->num_hints) {
373: struct zone_free_space_entry
374: *cur, **last = &entry->next;
375:
376: while ((cur = *last) != 0 &&
377: cur->length != entry->length)
378: last = &cur->next;
379:
380: hint->entry = cur;
381: }
382: else {
383: struct zone_free_space_entry
384: *cur, **last = &entry->next;
385:
386: while ((cur = *last) != 0 &&
387: cur->length < freespace->alloc_max)
388: last = &cur->next;
389:
390: hint->entry = cur;
391: }
392: }
393: }
394:
395: /*
396: * Update the hint for an entry which has been
397: * expanded from the front. In this case, the
398: * header has moved (and is no longer valid),
399: * so we need both the length and the address
400: * of the old entry. This could be accomplished
401: * with separate delete/insert operations, but
402: * this is much more efficient for several
403: * important cases.
404: */
405: static
406: void
407: zone_free_space_hint_prepend(
408: struct zone_free_space *freespace,
409: struct zone_free_space_entry *entry,
410: vm_size_t old_length,
411: void *old_entry
412: )
413: {
414: integer_t old_hash, new_hash;
415: struct zone_free_space_hint *hint;
416:
417: /*
418: * Calculate both the new and the old hash
419: * addresses, as well as the old hint structure,
420: * which we are likely to need.
421: */
422: new_hash = zone_free_space_hash(freespace, entry->length);
423: old_hash = zone_free_space_hash(freespace, old_length);
424: hint = zone_free_space_hint_from_hash(freespace, old_hash);
425:
426: /*
427: * Hash address has changed.
428: */
429: if (old_hash != new_hash) {
430: /*
431: * Old hint was valid, update it.
432: */
433: if (old_entry == hint->entry) {
434: if (old_hash < freespace->num_hints) {
435: struct zone_free_space_entry
436: *cur, **last = &entry->next;
437:
438: while ((cur = *last) != 0 &&
439: cur->length != old_length)
440: last = &cur->next;
441:
442: hint->entry = cur;
443: }
444: else {
445: struct zone_free_space_entry
446: *cur, **last = &entry->next;
447:
448: while ((cur = *last) != 0 &&
449: cur->length <
450: freespace->alloc_max)
451: last = &cur->next;
452:
453: hint->entry = cur;
454: }
455: }
456:
457: /*
458: * Insert a hint for the new entry.
459: */
460: hint = zone_free_space_hint_from_hash(freespace, new_hash);
461: if (hint->entry == 0 || entry < hint->entry)
462: hint->entry = entry;
463: }
464: /*
465: * If the hash address has not
466: * changed, always replace a valid
467: * old hint with the new one.
468: */
469: else if (old_entry == hint->entry)
470: hint->entry = entry;
471: }
472:
473: /*
474: * Update the hint for an entry which has been
475: * expanded on the end. This could be accomplished
476: * with separate delete/insert operations, but
477: * this is much more efficient for several
478: * important cases.
479: */
480: static
481: void
482: zone_free_space_hint_append(
483: struct zone_free_space *freespace,
484: struct zone_free_space_entry *entry,
485: vm_size_t old_length
486: )
487: {
488: integer_t old_hash, new_hash;
489: struct zone_free_space_hint *hint;
490:
491: /*
492: * Calculate both the new and the old hash
493: * addresses.
494: */
495: new_hash = zone_free_space_hash(freespace, entry->length);
496: old_hash = zone_free_space_hash(freespace, old_length);
497:
498: if (old_hash != new_hash) {
499: hint = zone_free_space_hint_from_hash(freespace, old_hash);
500: /*
501: * Old hint was valid, update it.
502: */
503: if (entry == hint->entry) {
504: if (old_hash < freespace->num_hints) {
505: struct zone_free_space_entry
506: *cur, **last = &entry->next;
507:
508: while ((cur = *last) != 0 &&
509: cur->length != old_length)
510: last = &cur->next;
511:
512: hint->entry = cur;
513: }
514: else {
515: struct zone_free_space_entry
516: *cur, **last = &entry->next;
517:
518: while ((cur = *last) != 0 &&
519: cur->length <
520: freespace->alloc_max)
521: last = &cur->next;
522:
523: hint->entry = cur;
524: }
525: }
526:
527: /*
528: * Insert a hint for the new entry.
529: */
530: hint = zone_free_space_hint_from_hash(freespace, new_hash);
531: if (hint->entry == 0 || entry < hint->entry)
532: hint->entry = entry;
533: }
534: }
535:
536: /*
537: * Locate a suitably sized entry, using the
538: * allocation hints. The entry is removed
539: * from the hint table, which is also updated
540: * accordingly. The entry's position in the
541: * free list is not affected.
542: */
543: static
544: struct zone_free_space_entry *
545: zone_free_space_lookup(
546: struct zone_free_space *freespace,
547: vm_size_t size
548: )
549: {
550: integer_t hash;
551: struct zone_free_space_hint *hint;
552: struct zone_free_space_entry *entry;
553:
554: /*
555: * Perform a couple of quick checks:
556: * 1) an empty free list or
557: * 2) a suitable first entry
558: */
559: if ((entry = freespace->entries) == 0)
560: return (entry);
561: else if (entry->length >= size) {
562: zone_free_space_hint_delete(freespace, entry);
563:
564: return (entry);
565: }
566:
567: /*
568: * Calculate the hash address and hint
569: * entry corresponding to the request
570: * size. We start there and move on to
571: * the larger hints if needed.
572: */
573: hash = zone_free_space_hash(freespace, size);
574: hint = zone_free_space_hint_from_hash(freespace, hash);
575:
576: /*
577: * This loop checks the exact sized hints
578: * (hash to [num_hints - 1]). If we encounter
579: * a valid hint, we return that entry, after
580: * first searching ahead in the free list to
581: * replace it. If we come to the end of
582: * the free list while searching, we end up
583: * invalidating this hint.
584: */
585: while (hash < freespace->num_hints) {
586: if ((entry = hint->entry) != 0) {
587: struct zone_free_space_entry
588: *cur, **last = &entry->next;
589:
590: while ((cur = *last) != 0 &&
591: cur->length != entry->length)
592: last = &cur->next;
593:
594: hint->entry = cur;
595:
596: return (entry);
597: }
598:
599: hash++; hint++;
600: }
601:
602: /*
603: * Now check the last bucket.
604: */
605: if ((entry = hint->entry) != 0) {
606: struct zone_free_space_entry
607: *cur, **last = &entry->next;
608:
609: /*
610: * The last bucket contains the lowest
611: * addressed entry >= alloc_max, which
612: * isn't necessarily big enough for this
613: * request. If it isn't big enough, then
614: * search ahead in the free list for a
615: * suitable entry to return. In this case
616: * we also leave the current hint alone
617: * since we aren't going to use it.
618: */
619: if (entry->length < size) {
620: while ((cur = *last) != 0 &&
621: cur->length < size)
622: last = &cur->next;
623:
624: return (cur);
625: }
626:
627: while ((cur = *last) != 0 &&
628: cur->length < freespace->alloc_max)
629: last = &cur->next;
630:
631: hint->entry = cur;
632: }
633:
634: return (entry);
635: }
636:
637: /*
638: * Protects first_zone, last_zone, num_zones,
639: * and the next_zone field of zones.
640: */
641: decl_simple_lock_data(,all_zones_lock)
642: zone_t first_zone;
643: zone_t *last_zone;
644: int num_zones;
645:
646: /*
647: * zinit initializes a new zone. The zone data structures themselves
648: * are stored in a zone, which is initially a static structure that
649: * is initialized by zone_init.
650: */
651: zone_t zinit(size, max, alloc, pageable, name)
652: vm_size_t size; /* the size of an element */
653: vm_size_t max; /* maximum memory to use */
654: vm_size_t alloc; /* allocation size */
655: boolean_t pageable; /* is this zone pageable? */
656: char *name; /* a name for the zone */
657: {
658: register zone_t z;
659:
660: if (zone_zone == ZONE_NULL)
661: z = (zone_t) zget_space(
662: zone_default_space,
663: sizeof(struct zone),
664: FALSE);
665: else
666: z = (zone_t) zalloc(zone_zone);
667: if (z == ZONE_NULL)
668: panic("zinit");
669:
670: if (alloc == 0)
671: alloc = PAGE_SIZE;
672:
673: if (size == 0)
674: size = sizeof(z->free_elements);
675:
676: size = ((size + (ZONE_MIN_ALLOC - 1)) & ~(ZONE_MIN_ALLOC - 1));
677:
678: /*
679: * Round off all the parameters appropriately.
680: */
681:
682: if ((max = round_page(max)) < (alloc = round_page(alloc)))
683: max = alloc;
684:
685: z->last_insert = z->free_elements = 0;
686: z->cur_size = 0;
687: z->max_size = max;
688: z->elem_size = size;
689:
1.1.1.2 ! root 690: #if DIAGNOSTIC
! 691: z->lowest = (vm_offset_t)(-1);
! 692: z->highest = (vm_offset_t)0;
! 693: #endif
1.1 root 694: z->alloc_size = alloc;
695: z->pageable = pageable;
696: z->zone_name = name;
697: z->count = 0;
698: z->doing_alloc = FALSE;
699: z->exhaustible = z->sleepable = FALSE;
700: z->expandable = TRUE;
701: lock_zone_init(z);
702: zone_free_space_select(z);
703:
704: /*
705: * Add the zone to the all-zones list.
706: */
707:
708: z->next_zone = ZONE_NULL;
709: simple_lock(simple_lock_addr(all_zones_lock));
710: *last_zone = z;
711: last_zone = &z->next_zone;
712: num_zones++;
713: simple_unlock(simple_lock_addr(all_zones_lock));
714:
715: return(z);
716: }
717:
718: /*
719: * Cram the given memory into the specified zone.
720: */
721: void zcram(zone, newmem, size)
722: register zone_t zone;
723: vm_offset_t newmem;
724: vm_size_t size;
725: {
726: register vm_size_t elem_size;
727:
728: if (newmem == (vm_offset_t) 0) {
729: panic("zcram - memory at zero");
730: }
731: elem_size = zone->elem_size;
732:
733: lock_zone(zone);
734: while (size >= elem_size) {
735: ADD_TO_ZONE(zone, newmem);
736: zone->count++; /* compensate for ADD_TO_ZONE */
737: size -= elem_size;
738: newmem += elem_size;
739: zone->cur_size += elem_size;
740: }
741: unlock_zone(zone);
742: }
743:
744: /*
745: * Allocate (return) a new zone element from a new memory
746: * region. Remaining memory from the new region is added
747: * to the specified freelist. Before generating the new
748: * element, an attempt is made to combine the new region
749: * with an existing entry. New elements are always taken
750: * from the front of a free region.
751: */
752: vm_offset_t zone_free_space_add(freespace, size, new_space, space_to_add)
753: struct zone_free_space *freespace;
754: vm_size_t size;
755: vm_offset_t new_space;
756: vm_size_t space_to_add;
757: {
758: struct zone_free_space_entry *cur, **last;
759:
760: if (freespace == 0)
761: freespace = zone_default_space;
762:
763: /*
764: * Search the free list for an existing
765: * abutting entry.
766: */
767: last = &freespace->entries;
768: while ((cur = *last) != 0 &&
769: (vm_offset_t)cur < new_space &&
770: ((vm_offset_t)cur + cur->length) != new_space)
771: last = &cur->next;
772:
773: if (cur == 0 || ((vm_offset_t)cur + cur->length) < new_space) {
774: /*
775: * No entry was found to combine with.
776: * Take the new element from the front
777: * of the new region, and insert the
778: * remainder as a new entry.
779: */
780: if ((space_to_add - size) >= ZONE_MIN_ALLOC) {
781: /*
782: * If we are not at the end of
783: * the free list, then insert the
784: * new entry after the current entry.
785: */
786: if (cur != 0)
787: last = &cur->next;
788: (vm_offset_t)cur = new_space + size;
789: cur->length = space_to_add - size;
790: if (cur->next = *last)
791: cur->next->pred = &cur->next;
792: cur->pred = last;
793: *last = cur;
794: freespace->num_entries++;
795:
796: /*
797: * Insert this entry into the hint
798: * table.
799: */
800: zone_free_space_hint_insert(freespace, cur);
801: }
802: }
803: else
804: if (((vm_offset_t)cur + cur->length) == new_space) {
805: struct zone_free_space_entry *new;
806:
807: /*
808: * Delete the existing entry from the
809: * hint table. It is very likely that
810: * the hash address will be changing
811: * anyways.
812: */
813: zone_free_space_hint_delete(freespace, cur);
814:
815: /*
816: * Combine the new region with an existing
817: * entry, and take the new element from the
818: * front of the aggregate region. Create a
819: * new entry for the remainder, and insert it
820: * in place of the existing entry.
821: */
822: new_space = (vm_offset_t)cur;
823: (vm_offset_t)new = (vm_offset_t)cur + size;
824: new->length = cur->length + space_to_add - size;
825: if (new->next = cur->next)
826: new->next->pred = &new->next;
827: new->pred = last;
828: *last = new;
829:
830: /*
831: * Insert this entry into the hint
832: * table.
833: */
834: zone_free_space_hint_insert(freespace, new);
835: }
836:
837: return (new_space);
838: }
839:
840: #if 0
841: /* NOT USED and OUTDATED */
842: /*
843: * Ensure that no portion of the specified region
844: * is represented on the free list (either wholly
845: * or in part).
846: */
847: void zone_free_space_remove(freespace, address, size)
848: struct zone_free_space *freespace;
849: vm_offset_t address;
850: vm_size_t size;
851: {
852: struct zone_free_space_entry
853: *cur, **last;
854:
855: if (freespace == 0)
856: freespace = zone_default_space;
857:
858: /*
859: * Search the free list, looking for
860: * the first suitable entry.
861: */
862: last = &freespace->entries;
863: while ((cur = *last) != 0) {
864: if ((vm_offset_t)cur >= address) {
865: /*
866: * Entry is above (and does not
867: * overlap) the region. (skip)
868: */
869: if ((vm_offset_t)cur >= (address + size))
870: last = &cur->next;
871: else {
872: /*
873: * Entry is entirely contained
874: * within the region. (remove)
875: */
876: if (((vm_offset_t)cur + cur->length) <=
877: (address + size)) {
878: *last = cur->next;
879: freespace->num_entries--;
880: }
881: else {
882: struct zone_free_space_entry *new;
883:
884: /*
885: * Entry overlaps the end
886: * of the region. (clip)
887: */
888: (vm_offset_t)new = address + size;
889: new->length = cur->length -
890: ((vm_offset_t)new -
891: (vm_offset_t)cur);
892: new->next = cur->next;
893: *last = new;
894: }
895:
896: break;
897: }
898: }
899: else {
900: /*
901: * Entry is entirely below the
902: * region. (skip)
903: */
904: if (((vm_offset_t)cur + cur->length) <= address)
905: last = &cur->next;
906: else {
907: /*
908: * Entry overlaps the front
909: * of the region. (clip)
910: */
911: cur->length = address - (vm_offset_t)cur;
912: break;
913: }
914: }
915: }
916: }
917: #endif
918:
919: /*
920: * Return the elements on the zone free list back
921: * to the free space pool, coalescing with existing
922: * space where possible. Caller must hold the
923: * zget_space_lock, as well as the lock on the zone.
924: */
925: void zone_collect(zone)
926: struct zone *zone;
927: {
928: struct zone_free_space_entry *cur, **last, *new;
929: vm_offset_t *free, *elem, next;
930: vm_size_t elem_size = zone->elem_size;
931: struct zone_free_space *freespace = zone->free_space;
932:
933: if (!zone_collectable(zone))
934: return;
935:
936: last = &freespace->entries;
937:
938: free = &zone->free_elements;
939: while (((vm_offset_t)elem = *free) != 0) {
940: zone->cur_size -= elem_size;
941: next = *elem;
942:
943: while ((cur = *last) != 0 &&
944: ((vm_offset_t)cur +
945: cur->length) < (vm_offset_t)elem)
946: last = &cur->next;
947: /*
948: * Either at end of (maybe empty)
949: * list, or new entry before current
950: * entry.
951: */
952: if (cur == 0 || ((vm_offset_t)elem + elem_size) <
953: (vm_offset_t)cur) {
954: (vm_offset_t)new = (vm_offset_t)elem;
955: new->length = elem_size;
956: if (new->next = cur)
957: cur->pred = &new->next;
958: new->pred = last;
959: *last = new;
960: freespace->num_entries++;
961:
962: /*
963: * Insert this entry into the hint
964: * table.
965: */
966: zone_free_space_hint_insert(freespace, new);
967: }
968: else
969: /*
970: * Prepend element to current entry
971: */
972: if (((vm_offset_t)elem + elem_size) == (vm_offset_t)cur) {
973: vm_size_t old_length = cur->length;
974:
975: (vm_offset_t)new = (vm_offset_t)elem;
976: new->length = cur->length + elem_size;
977: if (new->next = cur->next)
978: new->next->pred = &new->next;
979: new->pred = last;
980: *last = new;
981:
982: /*
983: * Update the hint table.
984: */
985: zone_free_space_hint_prepend(freespace,
986: new, old_length, cur);
987: }
988: else
989: /*
990: * Append element to current entry.
991: */
992: if (((vm_offset_t)cur + cur->length) == (vm_offset_t)elem) {
993: vm_size_t old_length = cur->length;
994:
995: cur->length += elem_size;
996: /*
997: * Coalesce the current entry with the
998: * following one if we are filling the
999: * gap between them.
1000: */
1001: if (((vm_offset_t)cur + cur->length) ==
1002: (vm_offset_t)cur->next) {
1003:
1004: /*
1005: * Delete the obsolete entry
1006: * from the hint table.
1007: */
1008: zone_free_space_hint_delete(
1009: freespace, cur->next);
1010:
1011: cur->length += cur->next->length;
1012: if (cur->next = cur->next->next)
1013: cur->next->pred = &cur->next;
1014: freespace->num_entries--;
1015: }
1016:
1017: /*
1018: * Update the hint table.
1019: */
1020: zone_free_space_hint_append(freespace,
1021: cur, old_length);
1022: }
1023: else
1024: /* WTF?? */;
1025:
1026: *free = next;
1027: }
1028:
1029: zone->last_insert = 0;
1030: }
1031:
1032: /*
1033: * Scan the collectable free space free lists
1034: * and gather up pages which can be returned to
1035: * the system. Caller must hold the zget_space_lock.
1036: */
1037: struct zone_free_space_entry *
1038: zone_free_space_reclaim(void)
1039: {
1040: struct zone_free_space_entry **last, *cur, *pages = 0;
1041: struct zone_free_space **f = &zone_free_space[0];
1042: int i;
1043:
1044: for (i = 1; i < zone_free_space_count; i++) {
1045: last = &(*++f)->entries;
1046: while ((cur = *last) != 0) {
1047: if (cur->length >= PAGE_SIZE) {
1048: vm_offset_t start, end;
1049:
1050: start = round_page((vm_offset_t)cur);
1051: end = trunc_page(
1052: (vm_offset_t)cur + cur->length);
1053: if (start < end &&
1054: start >= zone_min &&
1055: end <= zone_max) {
1056: struct zone_free_space_entry *tmp;
1057:
1058: /*
1059: * Delete this entry from the hint
1060: * table. Space which is leftover
1061: * after clipping will be added back
1062: * normally after the entries are
1063: * created.
1064: */
1065: zone_free_space_hint_delete(*f, cur);
1066:
1067: /*
1068: * If the region does not end on a
1069: * page boundary, create a new
1070: * trailing entry.
1071: */
1072: if (((vm_offset_t)cur + cur->length) != end) {
1073: (vm_offset_t)tmp = end;
1074: tmp->length = (vm_offset_t)cur +
1075: cur->length - end;
1076: if (tmp->next = cur->next)
1077: tmp->next->pred = &tmp->next;
1078:
1079: /*
1080: * Now, if the region does begin
1081: * on a page boundary, just remove
1082: * the current entry.
1083: */
1084: if ((vm_offset_t)cur == start) {
1085: *last = tmp;
1086: tmp->pred = last;
1087: }
1088: /*
1089: * Otherwise, adjust the current
1090: * entry, and link it to the new
1091: * one. Do not forget to account
1092: * for the new entry.
1093: */
1094: else {
1095: cur->length = start -
1096: (vm_offset_t)cur;
1097: cur->next = tmp;
1098: tmp->pred = &cur->next;
1099: (*f)->num_entries++;
1100:
1101: /*
1102: * Reinsert the leading
1103: * entry into the hint
1104: * table.
1105: */
1106: zone_free_space_hint_insert(
1107: *f, cur);
1108: }
1109:
1110: /*
1111: * Insert the new trailing entry
1112: * into the hint table.
1113: */
1114: zone_free_space_hint_insert(*f, tmp);
1115: }
1116: /*
1117: * If the region does not begin on a
1118: * page boundary (but the end does),
1119: * adjust the current entry.
1120: */
1121: else if ((vm_offset_t)cur != start) {
1122: cur->length = start - (vm_offset_t)cur;
1123:
1124: /*
1125: * Reinsert the entry into
1126: * the hint table.
1127: */
1128: zone_free_space_hint_insert(*f, cur);
1129: }
1130: /*
1131: * If no clipping is required, just
1132: * remove the current entry.
1133: */
1134: else {
1135: if (*last = cur->next)
1136: cur->next->pred = last;
1137: (*f)->num_entries--;
1138: }
1139:
1140: /*
1141: * Add the new page aligned region
1142: * to the list of pages to be freed.
1143: * Continue on with the next entry.
1144: */
1145: (vm_offset_t)tmp = start;
1146: tmp->length = end - start;
1147: tmp->next = pages;
1148: pages = tmp;
1149: continue;
1150: }
1151: }
1152:
1153: /*
1154: * Skip this entry.
1155: */
1156: last = &cur->next;
1157: }
1158: }
1159:
1160: return (pages);
1161: }
1162:
1163: /*
1164: * Contiguous space allocator for non-paged zones. Allocates "size" amount
1165: * of memory from zone_map.
1166: */
1167:
1168: vm_offset_t zget_space(freespace, size, canblock)
1169: struct zone_free_space *freespace;
1170: vm_size_t size;
1171: boolean_t canblock;
1172: {
1173: vm_offset_t new_space = 0;
1174: vm_offset_t result;
1175: vm_size_t space_to_add;
1176: struct zone_free_space_entry
1177: *cur, **last;
1178:
1179: if (freespace == 0)
1180: freespace = zone_default_space;
1181:
1182: /*
1183: * Round up all requests (even 0) to
1184: * our minimum allocation unit.
1185: */
1186: if (size > ZONE_MIN_ALLOC)
1187: size = ((size + (ZONE_MIN_ALLOC - 1)) & ~(ZONE_MIN_ALLOC - 1));
1188: else
1189: size = ZONE_MIN_ALLOC;
1190:
1191: simple_lock(simple_lock_addr(zget_space_lock));
1192: for (;;) {
1193: if ((cur = zone_free_space_lookup(freespace, size)) != 0) {
1194: last = cur->pred;
1195:
1196: /*
1197: * The entry which was found has been
1198: * removed from the hint table, but
1199: * remains in the free list. Trim off
1200: * the space to be returned.
1201: */
1202: if ((cur->length - size) < ZONE_MIN_ALLOC) {
1203: /*
1204: * This is a real lose if it
1205: * happens in a collectable
1206: * space, since the memory
1207: * will be lost, making it
1208: * impossible to reclaim the
1209: * page later.
1210: */
1211: if (*last = cur->next)
1212: cur->next->pred = last;
1213: freespace->num_entries--;
1214: }
1215: else {
1216: struct zone_free_space_entry *new;
1217:
1218: /*
1219: * Create a new entry for the
1220: * remaining space and position
1221: * on the free list.
1222: */
1223: (vm_offset_t)new = (vm_offset_t)cur + size;
1224: new->length = cur->length - size;
1225: if (new->next = cur->next)
1226: new->next->pred = &new->next;
1227: new->pred = last;
1228: *last = new;
1229:
1230: /*
1231: * After trimming the entry, reinsert
1232: * it back into the hint table.
1233: */
1234: zone_free_space_hint_insert(freespace, new);
1235: }
1236: result = (vm_offset_t)cur;
1237: break;
1238: }
1239: else if (new_space == 0) {
1240: /*
1241: * Add at least one page to allocation area.
1242: */
1243:
1244: space_to_add = round_page(size);
1245:
1246: if (zdata_size >= space_to_add) {
1247: zdata_size -= space_to_add;
1248: result = zone_free_space_add(
1249: freespace,
1250: size,
1251: zdata + zdata_size,
1252: space_to_add);
1253: break;
1254: }
1255:
1256: /*
1257: * Memory cannot be wired down while holding
1258: * any locks that the pageout daemon might
1259: * need to free up pages. [Making the zget_space
1260: * lock a complex lock does not help in this
1261: * regard.]
1262: *
1263: * Unlock and allocate memory. Because several
1264: * threads might try to do this at once, don't
1265: * use the memory before checking for available
1266: * space again.
1267: */
1268:
1269: simple_unlock(simple_lock_addr(zget_space_lock));
1270: {
1271: kern_return_t kr;
1272:
1273: kr = kmem_alloc_zone(zone_map,
1274: &new_space, space_to_add,
1275: canblock);
1276: if (kr != KERN_SUCCESS) {
1277: if (kr == KERN_NO_SPACE)
1278: panic("zget_space");
1279:
1280: return(0);
1281: }
1282: }
1283:
1284: simple_lock(simple_lock_addr(zget_space_lock));
1285: continue;
1286: }
1287: else {
1288: /*
1289: * Memory was allocated in a previous iteration.
1290: */
1291:
1292: result = zone_free_space_add(
1293: freespace,
1294: size,
1295: new_space,
1296: space_to_add);
1297: new_space = 0;
1298: break;
1299: }
1300: }
1301: simple_unlock(simple_lock_addr(zget_space_lock));
1302:
1303: if (new_space != 0)
1304: kmem_free(zone_map, new_space, space_to_add);
1305:
1306: return(result);
1307: }
1308:
1309: static
1310: struct zone_free_space *
1311: zone_free_space_alloc(alloc_unit, alloc_max)
1312: vm_size_t alloc_unit;
1313: vm_size_t alloc_max;
1314: {
1315: struct zone_free_space *freespace, **f;
1316:
1317: if (zone_free_space_count >=
1318: (sizeof (zone_free_space) / sizeof (freespace)))
1319: return (0);
1320:
1321: f = &zone_free_space[zone_free_space_count++];
1322:
1323: (vm_offset_t)freespace = zget_space(
1324: zone_default_space,
1325: sizeof (struct zone_free_space),
1326: FALSE);
1327: freespace->alloc_unit = alloc_unit;
1328: freespace->alloc_max = alloc_max;
1329:
1330: freespace->entries = 0;
1331: freespace->num_entries = 0;
1332:
1333: freespace->hash_shift = 0;
1334: while (!(alloc_unit & 01)) {
1335: freespace->hash_shift++; alloc_unit >>= 1;
1336: }
1337:
1338: freespace->num_hints = freespace->alloc_max >> freespace->hash_shift;
1339: (vm_offset_t)freespace->hints =
1340: zget_space(
1341: zone_default_space,
1342: freespace->num_hints *
1343: sizeof (struct zone_free_space_hint),
1344: FALSE);
1345: bzero(freespace->hints, freespace->num_hints *
1346: sizeof (struct zone_free_space_hint));
1347:
1348: *f = freespace;
1349:
1350: return (freespace);
1351: }
1352:
1353: static
1354: void
1355: zone_free_space_select(zone)
1356: zone_t zone;
1357: {
1358: struct zone_free_space **f = &zone_free_space[1];
1359: vm_size_t elem_size;
1360: int i;
1361:
1362: if (zone->cur_size > 0)
1363: return;
1364:
1365: for (i = 1; i < zone_free_space_count; i++) {
1366: elem_size = ((zone->elem_size + ((*f)->alloc_unit - 1))
1367: & ~((*f)->alloc_unit - 1));
1368: if (elem_size <= (*f)->alloc_max) {
1369: zone->elem_size = elem_size;
1370: zone->free_space = *f;
1371: break;
1372: }
1373:
1374: f++;
1375: }
1376: }
1377:
1378: /*
1379: * Initialize the "zone of zones" which uses fixed memory allocated
1380: * earlier in memory initialization. zone_bootstrap is called
1381: * before zone_init.
1382: */
1383: void zone_bootstrap(void)
1384: {
1385: simple_lock_init(simple_lock_addr(all_zones_lock));
1386: first_zone = ZONE_NULL;
1387: last_zone = &first_zone;
1388: num_zones = 0;
1389:
1390: if (sizeof (struct zone_free_space_entry) > ZONE_MIN_ALLOC)
1391: panic("zone_bootstrap");
1392:
1393: simple_lock_init(simple_lock_addr(zget_space_lock));
1394:
1395: _zone_default_space.hints = &_zone_default_space_hint;
1396: _zone_default_space.num_hints = 1;
1397:
1398: zone_free_space[0] = &_zone_default_space;
1399: zone_free_space_count = 1;
1400:
1401: zone_zone = ZONE_NULL;
1402: zone_zone = zinit(sizeof(struct zone), 128 * sizeof(struct zone),
1403: sizeof(struct zone), FALSE, "zones");
1404:
1405: zone_free_space_alloc(16, 96);
1406: zone_free_space_alloc(128, 768);
1407: zone_free_space_alloc(1024, PAGE_SIZE);
1408: }
1409:
1410: vm_size_t
1411: zone_map_sizer(void)
1412: {
1413: #if defined(__ppc__)
1414: vm_size_t map_size = mem_size / 4;
1415: #else
1416: vm_size_t map_size = mem_size / 8;
1417: #endif
1418:
1419: if (map_size < zone_map_size_min)
1420: map_size = zone_map_size_min;
1421: else
1422: if (map_size > zone_map_size_max)
1423: map_size = zone_map_size_max;
1424:
1425: return (map_size);
1426: }
1427:
1428: void zone_init(void)
1429: {
1430: zone_map_size = zone_map_sizer();
1431:
1432: zone_map = kmem_suballoc(kernel_map, &zone_min, &zone_max,
1433: zone_map_size, FALSE);
1434: }
1435:
1.1.1.2 ! root 1436: void
! 1437: check_zone(zone, elem)
! 1438: zone_t zone;
! 1439: vm_offset_t elem;
! 1440: {
! 1441: vm_offset_t this;
! 1442: /*
! 1443: * check the zone's consistency
! 1444: */
! 1445: #if !DIAGNOSTIC
! 1446: if (!elem)
! 1447: return;
! 1448: #endif
! 1449: for (this = zone->free_elements;
! 1450: this != 0;
! 1451: this = * (vm_offset_t *) this)
! 1452: if (this == elem)
! 1453: panic("check_zone: argument already free");
! 1454: #if DIAGNOSTIC
! 1455: else if (this < zone->lowest || this > zone->highest)
! 1456: panic("check_zone: item out of range");
! 1457: #endif
! 1458: }
1.1 root 1459:
1460: /*
1461: * zalloc returns an element from the specified zone.
1462: */
1463: static
1464: vm_offset_t zalloc_canblock(zone, canblock)
1465: register zone_t zone;
1466: boolean_t canblock;
1467: {
1468: vm_offset_t addr;
1469:
1470: if (zone == ZONE_NULL)
1471: panic ("zalloc: null zone");
1472:
1473: lock_zone(zone);
1.1.1.2 ! root 1474: if (zone_check & 8)
! 1475: check_zone(zone, 0);
1.1 root 1476: REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
1477: while (addr == 0) {
1478: /*
1479: * If nothing was there, try to get more
1480: */
1481: if (zone->doing_alloc) {
1482: /*
1483: * Someone is allocating memory for this zone.
1484: * Wait for it to show up, then try again.
1485: */
1486: if (!canblock) {
1487: unlock_zone(zone);
1488: return(0);
1489: }
1490: assert_wait((event_t)&zone->doing_alloc, TRUE);
1491: /* XXX say wakeup needed */
1492: unlock_zone(zone);
1493: thread_block_with_continuation((void (*)()) 0);
1494: lock_zone(zone);
1495: }
1496: else {
1497: if ((zone->cur_size + (zone->pageable ?
1498: zone->alloc_size : zone->elem_size)) >
1499: zone->max_size) {
1500: if (zone->exhaustible)
1501: break;
1502:
1503: if (zone->expandable) {
1504: /*
1505: * We're willing to overflow certain
1506: * zones, but not without complaining.
1507: *
1508: * This is best used in conjunction
1509: * with the collecatable flag. What we
1510: * want is an assurance we can get the
1511: * memory back, assuming there's no
1512: * leak.
1513: */
1514: zone->max_size += (zone->max_size >> 1);
1515: } else if (!zone_ignore_overflow) {
1516: unlock_zone(zone);
1517: if (!canblock)
1518: return(0);
1519: printf("zone \"%s\" empty.\n",
1520: zone->zone_name);
1521: panic("zalloc");
1522: }
1523: }
1524:
1525: if (zone->pageable)
1526: zone->doing_alloc = TRUE;
1527: unlock_zone(zone);
1528:
1529: if (zone->pageable) {
1530: if (kmem_alloc_pageable(zone_map, &addr,
1531: zone->alloc_size)
1532: != KERN_SUCCESS)
1533: panic("zalloc");
1534: zcram(zone, addr, zone->alloc_size);
1535: lock_zone(zone);
1536: zone->doing_alloc = FALSE;
1537: /* XXX check before doing this */
1538: thread_wakeup((event_t)&zone->doing_alloc);
1539:
1.1.1.2 ! root 1540: if (zone_check & 8)
! 1541: check_zone(zone, 0);
1.1 root 1542: REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
1543: } else {
1544: addr = zget_space(
1545: zone->free_space,
1546: zone->elem_size,
1547: canblock);
1548: if (addr == 0) {
1549: if (!canblock)
1550: return(0);
1551: panic("zalloc");
1552: }
1553:
1554: lock_zone(zone);
1555: zone->count++;
1556: zone->cur_size += zone->elem_size;
1.1.1.2 ! root 1557: WATERMARK_ZONE(zone, addr);
1.1 root 1558: unlock_zone(zone);
1559: return(addr);
1560: }
1561: }
1562: }
1563:
1564: unlock_zone(zone);
1565: return(addr);
1566: }
1567:
1568: vm_offset_t zalloc(zone)
1569: register zone_t zone;
1570: {
1571: return (zalloc_canblock(zone, TRUE));
1572: }
1573:
1574: vm_offset_t zalloc_noblock(zone)
1575: register zone_t zone;
1576: {
1577: return (zalloc_canblock(zone, FALSE));
1578: }
1579:
1580:
1581: /*
1582: * zget returns an element from the specified zone
1583: * and immediately returns nothing if there is nothing there.
1584: *
1585: * This form should be used when you can not block (like when
1586: * processing an interrupt).
1587: */
1588: vm_offset_t zget(zone)
1589: register zone_t zone;
1590: {
1591: register vm_offset_t addr;
1592:
1593: if (zone == ZONE_NULL)
1594: panic ("zalloc: null zone");
1595:
1596: lock_zone(zone);
1.1.1.2 ! root 1597: if (zone_check & 4)
! 1598: check_zone(zone, 0);
1.1 root 1599: REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
1600: unlock_zone(zone);
1601:
1602: return(addr);
1603: }
1604:
1.1.1.2 ! root 1605: void
! 1606: zfree(zone, elem)
1.1 root 1607: register zone_t zone;
1608: vm_offset_t elem;
1609: {
1610: lock_zone(zone);
1.1.1.2 ! root 1611: #if DIAGNOSTIC
! 1612: if (elem < zone->lowest || elem > zone->highest)
! 1613: panic("zfree: argument out of range");
! 1614: #endif
! 1615: if (zone_check & 2)
! 1616: check_zone(zone, elem);
1.1 root 1617: ADD_TO_ZONE(zone, elem);
1618: unlock_zone(zone);
1619: }
1620:
1621: void zcollectable(zone)
1622: zone_t zone;
1623: {
1624: /* zones are collectable by default
1625: * and cannot later be changed back to collectable */
1626: }
1627:
1628: void zchange(zone, pageable, sleepable, exhaustible, collectable)
1629: zone_t zone;
1630: boolean_t pageable;
1631: boolean_t sleepable;
1632: boolean_t exhaustible;
1633: boolean_t collectable;
1634: {
1635: zone->pageable = pageable;
1636: zone->sleepable = sleepable;
1637: zone->exhaustible = exhaustible;
1638: /* zones are collectable by default
1639: * and later can only be changed to non-collectable */
1640: if (!collectable)
1641: zone->free_space = zone_default_space;
1642: lock_zone_init(zone);
1643: }
1644:
1645: #if MACH_DEBUG
1646: kern_return_t host_zone_info(host, namesp, namesCntp, infop, infoCntp)
1647: host_t host;
1648: zone_name_array_t *namesp;
1649: unsigned int *namesCntp;
1650: zone_info_array_t *infop;
1651: unsigned int *infoCntp;
1652: {
1653: zone_name_t *names;
1654: vm_offset_t names_addr;
1655: vm_size_t names_size = 0; /*'=0' to quiet gcc warnings */
1656: zone_info_t *info;
1657: vm_offset_t info_addr;
1658: vm_size_t info_size = 0; /*'=0' to quiet gcc warnings */
1659: unsigned int max_zones, i;
1660: zone_t z;
1661: kern_return_t kr;
1662:
1663: if (host == HOST_NULL)
1664: return KERN_INVALID_HOST;
1665:
1666: /*
1667: * We assume that zones aren't freed once allocated.
1668: * We won't pick up any zones that are allocated later.
1669: */
1670:
1671: simple_lock(simple_lock_addr(all_zones_lock));
1672: max_zones = num_zones;
1673: z = first_zone;
1674: simple_unlock(simple_lock_addr(all_zones_lock));
1675:
1676: if (max_zones <= *namesCntp) {
1677: /* use in-line memory */
1678:
1679: names = *namesp;
1680: } else {
1681: names_size = round_page(max_zones * sizeof *names);
1682: kr = kmem_alloc_pageable(ipc_kernel_map,
1683: &names_addr, names_size);
1684: if (kr != KERN_SUCCESS)
1685: return kr;
1686:
1687: names = (zone_name_t *) names_addr;
1688: }
1689:
1690: if (max_zones <= *infoCntp) {
1691: /* use in-line memory */
1692:
1693: info = *infop;
1694: } else {
1695: info_size = round_page(max_zones * sizeof *info);
1696: kr = kmem_alloc_pageable(ipc_kernel_map,
1697: &info_addr, info_size);
1698: if (kr != KERN_SUCCESS) {
1699: if (names != *namesp)
1700: kmem_free(ipc_kernel_map,
1701: names_addr, names_size);
1702: return kr;
1703: }
1704:
1705: info = (zone_info_t *) info_addr;
1706: }
1707:
1708: for (i = 0; i < max_zones; i++) {
1709: zone_name_t *zn = &names[i];
1710: zone_info_t *zi = &info[i];
1711: struct zone zcopy;
1712:
1713: assert(z != ZONE_NULL);
1714:
1715: lock_zone(z);
1716: zcopy = *z;
1717: unlock_zone(z);
1718:
1719: simple_lock(simple_lock_addr(all_zones_lock));
1720: z = z->next_zone;
1721: simple_unlock(simple_lock_addr(all_zones_lock));
1722:
1723: /* assuming here the name data is static */
1724: (void) strncpy(zn->zn_name, zcopy.zone_name,
1725: sizeof zn->zn_name);
1726:
1727: zi->zi_count = zcopy.count;
1728: zi->zi_cur_size = zcopy.cur_size;
1729: zi->zi_max_size = zcopy.max_size;
1730: zi->zi_elem_size = zcopy.elem_size;
1731: zi->zi_alloc_size = zcopy.alloc_size;
1732: zi->zi_pageable = zcopy.pageable;
1733: zi->zi_sleepable = zcopy.sleepable;
1734: zi->zi_exhaustible = zcopy.exhaustible;
1735: zi->zi_collectable = zone_collectable(&zcopy);
1736: }
1737:
1738: if (names != *namesp) {
1739: vm_size_t used;
1740: #if MACH_OLD_VM_COPY
1741: #else
1742: vm_map_copy_t copy;
1743: #endif
1744:
1745: used = max_zones * sizeof *names;
1746:
1747: if (used != names_size)
1748: bzero((char *) (names_addr + used), names_size - used);
1749:
1750: #if MACH_OLD_VM_COPY
1751: kr = vm_move(
1752: ipc_kernel_map, names_addr,
1753: ipc_soft_map, names_size,
1754: TRUE, &names_addr);
1755: assert(kr == KERN_SUCCESS);
1756:
1757: *namesp = (zone_name_t *) names_addr;
1758: #else
1759: kr = vm_map_copyin(ipc_kernel_map, names_addr, names_size,
1760: TRUE, ©);
1761: assert(kr == KERN_SUCCESS);
1762:
1763: *namesp = (zone_name_t *) copy;
1764: #endif
1765: }
1766: *namesCntp = max_zones;
1767:
1768: if (info != *infop) {
1769: vm_size_t used;
1770: #if MACH_OLD_VM_COPY
1771: #else
1772: vm_map_copy_t copy;
1773: #endif
1774:
1775: used = max_zones * sizeof *info;
1776:
1777: if (used != info_size)
1778: bzero((char *) (info_addr + used), info_size - used);
1779:
1780: #if MACH_OLD_VM_COPY
1781: kr = vm_move(
1782: ipc_kernel_map, info_addr,
1783: ipc_soft_map, info_size,
1784: TRUE, &info_addr);
1785: assert(kr == KERN_SUCCESS);
1786:
1787: *infop = (zone_info_t *) info_addr;
1788: #else
1789: kr = vm_map_copyin(ipc_kernel_map, info_addr, info_size,
1790: TRUE, ©);
1791: assert(kr == KERN_SUCCESS);
1792:
1793: *infop = (zone_info_t *) copy;
1794: #endif
1795: }
1796: *infoCntp = max_zones;
1797:
1798: return KERN_SUCCESS;
1799: }
1800:
1801: kern_return_t host_zone_free_space_info(
1802: host,
1803: infop, infoCnt,
1804: chunksp, chunksCnt)
1805: host_t host;
1806: zone_free_space_info_array_t *infop;
1807: mach_msg_type_number_t *infoCnt;
1808: zone_free_space_chunk_array_t *chunksp;
1809: mach_msg_type_number_t *chunksCnt;
1810: {
1811: kern_return_t kr;
1812: vm_size_t size1, size2;
1813: vm_offset_t addr1, addr2;
1814: vm_offset_t memory1, memory2;
1815: mach_msg_type_number_t
1816: actual1, actual2;
1817: zone_free_space_info_t
1818: *info;
1819: zone_free_space_chunk_t
1820: *chunk;
1821: struct zone_free_space_entry
1822: **last, *cur;
1823: struct zone_free_space
1824: *freespace;
1825: int i;
1826:
1827: if (host == HOST_NULL)
1828: return KERN_INVALID_HOST;
1829:
1830: size1 = size2 = 0;
1831:
1832: for (;;) {
1833: vm_size_t size1_needed, size2_needed;
1834:
1835: size1_needed = size2_needed = 0;
1836:
1837: simple_lock(simple_lock_addr(zget_space_lock));
1838:
1839: actual1 = actual2 = 0;
1840:
1841: for (actual1 = 0; actual1 < zone_free_space_count; actual1++)
1842: actual2 += zone_free_space[actual1]->num_entries;
1843:
1844: if (actual1 > *infoCnt)
1845: size1_needed = round_page(
1846: actual1 * sizeof (**infop));
1847:
1848: if (actual2 > *chunksCnt)
1849: size2_needed = round_page(
1850: actual2 * sizeof (**chunksp));
1851:
1852: if (size1_needed <= size1 &&
1853: size2_needed <= size2)
1854: break;
1855:
1856: simple_unlock(simple_lock_addr(zget_space_lock));
1857:
1858: if (size1 < size1_needed) {
1859: if (size1 != 0)
1860: kmem_free(ipc_kernel_map, addr1, size1);
1861: size1 = size1_needed;
1862:
1863: kr = kmem_alloc_pageable(
1864: ipc_kernel_map, &addr1, size1);
1865: if (kr != KERN_SUCCESS) {
1866: if (size2 != 0)
1867: kmem_free(
1868: ipc_kernel_map, addr2, size2);
1869: return KERN_RESOURCE_SHORTAGE;
1870: }
1871: #if MACH_OLD_VM_COPY
1872: kr = vm_map_pageable(
1873: ipc_kernel_map, addr1, addr1 + size1, FALSE);
1874: assert(kr == KERN_SUCCESS);
1875: #else
1876: kr = vm_map_pageable(
1877: ipc_kernel_map, addr1, addr1 + size1,
1878: VM_PROT_READ|VM_PROT_WRITE);
1879: assert(kr == KERN_SUCCESS);
1880: #endif
1881: }
1882:
1883: if (size2 < size2_needed) {
1884: if (size2 != 0)
1885: kmem_free(ipc_kernel_map, addr2, size2);
1886: size2 = size2_needed;
1887:
1888: kr = kmem_alloc_pageable(
1889: ipc_kernel_map, &addr2, size2);
1890: if (kr != KERN_SUCCESS) {
1891: if (size1 != 0)
1892: kmem_free(
1893: ipc_kernel_map, addr1, size1);
1894: return KERN_RESOURCE_SHORTAGE;
1895: }
1896: #if MACH_OLD_VM_COPY
1897: kr = vm_map_pageable(
1898: ipc_kernel_map, addr2, addr2 + size2, FALSE);
1899: assert(kr == KERN_SUCCESS);
1900: #else
1901: kr = vm_map_pageable(
1902: ipc_kernel_map, addr2, addr2 + size2,
1903: VM_PROT_READ|VM_PROT_WRITE);
1904: assert(kr == KERN_SUCCESS);
1905: #endif
1906: }
1907: }
1908:
1909: if (size1 != 0)
1910: info = (zone_free_space_info_t *)addr1;
1911: else
1912: info = *infop;
1913:
1914: if (size2 != 0)
1915: chunk = (zone_free_space_chunk_t *)addr2;
1916: else
1917: chunk = *chunksp;
1918:
1919: for (i = 0; i < actual1; i++) {
1920: freespace = zone_free_space[i];
1921:
1922: info->zf_alloc_unit = freespace->alloc_unit;
1923: info->zf_alloc_max = freespace->alloc_max;
1924: info->zf_num_chunks = freespace->num_entries;
1925: info++;
1926:
1927: last = &freespace->entries;
1928: while ((cur = *last) != 0) {
1929: chunk->zf_address = (vm_offset_t)cur;
1930: chunk->zf_length = cur->length;
1931: chunk++;
1932:
1933: last = &cur->next;
1934: }
1935: }
1936:
1937: simple_unlock(simple_lock_addr(zget_space_lock));
1938:
1939: if (actual1 != 0 && size1 != 0) {
1940: vm_size_t size_used;
1941:
1942: size_used = round_page(actual1 * sizeof (**infop));
1943:
1944: #if MACH_OLD_VM_COPY
1945: kr = vm_map_pageable(
1946: ipc_kernel_map, addr1, addr1 + size_used, TRUE);
1947: assert(kr == KERN_SUCCESS);
1948:
1949: kr = vm_move(
1950: ipc_kernel_map, addr1,
1951: ipc_soft_map, size_used,
1952: TRUE, &memory1);
1953: assert(kr == KERN_SUCCESS);
1954: #else
1955: kr = vm_map_pageable(
1956: ipc_kernel_map,
1957: addr1, addr1 + size_used, VM_PROT_NONE);
1958: assert(kr == KERN_SUCCESS);
1959:
1960: kr = vm_map_copyin(
1961: ipc_kernel_map, addr1, size_used,
1962: TRUE, &memory1);
1963: assert(kr == KERN_SUCCESS);
1964: #endif
1965:
1966: if (size_used != size1)
1967: kmem_free(
1968: ipc_kernel_map,
1969: addr1 + size_used, size1 - size_used);
1970:
1971: *infop = (zone_free_space_info_t *)memory1;
1972: }
1973: else if (actual1 == 0) {
1974: #if MACH_OLD_VM_COPY
1975: *infop = (zone_free_space_info_t *)0;
1976: #else
1977: *infop = (zone_free_space_info_t *)VM_MAP_COPY_NULL;
1978: #endif
1979:
1980: if (size1 != 0)
1981: kmem_free(ipc_kernel_map, addr1, size1);
1982: }
1983:
1984: *infoCnt = actual1;
1985:
1986: if (actual2 != 0 && size2 != 0) {
1987: vm_size_t size_used;
1988:
1989: size_used = round_page(actual2 * sizeof (**chunksp));
1990:
1991: #if MACH_OLD_VM_COPY
1992: kr = vm_map_pageable(
1993: ipc_kernel_map, addr2, addr2 + size_used, TRUE);
1994: assert(kr == KERN_SUCCESS);
1995:
1996: kr = vm_move(
1997: ipc_kernel_map, addr2,
1998: ipc_soft_map, size_used,
1999: TRUE, &memory2);
2000: assert(kr == KERN_SUCCESS);
2001: #else
2002: kr = vm_map_pageable(
2003: ipc_kernel_map,
2004: addr2, addr2 + size2, VM_PROT_NONE);
2005: assert(kr == KERN_SUCCESS);
2006:
2007: kr = vm_map_copyin(
2008: ipc_kernel_map, addr2, size_used,
2009: TRUE, &memory2);
2010: assert(kr == KERN_SUCCESS);
2011: #endif
2012:
2013: if (size_used != size2)
2014: kmem_free(
2015: ipc_kernel_map,
2016: addr2 + size_used, size2 - size_used);
2017:
2018: *chunksp = (zone_free_space_chunk_t *)memory2;
2019: }
2020: else if (actual2 == 0) {
2021: #if MACH_OLD_VM_COPY
2022: *chunksp = (zone_free_space_chunk_t *)0;
2023: #else
2024: *chunksp = (zone_free_space_chunk_t *)VM_MAP_COPY_NULL;
2025: #endif
2026:
2027: if (size2 != 0)
2028: kmem_free(ipc_kernel_map, addr2, size2);
2029: }
2030:
2031: *chunksCnt = actual2;
2032:
2033: return KERN_SUCCESS;
2034: }
2035:
2036: kern_return_t host_zone_collect(
2037: host_t host,
2038: boolean_t collect_zones,
2039: boolean_t reclaim_pages)
2040: {
2041: struct zone_free_space_entry
2042: *cur, *pages = 0;
2043: zone_t z;
2044: int max_zones, i;
2045:
2046: if (host == HOST_NULL)
2047: return KERN_INVALID_HOST;
2048:
2049: if (!collect_zones)
2050: return KERN_SUCCESS;
2051:
2052: simple_lock(simple_lock_addr(zget_space_lock));
2053:
2054: simple_lock(simple_lock_addr(all_zones_lock));
2055: max_zones = num_zones;
2056: z = first_zone;
2057: simple_unlock(simple_lock_addr(all_zones_lock));
2058:
2059: for (i = 0; i < max_zones; i++) {
2060: assert(z != ZONE_NULL);
2061: /* run this at splhigh so that interupt routines that use zones
2062: can not interupt while their zone is locked */
2063: lock_zone(z);
2064:
2065: if (!z->pageable && zone_collectable(z))
2066: zone_collect(z);
2067:
2068: unlock_zone(z);
2069: simple_lock(simple_lock_addr(all_zones_lock));
2070: z = z->next_zone;
2071: simple_unlock(simple_lock_addr(all_zones_lock));
2072: }
2073:
2074: if (reclaim_pages)
2075: pages = zone_free_space_reclaim();
2076:
2077: simple_unlock(simple_lock_addr(zget_space_lock));
2078:
2079: /*
2080: * Return any reclaimed pages to
2081: * the system.
2082: */
2083: while ((cur = pages) != 0) {
2084: pages = cur->next;
2085: kmem_free(zone_map, (vm_offset_t)cur, cur->length);
2086: }
2087:
2088: return KERN_SUCCESS;
2089: }
2090: #endif MACH_DEBUG
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.