![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to dp_backing_store.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / default_pager|
Return to dp_backing_store.c CVS log | Up to [Apple XNU] / XNU / osfmk / default_pager |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50:
51: /*
52: * Default Pager.
53: * Paging File Management.
54: */
55:
56: #include <mach/memory_object_server.h>
57: #include "default_pager_internal.h"
58: #include <default_pager/default_pager_alerts.h>
59: #include <ipc/ipc_port.h>
60: #include <ipc/ipc_space.h>
61: #include <kern/queue.h>
62: #include <kern/counters.h>
63: #include <kern/sched_prim.h>
64: #include <vm/vm_kern.h>
65: #include <vm/vm_pageout.h>
66: /* CDY CDY */
67: #include <vm/vm_map.h>
68:
69: /* MAXPHYS derived from bsd/bsd/ppc/param.h, we need a */
70: /* universal originating in the kernel, or a formal means of exporting */
71: /* from the bsd component */
72:
73: #define MAXPHYS (64 * 1024)
74: int physical_transfer_cluster_count = 0;
75:
76: #define VM_SUPER_CLUSTER 0x10000
77:
78: /*
79: * 0 means no shift to pages, so == 1 page/cluster. 1 would mean
80: * 2 pages/cluster, 2 means 4 pages/cluster, and so on.
81: */
82: #define VSTRUCT_DEF_CLSHIFT 2
83: int vstruct_def_clshift = VSTRUCT_DEF_CLSHIFT;
84: int default_pager_clsize = 0;
85:
86: /* statistics */
87: unsigned int clustered_writes[MAX_CLUSTER_SIZE+1];
88: unsigned int clustered_reads[MAX_CLUSTER_SIZE+1];
89:
90: /*
91: * Globals used for asynchronous paging operations:
92: * vs_async_list: head of list of to-be-completed I/O ops
93: * async_num_queued: number of pages completed, but not yet
94: * processed by async thread.
95: * async_requests_out: number of pages of requests not completed.
96: */
97:
98: #if 0
99: struct vs_async *vs_async_list;
100: int async_num_queued;
101: int async_requests_out;
102: #endif
103:
104:
105: #define VS_ASYNC_REUSE 1
106: struct vs_async *vs_async_free_list;
107:
108: mutex_t default_pager_async_lock; /* Protects globals above */
109:
110:
111: int vs_alloc_async_failed = 0; /* statistics */
112: int vs_alloc_async_count = 0; /* statistics */
113: struct vs_async *vs_alloc_async(void); /* forward */
114: void vs_free_async(struct vs_async *vsa); /* forward */
115:
116:
117: #define VS_ALLOC_ASYNC() vs_alloc_async()
118: #define VS_FREE_ASYNC(vsa) vs_free_async(vsa)
119:
120: #define VS_ASYNC_LOCK() mutex_lock(&default_pager_async_lock)
121: #define VS_ASYNC_UNLOCK() mutex_unlock(&default_pager_async_lock)
122: #define VS_ASYNC_LOCK_INIT() mutex_init(&default_pager_async_lock, \
123: ETAP_IO_DEV_PAGEH)
124: #define VS_ASYNC_LOCK_ADDR() (&default_pager_async_lock)
125: /*
126: * Paging Space Hysteresis triggers and the target notification port
127: *
128: */
129:
130: unsigned int minimum_pages_remaining = 0;
131: unsigned int maximum_pages_free = 0;
132: ipc_port_t min_pages_trigger_port = NULL;
133: ipc_port_t max_pages_trigger_port = NULL;
134:
135: boolean_t bs_low = FALSE;
136:
137:
138:
139: /*
140: * Object sizes are rounded up to the next power of 2,
141: * unless they are bigger than a given maximum size.
142: */
143: vm_size_t max_doubled_size = 4 * 1024 * 1024; /* 4 meg */
144:
145: /*
146: * List of all backing store and segments.
147: */
148: struct backing_store_list_head backing_store_list;
149: paging_segment_t paging_segments[MAX_NUM_PAGING_SEGMENTS];
150: mutex_t paging_segments_lock;
151: int paging_segment_max = 0;
152: int paging_segment_count = 0;
153: int ps_select_array[BS_MAXPRI+1] = { -1,-1,-1,-1,-1 };
154:
155:
156: /*
157: * Total pages free in system
158: * This differs from clusters committed/avail which is a measure of the
159: * over commitment of paging segments to backing store. An idea which is
160: * likely to be deprecated.
161: */
162: unsigned int dp_pages_free = 0;
163: unsigned int cluster_transfer_minimum = 100;
164:
165: kern_return_t ps_write_file(paging_segment_t, vm_offset_t, vm_offset_t, unsigned int); /* forward */
166:
167: default_pager_thread_t *
168: get_read_buffer()
169: {
170: int i;
171:
172: DPT_LOCK(dpt_lock);
173: while(TRUE) {
174: for (i=0; i<default_pager_internal_count; i++) {
175: if(dpt_array[i]->checked_out == FALSE) {
176: dpt_array[i]->checked_out = TRUE;
177: DPT_UNLOCK(dpt_lock);
178: return dpt_array[i];
179: }
180: }
181: assert_wait(&dpt_array, THREAD_UNINT);
182: DPT_UNLOCK(dpt_lock);
183: thread_block((void(*)(void))0);
184: }
185: }
186:
187: void
188: bs_initialize(void)
189: {
190: int i;
191:
192: /*
193: * List of all backing store.
194: */
195: BSL_LOCK_INIT();
196: queue_init(&backing_store_list.bsl_queue);
197: PSL_LOCK_INIT();
198:
199: VS_ASYNC_LOCK_INIT();
200: #if VS_ASYNC_REUSE
201: vs_async_free_list = NULL;
202: #endif /* VS_ASYNC_REUSE */
203:
204: for (i = 0; i < MAX_CLUSTER_SIZE+1; i++) {
205: clustered_writes[i] = 0;
206: clustered_reads[i] = 0;
207: }
208:
209: }
210:
211: /*
212: * When things do not quite workout...
213: */
214: void bs_no_paging_space(boolean_t); /* forward */
215:
216: void
217: bs_no_paging_space(
218: boolean_t out_of_memory)
219: {
220: static char here[] = "bs_no_paging_space";
221:
222: if (out_of_memory)
223: dprintf(("*** OUT OF MEMORY ***\n"));
224: panic("bs_no_paging_space: NOT ENOUGH PAGING SPACE");
225: }
226:
227: void bs_more_space(int); /* forward */
228: void bs_commit(int); /* forward */
229:
230: boolean_t user_warned = FALSE;
231: unsigned int clusters_committed = 0;
232: unsigned int clusters_available = 0;
233: unsigned int clusters_committed_peak = 0;
234:
235: void
236: bs_more_space(
237: int nclusters)
238: {
239: BSL_LOCK();
240: /*
241: * Account for new paging space.
242: */
243: clusters_available += nclusters;
244:
245: if (clusters_available >= clusters_committed) {
246: if (verbose && user_warned) {
247: printf("%s%s - %d excess clusters now.\n",
248: my_name,
249: "paging space is OK now",
250: clusters_available - clusters_committed);
251: user_warned = FALSE;
252: clusters_committed_peak = 0;
253: }
254: } else {
255: if (verbose && user_warned) {
256: printf("%s%s - still short of %d clusters.\n",
257: my_name,
258: "WARNING: paging space over-committed",
259: clusters_committed - clusters_available);
260: clusters_committed_peak -= nclusters;
261: }
262: }
263: BSL_UNLOCK();
264:
265: return;
266: }
267:
268: void
269: bs_commit(
270: int nclusters)
271: {
272: BSL_LOCK();
273: clusters_committed += nclusters;
274: if (clusters_committed > clusters_available) {
275: if (verbose && !user_warned) {
276: user_warned = TRUE;
277: printf("%s%s - short of %d clusters.\n",
278: my_name,
279: "WARNING: paging space over-committed",
280: clusters_committed - clusters_available);
281: }
282: if (clusters_committed > clusters_committed_peak) {
283: clusters_committed_peak = clusters_committed;
284: }
285: } else {
286: if (verbose && user_warned) {
287: printf("%s%s - was short of up to %d clusters.\n",
288: my_name,
289: "paging space is OK now",
290: clusters_committed_peak - clusters_available);
291: user_warned = FALSE;
292: clusters_committed_peak = 0;
293: }
294: }
295: BSL_UNLOCK();
296:
297: return;
298: }
299:
300: int default_pager_info_verbose = 1;
301:
302: void
303: bs_global_info(
304: vm_size_t *totalp,
305: vm_size_t *freep)
306: {
307: vm_size_t pages_total, pages_free;
308: paging_segment_t ps;
309: int i;
310: static char here[] = "bs_global_info";
311:
312: PSL_LOCK();
313: pages_total = pages_free = 0;
314: for (i = 0; i <= paging_segment_max; i++) {
315: ps = paging_segments[i];
316: if (ps == PAGING_SEGMENT_NULL)
317: continue;
318:
319: /*
320: * no need to lock: by the time this data
321: * gets back to any remote requestor it
322: * will be obsolete anyways
323: */
324: pages_total += ps->ps_pgnum;
325: pages_free += ps->ps_clcount << ps->ps_clshift;
326: DEBUG(DEBUG_BS_INTERNAL,
327: ("segment #%d: %d total, %d free\n",
328: i, ps->ps_pgnum, ps->ps_clcount << ps->ps_clshift));
329: }
330: *totalp = pages_total;
331: *freep = pages_free;
332: if (verbose && user_warned && default_pager_info_verbose) {
333: if (clusters_available < clusters_committed) {
334: printf("%s %d clusters committed, %d available.\n",
335: my_name,
336: clusters_committed,
337: clusters_available);
338: }
339: }
340: PSL_UNLOCK();
341: }
342:
343: backing_store_t backing_store_alloc(void); /* forward */
344:
345: backing_store_t
346: backing_store_alloc(void)
347: {
348: backing_store_t bs;
349: static char here[] = "backing_store_alloc";
350:
351: bs = (backing_store_t) kalloc(sizeof (struct backing_store));
352: if (bs == BACKING_STORE_NULL)
353: panic("backing_store_alloc: no memory");
354:
355: BS_LOCK_INIT(bs);
356: bs->bs_port = MACH_PORT_NULL;
357: bs->bs_priority = 0;
358: bs->bs_clsize = 0;
359: bs->bs_pages_total = 0;
360: bs->bs_pages_in = 0;
361: bs->bs_pages_in_fail = 0;
362: bs->bs_pages_out = 0;
363: bs->bs_pages_out_fail = 0;
364:
365: return bs;
366: }
367:
368: backing_store_t backing_store_lookup(MACH_PORT_FACE); /* forward */
369:
370: /* Even in both the component space and external versions of this pager, */
371: /* backing_store_lookup will be called from tasks in the application space */
372: backing_store_t
373: backing_store_lookup(
374: MACH_PORT_FACE port)
375: {
376: backing_store_t bs;
377:
378: /*
379: port is currently backed with a vs structure in the alias field
380: we could create an ISBS alias and a port_is_bs call but frankly
381: I see no reason for the test, the bs->port == port check below
382: will work properly on junk entries.
383:
384: if ((port == MACH_PORT_NULL) || port_is_vs(port))
385: */
386: if ((port == MACH_PORT_NULL))
387: return BACKING_STORE_NULL;
388:
389: BSL_LOCK();
390: queue_iterate(&backing_store_list.bsl_queue, bs, backing_store_t,
391: bs_links) {
392: BS_LOCK(bs);
393: if (bs->bs_port == port) {
394: BSL_UNLOCK();
395: /* Success, return it locked. */
396: return bs;
397: }
398: BS_UNLOCK(bs);
399: }
400: BSL_UNLOCK();
401: return BACKING_STORE_NULL;
402: }
403:
404: void backing_store_add(backing_store_t); /* forward */
405:
406: void
407: backing_store_add(
408: backing_store_t bs)
409: {
410: MACH_PORT_FACE port = bs->bs_port;
411: MACH_PORT_FACE pset = default_pager_default_set;
412: kern_return_t kr = KERN_SUCCESS;
413: static char here[] = "backing_store_add";
414:
415: if (kr != KERN_SUCCESS)
416: panic("backing_store_add: add to set");
417:
418: }
419:
420: /*
421: * Set up default page shift, but only if not already
422: * set and argument is within range.
423: */
424: boolean_t
425: bs_set_default_clsize(unsigned int npages)
426: {
427: switch(npages){
428: case 1:
429: case 2:
430: case 4:
431: case 8:
432: if (default_pager_clsize == 0) /* if not yet set */
433: vstruct_def_clshift = local_log2(npages);
434: return(TRUE);
435: }
436: return(FALSE);
437: }
438:
439: int bs_get_global_clsize(int clsize); /* forward */
440:
441: int
442: bs_get_global_clsize(
443: int clsize)
444: {
445: int i;
446: MACH_PORT_FACE DMM;
447: kern_return_t kr;
448: static char here[] = "bs_get_global_clsize";
449:
450: /*
451: * Only allow setting of cluster size once. If called
452: * with no cluster size (default), we use the compiled-in default
453: * for the duration. The same cluster size is used for all
454: * paging segments.
455: */
456: if (default_pager_clsize == 0) {
457: if (norma_mk) {
458: /*
459: * On NORMA, don't use clustered paging because
460: * XMM can't handle it.
461: */
462: vstruct_def_clshift = 0;
463: }
464: /*
465: * Keep cluster size in bit shift because it's quicker
466: * arithmetic, and easier to keep at a power of 2.
467: */
468: if (clsize != NO_CLSIZE) {
469: for (i = 0; (1 << i) < clsize; i++);
470: if (i > MAX_CLUSTER_SHIFT)
471: i = MAX_CLUSTER_SHIFT;
472: vstruct_def_clshift = i;
473: }
474: default_pager_clsize = (1 << vstruct_def_clshift);
475:
476: /*
477: * Let the user know the new (and definitive) cluster size.
478: */
479: if (verbose)
480: printf("%scluster size = %d page%s\n",
481: my_name, default_pager_clsize,
482: (default_pager_clsize == 1) ? "" : "s");
483: /*
484: * Let the kernel know too, in case it hasn't used the
485: * default value provided in main() yet.
486: */
487: DMM = default_pager_default_port;
488: clsize = default_pager_clsize * vm_page_size; /* in bytes */
489: kr = host_default_memory_manager(default_pager_host_port,
490: &DMM,
491: clsize);
492: if (kr != KERN_SUCCESS) {
493: panic("bs_get_global_cl_size:host_default_memory_manager");
494: }
495: if (DMM != default_pager_default_port) {
496: panic("bs_get_global_cl_size:there is another default pager");
497: }
498: }
499: ASSERT(default_pager_clsize > 0 &&
500: (default_pager_clsize & (default_pager_clsize - 1)) == 0);
501:
502: return default_pager_clsize;
503: }
504:
505: kern_return_t
506: default_pager_backing_store_create(
507: MACH_PORT_FACE pager,
508: int priority,
509: int clsize, /* in bytes */
510: MACH_PORT_FACE *backing_store)
511: {
512: backing_store_t bs;
513: MACH_PORT_FACE port;
514: kern_return_t kr;
515: struct vstruct_alias *alias_struct;
516: static char here[] = "default_pager_backing_store_create";
517:
518: if (pager != default_pager_default_port)
519: return KERN_INVALID_ARGUMENT;
520:
521: bs = backing_store_alloc();
522: port = ipc_port_alloc_kernel();
523: ipc_port_make_send(port);
524: assert (port != IP_NULL);
525:
526: DEBUG(DEBUG_BS_EXTERNAL,
527: ("priority=%d clsize=%d bs_port=0x%x\n",
528: priority, clsize, (int) backing_store));
529:
530: alias_struct = (struct vstruct_alias *)
531: kalloc(sizeof (struct vstruct_alias));
532: if(alias_struct != NULL) {
533: alias_struct->vs = (struct vstruct *)bs;
534: alias_struct->name = ISVS;
535: port->alias = (int) alias_struct;
536: }
537: else {
538: ipc_port_dealloc_kernel((MACH_PORT_FACE)(port));
539: kfree((vm_offset_t)bs, sizeof (struct backing_store));
540: return KERN_RESOURCE_SHORTAGE;
541: }
542:
543: bs->bs_port = port;
544: if (priority == DEFAULT_PAGER_BACKING_STORE_MAXPRI)
545: priority = BS_MAXPRI;
546: else if (priority == BS_NOPRI)
547: priority = BS_MAXPRI;
548: else
549: priority = BS_MINPRI;
550: bs->bs_priority = priority;
551:
552: bs->bs_clsize = bs_get_global_clsize(atop(clsize));
553:
554: BSL_LOCK();
555: queue_enter(&backing_store_list.bsl_queue, bs, backing_store_t,
556: bs_links);
557: BSL_UNLOCK();
558:
559: backing_store_add(bs);
560:
561: *backing_store = port;
562: return KERN_SUCCESS;
563: }
564:
565: kern_return_t
566: default_pager_backing_store_info(
567: MACH_PORT_FACE backing_store,
568: backing_store_flavor_t flavour,
569: backing_store_info_t info,
570: mach_msg_type_number_t *size)
571: {
572: backing_store_t bs;
573: backing_store_basic_info_t basic;
574: int i;
575: paging_segment_t ps;
576:
577: if (flavour != BACKING_STORE_BASIC_INFO ||
578: *size < BACKING_STORE_BASIC_INFO_COUNT)
579: return KERN_INVALID_ARGUMENT;
580:
581: basic = (backing_store_basic_info_t)info;
582: *size = BACKING_STORE_BASIC_INFO_COUNT;
583:
584: VSTATS_LOCK(&global_stats.gs_lock);
585: basic->pageout_calls = global_stats.gs_pageout_calls;
586: basic->pagein_calls = global_stats.gs_pagein_calls;
587: basic->pages_in = global_stats.gs_pages_in;
588: basic->pages_out = global_stats.gs_pages_out;
589: basic->pages_unavail = global_stats.gs_pages_unavail;
590: basic->pages_init = global_stats.gs_pages_init;
591: basic->pages_init_writes= global_stats.gs_pages_init_writes;
592: VSTATS_UNLOCK(&global_stats.gs_lock);
593:
594: if ((bs = backing_store_lookup(backing_store)) == BACKING_STORE_NULL)
595: return KERN_INVALID_ARGUMENT;
596:
597: basic->bs_pages_total = bs->bs_pages_total;
598: PSL_LOCK();
599: bs->bs_pages_free = 0;
600: for (i = 0; i <= paging_segment_max; i++) {
601: ps = paging_segments[i];
602: if (ps != PAGING_SEGMENT_NULL && ps->ps_bs == bs) {
603: PS_LOCK(ps);
604: bs->bs_pages_free += ps->ps_clcount << ps->ps_clshift;
605: PS_UNLOCK(ps);
606: }
607: }
608: PSL_UNLOCK();
609: basic->bs_pages_free = bs->bs_pages_free;
610: basic->bs_pages_in = bs->bs_pages_in;
611: basic->bs_pages_in_fail = bs->bs_pages_in_fail;
612: basic->bs_pages_out = bs->bs_pages_out;
613: basic->bs_pages_out_fail= bs->bs_pages_out_fail;
614:
615: basic->bs_priority = bs->bs_priority;
616: basic->bs_clsize = ptoa(bs->bs_clsize); /* in bytes */
617:
618: BS_UNLOCK(bs);
619:
620: return KERN_SUCCESS;
621: }
622:
623: int ps_delete(paging_segment_t); /* forward */
624:
625: int
626: ps_delete(
627: paging_segment_t ps)
628: {
629: vstruct_t vs;
630: kern_return_t error = KERN_SUCCESS;
631: int vs_count;
632:
633: VSL_LOCK(); /* get the lock on the list of vs's */
634:
635: /* The lock relationship and sequence is farily complicated */
636: /* this code looks at a live list, locking and unlocking the list */
637: /* as it traverses it. It depends on the locking behavior of */
638: /* default_pager_no_senders. no_senders always locks the vstruct */
639: /* targeted for removal before locking the vstruct list. However */
640: /* it will remove that member of the list without locking its */
641: /* neighbors. We can be sure when we hold a lock on a vstruct */
642: /* it cannot be removed from the list but we must hold the list */
643: /* lock to be sure that its pointers to its neighbors are valid. */
644: /* Also, we can hold off destruction of a vstruct when the list */
645: /* lock and the vs locks are not being held by bumping the */
646: /* vs_async_pending count. */
647:
648: /* we will choose instead to hold a send right */
649: vs_count = vstruct_list.vsl_count;
650: vs = (vstruct_t) queue_first((queue_entry_t)&(vstruct_list.vsl_queue));
651: if(vs == (vstruct_t)&vstruct_list) {
652: VSL_UNLOCK();
653: return KERN_SUCCESS;
654: }
655: VS_LOCK(vs);
656: vs_async_wait(vs); /* wait for any pending async writes */
657: if ((vs_count != 0) && (vs != NULL))
658: vs->vs_async_pending += 1; /* hold parties calling */
659: /* vs_async_wait */
660: VS_UNLOCK(vs);
661: VSL_UNLOCK();
662: while((vs_count != 0) && (vs != NULL)) {
663: /* We take the count of AMO's before beginning the */
664: /* transfer of of the target segment. */
665: /* We are guaranteed that the target segment cannot get */
666: /* more users. We also know that queue entries are */
667: /* made at the back of the list. If some of the entries */
668: /* we would check disappear while we are traversing the */
669: /* list then we will either check new entries which */
670: /* do not have any backing store in the target segment */
671: /* or re-check old entries. This might not be optimal */
672: /* but it will always be correct. The alternative is to */
673: /* take a snapshot of the list. */
674: vstruct_t next_vs;
675:
676: if(dp_pages_free < cluster_transfer_minimum)
677: error = KERN_FAILURE;
678: else {
679: vm_object_t transfer_object;
680: upl_t page_list;
681:
682: transfer_object = vm_object_allocate(VM_SUPER_CLUSTER);
683: error = vm_fault_list_request(transfer_object,
684: 0, VM_SUPER_CLUSTER, &page_list, NULL,
685: 0, UPL_NO_SYNC | UPL_CLEAN_IN_PLACE);
686: if(error == KERN_SUCCESS)
687: error = ps_vstruct_transfer_from_segment(
688: vs, ps, page_list);
689: upl_commit(page_list, NULL);
690: vm_object_deallocate(transfer_object);
691: }
692: if(error) {
693: VS_LOCK(vs);
694: vs->vs_async_pending -= 1; /* release vs_async_wait */
695: if (vs->vs_async_pending == 0) {
696: VS_UNLOCK(vs);
697: thread_wakeup(&vs->vs_waiting_async);
698: } else {
699: VS_UNLOCK(vs);
700: }
701: return KERN_FAILURE;
702: }
703:
704: VSL_LOCK();
705: next_vs = (vstruct_t) queue_next(&(vs->vs_links));
706: if((next_vs != (vstruct_t)&vstruct_list) &&
707: (vs != next_vs) && (vs_count != 1)) {
708: VS_LOCK(next_vs);
709: next_vs->vs_async_pending += 1; /* hold parties */
710: /* calling vs_async_wait */
711: VS_UNLOCK(next_vs);
712: }
713: VSL_UNLOCK();
714: VS_LOCK(vs);
715: vs->vs_async_pending -= 1;
716: if (vs->vs_async_pending == 0) {
717: VS_UNLOCK(vs);
718: thread_wakeup(&vs->vs_waiting_async);
719: } else {
720: VS_UNLOCK(vs);
721: }
722: if((vs == next_vs) || (next_vs == (vstruct_t)&vstruct_list))
723: vs = NULL;
724: else
725: vs = next_vs;
726: vs_count--;
727: }
728: return KERN_SUCCESS;
729: }
730:
731:
732: kern_return_t
733: default_pager_backing_store_delete(
734: MACH_PORT_FACE backing_store)
735: {
736: backing_store_t bs;
737: int i;
738: paging_segment_t ps;
739: int error;
740: int interim_pages_removed = 0;
741: kern_return_t kr;
742: static char here[] = "default_pager_backing_store_delete";
743:
744: if ((bs = backing_store_lookup(backing_store)) == BACKING_STORE_NULL)
745: return KERN_INVALID_ARGUMENT;
746:
747: #if 0
748: /* not implemented */
749: BS_UNLOCK(bs);
750: return KERN_FAILURE;
751: #endif
752:
753: restart:
754: PSL_LOCK();
755: error = KERN_SUCCESS;
756: for (i = 0; i <= paging_segment_max; i++) {
757: ps = paging_segments[i];
758: if (ps != PAGING_SEGMENT_NULL &&
759: ps->ps_bs == bs &&
760: ! ps->ps_going_away) {
761: PS_LOCK(ps);
762: /* disable access to this segment */
763: ps->ps_going_away = TRUE;
764: PS_UNLOCK(ps);
765: /*
766: * The "ps" segment is "off-line" now,
767: * we can try and delete it...
768: */
769: if(dp_pages_free < (cluster_transfer_minimum
770: + (ps->ps_clcount << ps->ps_clshift))) {
771: error = KERN_FAILURE;
772: PSL_UNLOCK();
773: }
774: else {
775: dp_pages_free -=
776: ps->ps_clcount << ps->ps_clshift;
777: interim_pages_removed +=
778: ps->ps_clcount << ps->ps_clshift;
779: PSL_UNLOCK();
780: error = ps_delete(ps);
781: }
782: if (error != KERN_SUCCESS) {
783: /*
784: * We couldn't delete the segment,
785: * probably because there's not enough
786: * virtual memory left.
787: * Re-enable all the segments.
788: */
789: PSL_LOCK();
790: break;
791: }
792: goto restart;
793: }
794: }
795:
796: if (error != KERN_SUCCESS) {
797: for (i = 0; i <= paging_segment_max; i++) {
798: ps = paging_segments[i];
799: if (ps != PAGING_SEGMENT_NULL &&
800: ps->ps_bs == bs &&
801: ps->ps_going_away) {
802: PS_LOCK(ps);
803: /* re-enable access to this segment */
804: ps->ps_going_away = FALSE;
805: PS_UNLOCK(ps);
806: }
807: }
808: dp_pages_free += interim_pages_removed;
809: PSL_UNLOCK();
810: BS_UNLOCK(bs);
811: return error;
812: }
813:
814: for (i = 0; i <= paging_segment_max; i++) {
815: ps = paging_segments[i];
816: if (ps != PAGING_SEGMENT_NULL &&
817: ps->ps_bs == bs) {
818: if(ps->ps_going_away) {
819: paging_segments[i] = PAGING_SEGMENT_NULL;
820: paging_segment_count--;
821: PS_LOCK(ps);
822: kfree((vm_offset_t)ps->ps_bmap,
823: RMAPSIZE(ps->ps_ncls));
824: kfree((vm_offset_t)ps, sizeof *ps);
825: }
826: }
827: }
828:
829: /* Scan the entire ps array separately to make certain we find the */
830: /* proper paging_segment_max */
831: for (i = 0; i < MAX_NUM_PAGING_SEGMENTS; i++) {
832: if(paging_segments[i] != PAGING_SEGMENT_NULL)
833: paging_segment_max = i;
834: }
835:
836: PSL_UNLOCK();
837:
838: /*
839: * All the segments have been deleted.
840: * We can remove the backing store.
841: */
842:
843: /*
844: * Disable lookups of this backing store.
845: */
846: if((void *)bs->bs_port->alias != NULL)
847: kfree((vm_offset_t) bs->bs_port->alias,
848: sizeof (struct vstruct_alias));
849: pager_mux_hash_delete((ipc_port_t) (bs->bs_port));
850: ipc_port_dealloc_kernel((ipc_port_t) (bs->bs_port));
851: bs->bs_port = MACH_PORT_NULL;
852: BS_UNLOCK(bs);
853:
854: /*
855: * Remove backing store from backing_store list.
856: */
857: BSL_LOCK();
858: queue_remove(&backing_store_list.bsl_queue, bs, backing_store_t,
859: bs_links);
860: BSL_UNLOCK();
861:
862: /*
863: * Free the backing store structure.
864: */
865: kfree((vm_offset_t)bs, sizeof *bs);
866:
867: return KERN_SUCCESS;
868: }
869:
870: int ps_enter(paging_segment_t); /* forward */
871:
872: int
873: ps_enter(
874: paging_segment_t ps)
875: {
876: int i;
877:
878: PSL_LOCK();
879:
880: for (i = 0; i < MAX_NUM_PAGING_SEGMENTS; i++) {
881: if (paging_segments[i] == PAGING_SEGMENT_NULL)
882: break;
883: }
884:
885: if (i < MAX_NUM_PAGING_SEGMENTS) {
886: paging_segments[i] = ps;
887: if (i > paging_segment_max)
888: paging_segment_max = i;
889: paging_segment_count++;
890: if ((ps_select_array[ps->ps_bs->bs_priority] == BS_NOPRI) ||
891: (ps_select_array[ps->ps_bs->bs_priority] == BS_FULLPRI))
892: ps_select_array[ps->ps_bs->bs_priority] = 0;
893: i = 0;
894: } else {
895: PSL_UNLOCK();
896: return KERN_RESOURCE_SHORTAGE;
897: }
898:
899: PSL_UNLOCK();
900: return i;
901: }
902:
903: #ifdef DEVICE_PAGING
904: kern_return_t
905: default_pager_add_segment(
906: MACH_PORT_FACE backing_store,
907: MACH_PORT_FACE device,
908: recnum_t offset,
909: recnum_t count,
910: int record_size)
911: {
912: backing_store_t bs;
913: paging_segment_t ps;
914: int i;
915: int error;
916: static char here[] = "default_pager_add_segment";
917:
918: if ((bs = backing_store_lookup(backing_store))
919: == BACKING_STORE_NULL)
920: return KERN_INVALID_ARGUMENT;
921:
922: PSL_LOCK();
923: for (i = 0; i <= paging_segment_max; i++) {
924: ps = paging_segments[i];
925: if (ps == PAGING_SEGMENT_NULL)
926: continue;
927:
928: /*
929: * Check for overlap on same device.
930: */
931: if (!(ps->ps_device != device
932: || offset >= ps->ps_offset + ps->ps_recnum
933: || offset + count <= ps->ps_offset)) {
934: PSL_UNLOCK();
935: BS_UNLOCK(bs);
936: return KERN_INVALID_ARGUMENT;
937: }
938: }
939: PSL_UNLOCK();
940:
941: /*
942: * Set up the paging segment
943: */
944: ps = (paging_segment_t) kalloc(sizeof (struct paging_segment));
945: if (ps == PAGING_SEGMENT_NULL) {
946: BS_UNLOCK(bs);
947: return KERN_RESOURCE_SHORTAGE;
948: }
949:
950: ps->ps_segtype = PS_PARTITION;
951: ps->ps_device = device;
952: ps->ps_offset = offset;
953: ps->ps_record_shift = local_log2(vm_page_size / record_size);
954: ps->ps_recnum = count;
955: ps->ps_pgnum = count >> ps->ps_record_shift;
956:
957: ps->ps_pgcount = ps->ps_pgnum;
958: ps->ps_clshift = local_log2(bs->bs_clsize);
959: ps->ps_clcount = ps->ps_ncls = ps->ps_pgcount >> ps->ps_clshift;
960: ps->ps_hint = 0;
961:
962: PS_LOCK_INIT(ps);
963: ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
964: if (!ps->ps_bmap) {
965: kfree((vm_offset_t)ps, sizeof *ps);
966: BS_UNLOCK(bs);
967: return KERN_RESOURCE_SHORTAGE;
968: }
969: for (i = 0; i < ps->ps_ncls; i++) {
970: clrbit(ps->ps_bmap, i);
971: }
972:
973: ps->ps_going_away = FALSE;
974: ps->ps_bs = bs;
975:
976: if ((error = ps_enter(ps)) != 0) {
977: kfree((vm_offset_t)ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
978: kfree((vm_offset_t)ps, sizeof *ps);
979: BS_UNLOCK(bs);
980: return KERN_RESOURCE_SHORTAGE;
981: }
982:
983: bs->bs_pages_free += ps->ps_clcount << ps->ps_clshift;
984: bs->bs_pages_total += ps->ps_clcount << ps->ps_clshift;
985: BS_UNLOCK(bs);
986:
987: PSL_LOCK();
988: dp_pages_free += ps->ps_pgcount;
989: PSL_UNLOCK();
990:
991: bs_more_space(ps->ps_clcount);
992:
993: DEBUG(DEBUG_BS_INTERNAL,
994: ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
995: device, offset, count, record_size,
996: ps->ps_record_shift, ps->ps_pgnum));
997:
998: return KERN_SUCCESS;
999: }
1000:
1001: boolean_t
1002: bs_add_device(
1003: char *dev_name,
1004: MACH_PORT_FACE master)
1005: {
1006: security_token_t null_security_token = {
1007: { 0, 0 }
1008: };
1009: MACH_PORT_FACE device;
1010: int info[DEV_GET_SIZE_COUNT];
1011: mach_msg_type_number_t info_count;
1012: MACH_PORT_FACE bs = MACH_PORT_NULL;
1013: unsigned int rec_size;
1014: recnum_t count;
1015: int clsize;
1016: MACH_PORT_FACE reply_port;
1017:
1018: if (ds_device_open_sync(master, MACH_PORT_NULL, D_READ | D_WRITE,
1019: null_security_token, dev_name, &device))
1020: return FALSE;
1021:
1022: info_count = DEV_GET_SIZE_COUNT;
1023: if (!ds_device_get_status(device, DEV_GET_SIZE, info, &info_count)) {
1024: rec_size = info[DEV_GET_SIZE_RECORD_SIZE];
1025: count = info[DEV_GET_SIZE_DEVICE_SIZE] / rec_size;
1026: clsize = bs_get_global_clsize(0);
1027: if (!default_pager_backing_store_create(
1028: default_pager_default_port,
1029: DEFAULT_PAGER_BACKING_STORE_MAXPRI,
1030: (clsize * vm_page_size),
1031: &bs)) {
1032: if (!default_pager_add_segment(bs, device,
1033: 0, count, rec_size)) {
1034: return TRUE;
1035: }
1036: ipc_port_release_receive(bs);
1037: }
1038: }
1039:
1040: ipc_port_release_send(device);
1041: return FALSE;
1042: }
1043: #endif /* DEVICE_PAGING */
1044:
1045: #if VS_ASYNC_REUSE
1046:
1047: struct vs_async *
1048: vs_alloc_async(void)
1049: {
1050: struct vs_async *vsa;
1051: MACH_PORT_FACE reply_port;
1052: kern_return_t kr;
1053:
1054: VS_ASYNC_LOCK();
1055: if (vs_async_free_list == NULL) {
1056: VS_ASYNC_UNLOCK();
1057: vsa = (struct vs_async *) kalloc(sizeof (struct vs_async));
1058: if (vsa != NULL) {
1059: /*
1060: * Try allocating a reply port named after the
1061: * address of the vs_async structure.
1062: */
1063: struct vstruct_alias *alias_struct;
1064:
1065: reply_port = ipc_port_alloc_kernel();
1066: alias_struct = (struct vstruct_alias *)
1067: kalloc(sizeof (struct vstruct_alias));
1068: if(alias_struct != NULL) {
1069: alias_struct->vs = (struct vstruct *)vsa;
1070: alias_struct->name = ISVS;
1071: reply_port->alias = (int) alias_struct;
1072: vsa->reply_port = reply_port;
1073: vs_alloc_async_count++;
1074: }
1075: else {
1076: vs_alloc_async_failed++;
1077: ipc_port_dealloc_kernel((MACH_PORT_FACE)
1078: (reply_port));
1079: kfree((vm_offset_t)vsa,
1080: sizeof (struct vs_async));
1081: vsa = NULL;
1082: }
1083: }
1084: } else {
1085: vsa = vs_async_free_list;
1086: vs_async_free_list = vs_async_free_list->vsa_next;
1087: VS_ASYNC_UNLOCK();
1088: }
1089:
1090: return vsa;
1091: }
1092:
1093: void
1094: vs_free_async(
1095: struct vs_async *vsa)
1096: {
1097: VS_ASYNC_LOCK();
1098: vsa->vsa_next = vs_async_free_list;
1099: vs_async_free_list = vsa;
1100: VS_ASYNC_UNLOCK();
1101: }
1102:
1103: #else /* VS_ASYNC_REUSE */
1104:
1105: struct vs_async *
1106: vs_alloc_async(void)
1107: {
1108: struct vs_async *vsa;
1109: MACH_PORT_FACE reply_port;
1110: kern_return_t kr;
1111:
1112: vsa = (struct vs_async *) kalloc(sizeof (struct vs_async));
1113: if (vsa != NULL) {
1114: /*
1115: * Try allocating a reply port named after the
1116: * address of the vs_async structure.
1117: */
1118: reply_port = ipc_port_alloc_kernel();
1119: alias_struct = (vstruct_alias *)
1120: kalloc(sizeof (struct vstruct_alias));
1121: if(alias_struct != NULL) {
1122: alias_struct->vs = reply_port;
1123: alias_struct->name = ISVS;
1124: reply_port->alias = (int) vsa;
1125: vsa->reply_port = reply_port;
1126: vs_alloc_async_count++;
1127: }
1128: else {
1129: vs_alloc_async_failed++;
1130: ipc_port_dealloc_kernel((MACH_PORT_FACE)
1131: (reply_port));
1132: kfree((vm_offset_t) vsa,
1133: sizeof (struct vs_async));
1134: vsa = NULL;
1135: }
1136: }
1137:
1138: return vsa;
1139: }
1140:
1141: void
1142: vs_free_async(
1143: struct vs_async *vsa)
1144: {
1145: static char here[] = "vs_free_async";
1146: MACH_PORT_FACE reply_port;
1147: kern_return_t kr;
1148:
1149: reply_port = vsa->reply_port;
1150: kfree((vm_offset_t) reply_port->alias, sizeof (struct vstuct_alias));
1151: kfree((vm_offset_t) vsa, sizeof (struct vs_async));
1152: pager_mux_hash_delete(reply_port);
1153: ipc_port_dealloc_kernel((MACH_PORT_FACE) (reply_port));
1154: #if 0
1155: VS_ASYNC_LOCK();
1156: vs_alloc_async_count--;
1157: VS_ASYNC_UNLOCK();
1158: #endif
1159: }
1160:
1161: #endif /* VS_ASYNC_REUSE */
1162:
1163: vstruct_t
1164: ps_vstruct_create(
1165: vm_size_t size)
1166: {
1167: vstruct_t vs;
1168: int i;
1169: static char here[] = "ps_vstruct_create";
1170:
1171: vs = (vstruct_t) kalloc(sizeof (struct vstruct));
1172: if (vs == VSTRUCT_NULL) {
1173: return VSTRUCT_NULL;
1174: }
1175:
1176: VS_LOCK_INIT(vs);
1177:
1178: /*
1179: * The following fields will be provided later.
1180: */
1181: vs->vs_mem_obj_port = MACH_PORT_NULL;
1182: vs->vs_seqno = 0;
1183: vs->vs_control_port = MACH_PORT_NULL;
1184: vs->vs_control_refs = 0;
1185: vs->vs_object_name = MACH_PORT_NULL;
1186: vs->vs_name_refs = 0;
1187:
1188: #ifdef MACH_KERNEL
1189: vs->vs_waiting_seqno = FALSE;
1190: vs->vs_waiting_read = FALSE;
1191: vs->vs_waiting_write = FALSE;
1192: vs->vs_waiting_refs = FALSE;
1193: vs->vs_waiting_async = FALSE;
1194: #else
1195: mutex_init(&vs->vs_waiting_seqno, ETAP_DPAGE_VSSEQNO);
1196: mutex_init(&vs->vs_waiting_read, ETAP_DPAGE_VSREAD);
1197: mutex_init(&vs->vs_waiting_write, ETAP_DPAGE_VSWRITE);
1198: mutex_init(&vs->vs_waiting_refs, ETAP_DPAGE_VSREFS);
1199: mutex_init(&vs->vs_waiting_async, ETAP_DPAGE_VSASYNC);
1200: #endif
1201:
1202: vs->vs_readers = 0;
1203: vs->vs_writers = 0;
1204:
1205: vs->vs_errors = 0;
1206:
1207: vs->vs_clshift = local_log2(bs_get_global_clsize(0));
1208: vs->vs_size = ((atop(round_page(size)) - 1) >> vs->vs_clshift) + 1;
1209: vs->vs_async_pending = 0;
1210:
1211: /*
1212: * Allocate the pmap, either CLMAP_SIZE or INDIRECT_CLMAP_SIZE
1213: * depending on the size of the memory object.
1214: */
1215: if (INDIRECT_CLMAP(vs->vs_size)) {
1216: vs->vs_imap = (struct vs_map **)
1217: kalloc(INDIRECT_CLMAP_SIZE(vs->vs_size));
1218: vs->vs_indirect = TRUE;
1219: } else {
1220: vs->vs_dmap = (struct vs_map *)
1221: kalloc(CLMAP_SIZE(vs->vs_size));
1222: vs->vs_indirect = FALSE;
1223: }
1224: vs->vs_xfer_pending = FALSE;
1225: DEBUG(DEBUG_VS_INTERNAL,
1226: ("map=0x%x, indirect=%d\n", (int) vs->vs_dmap, vs->vs_indirect));
1227:
1228: /*
1229: * Check to see that we got the space.
1230: */
1231: if (!vs->vs_dmap) {
1232: kfree((vm_offset_t)vs, sizeof *vs);
1233: return VSTRUCT_NULL;
1234: }
1235:
1236: /*
1237: * Zero the indirect pointers, or clear the direct pointers.
1238: */
1239: if (vs->vs_indirect)
1240: memset(vs->vs_imap, 0,
1241: INDIRECT_CLMAP_SIZE(vs->vs_size));
1242: else
1243: for (i = 0; i < vs->vs_size; i++)
1244: VSM_CLR(vs->vs_dmap[i]);
1245:
1246: VS_MAP_LOCK_INIT(vs);
1247:
1248: bs_commit(vs->vs_size);
1249:
1250: return vs;
1251: }
1252:
1253: paging_segment_t ps_select_segment(int, int *); /* forward */
1254:
1255: paging_segment_t
1256: ps_select_segment(
1257: int shift,
1258: int *psindex)
1259: {
1260: paging_segment_t ps;
1261: int i;
1262: int j;
1263: static char here[] = "ps_select_segment";
1264:
1265: /*
1266: * Optimize case where there's only one segment.
1267: * paging_segment_max will index the one and only segment.
1268: */
1269:
1270: PSL_LOCK();
1271: if (paging_segment_count == 1) {
1272: paging_segment_t lps; /* used to avoid extra PS_UNLOCK */
1273:
1274: ps = paging_segments[paging_segment_max];
1275: *psindex = paging_segment_max;
1276: PS_LOCK(ps);
1277: if (ps->ps_going_away) {
1278: /* this segment is being turned off */
1279: lps = PAGING_SEGMENT_NULL;
1280: } else {
1281: ASSERT(ps->ps_clshift >= shift);
1282: if (ps->ps_clcount) {
1283: ps->ps_clcount--;
1284: dp_pages_free -= 1 << ps->ps_clshift;
1285: if(min_pages_trigger_port &&
1286: (dp_pages_free < minimum_pages_remaining)) {
1287: default_pager_space_alert(
1288: min_pages_trigger_port,
1289: HI_WAT_ALERT);
1290: min_pages_trigger_port = NULL;
1291: bs_low = TRUE;
1292: }
1293: lps = ps;
1294: } else
1295: lps = PAGING_SEGMENT_NULL;
1296: }
1297: PS_UNLOCK(ps);
1298: PSL_UNLOCK();
1299: return lps;
1300: }
1301:
1302: if (paging_segment_count == 0) {
1303: PSL_UNLOCK();
1304: return PAGING_SEGMENT_NULL;
1305: }
1306:
1307: for (i = BS_MAXPRI;
1308: i >= BS_MINPRI; i--) {
1309: int start_index;
1310:
1311: if ((ps_select_array[i] == BS_NOPRI) ||
1312: (ps_select_array[i] == BS_FULLPRI))
1313: continue;
1314: start_index = ps_select_array[i];
1315:
1316: if(!(paging_segments[start_index])) {
1317: j = start_index+1;
1318: physical_transfer_cluster_count = 0;
1319: }
1320: else if ((physical_transfer_cluster_count+1) == (MAXPHYS >>
1321: (((paging_segments[start_index])->ps_clshift)
1322: + page_shift))) {
1323: physical_transfer_cluster_count = 0;
1324: j = start_index + 1;
1325: } else {
1326: physical_transfer_cluster_count+=1;
1327: j = start_index;
1328: if(start_index == 0)
1329: start_index = paging_segment_max;
1330: else
1331: start_index = start_index - 1;
1332: }
1333:
1334: while (1) {
1335: if (j > paging_segment_max)
1336: j = 0;
1337: if ((ps = paging_segments[j]) &&
1338: (ps->ps_bs->bs_priority == i)) {
1339: /*
1340: * Force the ps cluster size to be
1341: * >= that of the vstruct.
1342: */
1343: PS_LOCK(ps);
1344: if (ps->ps_going_away) {
1345: /* this segment is being turned off */
1346: } else if ((ps->ps_clcount) &&
1347: (ps->ps_clshift >= shift)) {
1348: ps->ps_clcount--;
1349: dp_pages_free -= 1 << ps->ps_clshift;
1350: if(min_pages_trigger_port &&
1351: (dp_pages_free <
1352: minimum_pages_remaining)) {
1353: default_pager_space_alert(
1354: min_pages_trigger_port,
1355: HI_WAT_ALERT);
1356: min_pages_trigger_port = NULL;
1357: }
1358: PS_UNLOCK(ps);
1359: /*
1360: * found one, quit looking.
1361: */
1362: ps_select_array[i] = j;
1363: PSL_UNLOCK();
1364: *psindex = j;
1365: return ps;
1366: }
1367: PS_UNLOCK(ps);
1368: }
1369: if (j == start_index) {
1370: /*
1371: * none at this priority -- mark it full
1372: */
1373: ps_select_array[i] = BS_FULLPRI;
1374: break;
1375: }
1376: j++;
1377: }
1378: }
1379: PSL_UNLOCK();
1380: return PAGING_SEGMENT_NULL;
1381: }
1382:
1383: vm_offset_t ps_allocate_cluster(vstruct_t, int *, paging_segment_t); /*forward*/
1384:
1385: vm_offset_t
1386: ps_allocate_cluster(
1387: vstruct_t vs,
1388: int *psindex,
1389: paging_segment_t use_ps)
1390: {
1391: int byte_num;
1392: int bit_num = 0;
1393: paging_segment_t ps;
1394: vm_offset_t cluster;
1395: static char here[] = "ps_allocate_cluster";
1396:
1397: /*
1398: * Find best paging segment.
1399: * ps_select_segment will decrement cluster count on ps.
1400: * Must pass cluster shift to find the most appropriate segment.
1401: */
1402: /* NOTE: The addition of paging segment delete capability threatened
1403: * to seriously complicate the treatment of paging segments in this
1404: * module and the ones that call it (notably ps_clmap), because of the
1405: * difficulty in assuring that the paging segment would continue to
1406: * exist between being unlocked and locked. This was
1407: * avoided because all calls to this module are based in either
1408: * dp_memory_object calls which rely on the vs lock, or by
1409: * the transfer function which is part of the segment delete path.
1410: * The transfer function which is part of paging segment delete is
1411: * protected from multiple callers by the backing store lock.
1412: * The paging segment delete function treats mappings to a paging
1413: * segment on a vstruct by vstruct basis, locking the vstruct targeted
1414: * while data is transferred to the remaining segments. This is in
1415: * line with the view that incomplete or in-transition mappings between
1416: * data, a vstruct, and backing store are protected by the vs lock.
1417: * This and the ordering of the paging segment "going_away" bit setting
1418: * protects us.
1419: */
1420: if (use_ps != PAGING_SEGMENT_NULL) {
1421: ps = use_ps;
1422: PSL_LOCK();
1423: PS_LOCK(ps);
1424: ps->ps_clcount--;
1425: dp_pages_free -= 1 << ps->ps_clshift;
1426: PSL_UNLOCK();
1427: if(min_pages_trigger_port &&
1428: (dp_pages_free < minimum_pages_remaining)) {
1429: default_pager_space_alert(
1430: min_pages_trigger_port,
1431: HI_WAT_ALERT);
1432: min_pages_trigger_port = NULL;
1433: }
1434: PS_UNLOCK(ps);
1435: } else if ((ps = ps_select_segment(vs->vs_clshift, psindex)) ==
1436: PAGING_SEGMENT_NULL) {
1437: #if 0
1438: bs_no_paging_space(TRUE);
1439: #endif
1440: #if 0
1441: if (verbose)
1442: #endif
1443: dprintf(("no space in available paging segments; "
1444: "swapon suggested\n"));
1445: return (vm_offset_t) -1;
1446: }
1447: ASSERT(ps->ps_clcount != 0);
1448:
1449: /*
1450: * Look for an available cluster. At the end of the loop,
1451: * byte_num is the byte offset and bit_num is the bit offset of the
1452: * first zero bit in the paging segment bitmap.
1453: */
1454: PS_LOCK(ps);
1455: byte_num = ps->ps_hint;
1456: for (; byte_num < howmany(ps->ps_ncls, NBBY); byte_num++) {
1457: if (*(ps->ps_bmap + byte_num) != BYTEMASK) {
1458: for (bit_num = 0; bit_num < NBBY; bit_num++) {
1459: if (isclr((ps->ps_bmap + byte_num), bit_num))
1460: break;
1461: }
1462: ASSERT(bit_num != NBBY);
1463: break;
1464: }
1465: }
1466: ps->ps_hint = byte_num;
1467: cluster = (byte_num*NBBY) + bit_num;
1468:
1469: /* Space was reserved, so this must be true */
1470: ASSERT(cluster < ps->ps_ncls);
1471:
1472: setbit(ps->ps_bmap, cluster);
1473: PS_UNLOCK(ps);
1474:
1475: return cluster;
1476: }
1477:
1478: void ps_deallocate_cluster(paging_segment_t, vm_offset_t); /* forward */
1479:
1480: void
1481: ps_deallocate_cluster(
1482: paging_segment_t ps,
1483: vm_offset_t cluster)
1484: {
1485:
1486: if (cluster >= (vm_offset_t) ps->ps_ncls)
1487: panic("ps_deallocate_cluster: Invalid cluster number");
1488:
1489: /*
1490: * Lock the paging segment, clear the cluster's bitmap and increment the
1491: * number of free cluster.
1492: */
1493: PSL_LOCK();
1494: PS_LOCK(ps);
1495: clrbit(ps->ps_bmap, cluster);
1496: ++ps->ps_clcount;
1497: dp_pages_free += 1 << ps->ps_clshift;
1498: PSL_UNLOCK();
1499: if(max_pages_trigger_port && (dp_pages_free > maximum_pages_free)) {
1500: default_pager_space_alert(max_pages_trigger_port, LO_WAT_ALERT);
1501: max_pages_trigger_port = NULL;
1502: }
1503:
1504: /*
1505: * Move the hint down to the freed cluster if it is
1506: * less than the current hint.
1507: */
1508: if ((cluster/NBBY) < ps->ps_hint) {
1509: ps->ps_hint = (cluster/NBBY);
1510: }
1511:
1512: PS_UNLOCK(ps);
1513:
1514: /*
1515: * If we're freeing space on a full priority, reset the array.
1516: */
1517: PSL_LOCK();
1518: if (ps_select_array[ps->ps_bs->bs_priority] == BS_FULLPRI)
1519: ps_select_array[ps->ps_bs->bs_priority] = 0;
1520: PSL_UNLOCK();
1521:
1522: return;
1523: }
1524:
1525: void ps_dealloc_vsmap(struct vs_map *, vm_size_t); /* forward */
1526:
1527: void
1528: ps_dealloc_vsmap(
1529: struct vs_map *vsmap,
1530: vm_size_t size)
1531: {
1532: int i;
1533: for (i = 0; i < size; i++)
1534: if (!VSM_ISCLR(vsmap[i]) && !VSM_ISERR(vsmap[i]))
1535: ps_deallocate_cluster(VSM_PS(vsmap[i]),
1536: VSM_CLOFF(vsmap[i]));
1537: }
1538:
1539: void
1540: ps_vstruct_dealloc(
1541: vstruct_t vs)
1542: {
1543: int i;
1544: static char here[] = "ps_vstruct_dealloc";
1545:
1546: VS_MAP_LOCK(vs);
1547:
1548: /*
1549: * If this is an indirect structure, then we walk through the valid
1550: * (non-zero) indirect pointers and deallocate the clusters
1551: * associated with each used map entry (via ps_dealloc_vsmap).
1552: * When all of the clusters in an indirect block have been
1553: * freed, we deallocate the block. When all of the indirect
1554: * blocks have been deallocated we deallocate the memory
1555: * holding the indirect pointers.
1556: */
1557: if (vs->vs_indirect) {
1558: for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
1559: if (vs->vs_imap[i] != NULL) {
1560: ps_dealloc_vsmap(vs->vs_imap[i], CLMAP_ENTRIES);
1561: kfree((vm_offset_t)vs->vs_imap[i],
1562: CLMAP_THRESHOLD);
1563: }
1564: }
1565: kfree((vm_offset_t)vs->vs_imap,
1566: INDIRECT_CLMAP_SIZE(vs->vs_size));
1567: } else {
1568: /*
1569: * Direct map. Free used clusters, then memory.
1570: */
1571: ps_dealloc_vsmap(vs->vs_dmap, vs->vs_size);
1572: kfree((vm_offset_t)vs->vs_dmap, CLMAP_SIZE(vs->vs_size));
1573: }
1574: VS_MAP_UNLOCK(vs);
1575:
1576: bs_commit(- vs->vs_size);
1577:
1578: ip_lock(vs_to_port(vs));
1579: (vs_to_port(vs))->ip_destination = 0;
1580: (vs_to_port(vs))->ip_receiver_name = MACH_PORT_NULL;
1581: imq_lock(&vs_to_port(vs)->ip_messages);
1582: (vs_to_port(vs))->ip_mscount = 0;
1583: (vs_to_port(vs))->ip_messages.imq_seqno = 0;
1584: imq_unlock(&vs_to_port(vs)->ip_messages);
1585: ip_unlock(vs_to_port(vs));
1586: pager_mux_hash_delete((ipc_port_t) vs_to_port(vs));
1587: ipc_port_release_receive(vs_to_port(vs));
1588: /*
1589: * Do this *after* deallocating the port name
1590: */
1591: kfree((vm_offset_t)vs, sizeof *vs);
1592: }
1593:
1594: int ps_map_extend(vstruct_t, int); /* forward */
1595:
1596: int ps_map_extend(
1597: vstruct_t vs,
1598: int new_size)
1599: {
1600: struct vs_map **new_imap;
1601: struct vs_map *new_dmap = NULL;
1602: int newdsize;
1603: int i;
1604: void *old_map = NULL;
1605: int old_map_size = 0;
1606:
1607: if (vs->vs_size >= new_size) {
1608: /*
1609: * Someone has already done the work.
1610: */
1611: return 0;
1612: }
1613:
1614: /*
1615: * If the new size extends into the indirect range, then we have one
1616: * of two cases: we are going from indirect to indirect, or we are
1617: * going from direct to indirect. If we are going from indirect to
1618: * indirect, then it is possible that the new size will fit in the old
1619: * indirect map. If this is the case, then just reset the size of the
1620: * vstruct map and we are done. If the new size will not
1621: * fit into the old indirect map, then we have to allocate a new
1622: * indirect map and copy the old map pointers into this new map.
1623: *
1624: * If we are going from direct to indirect, then we have to allocate a
1625: * new indirect map and copy the old direct pages into the first
1626: * indirect page of the new map.
1627: * NOTE: allocating memory here is dangerous, as we're in the
1628: * pageout path.
1629: */
1630: if (INDIRECT_CLMAP(new_size)) {
1631: int new_map_size = INDIRECT_CLMAP_SIZE(new_size);
1632:
1633: /*
1634: * Get a new indirect map and zero it.
1635: */
1636: old_map_size = INDIRECT_CLMAP_SIZE(vs->vs_size);
1637: if (vs->vs_indirect &&
1638: (new_map_size == old_map_size)) {
1639: bs_commit(new_size - vs->vs_size);
1640: vs->vs_size = new_size;
1641: return 0;
1642: }
1643:
1644: new_imap = (struct vs_map **)kalloc(new_map_size);
1645: if (new_imap == NULL) {
1646: return -1;
1647: }
1648: memset(new_imap, 0, new_map_size);
1649:
1650: if (vs->vs_indirect) {
1651: /* Copy old entries into new map */
1652: memcpy(new_imap, vs->vs_imap, old_map_size);
1653: /* Arrange to free the old map */
1654: old_map = (void *) vs->vs_imap;
1655: newdsize = 0;
1656: } else { /* Old map was a direct map */
1657: /* Allocate an indirect page */
1658: if ((new_imap[0] = (struct vs_map *)
1659: kalloc(CLMAP_THRESHOLD)) == NULL) {
1660: kfree((vm_offset_t)new_imap, new_map_size);
1661: return -1;
1662: }
1663: new_dmap = new_imap[0];
1664: newdsize = CLMAP_ENTRIES;
1665: }
1666: } else {
1667: new_imap = NULL;
1668: newdsize = new_size;
1669: /*
1670: * If the new map is a direct map, then the old map must
1671: * also have been a direct map. All we have to do is
1672: * to allocate a new direct map, copy the old entries
1673: * into it and free the old map.
1674: */
1675: if ((new_dmap = (struct vs_map *)
1676: kalloc(CLMAP_SIZE(new_size))) == NULL) {
1677: return -1;
1678: }
1679: }
1680: if (newdsize) {
1681:
1682: /* Free the old map */
1683: old_map = (void *) vs->vs_dmap;
1684: old_map_size = CLMAP_SIZE(vs->vs_size);
1685:
1686: /* Copy info from the old map into the new map */
1687: memcpy(new_dmap, vs->vs_dmap, old_map_size);
1688:
1689: /* Initialize the rest of the new map */
1690: for (i = vs->vs_size; i < newdsize; i++)
1691: VSM_CLR(new_dmap[i]);
1692: }
1693: if (new_imap) {
1694: vs->vs_imap = new_imap;
1695: vs->vs_indirect = TRUE;
1696: } else
1697: vs->vs_dmap = new_dmap;
1698: bs_commit(new_size - vs->vs_size);
1699: vs->vs_size = new_size;
1700: if (old_map)
1701: kfree((vm_offset_t)old_map, old_map_size);
1702: return 0;
1703: }
1704:
1705: vm_offset_t
1706: ps_clmap(
1707: vstruct_t vs,
1708: vm_offset_t offset,
1709: struct clmap *clmap,
1710: int flag,
1711: vm_size_t size,
1712: int error)
1713: {
1714: vm_offset_t cluster; /* The cluster of offset. */
1715: vm_offset_t newcl; /* The new cluster allocated. */
1716: vm_offset_t newoff;
1717: int i;
1718: struct vs_map *vsmap;
1719: static char here[] = "ps_clmap";
1720:
1721: VS_MAP_LOCK(vs);
1722:
1723: ASSERT(vs->vs_dmap);
1724: cluster = atop(offset) >> vs->vs_clshift;
1725:
1726: /*
1727: * Initialize cluster error value
1728: */
1729: clmap->cl_error = 0;
1730:
1731: /*
1732: * If the object has grown, extend the page map.
1733: */
1734: if (cluster >= vs->vs_size) {
1735: if (flag == CL_FIND) {
1736: /* Do not allocate if just doing a lookup */
1737: VS_MAP_UNLOCK(vs);
1738: return (vm_offset_t) -1;
1739: }
1740: if (ps_map_extend(vs, cluster + 1)) {
1741: VS_MAP_UNLOCK(vs);
1742: return (vm_offset_t) -1;
1743: }
1744: }
1745:
1746: /*
1747: * Look for the desired cluster. If the map is indirect, then we
1748: * have a two level lookup. First find the indirect block, then
1749: * find the actual cluster. If the indirect block has not yet
1750: * been allocated, then do so. If the cluster has not yet been
1751: * allocated, then do so.
1752: *
1753: * If any of the allocations fail, then return an error.
1754: * Don't allocate if just doing a lookup.
1755: */
1756: if (vs->vs_indirect) {
1757: long ind_block = cluster/CLMAP_ENTRIES;
1758:
1759: /* Is the indirect block allocated? */
1760: vsmap = vs->vs_imap[ind_block];
1761: if (vsmap == NULL) {
1762: if (flag == CL_FIND) {
1763: VS_MAP_UNLOCK(vs);
1764: return (vm_offset_t) -1;
1765: }
1766:
1767: /* Allocate the indirect block */
1768: vsmap = (struct vs_map *) kalloc(CLMAP_THRESHOLD);
1769: if (vsmap == NULL) {
1770: VS_MAP_UNLOCK(vs);
1771: return (vm_offset_t) -1;
1772: }
1773: /* Initialize the cluster offsets */
1774: for (i = 0; i < CLMAP_ENTRIES; i++)
1775: VSM_CLR(vsmap[i]);
1776: vs->vs_imap[ind_block] = vsmap;
1777: }
1778: } else
1779: vsmap = vs->vs_dmap;
1780:
1781: ASSERT(vsmap);
1782: vsmap += cluster%CLMAP_ENTRIES;
1783:
1784: /*
1785: * At this point, vsmap points to the struct vs_map desired.
1786: *
1787: * Look in the map for the cluster, if there was an error on a
1788: * previous write, flag it and return. If it is not yet
1789: * allocated, then allocate it, if we're writing; if we're
1790: * doing a lookup and the cluster's not allocated, return error.
1791: */
1792: if (VSM_ISERR(*vsmap)) {
1793: clmap->cl_error = VSM_GETERR(*vsmap);
1794: VS_MAP_UNLOCK(vs);
1795: return (vm_offset_t) -1;
1796: } else if (VSM_ISCLR(*vsmap)) {
1797: int psindex;
1798:
1799: if (flag == CL_FIND) {
1800: /*
1801: * If there's an error and the entry is clear, then
1802: * we've run out of swap space. Record the error
1803: * here and return.
1804: */
1805: if (error) {
1806: VSM_SETERR(*vsmap, error);
1807: }
1808: VS_MAP_UNLOCK(vs);
1809: return (vm_offset_t) -1;
1810: } else {
1811: /*
1812: * Attempt to allocate a cluster from the paging segment
1813: */
1814: newcl = ps_allocate_cluster(vs, &psindex,
1815: PAGING_SEGMENT_NULL);
1816: if (newcl == -1) {
1817: VS_MAP_UNLOCK(vs);
1818: return (vm_offset_t) -1;
1819: }
1820: VSM_CLR(*vsmap);
1821: VSM_SETCLOFF(*vsmap, newcl);
1822: VSM_SETPS(*vsmap, psindex);
1823: }
1824: } else
1825: newcl = VSM_CLOFF(*vsmap);
1826:
1827: /*
1828: * Fill in pertinent fields of the clmap
1829: */
1830: clmap->cl_ps = VSM_PS(*vsmap);
1831: clmap->cl_numpages = VSCLSIZE(vs);
1832: clmap->cl_bmap.clb_map = (unsigned int) VSM_BMAP(*vsmap);
1833:
1834: /*
1835: * Byte offset in paging segment is byte offset to cluster plus
1836: * byte offset within cluster. It looks ugly, but should be
1837: * relatively quick.
1838: */
1839: ASSERT(trunc_page(offset) == offset);
1840: newcl = ptoa(newcl) << vs->vs_clshift;
1841: newoff = offset & ((1<<(vm_page_shift + vs->vs_clshift)) - 1);
1842: if (flag == CL_ALLOC) {
1843: /*
1844: * set bits in the allocation bitmap according to which
1845: * pages were requested. size is in bytes.
1846: */
1847: i = atop(newoff);
1848: while ((size > 0) && (i < VSCLSIZE(vs))) {
1849: VSM_SETALLOC(*vsmap, i);
1850: i++;
1851: size -= vm_page_size;
1852: }
1853: }
1854: clmap->cl_alloc.clb_map = (unsigned int) VSM_ALLOC(*vsmap);
1855: if (newoff) {
1856: /*
1857: * Offset is not cluster aligned, so number of pages
1858: * and bitmaps must be adjusted
1859: */
1860: clmap->cl_numpages -= atop(newoff);
1861: CLMAP_SHIFT(clmap, vs);
1862: CLMAP_SHIFTALLOC(clmap, vs);
1863: }
1864:
1865: /*
1866: *
1867: * The setting of valid bits and handling of write errors
1868: * must be done here, while we hold the lock on the map.
1869: * It logically should be done in ps_vs_write_complete().
1870: * The size and error information has been passed from
1871: * ps_vs_write_complete(). If the size parameter is non-zero,
1872: * then there is work to be done. If error is also non-zero,
1873: * then the error number is recorded in the cluster and the
1874: * entire cluster is in error.
1875: */
1876: if (size && flag == CL_FIND) {
1877: vm_offset_t off = (vm_offset_t) 0;
1878:
1879: if (!error) {
1880: for (i = VSCLSIZE(vs) - clmap->cl_numpages; size > 0;
1881: i++) {
1882: VSM_SETPG(*vsmap, i);
1883: size -= vm_page_size;
1884: }
1885: ASSERT(i <= VSCLSIZE(vs));
1886: } else {
1887: BS_STAT(clmap->cl_ps->ps_bs,
1888: clmap->cl_ps->ps_bs->bs_pages_out_fail +=
1889: atop(size));
1890: off = VSM_CLOFF(*vsmap);
1891: VSM_SETERR(*vsmap, error);
1892: }
1893: /*
1894: * Deallocate cluster if error, and no valid pages
1895: * already present.
1896: */
1897: if (off != (vm_offset_t) 0)
1898: ps_deallocate_cluster(clmap->cl_ps, off);
1899: VS_MAP_UNLOCK(vs);
1900: return (vm_offset_t) 0;
1901: } else
1902: VS_MAP_UNLOCK(vs);
1903:
1904: DEBUG(DEBUG_VS_INTERNAL,
1905: ("returning 0x%X,vs=0x%X,vsmap=0x%X,flag=%d\n",
1906: newcl+newoff, (int) vs, (int) vsmap, flag));
1907: DEBUG(DEBUG_VS_INTERNAL,
1908: (" clmap->cl_ps=0x%X,cl_numpages=%d,clbmap=0x%x,cl_alloc=%x\n",
1909: (int) clmap->cl_ps, clmap->cl_numpages,
1910: (int) clmap->cl_bmap.clb_map, (int) clmap->cl_alloc.clb_map));
1911:
1912: return (newcl + newoff);
1913: }
1914:
1915: void ps_clunmap(vstruct_t, vm_offset_t, vm_size_t); /* forward */
1916:
1917: void
1918: ps_clunmap(
1919: vstruct_t vs,
1920: vm_offset_t offset,
1921: vm_size_t length)
1922: {
1923: vm_offset_t cluster; /* The cluster number of offset */
1924: struct vs_map *vsmap;
1925: static char here[] = "ps_clunmap";
1926:
1927: VS_MAP_LOCK(vs);
1928:
1929: /*
1930: * Loop through all clusters in this range, freeing paging segment
1931: * clusters and map entries as encountered.
1932: */
1933: while (length > 0) {
1934: vm_offset_t newoff;
1935: int i;
1936:
1937: cluster = atop(offset) >> vs->vs_clshift;
1938: if (vs->vs_indirect) /* indirect map */
1939: vsmap = vs->vs_imap[cluster/CLMAP_ENTRIES];
1940: else
1941: vsmap = vs->vs_dmap;
1942: if (vsmap == NULL) {
1943: VS_MAP_UNLOCK(vs);
1944: return;
1945: }
1946: vsmap += cluster%CLMAP_ENTRIES;
1947: if (VSM_ISCLR(*vsmap)) {
1948: length -= vm_page_size;
1949: offset += vm_page_size;
1950: continue;
1951: }
1952: /*
1953: * We've got a valid mapping. Clear it and deallocate
1954: * paging segment cluster pages.
1955: * Optimize for entire cluster cleraing.
1956: */
1957: if (newoff = (offset&((1<<(vm_page_shift+vs->vs_clshift))-1))) {
1958: /*
1959: * Not cluster aligned.
1960: */
1961: ASSERT(trunc_page(newoff) == newoff);
1962: i = atop(newoff);
1963: } else
1964: i = 0;
1965: while ((i < VSCLSIZE(vs)) && (length > 0)) {
1966: VSM_CLRPG(*vsmap, i);
1967: VSM_CLRALLOC(*vsmap, i);
1968: length -= vm_page_size;
1969: offset += vm_page_size;
1970: i++;
1971: }
1972:
1973: /*
1974: * If map entry is empty, clear and deallocate cluster.
1975: */
1976: if (!VSM_ALLOC(*vsmap)) {
1977: ps_deallocate_cluster(VSM_PS(*vsmap),
1978: VSM_CLOFF(*vsmap));
1979: VSM_CLR(*vsmap);
1980: }
1981: }
1982:
1983: VS_MAP_UNLOCK(vs);
1984: }
1985:
1986: void ps_vs_write_complete(vstruct_t, vm_offset_t, vm_size_t, int); /* forward */
1987:
1988: void
1989: ps_vs_write_complete(
1990: vstruct_t vs,
1991: vm_offset_t offset,
1992: vm_size_t size,
1993: int error)
1994: {
1995: struct clmap clmap;
1996:
1997: /*
1998: * Get the struct vsmap for this cluster.
1999: * Use READ, even though it was written, because the
2000: * cluster MUST be present, unless there was an error
2001: * in the original ps_clmap (e.g. no space), in which
2002: * case, nothing happens.
2003: *
2004: * Must pass enough information to ps_clmap to allow it
2005: * to set the vs_map structure bitmap under lock.
2006: */
2007: (void) ps_clmap(vs, offset, &clmap, CL_FIND, size, error);
2008: }
2009:
2010: void vs_cl_write_complete(vstruct_t, paging_segment_t, vm_offset_t, vm_offset_t, vm_size_t, boolean_t, int); /* forward */
2011:
2012: void
2013: vs_cl_write_complete(
2014: vstruct_t vs,
2015: paging_segment_t ps,
2016: vm_offset_t offset,
2017: vm_offset_t addr,
2018: vm_size_t size,
2019: boolean_t async,
2020: int error)
2021: {
2022: static char here[] = "vs_cl_write_complete";
2023: kern_return_t kr;
2024:
2025: if (error) {
2026: /*
2027: * For internal objects, the error is recorded on a
2028: * per-cluster basis by ps_clmap() which is called
2029: * by ps_vs_write_complete() below.
2030: */
2031: dprintf(("write failed error = 0x%x\n", error));
2032: /* add upl_abort code here */
2033: } else
2034: GSTAT(global_stats.gs_pages_out += atop(size));
2035: /*
2036: * Notify the vstruct mapping code, so it can do its accounting.
2037: */
2038: ps_vs_write_complete(vs, offset, size, error);
2039:
2040: if (async) {
2041: VS_LOCK(vs);
2042: ASSERT(vs->vs_async_pending > 0);
2043: vs->vs_async_pending -= size;
2044: if (vs->vs_async_pending == 0) {
2045: VS_UNLOCK(vs);
2046: /* mutex_unlock(&vs->vs_waiting_async); */
2047: thread_wakeup(&vs->vs_waiting_async);
2048: } else {
2049: VS_UNLOCK(vs);
2050: }
2051: }
2052: }
2053:
2054: #ifdef DEVICE_PAGING
2055: kern_return_t device_write_reply(MACH_PORT_FACE, kern_return_t, io_buf_len_t);
2056:
2057: kern_return_t
2058: device_write_reply(
2059: MACH_PORT_FACE reply_port,
2060: kern_return_t device_code,
2061: io_buf_len_t bytes_written)
2062: {
2063: struct vs_async *vsa;
2064: static char here[] = "device_write_reply";
2065:
2066: vsa = (struct vs_async *)
2067: ((struct vstruct_alias *)(reply_port->alias))->vs;
2068:
2069: if (device_code == KERN_SUCCESS && bytes_written != vsa->vsa_size) {
2070: device_code = KERN_FAILURE;
2071: }
2072:
2073: vsa->vsa_error = device_code;
2074:
2075:
2076: ASSERT(vsa->vsa_vs != VSTRUCT_NULL);
2077: if(vsa->vsa_flags & VSA_TRANSFER) {
2078: /* revisit when async disk segments redone */
2079: if(vsa->vsa_error) {
2080: /* need to consider error condition. re-write data or */
2081: /* throw it away here. */
2082: vm_offset_t ioaddr;
2083: if(vm_map_copyout(kernel_map, &ioaddr,
2084: (vm_map_copy_t)vsa->vsa_addr) != KERN_SUCCESS)
2085: panic("vs_cluster_write: unable to copy source list\n");
2086: vm_deallocate(kernel_map, ioaddr, vsa->vsa_size);
2087: }
2088: ps_vs_write_complete(vsa->vsa_vs, vsa->vsa_offset,
2089: vsa->vsa_size, vsa->vsa_error);
2090: } else {
2091: vs_cl_write_complete(vsa->vsa_vs, vsa->vsa_ps, vsa->vsa_offset,
2092: vsa->vsa_addr, vsa->vsa_size, TRUE,
2093: vsa->vsa_error);
2094: }
2095: VS_FREE_ASYNC(vsa);
2096:
2097: return KERN_SUCCESS;
2098: }
2099:
2100: kern_return_t device_write_reply_inband(MACH_PORT_FACE, kern_return_t, io_buf_len_t);
2101: kern_return_t
2102: device_write_reply_inband(
2103: MACH_PORT_FACE reply_port,
2104: kern_return_t return_code,
2105: io_buf_len_t bytes_written)
2106: {
2107: panic("device_write_reply_inband: illegal");
2108: return KERN_SUCCESS;
2109: }
2110:
2111: kern_return_t device_read_reply(MACH_PORT_FACE, kern_return_t, io_buf_ptr_t, mach_msg_type_number_t);
2112: kern_return_t
2113: device_read_reply(
2114: MACH_PORT_FACE reply_port,
2115: kern_return_t return_code,
2116: io_buf_ptr_t data,
2117: mach_msg_type_number_t dataCnt)
2118: {
2119: struct vs_async *vsa;
2120: vsa = (struct vs_async *)
2121: ((struct vstruct_alias *)(reply_port->alias))->vs;
2122: vsa->vsa_addr = (vm_offset_t)data;
2123: vsa->vsa_size = (vm_size_t)dataCnt;
2124: vsa->vsa_error = return_code;
2125: thread_wakeup(&vsa->vsa_lock);
2126: return KERN_SUCCESS;
2127: }
2128:
2129: kern_return_t device_read_reply_inband(MACH_PORT_FACE, kern_return_t, io_buf_ptr_inband_t, mach_msg_type_number_t);
2130: kern_return_t
2131: device_read_reply_inband(
2132: MACH_PORT_FACE reply_port,
2133: kern_return_t return_code,
2134: io_buf_ptr_inband_t data,
2135: mach_msg_type_number_t dataCnt)
2136: {
2137: panic("device_read_reply_inband: illegal");
2138: return KERN_SUCCESS;
2139: }
2140:
2141: kern_return_t device_read_reply_overwrite(MACH_PORT_FACE, kern_return_t, io_buf_len_t);
2142: kern_return_t
2143: device_read_reply_overwrite(
2144: MACH_PORT_FACE reply_port,
2145: kern_return_t return_code,
2146: io_buf_len_t bytes_read)
2147: {
2148: panic("device_read_reply_overwrite: illegal\n");
2149: return KERN_SUCCESS;
2150: }
2151:
2152: kern_return_t device_open_reply(MACH_PORT_FACE, kern_return_t, MACH_PORT_FACE);
2153: kern_return_t
2154: device_open_reply(
2155: MACH_PORT_FACE reply_port,
2156: kern_return_t return_code,
2157: MACH_PORT_FACE device_port)
2158: {
2159: panic("device_open_reply: illegal\n");
2160: return KERN_SUCCESS;
2161: }
2162:
2163: kern_return_t ps_read_device(paging_segment_t, vm_offset_t, vm_offset_t *, unsigned int, unsigned int *); /* forward */
2164:
2165: kern_return_t
2166: ps_read_device(
2167: paging_segment_t ps,
2168: vm_offset_t offset,
2169: vm_offset_t *bufferp,
2170: unsigned int size,
2171: unsigned int *residualp)
2172: {
2173: kern_return_t kr;
2174: recnum_t dev_offset;
2175: unsigned int bytes_wanted;
2176: unsigned int bytes_read;
2177: unsigned int total_read;
2178: vm_offset_t dev_buffer;
2179: vm_offset_t buf_ptr;
2180: unsigned int records_read;
2181: static char here[] = "ps_read_device";
2182: struct vs_async *vsa;
2183: mutex_t vs_waiting_read_reply;
2184:
2185: device_t device;
2186: vm_map_copy_t device_data = NULL;
2187: default_pager_thread_t *dpt = NULL;
2188:
2189: device = dev_port_lookup(ps->ps_device);
2190: clustered_reads[atop(size)]++;
2191:
2192: dev_offset = (ps->ps_offset +
2193: (offset >> (vm_page_shift - ps->ps_record_shift)));
2194: bytes_wanted = size;
2195: total_read = 0;
2196: *bufferp = (vm_offset_t)NULL;
2197:
2198: do {
2199: vsa = VS_ALLOC_ASYNC();
2200: if (vsa) {
2201: vsa->vsa_vs = NULL;
2202: vsa->vsa_addr = 0;
2203: vsa->vsa_offset = 0;
2204: vsa->vsa_size = 0;
2205: vsa->vsa_ps = NULL;
2206: }
2207: mutex_init(&vsa->vsa_lock, ETAP_DPAGE_VSSEQNO);
2208: ip_lock(vsa->reply_port);
2209: vsa->reply_port->ip_sorights++;
2210: ip_reference(vsa->reply_port);
2211: ip_unlock(vsa->reply_port);
2212: kr = ds_device_read_common(device,
2213: vsa->reply_port,
2214: (mach_msg_type_name_t)
2215: MACH_MSG_TYPE_MOVE_SEND_ONCE,
2216: (dev_mode_t) 0,
2217: dev_offset,
2218: bytes_wanted,
2219: (IO_READ | IO_CALL),
2220: (io_buf_ptr_t *) &dev_buffer,
2221: (mach_msg_type_number_t *) &bytes_read);
2222: if(kr == MIG_NO_REPLY) {
2223: assert_wait(&vsa->vsa_lock, THREAD_UNINT);
2224: thread_block((void(*)(void))0);
2225:
2226: dev_buffer = vsa->vsa_addr;
2227: bytes_read = (unsigned int)vsa->vsa_size;
2228: kr = vsa->vsa_error;
2229: }
2230: VS_FREE_ASYNC(vsa);
2231: if (kr != KERN_SUCCESS || bytes_read == 0) {
2232: break;
2233: }
2234: total_read += bytes_read;
2235:
2236: /*
2237: * If we got the entire range, use the returned dev_buffer.
2238: */
2239: if (bytes_read == size) {
2240: *bufferp = (vm_offset_t)dev_buffer;
2241: break;
2242: }
2243:
2244: #if 1
2245: dprintf(("read only %d bytes out of %d\n",
2246: bytes_read, bytes_wanted));
2247: #endif
2248: if(dpt == NULL) {
2249: dpt = get_read_buffer();
2250: buf_ptr = dpt->dpt_buffer;
2251: *bufferp = (vm_offset_t)buf_ptr;
2252: }
2253: /*
2254: * Otherwise, copy the data into the provided buffer (*bufferp)
2255: * and append the rest of the range as it comes in.
2256: */
2257: memcpy((void *) buf_ptr, (void *) dev_buffer, bytes_read);
2258: buf_ptr += bytes_read;
2259: bytes_wanted -= bytes_read;
2260: records_read = (bytes_read >>
2261: (vm_page_shift - ps->ps_record_shift));
2262: dev_offset += records_read;
2263: DEBUG(DEBUG_VS_INTERNAL,
2264: ("calling vm_deallocate(addr=0x%X,size=0x%X)\n",
2265: dev_buffer, bytes_read));
2266: if (vm_deallocate(kernel_map, dev_buffer, bytes_read)
2267: != KERN_SUCCESS)
2268: Panic("dealloc buf");
2269: } while (bytes_wanted);
2270:
2271: *residualp = size - total_read;
2272: if((dev_buffer != *bufferp) && (total_read != 0)) {
2273: vm_offset_t temp_buffer;
2274: vm_allocate(kernel_map, &temp_buffer, total_read, TRUE);
2275: memcpy((void *) temp_buffer, (void *) *bufferp, total_read);
2276: if(vm_map_copyin_page_list(kernel_map, temp_buffer, total_read,
2277: VM_MAP_COPYIN_OPT_SRC_DESTROY |
2278: VM_MAP_COPYIN_OPT_STEAL_PAGES |
2279: VM_MAP_COPYIN_OPT_PMAP_ENTER,
2280: (vm_map_copy_t *)&device_data, FALSE))
2281: panic("ps_read_device: cannot copyin locally provided buffer\n");
2282: }
2283: else if((kr == KERN_SUCCESS) && (total_read != 0) && (dev_buffer != 0)){
2284: if(vm_map_copyin_page_list(kernel_map, dev_buffer, bytes_read,
2285: VM_MAP_COPYIN_OPT_SRC_DESTROY |
2286: VM_MAP_COPYIN_OPT_STEAL_PAGES |
2287: VM_MAP_COPYIN_OPT_PMAP_ENTER,
2288: (vm_map_copy_t *)&device_data, FALSE))
2289: panic("ps_read_device: cannot copyin backing store provided buffer\n");
2290: }
2291: else {
2292: device_data = NULL;
2293: }
2294: *bufferp = (vm_offset_t)device_data;
2295:
2296: if(dpt != NULL) {
2297: /* Free the receive buffer */
2298: dpt->checked_out = 0;
2299: thread_wakeup(&dpt_array);
2300: }
2301: return KERN_SUCCESS;
2302: }
2303:
2304: kern_return_t ps_write_device(paging_segment_t, vm_offset_t, vm_offset_t, unsigned int, struct vs_async *); /* forward */
2305:
2306: kern_return_t
2307: ps_write_device(
2308: paging_segment_t ps,
2309: vm_offset_t offset,
2310: vm_offset_t addr,
2311: unsigned int size,
2312: struct vs_async *vsa)
2313: {
2314: recnum_t dev_offset;
2315: io_buf_len_t bytes_to_write, bytes_written;
2316: recnum_t records_written;
2317: kern_return_t kr;
2318: MACH_PORT_FACE reply_port;
2319: static char here[] = "ps_write_device";
2320:
2321:
2322:
2323: clustered_writes[atop(size)]++;
2324:
2325: dev_offset = (ps->ps_offset +
2326: (offset >> (vm_page_shift - ps->ps_record_shift)));
2327: bytes_to_write = size;
2328:
2329: if (vsa) {
2330: /*
2331: * Asynchronous write.
2332: */
2333: reply_port = vsa->reply_port;
2334: ip_lock(reply_port);
2335: reply_port->ip_sorights++;
2336: ip_reference(reply_port);
2337: ip_unlock(reply_port);
2338: {
2339: device_t device;
2340: device = dev_port_lookup(ps->ps_device);
2341:
2342: vsa->vsa_addr = addr;
2343: kr=ds_device_write_common(device,
2344: reply_port,
2345: (mach_msg_type_name_t) MACH_MSG_TYPE_MOVE_SEND_ONCE,
2346: (dev_mode_t) 0,
2347: dev_offset,
2348: (io_buf_ptr_t) addr,
2349: size,
2350: (IO_WRITE | IO_CALL),
2351: &bytes_written);
2352: }
2353: if ((kr != KERN_SUCCESS) && (kr != MIG_NO_REPLY)) {
2354: if (verbose)
2355: dprintf(("%s0x%x, addr=0x%x,"
2356: "size=0x%x,offset=0x%x\n",
2357: "device_write_request returned ",
2358: kr, addr, size, offset));
2359: BS_STAT(ps->ps_bs,
2360: ps->ps_bs->bs_pages_out_fail += atop(size));
2361: /* do the completion notification to free resources */
2362: device_write_reply(reply_port, kr, 0);
2363: return PAGER_ERROR;
2364: }
2365: } else do {
2366: /*
2367: * Synchronous write.
2368: */
2369: {
2370: device_t device;
2371: device = dev_port_lookup(ps->ps_device);
2372: kr=ds_device_write_common(device,
2373: IP_NULL, 0,
2374: (dev_mode_t) 0,
2375: dev_offset,
2376: (io_buf_ptr_t) addr,
2377: size,
2378: (IO_WRITE | IO_SYNC | IO_KERNEL_BUF),
2379: &bytes_written);
2380: }
2381: if (kr != KERN_SUCCESS) {
2382: dprintf(("%s0x%x, addr=0x%x,size=0x%x,offset=0x%x\n",
2383: "device_write returned ",
2384: kr, addr, size, offset));
2385: BS_STAT(ps->ps_bs,
2386: ps->ps_bs->bs_pages_out_fail += atop(size));
2387: return PAGER_ERROR;
2388: }
2389: if (bytes_written & ((vm_page_size >> ps->ps_record_shift) - 1))
2390: Panic("fragmented write");
2391: records_written = (bytes_written >>
2392: (vm_page_shift - ps->ps_record_shift));
2393: dev_offset += records_written;
2394: #if 1
2395: if (bytes_written != bytes_to_write) {
2396: dprintf(("wrote only %d bytes out of %d\n",
2397: bytes_written, bytes_to_write));
2398: }
2399: #endif
2400: bytes_to_write -= bytes_written;
2401: addr += bytes_written;
2402: } while (bytes_to_write > 0);
2403:
2404: return PAGER_SUCCESS;
2405: }
2406:
2407:
2408: #else /* !DEVICE_PAGING */
2409:
2410: kern_return_t
2411: ps_read_device(
2412: paging_segment_t ps,
2413: vm_offset_t offset,
2414: vm_offset_t *bufferp,
2415: unsigned int size,
2416: unsigned int *residualp)
2417: {
2418: panic("ps_read_device not supported");
2419: }
2420:
2421: ps_write_device(
2422: paging_segment_t ps,
2423: vm_offset_t offset,
2424: vm_offset_t addr,
2425: unsigned int size,
2426: struct vs_async *vsa)
2427: {
2428: panic("ps_write_device not supported");
2429: }
2430:
2431: #endif /* DEVICE_PAGING */
2432: void pvs_object_data_provided(vstruct_t, upl_t, vm_offset_t, vm_size_t); /* forward */
2433:
2434: void
2435: pvs_object_data_provided(
2436: vstruct_t vs,
2437: upl_t page_list,
2438: vm_offset_t offset,
2439: vm_size_t size)
2440: {
2441: static char here[] = "pvs_object_data_provided";
2442:
2443: DEBUG(DEBUG_VS_INTERNAL,
2444: ("buffer=0x%x,offset=0x%x,size=0x%x\n",
2445: page_list, offset, size));
2446:
2447: ASSERT(size > 0);
2448: GSTAT(global_stats.gs_pages_in += atop(size));
2449:
2450:
2451: #if USE_PRECIOUS
2452: ps_clunmap(vs, offset, size);
2453: #endif /* USE_PRECIOUS */
2454:
2455: if(size < (page_list->size)) {
2456: upl_commit_range(page_list, 0, size, FALSE, NULL);
2457: } else {
2458: upl_commit(page_list, NULL);
2459: }
2460: }
2461:
2462: kern_return_t
2463: pvs_cluster_read(
2464: vstruct_t vs,
2465: vm_offset_t offset,
2466: vm_size_t cnt)
2467: {
2468: vm_offset_t actual_offset;
2469: vm_offset_t buffer;
2470: paging_segment_t ps;
2471: struct clmap clmap;
2472: upl_t page_list;
2473: kern_return_t error = KERN_SUCCESS;
2474: int size, size_wanted, i;
2475: unsigned int residual;
2476: unsigned int request_flags;
2477: int unavail_size;
2478: default_pager_thread_t *dpt;
2479: boolean_t dealloc;
2480: static char here[] = "pvs_cluster_read";
2481:
2482: /*
2483: * This loop will be executed once per cluster referenced.
2484: * Typically this means once, since it's unlikely that the
2485: * VM system will ask for anything spanning cluster boundaries.
2486: *
2487: * If there are holes in a cluster (in a paging segment), we stop
2488: * reading at the hole, inform the VM of any data read, inform
2489: * the VM of an unavailable range, then loop again, hoping to
2490: * find valid pages later in the cluster. This continues until
2491: * the entire range has been examined, and read, if present.
2492: */
2493: #if USE_PRECIOUS
2494: request_flags = UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_PRECIOUS;
2495: #else
2496: request_flags = UPL_NO_SYNC | UPL_CLEAN_IN_PLACE;
2497: #endif
2498: while (cnt && (error == KERN_SUCCESS)) {
2499: actual_offset = ps_clmap(vs, offset, &clmap, CL_FIND, 0, 0);
2500:
2501: if (actual_offset == (vm_offset_t) -1) {
2502:
2503: /*
2504: * Either a failure due to an error on a previous
2505: * write or a zero fill on demand page. In either case,
2506: * optimize to do one reply for all pages up to next
2507: * cluster boundary.
2508: */
2509: unsigned int local_size, clmask, clsize;
2510:
2511: clmask = (vm_page_size << vs->vs_clshift) - 1;
2512: clsize = vm_page_size << vs->vs_clshift;
2513: clmask = clsize - 1;
2514: local_size = clsize - (offset & clmask);
2515: ASSERT(local_size);
2516: local_size = MIN(local_size, cnt);
2517:
2518: upl_system_list_request((vm_object_t)
2519: vs->vs_control_port->ip_kobject,
2520: offset, local_size, local_size,
2521: &page_list, NULL, 0, request_flags);
2522: if (clmap.cl_error) {
2523: upl_abort(page_list, UPL_ABORT_ERROR);
2524: } else {
2525: upl_abort(page_list, UPL_ABORT_UNAVAILABLE);
2526: }
2527:
2528: cnt -= local_size;
2529: offset += local_size;
2530: continue;
2531: }
2532:
2533: /*
2534: * Count up contiguous available or unavailable
2535: * pages.
2536: */
2537: ps = CLMAP_PS(clmap);
2538: ASSERT(ps);
2539: size = 0;
2540: unavail_size = 0;
2541: for (i = 0;
2542: (size < cnt) && (unavail_size < cnt) &&
2543: (i < CLMAP_NPGS(clmap)); i++) {
2544: if (CLMAP_ISSET(clmap, i)) {
2545: if (unavail_size != 0)
2546: break;
2547: size += vm_page_size;
2548: BS_STAT(ps->ps_bs,
2549: ps->ps_bs->bs_pages_in++);
2550: } else {
2551: if (size != 0)
2552: break;
2553: unavail_size += vm_page_size;
2554: }
2555: }
2556: /*
2557: * Let VM system know about holes in clusters.
2558: */
2559: if (size == 0) {
2560: ASSERT(unavail_size);
2561: GSTAT(global_stats.gs_pages_unavail +=
2562: atop(unavail_size));
2563: upl_system_list_request((vm_object_t)
2564: vs->vs_control_port->ip_kobject,
2565: offset, unavail_size,
2566: unavail_size, &page_list, NULL, 0,
2567: request_flags);
2568: upl_abort(page_list, UPL_ABORT_UNAVAILABLE);
2569: cnt -= unavail_size;
2570: offset += unavail_size;
2571: continue;
2572: }
2573:
2574: upl_system_list_request((vm_object_t)
2575: vs->vs_control_port->ip_kobject,
2576: offset, size, size, &page_list,
2577: NULL, 0, request_flags);
2578: if(ps->ps_segtype == PS_PARTITION) {
2579: /*
2580: error = ps_read_device(ps, actual_offset, page_list,
2581: size, &residual);
2582: */
2583: } else {
2584: vm_offset_t ioaddr;
2585:
2586: upl_map(kernel_map, page_list, &ioaddr);
2587: error = ps_read_file(ps, actual_offset, ioaddr,
2588: size, &residual);
2589: upl_un_map(kernel_map, page_list);
2590: }
2591:
2592: /*
2593: * Adjust counts and send response to VM. Optimize for the
2594: * common case, i.e. no error and/or partial data.
2595: * If there was an error, then we need to error the entire
2596: * range, even if some data was successfully read.
2597: * If there was a partial read we may supply some
2598: * data and may error some as well. In all cases the
2599: * VM must receive some notification for every page in the
2600: * range.
2601: */
2602: if ((error == KERN_SUCCESS) && (residual == 0)) {
2603: /*
2604: * Got everything we asked for, supply the data to
2605: * the VM. Note that as a side effect of supplying
2606: * the data, the buffer holding the supplied data is
2607: * deallocated from the pager's address space.
2608: */
2609: pvs_object_data_provided(vs, page_list, offset, size);
2610: } else {
2611: size_wanted = size;
2612: if (error == KERN_SUCCESS) {
2613: if (residual == size) {
2614: /*
2615: * If a read operation returns no error
2616: * and no data moved, we turn it into
2617: * an error, assuming we're reading at
2618: * or beyong EOF.
2619: * Fall through and error the entire
2620: * range.
2621: */
2622: error = KERN_FAILURE;
2623: } else {
2624: /*
2625: * Otherwise, we have partial read. If
2626: * the part read is a integral number
2627: * of pages supply it. Otherwise round
2628: * it up to a page boundary, zero fill
2629: * the unread part, and supply it.
2630: * Fall through and error the remainder
2631: * of the range, if any.
2632: */
2633: int fill, lsize;
2634:
2635: fill = residual & ~vm_page_size;
2636: lsize = (size - residual) + fill;
2637: pvs_object_data_provided(vs, page_list,
2638: offset, lsize);
2639: cnt -= lsize;
2640: offset += lsize;
2641: if (size -= lsize) {
2642: error = KERN_FAILURE;
2643: }
2644: }
2645: }
2646:
2647: /*
2648: * If there was an error in any part of the range, tell
2649: * the VM. Deallocate the remainder of the buffer.
2650: * Note that error is explicitly checked again since
2651: * it can be modified above.
2652: */
2653: if (error != KERN_SUCCESS) {
2654: BS_STAT(ps->ps_bs,
2655: ps->ps_bs->bs_pages_in_fail +=
2656: atop(size));
2657: upl_abort(page_list, UPL_ABORT_RESTART);
2658: }
2659: }
2660: cnt -= size;
2661: offset += size;
2662:
2663: } /* END while (cnt && (error == 0)) */
2664: return error;
2665: }
2666:
2667: int vs_do_async_write = 1;
2668:
2669: kern_return_t
2670: vs_cluster_write(
2671: vstruct_t vs,
2672: vm_offset_t offset,
2673: vm_offset_t addr,
2674: vm_size_t cnt,
2675: boolean_t dp_internal)
2676: {
2677: vm_offset_t actual_offset; /* Offset within paging segment */
2678: vm_offset_t size;
2679: vm_offset_t transfer_size;
2680: vm_offset_t subx_size;
2681: int error = 0;
2682: struct clmap clmap;
2683: paging_segment_t ps;
2684: struct vs_async *vsa;
2685: vm_map_copy_t copy;
2686: static char here[] = "vs_cluster_write";
2687:
2688:
2689: upl_t upl;
2690:
2691: vm_offset_t mapped_data;
2692: vm_offset_t mobj_base_addr;
2693: vm_offset_t mobj_target_addr;
2694: int mobj_size;
2695: vm_offset_t vs_map_error_addr = 0;
2696: int page_index;
2697: int list_size;
2698: int cl_size;
2699:
2700: upl_page_info_t page_list[20];
2701: upl_page_info_t *pl;
2702:
2703:
2704: ps = PAGING_SEGMENT_NULL;
2705: if(!dp_internal) {
2706: int request_flags;
2707: int super_size;
2708:
2709: cl_size = (1 << vs->vs_clshift) * vm_page_size;
2710: if(bs_low) {
2711: super_size = cl_size;
2712: request_flags = UPL_NOBLOCK |
2713: UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM |
2714: UPL_NO_SYNC;
2715: } else {
2716: super_size = VM_SUPER_CLUSTER;
2717: request_flags = UPL_NOBLOCK | UPL_CLEAN_IN_PLACE |
2718: UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM |
2719: UPL_NO_SYNC;
2720: }
2721:
2722:
2723: pl = page_list;
2724: upl_system_list_request((vm_object_t)
2725: vs->vs_control_port->ip_kobject,
2726: offset, cnt, super_size,
2727: &upl, &pl,
2728: 20, request_flags);
2729:
2730: mobj_base_addr = upl->offset;
2731: list_size = upl->size;
2732:
2733: upl_map(kernel_map, upl, &mapped_data);
2734:
2735: /* Now parcel up the 64k transfer, do at most cluster size */
2736: /* at a time. */
2737: page_index = 0;
2738: mobj_target_addr = mobj_base_addr;
2739: for(transfer_size = list_size; transfer_size != 0;) {
2740: actual_offset = ps_clmap(vs, mobj_target_addr,
2741: &clmap, CL_ALLOC,
2742: transfer_size < cl_size ?
2743: transfer_size : cl_size, 0);
2744: if(actual_offset == (vm_offset_t) -1) {
2745: vs_map_error_addr = mobj_target_addr;
2746: error = 1;
2747: break;
2748: }
2749: cnt = MIN(transfer_size,
2750: CLMAP_NPGS(clmap) * vm_page_size);
2751: ps = CLMAP_PS(clmap);
2752: while(cnt > 0) {
2753: /* attempt to send entire cluster */
2754: subx_size = 0;
2755: while(cnt > 0) {
2756: /* do the biggest contiguous transfer of dirty */
2757: /* pages */
2758: if(UPL_DIRTY_PAGE(page_list, page_index) ||
2759: ((bs_low) &&
2760: UPL_PRECIOUS_PAGE(page_list, page_index))) {
2761: page_index++;
2762: subx_size += vm_page_size;
2763: cnt -= vm_page_size;
2764: } else {
2765: if(subx_size == 0) {
2766: actual_offset += vm_page_size;
2767: mobj_target_addr += vm_page_size;
2768: mapped_data += vm_page_size;
2769: transfer_size -= vm_page_size;
2770: page_index++;
2771: cnt -= vm_page_size;
2772: } else {
2773: break;
2774: }
2775: }
2776: }
2777: if(subx_size) {
2778: error = ps_write_file(ps, actual_offset,
2779: mapped_data, subx_size);
2780: if(error) {
2781: vs_map_error_addr = mobj_target_addr;
2782: }
2783: ps_vs_write_complete(vs, mobj_target_addr,
2784: subx_size, error);
2785: }
2786: if(error)
2787: break;
2788: actual_offset += subx_size;
2789: mobj_target_addr += subx_size;
2790: mapped_data += subx_size;
2791: transfer_size -= subx_size;
2792: subx_size = 0;
2793: }
2794: if(error)
2795: break;
2796: }
2797: upl_un_map(kernel_map, upl);
2798: if(error) {
2799: if(vs_map_error_addr > mobj_base_addr) {
2800: upl_commit_range(upl, 0,
2801: vs_map_error_addr - mobj_base_addr,
2802: FALSE, page_list);
2803: }
2804: upl_abort(upl, 0);
2805: } else {
2806: upl_commit(upl, page_list);
2807: }
2808: } else {
2809: assert(cnt <= (vm_page_size << vs->vs_clshift));
2810: mapped_data = addr;
2811: list_size = cnt;
2812:
2813: page_index = 0;
2814: /* The caller provides a mapped_data which is derived */
2815: /* from a temporary object. The targeted pages are */
2816: /* guaranteed to be set at offset 0 in the mapped_data */
2817: /* The actual offset however must still be derived */
2818: /* from the offset in the vs in question */
2819: mobj_base_addr = offset;
2820: mobj_target_addr = mobj_base_addr;
2821: for(transfer_size = list_size; transfer_size != 0;) {
2822: actual_offset = ps_clmap(vs, mobj_target_addr,
2823: &clmap, CL_ALLOC,
2824: transfer_size < cl_size ?
2825: transfer_size : cl_size, 0);
2826: if(actual_offset == (vm_offset_t) -1) {
2827: error = KERN_FAILURE;
2828: break;
2829: }
2830: cnt = MIN(transfer_size,
2831: CLMAP_NPGS(clmap) * vm_page_size);
2832: ps = CLMAP_PS(clmap);
2833: /* Assume that the caller has given us contiguous */
2834: /* pages */
2835: if(cnt) {
2836: error = ps_write_file(ps, actual_offset,
2837: mapped_data, cnt);
2838: ps_vs_write_complete(vs, mobj_target_addr,
2839: cnt, error);
2840: }
2841: actual_offset += cnt;
2842: mobj_target_addr += cnt;
2843: mapped_data += cnt;
2844: transfer_size -= cnt;
2845: cnt = 0;
2846:
2847: if(error)
2848: break;
2849: }
2850: }
2851: if(error)
2852: return KERN_FAILURE;
2853: else
2854: return KERN_SUCCESS;
2855: }
2856:
2857: vm_size_t
2858: ps_vstruct_allocated_size(
2859: vstruct_t vs)
2860: {
2861: int num_pages;
2862: struct vs_map *vsmap;
2863: int i, j, k;
2864:
2865: num_pages = 0;
2866: if (vs->vs_indirect) {
2867: /* loop on indirect maps */
2868: for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
2869: vsmap = vs->vs_imap[i];
2870: if (vsmap == NULL)
2871: continue;
2872: /* loop on clusters in this indirect map */
2873: for (j = 0; j < CLMAP_ENTRIES; j++) {
2874: if (VSM_ISCLR(vsmap[j]) ||
2875: VSM_ISERR(vsmap[j]))
2876: continue;
2877: /* loop on pages in this cluster */
2878: for (k = 0; k < VSCLSIZE(vs); k++) {
2879: if ((VSM_BMAP(vsmap[j])) & (1 << k))
2880: num_pages++;
2881: }
2882: }
2883: }
2884: } else {
2885: vsmap = vs->vs_dmap;
2886: if (vsmap == NULL)
2887: return 0;
2888: /* loop on clusters in the direct map */
2889: for (j = 0; j < CLMAP_ENTRIES; j++) {
2890: if (VSM_ISCLR(vsmap[j]) ||
2891: VSM_ISERR(vsmap[j]))
2892: continue;
2893: /* loop on pages in this cluster */
2894: for (k = 0; k < VSCLSIZE(vs); k++) {
2895: if ((VSM_BMAP(vsmap[j])) & (1 << k))
2896: num_pages++;
2897: }
2898: }
2899: }
2900:
2901: return ptoa(num_pages);
2902: }
2903:
2904: size_t
2905: ps_vstruct_allocated_pages(
2906: vstruct_t vs,
2907: default_pager_page_t *pages,
2908: size_t pages_size)
2909: {
2910: int num_pages;
2911: struct vs_map *vsmap;
2912: vm_offset_t offset;
2913: int i, j, k;
2914:
2915: num_pages = 0;
2916: offset = 0;
2917: if (vs->vs_indirect) {
2918: /* loop on indirect maps */
2919: for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
2920: vsmap = vs->vs_imap[i];
2921: if (vsmap == NULL) {
2922: offset += (vm_page_size * CLMAP_ENTRIES *
2923: VSCLSIZE(vs));
2924: continue;
2925: }
2926: /* loop on clusters in this indirect map */
2927: for (j = 0; j < CLMAP_ENTRIES; j++) {
2928: if (VSM_ISCLR(vsmap[j]) ||
2929: VSM_ISERR(vsmap[j])) {
2930: offset += vm_page_size * VSCLSIZE(vs);
2931: continue;
2932: }
2933: /* loop on pages in this cluster */
2934: for (k = 0; k < VSCLSIZE(vs); k++) {
2935: if ((VSM_BMAP(vsmap[j])) & (1 << k)) {
2936: num_pages++;
2937: if (num_pages < pages_size)
2938: pages++->dpp_offset =
2939: offset;
2940: }
2941: offset += vm_page_size;
2942: }
2943: }
2944: }
2945: } else {
2946: vsmap = vs->vs_dmap;
2947: if (vsmap == NULL)
2948: return 0;
2949: /* loop on clusters in the direct map */
2950: for (j = 0; j < CLMAP_ENTRIES; j++) {
2951: if (VSM_ISCLR(vsmap[j]) ||
2952: VSM_ISERR(vsmap[j])) {
2953: offset += vm_page_size * VSCLSIZE(vs);
2954: continue;
2955: }
2956: /* loop on pages in this cluster */
2957: for (k = 0; k < VSCLSIZE(vs); k++) {
2958: if ((VSM_BMAP(vsmap[j])) & (1 << k)) {
2959: num_pages++;
2960: if (num_pages < pages_size)
2961: pages++->dpp_offset = offset;
2962: }
2963: offset += vm_page_size;
2964: }
2965: }
2966: }
2967:
2968: return num_pages;
2969: }
2970:
2971:
2972: kern_return_t
2973: ps_vstruct_transfer_from_segment(
2974: vstruct_t vs,
2975: paging_segment_t segment,
2976: upl_t page_list)
2977: {
2978: struct vs_map *vsmap;
2979: struct vs_map old_vsmap;
2980: struct vs_map new_vsmap;
2981: int i, j, k;
2982:
2983: VS_LOCK(vs); /* block all work on this vstruct */
2984: /* can't allow the normal multiple write */
2985: /* semantic because writes may conflict */
2986: vs->vs_xfer_pending = TRUE;
2987: vs_wait_for_sync_writers(vs);
2988: vs_start_write(vs);
2989: vs_wait_for_readers(vs);
2990: /* we will unlock the vs to allow other writes while transferring */
2991: /* and will be guaranteed of the persistance of the vs struct */
2992: /* because the caller of ps_vstruct_transfer_from_segment bumped */
2993: /* vs_async_pending */
2994: /* OK we now have guaranteed no other parties are accessing this */
2995: /* vs. Now that we are also supporting simple lock versions of */
2996: /* vs_lock we cannot hold onto VS_LOCK as we may block below. */
2997: /* our purpose in holding it before was the multiple write case */
2998: /* we now use the boolean xfer_pending to do that. We can use */
2999: /* a boolean instead of a count because we have guaranteed single */
3000: /* file access to this code in its caller */
3001: VS_UNLOCK(vs);
3002: vs_changed:
3003: if (vs->vs_indirect) {
3004: int vsmap_size;
3005: int clmap_off;
3006: /* loop on indirect maps */
3007: for (i = 0; i < INDIRECT_CLMAP_ENTRIES(vs->vs_size); i++) {
3008: vsmap = vs->vs_imap[i];
3009: if (vsmap == NULL)
3010: continue;
3011: /* loop on clusters in this indirect map */
3012: clmap_off = (vm_page_size * CLMAP_ENTRIES *
3013: VSCLSIZE(vs) * i);
3014: if(i+1 == INDIRECT_CLMAP_ENTRIES(vs->vs_size))
3015: vsmap_size = vs->vs_size - (CLMAP_ENTRIES * i);
3016: else
3017: vsmap_size = CLMAP_ENTRIES;
3018: for (j = 0; j < vsmap_size; j++) {
3019: if (VSM_ISCLR(vsmap[j]) ||
3020: VSM_ISERR(vsmap[j]) ||
3021: (VSM_PS(vsmap[j]) != segment))
3022: continue;
3023: if(vs_cluster_transfer(vs,
3024: (vm_page_size * (j << vs->vs_clshift))
3025: + clmap_off,
3026: vm_page_size << vs->vs_clshift,
3027: page_list)
3028: != KERN_SUCCESS) {
3029: VS_LOCK(vs);
3030: vs->vs_xfer_pending = FALSE;
3031: VS_UNLOCK(vs);
3032: vs_finish_write(vs);
3033: return KERN_FAILURE;
3034: }
3035: /* allow other readers/writers during transfer*/
3036: VS_LOCK(vs);
3037: vs->vs_xfer_pending = FALSE;
3038: VS_UNLOCK(vs);
3039: vs_finish_write(vs);
3040: VS_LOCK(vs);
3041: vs->vs_xfer_pending = TRUE;
3042: VS_UNLOCK(vs);
3043: vs_wait_for_sync_writers(vs);
3044: vs_start_write(vs);
3045: vs_wait_for_readers(vs);
3046: if (!(vs->vs_indirect)) {
3047: goto vs_changed;
3048: }
3049: }
3050: }
3051: } else {
3052: vsmap = vs->vs_dmap;
3053: if (vsmap == NULL) {
3054: VS_LOCK(vs);
3055: vs->vs_xfer_pending = FALSE;
3056: VS_UNLOCK(vs);
3057: vs_finish_write(vs);
3058: return KERN_SUCCESS;
3059: }
3060: /* loop on clusters in the direct map */
3061: for (j = 0; j < vs->vs_size; j++) {
3062: if (VSM_ISCLR(vsmap[j]) ||
3063: VSM_ISERR(vsmap[j]) ||
3064: (VSM_PS(vsmap[j]) != segment))
3065: continue;
3066: if(vs_cluster_transfer(vs,
3067: vm_page_size * (j << vs->vs_clshift),
3068: vm_page_size << vs->vs_clshift,
3069: page_list) != KERN_SUCCESS) {
3070: VS_LOCK(vs);
3071: vs->vs_xfer_pending = FALSE;
3072: VS_UNLOCK(vs);
3073: vs_finish_write(vs);
3074: return KERN_FAILURE;
3075: }
3076: /* allow other readers/writers during transfer*/
3077: VS_LOCK(vs);
3078: vs->vs_xfer_pending = FALSE;
3079: VS_UNLOCK(vs);
3080: vs_finish_write(vs);
3081: VS_LOCK(vs);
3082: vs->vs_xfer_pending = TRUE;
3083: VS_UNLOCK(vs);
3084: vs_wait_for_sync_writers(vs);
3085: vs_start_write(vs);
3086: vs_wait_for_readers(vs);
3087: if (vs->vs_indirect) {
3088: goto vs_changed;
3089: }
3090: }
3091: }
3092:
3093: VS_LOCK(vs);
3094: vs->vs_xfer_pending = FALSE;
3095: VS_UNLOCK(vs);
3096: vs_finish_write(vs);
3097: return KERN_SUCCESS;
3098: }
3099:
3100:
3101:
3102: vs_map_t
3103: vs_get_map_entry(
3104: vstruct_t vs,
3105: vm_offset_t offset)
3106: {
3107: struct vs_map *vsmap;
3108: vm_offset_t cluster;
3109:
3110: cluster = atop(offset) >> vs->vs_clshift;
3111: if (vs->vs_indirect) {
3112: long ind_block = cluster/CLMAP_ENTRIES;
3113:
3114: /* Is the indirect block allocated? */
3115: vsmap = vs->vs_imap[ind_block];
3116: if(vsmap == (vs_map_t) NULL)
3117: return vsmap;
3118: } else
3119: vsmap = vs->vs_dmap;
3120: vsmap += cluster%CLMAP_ENTRIES;
3121: return vsmap;
3122: }
3123:
3124:
3125:
3126:
3127:
3128:
3129:
3130: kern_return_t
3131: vs_cluster_transfer(
3132: vstruct_t vs,
3133: vm_offset_t offset,
3134: vm_size_t cnt,
3135: upl_t page_list)
3136: {
3137: vm_offset_t actual_offset;
3138: paging_segment_t ps;
3139: struct clmap clmap;
3140: kern_return_t error = KERN_SUCCESS;
3141: int size, size_wanted, i;
3142: unsigned int residual;
3143: int unavail_size;
3144: default_pager_thread_t *dpt;
3145: boolean_t dealloc;
3146: struct vs_map *vsmap_ptr;
3147: struct vs_map read_vsmap;
3148: struct vs_map original_read_vsmap;
3149: struct vs_map write_vsmap;
3150:
3151: vm_offset_t ioaddr;
3152:
3153: static char here[] = "vs_cluster_transfer";
3154:
3155: /* vs_cluster_transfer reads in the pages of a cluster and
3156: * then writes these pages back to new backing store. The
3157: * segment the pages are being read from is assumed to have
3158: * been taken off-line and is no longer considered for new
3159: * space requests.
3160: */
3161:
3162: /*
3163: * This loop will be executed once per cluster referenced.
3164: * Typically this means once, since it's unlikely that the
3165: * VM system will ask for anything spanning cluster boundaries.
3166: *
3167: * If there are holes in a cluster (in a paging segment), we stop
3168: * reading at the hole, then loop again, hoping to
3169: * find valid pages later in the cluster. This continues until
3170: * the entire range has been examined, and read, if present. The
3171: * pages are written as they are read. If a failure occurs after
3172: * some pages are written the unmap call at the bottom of the loop
3173: * recovers the backing store and the old backing store remains
3174: * in effect.
3175: */
3176:
3177: upl_map(kernel_map, page_list, &ioaddr);
3178:
3179: VSM_CLR(write_vsmap);
3180: VSM_CLR(original_read_vsmap);
3181: while (cnt && (error == KERN_SUCCESS)) {
3182: vsmap_ptr = vs_get_map_entry(vs, offset);
3183: actual_offset = ps_clmap(vs, offset, &clmap, CL_FIND, 0, 0);
3184:
3185: if (actual_offset == (vm_offset_t) -1) {
3186:
3187: /*
3188: * Nothing left to write in this cluster at least
3189: * set write cluster information for any previous
3190: * write, clear for next cluster, if there is one
3191: */
3192: unsigned int local_size, clmask, clsize;
3193:
3194: clsize = vm_page_size << vs->vs_clshift;
3195: clmask = clsize - 1;
3196: local_size = clsize - (offset & clmask);
3197: ASSERT(local_size);
3198: local_size = MIN(local_size, cnt);
3199:
3200: /* This cluster has no data in it beyond what may */
3201: /* have been found on a previous iteration through */
3202: /* the loop "write_vsmap" */
3203: *vsmap_ptr = write_vsmap;
3204: VSM_CLR(write_vsmap);
3205: VSM_CLR(original_read_vsmap);
3206:
3207: cnt -= local_size;
3208: offset += local_size;
3209: continue;
3210: }
3211:
3212: /*
3213: * Count up contiguous available or unavailable
3214: * pages.
3215: */
3216: ps = CLMAP_PS(clmap);
3217: ASSERT(ps);
3218: size = 0;
3219: unavail_size = 0;
3220: for (i = 0;
3221: (size < cnt) && (unavail_size < cnt) &&
3222: (i < CLMAP_NPGS(clmap)); i++) {
3223: if (CLMAP_ISSET(clmap, i)) {
3224: if (unavail_size != 0)
3225: break;
3226: size += vm_page_size;
3227: BS_STAT(ps->ps_bs,
3228: ps->ps_bs->bs_pages_in++);
3229: } else {
3230: if (size != 0)
3231: break;
3232: unavail_size += vm_page_size;
3233: }
3234: }
3235:
3236: if (size == 0) {
3237: ASSERT(unavail_size);
3238: cnt -= unavail_size;
3239: offset += unavail_size;
3240: if((offset & ((vm_page_size << vs->vs_clshift) - 1))
3241: == 0) {
3242: /* There is no more to transfer in this
3243: cluster
3244: */
3245: *vsmap_ptr = write_vsmap;
3246: VSM_CLR(write_vsmap);
3247: VSM_CLR(original_read_vsmap);
3248: }
3249: continue;
3250: }
3251:
3252: if(VSM_ISCLR(original_read_vsmap))
3253: original_read_vsmap = *vsmap_ptr;
3254:
3255: if(ps->ps_segtype == PS_PARTITION) {
3256: /*
3257: error = ps_read_device(ps, actual_offset, &buffer,
3258: size, &residual);
3259: */
3260: } else {
3261: error = ps_read_file(ps, actual_offset, ioaddr,
3262: size, &residual);
3263: }
3264:
3265: read_vsmap = *vsmap_ptr;
3266:
3267:
3268: /*
3269: * Adjust counts and put data in new BS. Optimize for the
3270: * common case, i.e. no error and/or partial data.
3271: * If there was an error, then we need to error the entire
3272: * range, even if some data was successfully read.
3273: *
3274: */
3275: if ((error == KERN_SUCCESS) && (residual == 0)) {
3276: /*
3277: * Got everything we asked for, supply the data to
3278: * the new BS. Note that as a side effect of supplying
3279: * the data, the buffer holding the supplied data is
3280: * deallocated from the pager's address space unless
3281: * the write is unsuccessful.
3282: */
3283:
3284: /* note buffer will be cleaned up in all cases by */
3285: /* internal_cluster_write or if an error on write */
3286: /* the vm_map_copy_page_discard call */
3287: *vsmap_ptr = write_vsmap;
3288: if(vs_cluster_write(vs, offset,
3289: ioaddr, size, TRUE) != KERN_SUCCESS) {
3290: error = KERN_FAILURE;
3291: if(!(VSM_ISCLR(*vsmap_ptr))) {
3292: /* unmap the new backing store object */
3293: ps_clunmap(vs, offset, size);
3294: }
3295: /* original vsmap */
3296: *vsmap_ptr = original_read_vsmap;
3297: VSM_CLR(write_vsmap);
3298: } else {
3299: if((offset + size) &
3300: ((vm_page_size << vs->vs_clshift)
3301: - 1)) {
3302: /* There is more to transfer in this
3303: cluster
3304: */
3305: write_vsmap = *vsmap_ptr;
3306: *vsmap_ptr = read_vsmap;
3307: } else {
3308: /* discard the old backing object */
3309: write_vsmap = *vsmap_ptr;
3310: *vsmap_ptr = read_vsmap;
3311: ps_clunmap(vs, offset, size);
3312: *vsmap_ptr = write_vsmap;
3313: VSM_CLR(write_vsmap);
3314: VSM_CLR(original_read_vsmap);
3315: }
3316: }
3317: } else {
3318: size_wanted = size;
3319: if (error == KERN_SUCCESS) {
3320: if (residual == size) {
3321: /*
3322: * If a read operation returns no error
3323: * and no data moved, we turn it into
3324: * an error, assuming we're reading at
3325: * or beyond EOF.
3326: * Fall through and error the entire
3327: * range.
3328: */
3329: error = KERN_FAILURE;
3330: *vsmap_ptr = original_read_vsmap;
3331: VSM_CLR(write_vsmap);
3332: continue;
3333: } else {
3334: /*
3335: * Otherwise, we have partial read.
3336: * This is also considered an error
3337: * for the purposes of cluster transfer
3338: */
3339: error = KERN_FAILURE;
3340: *vsmap_ptr = original_read_vsmap;
3341: VSM_CLR(write_vsmap);
3342: continue;
3343: }
3344: }
3345:
3346: }
3347: cnt -= size;
3348: offset += size;
3349:
3350: } /* END while (cnt && (error == 0)) */
3351: if(!VSM_ISCLR(write_vsmap))
3352: *vsmap_ptr = write_vsmap;
3353:
3354: upl_un_map(kernel_map, page_list);
3355: return error;
3356: }
3357:
3358: kern_return_t
3359: default_pager_add_file(MACH_PORT_FACE backing_store,
3360: int *vp,
3361: int record_size,
3362: long size)
3363: {
3364: backing_store_t bs;
3365: paging_segment_t ps;
3366: int i;
3367: int error;
3368: static char here[] = "default_pager_add_file";
3369:
3370: if ((bs = backing_store_lookup(backing_store))
3371: == BACKING_STORE_NULL)
3372: return KERN_INVALID_ARGUMENT;
3373:
3374: PSL_LOCK();
3375: for (i = 0; i <= paging_segment_max; i++) {
3376: ps = paging_segments[i];
3377: if (ps == PAGING_SEGMENT_NULL)
3378: continue;
3379: if (ps->ps_segtype != PS_FILE)
3380: continue;
3381:
3382: /*
3383: * Check for overlap on same device.
3384: */
3385: if (ps->ps_vnode == (struct vnode *)vp) {
3386: PSL_UNLOCK();
3387: BS_UNLOCK(bs);
3388: return KERN_INVALID_ARGUMENT;
3389: }
3390: }
3391: PSL_UNLOCK();
3392:
3393: /*
3394: * Set up the paging segment
3395: */
3396: ps = (paging_segment_t) kalloc(sizeof (struct paging_segment));
3397: if (ps == PAGING_SEGMENT_NULL) {
3398: BS_UNLOCK(bs);
3399: return KERN_RESOURCE_SHORTAGE;
3400: }
3401:
3402: ps->ps_segtype = PS_FILE;
3403: ps->ps_vnode = (struct vnode *)vp;
3404: ps->ps_offset = 0;
3405: ps->ps_record_shift = local_log2(vm_page_size / record_size);
3406: ps->ps_recnum = size;
3407: ps->ps_pgnum = size >> ps->ps_record_shift;
3408:
3409: ps->ps_pgcount = ps->ps_pgnum;
3410: ps->ps_clshift = local_log2(bs->bs_clsize);
3411: ps->ps_clcount = ps->ps_ncls = ps->ps_pgcount >> ps->ps_clshift;
3412: ps->ps_hint = 0;
3413:
3414: PS_LOCK_INIT(ps);
3415: ps->ps_bmap = (unsigned char *) kalloc(RMAPSIZE(ps->ps_ncls));
3416: if (!ps->ps_bmap) {
3417: kfree((vm_offset_t)ps, sizeof *ps);
3418: BS_UNLOCK(bs);
3419: return KERN_RESOURCE_SHORTAGE;
3420: }
3421: for (i = 0; i < ps->ps_ncls; i++) {
3422: clrbit(ps->ps_bmap, i);
3423: }
3424:
3425: ps->ps_going_away = FALSE;
3426: ps->ps_bs = bs;
3427:
3428: if ((error = ps_enter(ps)) != 0) {
3429: kfree((vm_offset_t)ps->ps_bmap, RMAPSIZE(ps->ps_ncls));
3430: kfree((vm_offset_t)ps, sizeof *ps);
3431: BS_UNLOCK(bs);
3432: return KERN_RESOURCE_SHORTAGE;
3433: }
3434:
3435: bs->bs_pages_free += ps->ps_clcount << ps->ps_clshift;
3436: bs->bs_pages_total += ps->ps_clcount << ps->ps_clshift;
3437: PSL_LOCK();
3438: dp_pages_free += ps->ps_pgcount;
3439: PSL_UNLOCK();
3440:
3441: BS_UNLOCK(bs);
3442:
3443: bs_more_space(ps->ps_clcount);
3444:
3445: DEBUG(DEBUG_BS_INTERNAL,
3446: ("device=0x%x,offset=0x%x,count=0x%x,record_size=0x%x,shift=%d,total_size=0x%x\n",
3447: device, offset, size, record_size,
3448: ps->ps_record_shift, ps->ps_pgnum));
3449:
3450: return KERN_SUCCESS;
3451: }
3452:
3453:
3454:
3455: kern_return_t ps_read_file(paging_segment_t, vm_offset_t, vm_offset_t, unsigned int, unsigned int *); /* forward */
3456:
3457: kern_return_t
3458: ps_read_file(
3459: paging_segment_t ps,
3460: vm_offset_t offset,
3461: vm_offset_t ioaddr,
3462: unsigned int size,
3463: unsigned int *residualp)
3464: {
3465: kern_return_t kr = KERN_SUCCESS;
3466: vm_offset_t dev_offset;
3467: static char here[] = "ps_read_file";
3468:
3469: vm_map_copy_t device_data = NULL;
3470: int error = 0;
3471: int result;
3472:
3473:
3474: clustered_reads[atop(size)]++;
3475:
3476: dev_offset = ps->ps_offset + offset;
3477:
3478: error = pager_vnode_pagein(ps->ps_vnode,
3479: (vm_offset_t)ioaddr, dev_offset, (vm_size_t)size, NULL);
3480:
3481: /* The vnode_pagein semantic is somewhat at odds with the existing */
3482: /* device_read semantic. Partial reads are not experienced at this */
3483: /* level. It is up to the bit map code and cluster read code to */
3484: /* check that requested data locations are actually backed, and the */
3485: /* pagein code to either read all of the requested data or return an */
3486: /* error. */
3487:
3488: if(error)
3489: result = KERN_FAILURE;
3490: else {
3491: *residualp = 0;
3492: result = KERN_SUCCESS;
3493: }
3494:
3495: return result;
3496:
3497: }
3498:
3499: kern_return_t
3500: ps_write_file(
3501: paging_segment_t ps,
3502: vm_offset_t offset,
3503: vm_offset_t addr,
3504: unsigned int size)
3505: {
3506: vm_offset_t dev_offset;
3507: kern_return_t result;
3508: static char here[] = "ps_write_file";
3509:
3510: vm_offset_t ioaddr;
3511: int error = 0;
3512:
3513:
3514: clustered_writes[atop(size)]++;
3515: dev_offset = ps->ps_offset + offset;
3516:
3517: ioaddr = addr;
3518:
3519:
3520: if (ioaddr) {
3521: if(pager_vnode_pageout(ps->ps_vnode, ioaddr,
3522: (vm_offset_t) dev_offset, (vm_size_t)size, NULL)) {
3523: result = KERN_FAILURE;
3524: }
3525: else {
3526: result = KERN_SUCCESS;
3527: }
3528:
3529: }
3530: else {
3531: panic("ps_write_file: No data to copy out\n");
3532: result = KERN_SUCCESS;
3533: }
3534:
3535:
3536: return result;
3537: }
3538:
3539: kern_return_t
3540: default_pager_triggers(MACH_PORT_FACE default_pager,
3541: int hi_wat,
3542: int lo_wat,
3543: int flags,
3544: MACH_PORT_FACE trigger_port)
3545: {
3546:
3547: if(flags & HI_WAT_ALERT) {
3548: if(min_pages_trigger_port)
3549: ipc_port_release_send(min_pages_trigger_port);
3550: min_pages_trigger_port = trigger_port;
3551: minimum_pages_remaining = hi_wat/vm_page_size;
3552: bs_low = FALSE;
3553: }
3554: if(flags & LO_WAT_ALERT) {
3555: if(max_pages_trigger_port)
3556: ipc_port_release_send(max_pages_trigger_port);
3557: max_pages_trigger_port = trigger_port;
3558: maximum_pages_free = lo_wat/vm_page_size;
3559: }
3560: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.