![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mach_msg.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / ipc|
Return to mach_msg.c CVS log | Up to [Apple XNU] / XNU / osfmk / ipc |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50: /*
51: */
52: /*
53: * File: ipc/mach_msg.c
54: * Author: Rich Draves
55: * Date: 1989
56: *
57: * Exported message traps. See mach/message.h.
58: */
59:
60: #include <cpus.h>
61: #include <dipc.h>
62: #include <mach_rt.h>
63:
64: #include <mach/kern_return.h>
65: #include <mach/port.h>
66: #include <mach/message.h>
67: #include <mach/mig_errors.h>
68: #include <kern/assert.h>
69: #include <kern/counters.h>
70: #include <kern/cpu_number.h>
71: #include <kern/task.h>
72: #include <kern/thread.h>
73: #include <kern/lock.h>
74: #include <kern/sched_prim.h>
75: #include <kern/exception.h>
76: #include <kern/misc_protos.h>
77: #include <vm/vm_map.h>
78: #include <ipc/ipc_kmsg.h>
79: #include <ipc/ipc_mqueue.h>
80: #include <ipc/ipc_object.h>
81: #include <ipc/ipc_notify.h>
82: #include <ipc/ipc_port.h>
83: #include <ipc/ipc_pset.h>
84: #include <ipc/ipc_space.h>
85: #include <ipc/ipc_entry.h>
86: #include <kern/kalloc.h>
87: #include <kern/thread_swap.h>
88: #include <kern/processor.h>
89: #include <kern/rtmalloc.h>
90:
91: #include <kern/sf.h>
92: #include <kern/mk_sp.h> /* JMM - to support handoff hack */
93:
94:
95: /*
96: * Forward declarations
97: */
98:
99: mach_msg_return_t mach_msg_send(
100: mach_msg_header_t *msg,
101: mach_msg_option_t option,
102: mach_msg_size_t send_size,
103: mach_msg_timeout_t timeout,
104: mach_port_name_t notify);
105:
106: mach_msg_return_t mach_msg_receive(
107: mach_msg_header_t *msg,
108: mach_msg_option_t option,
109: mach_msg_size_t rcv_size,
110: mach_port_name_t rcv_name,
111: mach_msg_timeout_t timeout,
112: mach_port_name_t notify,
113: mach_msg_size_t slist_size);
114:
115: mach_msg_return_t msg_receive_error(
116: ipc_kmsg_t kmsg,
117: mach_msg_header_t *msg,
118: mach_msg_option_t option,
119: mach_port_seqno_t seqno,
120: ipc_space_t space);
121:
122: /* the size of each trailer has to be listed here for copyout purposes */
123: mach_msg_trailer_size_t trailer_size[] = {
124: sizeof(mach_msg_trailer_t),
125: sizeof(mach_msg_seqno_trailer_t),
126: sizeof(mach_msg_security_trailer_t) };
127:
128: security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
129:
130: mach_msg_format_0_trailer_t trailer_template = {
131: /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
132: /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
133: /* mach_port_seqno_t */ 0,
134: /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
135: };
136:
137: /*
138: * Routine: mach_msg_send
139: * Purpose:
140: * Send a message.
141: * Conditions:
142: * Nothing locked.
143: * Returns:
144: * MACH_MSG_SUCCESS Sent the message.
145: * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
146: * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
147: * MACH_SEND_INVALID_DATA Couldn't copy message data.
148: * MACH_SEND_INVALID_HEADER
149: * Illegal value in the message header bits.
150: * MACH_SEND_INVALID_DEST The space is dead.
151: * MACH_SEND_INVALID_NOTIFY Bad notify port.
152: * MACH_SEND_INVALID_DEST Can't copyin destination port.
153: * MACH_SEND_INVALID_REPLY Can't copyin reply port.
154: * MACH_SEND_TIMED_OUT Timeout expired without delivery.
155: * MACH_SEND_INTERRUPTED Delivery interrupted.
156: * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
157: * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
158: * MACH_SEND_NOTIFY_IN_PROGRESS
159: * This space has already forced a message to this port.
160: */
161:
162: mach_msg_return_t
163: mach_msg_send(
164: mach_msg_header_t *msg,
165: mach_msg_option_t option,
166: mach_msg_size_t send_size,
167: mach_msg_timeout_t timeout,
168: mach_port_name_t notify)
169: {
170: ipc_space_t space = current_space();
171: vm_map_t map = current_map();
172: ipc_kmsg_t kmsg;
173: mach_msg_return_t mr;
174:
175: mr = ipc_kmsg_get(msg, send_size, &kmsg, space);
176:
177: if (mr != MACH_MSG_SUCCESS)
178: return mr;
179:
180: if (option & MACH_SEND_CANCEL) {
181: if (notify == MACH_PORT_NULL)
182: mr = MACH_SEND_INVALID_NOTIFY;
183: else
184: mr = ipc_kmsg_copyin(kmsg, space, map, notify);
185: } else
186: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
187: if (mr != MACH_MSG_SUCCESS) {
188: ikm_free(kmsg);
189: return mr;
190: }
191:
192: mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, timeout);
193:
194: if (mr != MACH_MSG_SUCCESS) {
195: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
196: (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
197: }
198:
199: return mr;
200: }
201:
202: #define FREE_SCATTER_LIST(s, l, rt) \
203: MACRO_BEGIN \
204: if((s) != MACH_MSG_BODY_NULL) { \
205: KFREE(((vm_offset_t)(s)), (l), rt); \
206: } \
207: MACRO_END
208:
209: /*
210: * Routine: mach_msg_receive
211: * Purpose:
212: * Receive a message.
213: * Conditions:
214: * Nothing locked.
215: * Returns:
216: * MACH_MSG_SUCCESS Received a message.
217: * MACH_RCV_INVALID_NAME The name doesn't denote a right,
218: * or the denoted right is not receive or port set.
219: * MACH_RCV_IN_SET Receive right is a member of a set.
220: * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
221: * MACH_RCV_TIMED_OUT Timeout expired without a message.
222: * MACH_RCV_INTERRUPTED Reception interrupted.
223: * MACH_RCV_PORT_DIED Port/set died while receiving.
224: * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
225: * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
226: * MACH_RCV_INVALID_NOTIFY Bad notify port.
227: * MACH_RCV_HEADER_ERROR
228: */
229:
230: mach_msg_return_t
231: mach_msg_receive(
232: mach_msg_header_t *msg,
233: mach_msg_option_t option,
234: mach_msg_size_t rcv_size,
235: mach_port_name_t rcv_name,
236: mach_msg_timeout_t timeout,
237: mach_port_name_t notify,
238: mach_msg_size_t slist_size)
239: {
240: thread_t self = current_thread();
241: ipc_space_t space = current_space();
242: vm_map_t map = current_map();
243: ipc_object_t object;
244: ipc_mqueue_t mqueue;
245: ipc_kmsg_t kmsg;
246: mach_port_seqno_t seqno;
247: mach_msg_return_t mr;
248: mach_msg_body_t *slist;
249: mach_msg_format_0_trailer_t *trailer;
250: #if MACH_RT
251: boolean_t slist_rt;
252: #endif /* MACH_RT */
253:
254: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
255: if (mr != MACH_MSG_SUCCESS) {
256: return mr;
257: }
258: /* hold ref for object; mqueue is locked */
259:
260: #if MACH_RT
261: /* NOTE: This only works for true ports, and not port sets */
262: slist_rt = ipc_object_is_rt(object);
263: #endif /* MACH_RT */
264:
265: /*
266: * If MACH_RCV_OVERWRITE was specified, both receive_msg (msg)
267: * and receive_msg_size (slist_size) need to be non NULL.
268: */
269: if (option & MACH_RCV_OVERWRITE) {
270: if (slist_size < sizeof(mach_msg_base_t)) {
271: ipc_object_release(object);
272: return MACH_RCV_SCATTER_SMALL;
273: } else {
274: slist_size -= sizeof(mach_msg_header_t);
275: slist = (mach_msg_body_t *)KALLOC(slist_size, slist_rt);
276: if (slist == MACH_MSG_BODY_NULL ||
277: copyin((char *) (msg + 1), (char *)slist,
278: slist_size)) {
279: ipc_object_release(object);
280: return MACH_RCV_INVALID_DATA;
281: }
282: if ((slist->msgh_descriptor_count*
283: sizeof(mach_msg_descriptor_t)
284: + sizeof(mach_msg_size_t)) > slist_size) {
285: FREE_SCATTER_LIST(slist, slist_size, slist_rt);
286: ipc_object_release(object);
287: return MACH_RCV_INVALID_TYPE;
288: }
289: }
290: } else {
291: slist = MACH_MSG_BODY_NULL;
292: }
293:
294: self->ith_option = option;
295: self->ith_scatter_list = slist;
296: self->ith_scatter_list_size = slist_size;
297:
298: mr = ipc_mqueue_receive(mqueue, option, rcv_size,
299: timeout, THREAD_ABORTSAFE,
300: &kmsg, &seqno);
301:
302: ipc_object_release(object);
303:
304: if (mr != MACH_MSG_SUCCESS) {
305:
306: if (mr == MACH_RCV_TOO_LARGE || mr == MACH_RCV_SCATTER_SMALL) {
307: if (option & MACH_RCV_LARGE) {
308: /*
309: * We need to inform the user-level code that it needs more
310: * space. The value for how much space was returned in the
311: * kmsg pointer instead of the message (which was left on the
312: * queue).
313: */
314: if (copyout((char *) &kmsg,
315: (char *) &msg->msgh_size,
316: sizeof(mach_msg_size_t)))
317: mr = MACH_RCV_INVALID_DATA;
318: return mr;
319: }
320:
321: if (msg_receive_error(kmsg, msg, option, seqno, space)
322: == MACH_RCV_INVALID_DATA)
323: mr = MACH_RCV_INVALID_DATA;
324: }
325:
326: FREE_SCATTER_LIST(slist, slist_size, slist_rt);
327: return mr;
328: }
329:
330: trailer = (mach_msg_format_0_trailer_t *)
331: ((vm_offset_t)&kmsg->ikm_header +
332: round_msg(kmsg->ikm_header.msgh_size));
333: if (option & MACH_RCV_TRAILER_MASK) {
334: trailer->msgh_seqno = seqno;
335: trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
336: }
337:
338: if (option & MACH_RCV_NOTIFY) {
339: if (notify == MACH_PORT_NULL)
340: mr = MACH_RCV_INVALID_NOTIFY;
341: else
342: mr = ipc_kmsg_copyout(kmsg, space, map, notify, slist);
343: } else {
344: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
345: }
346: if (mr != MACH_MSG_SUCCESS) {
347: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
348: if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
349: trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
350: mr = MACH_RCV_INVALID_DATA;
351: }
352: else {
353: if (msg_receive_error(kmsg, msg, option, seqno, space)
354: == MACH_RCV_INVALID_DATA)
355: mr = MACH_RCV_INVALID_DATA;
356: }
357:
358: FREE_SCATTER_LIST(slist, slist_size, slist_rt);
359: return mr;
360: }
361: mr = ipc_kmsg_put(msg, kmsg,
362: kmsg->ikm_header.msgh_size + trailer->msgh_trailer_size);
363: FREE_SCATTER_LIST(slist, slist_size, slist_rt);
364:
365: return mr;
366: }
367:
368:
369: /*
370: * Toggle this to compile the hotpath in/out
371: * If compiled in, the run-time toggle "enable_hotpath" below
372: * eases testing & debugging
373: */
374: #define ENABLE_HOTPATH 1 /* Hacked on for now */
375:
376: #ifndef ENABLE_HOTPATH
377: #if NCPUS == 1 && !MACH_RT
378: #define ENABLE_HOTPATH 1
379: #else /* NCPUS == 1 && !MACH_RT */
380: #define ENABLE_HOTPATH 0
381: #endif /* NCPUS == 1 && !MACH_RT */
382: #endif
383:
384: #if ENABLE_HOTPATH
385: /*
386: * These counters allow tracing of hotpath behavior under test loads.
387: * A couple key counters are unconditional (see below).
388: */
389: #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
390: #if HOTPATH_DEBUG
391: #define HOT(expr) expr
392:
393: unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
394: unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
395: unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
396: unsigned int c_mmot_ikm_cache_miss = 0; /* Reasons to Fall Off: */
397: unsigned int c_mmot_kernel_send = 0; /* kernel server */
398: unsigned int c_mmot_cold_000 = 0; /* see below ... */
399: unsigned int c_mmot_smallsendsize = 0;
400: unsigned int c_mmot_oddsendsize = 0;
401: unsigned int c_mmot_bigsendsize = 0;
402: unsigned int c_mmot_copyinmsg_fail = 0;
403: unsigned int c_mmot_g_slow_copyin3 = 0;
404: unsigned int c_mmot_cold_006 = 0;
405: unsigned int c_mmot_cold_007 = 0;
406: unsigned int c_mmot_cold_008 = 0;
407: unsigned int c_mmot_cold_009 = 0;
408: unsigned int c_mmot_cold_010 = 0;
409: unsigned int c_mmot_cold_012 = 0;
410: unsigned int c_mmot_cold_013 = 0;
411: unsigned int c_mmot_cold_014 = 0;
412: unsigned int c_mmot_cold_016 = 0;
413: unsigned int c_mmot_cold_018 = 0;
414: unsigned int c_mmot_cold_019 = 0;
415: unsigned int c_mmot_cold_020 = 0;
416: unsigned int c_mmot_cold_021 = 0;
417: unsigned int c_mmot_cold_022 = 0;
418: unsigned int c_mmot_cold_023 = 0;
419: unsigned int c_mmot_cold_024 = 0;
420: unsigned int c_mmot_cold_025 = 0;
421: unsigned int c_mmot_cold_026 = 0;
422: unsigned int c_mmot_cold_027 = 0;
423: unsigned int c_mmot_hot_fSR_ok = 0;
424: unsigned int c_mmot_cold_029 = 0;
425: unsigned int c_mmot_cold_030 = 0;
426: unsigned int c_mmot_cold_031 = 0;
427: unsigned int c_mmot_cold_032 = 0;
428: unsigned int c_mmot_cold_033 = 0;
429: unsigned int c_mmot_bad_rcvr = 0;
430: unsigned int c_mmot_rcvr_swapped = 0;
431: unsigned int c_mmot_rcvr_locked = 0;
432: unsigned int c_mmot_rcvr_tswapped = 0;
433: unsigned int c_mmot_rcvr_freed = 0;
434: unsigned int c_mmot_g_slow_copyout6 = 0;
435: unsigned int c_mmot_g_slow_copyout5 = 0;
436: unsigned int c_mmot_cold_037 = 0;
437: unsigned int c_mmot_cold_038 = 0;
438: unsigned int c_mmot_cold_039 = 0;
439: unsigned int c_mmot_g_slow_copyout4 = 0;
440: unsigned int c_mmot_g_slow_copyout3 = 0;
441: unsigned int c_mmot_hot_ok1 = 0;
442: unsigned int c_mmot_hot_ok2 = 0;
443: unsigned int c_mmot_hot_ok3 = 0;
444: unsigned int c_mmot_g_slow_copyout1 = 0;
445: unsigned int c_mmot_g_slow_copyout2 = 0;
446: unsigned int c_mmot_getback_fast_copyin = 0;
447: unsigned int c_mmot_cold_048 = 0;
448: unsigned int c_mmot_getback_FastSR = 0;
449: unsigned int c_mmot_cold_050 = 0;
450: unsigned int c_mmot_cold_051 = 0;
451: unsigned int c_mmot_cold_052 = 0;
452: unsigned int c_mmot_cold_053 = 0;
453: unsigned int c_mmot_fastkernelreply = 0;
454: unsigned int c_mmot_cold_055 = 0;
455: unsigned int c_mmot_getback_fast_put = 0;
456: unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
457:
458: void db_mmot_zero_counters(void); /* forward; */
459: void db_mmot_show_counters(void); /* forward; */
460:
461: void /* Call from the debugger to clear all counters */
462: db_mmot_zero_counters(void)
463: {
464: register unsigned int *ip = &c_mmot_FIRST;
465: while (ip <= &c_mmot_LAST)
466: *ip++ = 0;
467: }
468:
469: void /* Call from the debugger to show all counters */
470: db_mmot_show_counters(void)
471: {
472: #define xx(str) printf("%s: %d\n", # str, str);
473:
474: xx(c_mmot_combined_S_R);
475: xx(c_mach_msg_trap_switch_fast);
476: xx(c_mmot_ikm_cache_miss);
477: xx(c_mmot_kernel_send);
478: xx(c_mmot_cold_000);
479: xx(c_mmot_smallsendsize);
480: xx(c_mmot_oddsendsize);
481: xx(c_mmot_bigsendsize);
482: xx(c_mmot_copyinmsg_fail);
483: xx(c_mmot_g_slow_copyin3);
484: xx(c_mmot_cold_006);
485: xx(c_mmot_cold_007);
486: xx(c_mmot_cold_008);
487: xx(c_mmot_cold_009);
488: xx(c_mmot_cold_010);
489: xx(c_mmot_cold_012);
490: xx(c_mmot_cold_013);
491: xx(c_mmot_cold_014);
492: xx(c_mmot_cold_016);
493: xx(c_mmot_cold_018);
494: xx(c_mmot_cold_019);
495: xx(c_mmot_cold_020);
496: xx(c_mmot_cold_021);
497: xx(c_mmot_cold_022);
498: xx(c_mmot_cold_023);
499: xx(c_mmot_cold_024);
500: xx(c_mmot_cold_025);
501: xx(c_mmot_cold_026);
502: xx(c_mmot_cold_027);
503: xx(c_mmot_hot_fSR_ok);
504: xx(c_mmot_cold_029);
505: xx(c_mmot_cold_030);
506: xx(c_mmot_cold_031);
507: xx(c_mmot_cold_032);
508: xx(c_mmot_cold_033);
509: xx(c_mmot_bad_rcvr);
510: xx(c_mmot_rcvr_swapped);
511: xx(c_mmot_rcvr_locked);
512: xx(c_mmot_rcvr_tswapped);
513: xx(c_mmot_rcvr_freed);
514: xx(c_mmot_g_slow_copyout6);
515: xx(c_mmot_g_slow_copyout5);
516: xx(c_mmot_cold_037);
517: xx(c_mmot_cold_038);
518: xx(c_mmot_cold_039);
519: xx(c_mmot_g_slow_copyout4);
520: xx(c_mmot_g_slow_copyout3);
521: xx(c_mmot_g_slow_copyout1);
522: xx(c_mmot_hot_ok3);
523: xx(c_mmot_hot_ok2);
524: xx(c_mmot_hot_ok1);
525: xx(c_mmot_g_slow_copyout2);
526: xx(c_mmot_getback_fast_copyin);
527: xx(c_mmot_cold_048);
528: xx(c_mmot_getback_FastSR);
529: xx(c_mmot_cold_050);
530: xx(c_mmot_cold_051);
531: xx(c_mmot_cold_052);
532: xx(c_mmot_cold_053);
533: xx(c_mmot_fastkernelreply);
534: xx(c_mmot_cold_055);
535: xx(c_mmot_getback_fast_put);
536:
537: #undef xx
538: }
539:
540: #else /* !HOTPATH_DEBUG */
541:
542: /*
543: * Duplicate just these few so we can always do a quick sanity check
544: */
545: unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
546: unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
547: unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
548: #define HOT(expr) /* no optional counters */
549:
550: #endif /* !HOTPATH_DEBUG */
551:
552: boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
553: #endif /* HOTPATH_ENABLE */
554:
555: /*
556: * Routine: mach_msg_overwrite_trap [mach trap]
557: * Purpose:
558: * Possibly send a message; possibly receive a message.
559: * Conditions:
560: * Nothing locked.
561: * Returns:
562: * All of mach_msg_send and mach_msg_receive error codes.
563: */
564:
565: mach_msg_return_t
566: mach_msg_overwrite_trap(
567: mach_msg_header_t *msg,
568: mach_msg_option_t option,
569: mach_msg_size_t send_size,
570: mach_msg_size_t rcv_size,
571: mach_port_name_t rcv_name,
572: mach_msg_timeout_t timeout,
573: mach_port_name_t notify,
574: mach_msg_header_t *rcv_msg,
575: mach_msg_size_t scatter_list_size)
576: {
577: register mach_msg_header_t *hdr;
578: mach_msg_return_t mr = MACH_MSG_SUCCESS;
579: /* mask out some of the options before entering the hot path */
580: mach_msg_option_t masked_option =
581: option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
582: int i;
583:
584: #if ENABLE_HOTPATH
585: /* BEGINNING OF HOT PATH */
586: if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
587: register thread_t self = current_thread();
588: register mach_msg_format_0_trailer_t *trailer;
589:
590: ipc_space_t space = current_act()->task->itk_space;
591: ipc_kmsg_t kmsg;
592: register ipc_port_t dest_port;
593: ipc_object_t rcv_object;
594: register ipc_mqueue_t rcv_mqueue;
595: mach_msg_size_t reply_size;
596: ipc_kmsg_t rcv_kmsg;
597:
598: c_mmot_combined_S_R++;
599: self->ith_scatter_list = MACH_MSG_BODY_NULL;
600:
601: /*
602: * This case is divided into ten sections, each
603: * with a label. There are five optimized
604: * sections and six unoptimized sections, which
605: * do the same thing but handle all possible
606: * cases and are slower.
607: *
608: * The five sections for an RPC are
609: * 1) Get request message into a buffer.
610: * (fast_get or slow_get)
611: * 2) Copyin request message and rcv_name.
612: * (fast_copyin or slow_copyin)
613: * 3) Enqueue request and dequeue reply.
614: * (fast_send_receive or
615: * slow_send and slow_receive)
616: * 4) Copyout reply message.
617: * (fast_copyout or slow_copyout)
618: * 5) Put reply message to user's buffer.
619: * (fast_put or slow_put)
620: *
621: * Keep the locking hierarchy firmly in mind.
622: * (First spaces, then ports, then port sets,
623: * then message queues.) Only a non-blocking
624: * attempt can be made to acquire locks out of
625: * order, or acquire two locks on the same level.
626: * Acquiring two locks on the same level will
627: * fail if the objects are really the same,
628: * unless simple locking is disabled. This is OK,
629: * because then the extra unlock does nothing.
630: *
631: * There are two major reasons these RPCs can't use
632: * ipc_thread_switch, and use slow_send/slow_receive:
633: * 1) Kernel RPCs.
634: * 2) Servers fall behind clients, so
635: * client doesn't find a blocked server thread and
636: * server finds waiting messages and can't block.
637: */
638:
639: fast_get:
640: /*
641: * optimized ipc_kmsg_get
642: *
643: * No locks, references, or messages held.
644: * We must clear ikm_cache before copyinmsg.
645: */
646:
647: if ((send_size < sizeof(mach_msg_header_t)) ||
648: (send_size & 3) ||
649: (send_size > (IKM_SAVED_MSG_SIZE - MAX_TRAILER_SIZE)) ||
650: !ikm_cache_get(&kmsg)) {
651: #if HOTPATH_DEBUG
652: if (send_size < sizeof(mach_msg_header_t)) {
653: HOT(c_mmot_smallsendsize++);
654: } else if (send_size & 3) {
655: HOT(c_mmot_oddsendsize++);
656: } else if (send_size >
657: (IKM_SAVED_MSG_SIZE - MAX_TRAILER_SIZE)) {
658: HOT(c_mmot_bigsendsize++);
659: } else if (!kmsg) {
660: HOT(c_mmot_ikm_cache_miss++);
661: }
662: #endif
663: goto slow_get;
664: }
665:
666: ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
667:
668: hdr = &kmsg->ikm_header;
669: if (copyinmsg((char *) msg, (char *) hdr, send_size)) {
670: if (kmsg->ikm_size != IKM_SAVED_KMSG_SIZE ||
671: KMSG_IS_RT(kmsg) || !ikm_cache_put(kmsg))
672: ikm_free(kmsg);
673: HOT(c_mmot_copyinmsg_fail++);
674: goto slow_get;
675: }
676:
677: hdr->msgh_size = send_size;
678:
679: /* naturally align the message before tacking on the trailer */
680: trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
681: send_size);
682: bcopy((char *)&trailer_template, (char *)trailer,
683: sizeof(trailer_template));
684: trailer->msgh_sender = current_act()->task->sec_token;
685:
686: fast_copyin:
687: /*
688: * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
689: *
690: * We have the request message data in kmsg.
691: * Must still do copyin, send, receive, etc.
692: *
693: * If the message isn't simple, we can't combine
694: * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
695: * because copyin of the message body might
696: * affect rcv_name.
697: */
698:
699: switch (hdr->msgh_bits) {
700: case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
701: MACH_MSG_TYPE_MAKE_SEND_ONCE): {
702: register ipc_entry_t table;
703: register ipc_entry_num_t size;
704: register ipc_port_t reply_port;
705:
706: /* sending a request message */
707:
708: {
709: register mach_port_index_t index;
710: register mach_port_gen_t gen;
711:
712: {
713: register mach_port_name_t reply_name =
714: (mach_port_name_t)hdr->msgh_local_port;
715:
716: if (reply_name != rcv_name) {
717: HOT(c_mmot_g_slow_copyin3++);
718: goto slow_copyin;
719: }
720:
721: /* optimized ipc_entry_lookup of reply_name */
722:
723: index = MACH_PORT_INDEX(reply_name);
724: gen = MACH_PORT_GEN(reply_name);
725:
726: is_read_lock(space);
727: assert(space->is_active);
728:
729: size = space->is_table_size;
730: table = space->is_table;
731:
732: {
733: register ipc_entry_t entry;
734: register ipc_entry_bits_t bits;
735:
736: if (index < size) {
737: entry = &table[index];
738: bits = entry->ie_bits;
739: if (IE_BITS_GEN(bits) != gen ||
740: (bits & IE_BITS_COLLISION)) {
741: entry = IE_NULL;
742: }
743: } else {
744: entry = IE_NULL;
745: }
746: if (entry == IE_NULL) {
747: entry = ipc_entry_lookup(space, reply_name);
748: if (entry == IE_NULL) {
749: HOT(c_mmot_cold_006++);
750: goto abort_request_copyin;
751: }
752: bits = entry->ie_bits;
753: }
754:
755: /* check type bit */
756:
757: if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
758: HOT(c_mmot_cold_007++);
759: goto abort_request_copyin;
760: }
761:
762: reply_port = (ipc_port_t) entry->ie_object;
763: assert(reply_port != IP_NULL);
764: }
765: }
766: }
767:
768: /* optimized ipc_entry_lookup of dest_name */
769:
770: {
771: register mach_port_index_t index;
772: register mach_port_gen_t gen;
773:
774: {
775: register mach_port_name_t dest_name =
776: (mach_port_name_t)hdr->msgh_remote_port;
777:
778: index = MACH_PORT_INDEX(dest_name);
779: gen = MACH_PORT_GEN(dest_name);
780:
781: {
782: register ipc_entry_t entry;
783: register ipc_entry_bits_t bits;
784:
785: if (index < size) {
786: entry = &table[index];
787: bits = entry->ie_bits;
788: if (IE_BITS_GEN(bits) != gen ||
789: (bits & IE_BITS_COLLISION)) {
790: entry = IE_NULL;
791: }
792: } else {
793: entry = IE_NULL;
794: }
795: if (entry == IE_NULL) {
796: entry = ipc_entry_lookup(space, dest_name);
797: if (entry == IE_NULL) {
798: HOT(c_mmot_cold_008++);
799: goto abort_request_copyin;
800: }
801: bits = entry->ie_bits;
802: }
803:
804: /* check type bit */
805:
806: if (! (bits & MACH_PORT_TYPE_SEND)) {
807: HOT(c_mmot_cold_009++);
808: goto abort_request_copyin;
809: }
810:
811: assert(IE_BITS_UREFS(bits) > 0);
812:
813: dest_port = (ipc_port_t) entry->ie_object;
814: assert(dest_port != IP_NULL);
815: }
816: }
817: }
818:
819: /*
820: * To do an atomic copyin, need simultaneous
821: * locks on both ports and the space. If
822: * dest_port == reply_port, and simple locking is
823: * enabled, then we will abort. Otherwise it's
824: * OK to unlock twice.
825: */
826:
827: ip_lock(dest_port);
828: if (!ip_active(dest_port) ||
829: !ip_lock_try(reply_port)) {
830: ip_unlock(dest_port);
831: HOT(c_mmot_cold_010++);
832: goto abort_request_copyin;
833: }
834: is_read_unlock(space);
835:
836: assert(dest_port->ip_srights > 0);
837: dest_port->ip_srights++;
838: ip_reference(dest_port);
839:
840: assert(ip_active(reply_port));
841: assert(reply_port->ip_receiver_name ==
842: (mach_port_name_t)hdr->msgh_local_port);
843: assert(reply_port->ip_receiver == space);
844:
845: reply_port->ip_sorights++;
846: ip_reference(reply_port);
847:
848: hdr->msgh_bits =
849: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
850: MACH_MSG_TYPE_PORT_SEND_ONCE);
851: hdr->msgh_remote_port = dest_port;
852: hdr->msgh_local_port = reply_port;
853:
854: /* make sure we can queue to the destination */
855:
856: if (dest_port->ip_receiver == ipc_space_kernel) {
857: /*
858: * The kernel server has a reference to
859: * the reply port, which it hands back
860: * to us in the reply message. We do
861: * not need to keep another reference to
862: * it.
863: */
864: ip_unlock(reply_port);
865:
866: assert(ip_active(dest_port));
867: dest_port->ip_messages.imq_seqno++;
868: ip_unlock(dest_port);
869: goto kernel_send;
870: }
871:
872: if (imq_full(&dest_port->ip_messages)) {
873: HOT(c_mmot_cold_013++);
874: goto abort_request_send_receive;
875: }
876:
877: /* optimized ipc_mqueue_copyin */
878:
879: if (reply_port->ip_pset_count != 0) {
880: HOT(c_mmot_cold_014++);
881: goto abort_request_send_receive;
882: }
883:
884: rcv_object = (ipc_object_t) reply_port;
885: io_reference(rcv_object);
886: rcv_mqueue = &reply_port->ip_messages;
887: imq_lock(rcv_mqueue);
888: io_unlock(rcv_object);
889: HOT(c_mmot_hot_fSR_ok++);
890: goto fast_send_receive;
891:
892: abort_request_copyin:
893: is_read_unlock(space);
894: goto slow_copyin;
895:
896: abort_request_send_receive:
897: ip_unlock(dest_port);
898: ip_unlock(reply_port);
899: goto slow_send;
900: }
901:
902: case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
903: register ipc_entry_num_t size;
904: register ipc_entry_t table;
905:
906: /* sending a reply message */
907:
908: {
909: register mach_port_name_t reply_name =
910: (mach_port_name_t)hdr->msgh_local_port;
911:
912: if (reply_name != MACH_PORT_NULL) {
913: HOT(c_mmot_cold_018++);
914: goto slow_copyin;
915: }
916: }
917:
918: is_write_lock(space);
919: assert(space->is_active);
920:
921: /* optimized ipc_entry_lookup */
922:
923: size = space->is_table_size;
924: table = space->is_table;
925:
926: {
927: register ipc_entry_t entry;
928: register mach_port_gen_t gen;
929: register mach_port_index_t index;
930: ipc_table_index_t *requests;
931:
932: {
933: register mach_port_name_t dest_name =
934: (mach_port_name_t)hdr->msgh_remote_port;
935:
936: index = MACH_PORT_INDEX(dest_name);
937: gen = MACH_PORT_GEN(dest_name);
938: }
939:
940: if (index >= size) {
941: HOT(c_mmot_cold_019++);
942: goto abort_reply_dest_copyin;
943: }
944:
945: entry = &table[index];
946:
947: /* check generation, collision bit, and type bit */
948:
949: if ((entry->ie_bits & (IE_BITS_GEN_MASK|
950: IE_BITS_COLLISION|
951: MACH_PORT_TYPE_SEND_ONCE)) !=
952: (gen | MACH_PORT_TYPE_SEND_ONCE)) {
953: HOT(c_mmot_cold_020++);
954: goto abort_reply_dest_copyin;
955: }
956:
957: /* optimized ipc_right_copyin */
958:
959: assert(IE_BITS_TYPE(entry->ie_bits) ==
960: MACH_PORT_TYPE_SEND_ONCE);
961: assert(IE_BITS_UREFS(entry->ie_bits) == 1);
962:
963: if (entry->ie_request != 0) {
964: HOT(c_mmot_cold_021++);
965: goto abort_reply_dest_copyin;
966: }
967:
968: dest_port = (ipc_port_t) entry->ie_object;
969: assert(dest_port != IP_NULL);
970:
971: ip_lock(dest_port);
972: if (!ip_active(dest_port)) {
973: ip_unlock(dest_port);
974: HOT(c_mmot_cold_022++);
975: goto abort_reply_dest_copyin;
976: }
977:
978: assert(dest_port->ip_sorights > 0);
979:
980: /* optimized ipc_entry_dealloc */
981:
982:
983: entry->ie_bits = gen;
984: entry->ie_next = table->ie_next;
985: table->ie_next = index;
986: entry->ie_object = IO_NULL;
987: }
988:
989: hdr->msgh_bits =
990: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
991: 0);
992: hdr->msgh_remote_port = dest_port;
993:
994: /* make sure we can queue to the destination */
995:
996: assert(dest_port->ip_receiver != ipc_space_kernel);
997:
998: /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
999:
1000: {
1001: register ipc_entry_t entry;
1002: register ipc_entry_bits_t bits;
1003:
1004: {
1005: register mach_port_index_t index;
1006: register mach_port_gen_t gen;
1007:
1008: index = MACH_PORT_INDEX(rcv_name);
1009: gen = MACH_PORT_GEN(rcv_name);
1010:
1011: if (index < size) {
1012: entry = &table[index];
1013: bits = entry->ie_bits;
1014: if (IE_BITS_GEN(bits) != gen ||
1015: (bits & IE_BITS_COLLISION)) {
1016: entry = IE_NULL;
1017: }
1018: } else {
1019: entry = IE_NULL;
1020: }
1021: if (entry == IE_NULL) {
1022: entry = ipc_entry_lookup(space, rcv_name);
1023: if (entry == IE_NULL) {
1024: HOT(c_mmot_cold_024++);
1025: goto abort_reply_rcv_copyin;
1026: }
1027: bits = entry->ie_bits;
1028: }
1029:
1030: }
1031:
1032: /* check type bits; looking for receive or set */
1033:
1034: if (bits & MACH_PORT_TYPE_PORT_SET) {
1035: register ipc_pset_t rcv_pset;
1036:
1037: rcv_pset = (ipc_pset_t) entry->ie_object;
1038: assert(rcv_pset != IPS_NULL);
1039:
1040: ips_lock(rcv_pset);
1041: assert(ips_active(rcv_pset));
1042:
1043: rcv_object = (ipc_object_t) rcv_pset;
1044: rcv_mqueue = &rcv_pset->ips_messages;
1045: } else if (bits & MACH_PORT_TYPE_RECEIVE) {
1046: register ipc_port_t rcv_port;
1047:
1048: rcv_port = (ipc_port_t) entry->ie_object;
1049: assert(rcv_port != IP_NULL);
1050:
1051: if (!ip_lock_try(rcv_port)) {
1052: HOT(c_mmot_cold_025++);
1053: goto abort_reply_rcv_copyin;
1054: }
1055: assert(ip_active(rcv_port));
1056:
1057: if (rcv_port->ip_pset_count != 0) {
1058: ip_unlock(rcv_port);
1059: HOT(c_mmot_cold_026++);
1060: goto abort_reply_rcv_copyin;
1061: }
1062:
1063: rcv_object = (ipc_object_t) rcv_port;
1064: rcv_mqueue = &rcv_port->ip_messages;
1065: } else {
1066: HOT(c_mmot_cold_027++);
1067: goto abort_reply_rcv_copyin;
1068: }
1069: }
1070:
1071: is_write_unlock(space);
1072: io_reference(rcv_object);
1073: imq_lock(rcv_mqueue);
1074: io_unlock(rcv_object);
1075: HOT(c_mmot_hot_fSR_ok++);
1076: goto fast_send_receive;
1077:
1078: abort_reply_dest_copyin:
1079: is_write_unlock(space);
1080: HOT(c_mmot_cold_029++);
1081: goto slow_copyin;
1082:
1083: abort_reply_rcv_copyin:
1084: ip_unlock(dest_port);
1085: is_write_unlock(space);
1086: HOT(c_mmot_cold_030++);
1087: goto slow_send;
1088: }
1089:
1090: default:
1091: HOT(c_mmot_cold_031++);
1092: goto slow_copyin;
1093: }
1094: /*NOTREACHED*/
1095:
1096: fast_send_receive:
1097: /*
1098: * optimized ipc_mqueue_send/ipc_mqueue_receive
1099: *
1100: * Finished get/copyin of kmsg and copyin of rcv_name.
1101: * space is unlocked, dest_port is locked,
1102: * we can queue kmsg to dest_port,
1103: * rcv_mqueue is locked, rcv_object holds a ref,
1104: * if rcv_object is a port it isn't in a port set
1105: */
1106:
1107: assert(ip_active(dest_port));
1108: assert(dest_port->ip_receiver != ipc_space_kernel);
1109: assert(!imq_full(&dest_port->ip_messages) ||
1110: (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1111: MACH_MSG_TYPE_PORT_SEND_ONCE));
1112: assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1113:
1114: {
1115: register ipc_mqueue_t dest_mqueue;
1116: wait_queue_t waitq;
1117: thread_t receiver;
1118: spl_t s;
1119: #if THREAD_SWAPPER
1120: thread_act_t rcv_act;
1121: #endif
1122:
1123: dest_mqueue = &dest_port->ip_messages;
1124:
1125: if (!imq_lock_try(dest_mqueue)) {
1126: abort_send_receive:
1127: ip_unlock(dest_port);
1128: imq_unlock(rcv_mqueue);
1129: ipc_object_release(rcv_object);
1130: HOT(c_mmot_cold_032++);
1131: goto slow_send;
1132: }
1133: if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1134: imq_unlock(dest_mqueue);
1135: HOT(c_mmot_cold_033++);
1136: goto abort_send_receive;
1137: }
1138:
1139: waitq = &dest_mqueue->imq_wait_queue;
1140: s = splsched();
1141: wait_queue_peek_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1142: /* queue still locked, thread locked - but still on q */
1143:
1144: if (receiver == THREAD_NULL) {
1145: splx(s);
1146: imq_unlock(dest_mqueue);
1147: HOT(c_mmot_cold_033++);
1148: goto abort_send_receive;
1149: }
1150:
1151: assert(receiver->wait_queue == waitq);
1152: assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1153:
1154: /*
1155: * See if it is still running on another processor (trying to
1156: * block itself). If so, fall off.
1157: *
1158: * JMM - We have an opportunity here. Since the thread is locked
1159: * and we find it runnable, it must still be trying to get into
1160: * thread_block on itself. We could just "hand him the message"
1161: * and let him go (thread_go_locked()) and then fall down into a
1162: * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
1163: * runs afoul of that. Clean this up!
1164: */
1165: if ((receiver->state & TH_RUN) != TH_WAIT) {
1166: assert(NCPUS > 1);
1167: HOT(c_mmot_cold_033++);
1168: fall_off:
1169: thread_unlock(receiver);
1170: if (waitq != &dest_mqueue->imq_wait_queue)
1171: wait_queue_unlock(waitq);
1172: splx(s);
1173: imq_unlock(dest_mqueue);
1174: goto abort_send_receive;
1175: }
1176:
1177: /*
1178: * Check that the receiver can stay on the hot path.
1179: */
1180: if ((send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option)
1181: > receiver->ith_msize) ||
1182: (receiver->ith_option & MACH_RCV_NOTIFY) ||
1183: (receiver->ith_scatter_list != MACH_MSG_BODY_NULL)) {
1184: /*
1185: * The receiver can't accept the message.
1186: */
1187: HOT(c_mmot_bad_rcvr++);
1188: goto fall_off;
1189: }
1190:
1191: #if THREAD_SWAPPER
1192: /*
1193: * Receiver looks okay -- is it swapped in?
1194: */
1195: rpc_lock(receiver);
1196: rcv_act = receiver->top_act;
1197: if (rcv_act->swap_state != TH_SW_IN &&
1198: rcv_act->swap_state != TH_SW_UNSWAPPABLE) {
1199: rpc_unlock(receiver);
1200: HOT(c_mmot_rcvr_swapped++);
1201: goto fall_off;
1202: }
1203:
1204: /*
1205: * Make sure receiver stays swapped in (if we can).
1206: */
1207: if (!act_lock_try(rcv_act)) { /* out of order! */
1208: rpc_unlock(receiver);
1209: HOT(c_mmot_rcvr_locked++);
1210: goto fall_off;
1211: }
1212:
1213: /*
1214: * Check for task swapping in progress affecting
1215: * receiver. Since rcv_act is attached to a shuttle,
1216: * its swap_state is covered by shuttle's thread_lock()
1217: * (sigh).
1218: */
1219: if ((rcv_act->swap_state != TH_SW_IN &&
1220: rcv_act->swap_state != TH_SW_UNSWAPPABLE) ||
1221: rcv_act->ast & AST_SWAPOUT) {
1222: act_unlock(rcv_act);
1223: rpc_unlock(receiver);
1224: HOT(c_mmot_rcvr_tswapped++);
1225: goto fall_off;
1226: }
1227:
1228: /*
1229: * We don't need to make receiver unswappable here -- holding
1230: * act_lock() of rcv_act is sufficient to prevent either thread
1231: * or task swapping from changing its state (see swapout_scan(),
1232: * task_swapout()). Don't release lock till receiver's state
1233: * is consistent. Its task may then be marked for swapout,
1234: * but that's life.
1235: */
1236: rpc_unlock(receiver);
1237: /*
1238: * NB: act_lock(rcv_act) still held
1239: */
1240: #endif /* THREAD_SWAPPER */
1241:
1242: /* At this point we are committed to do the "handoff". */
1243: c_mach_msg_trap_switch_fast++;
1244:
1245: /*
1246: * JMM - Go ahead and pull the receiver from the runq. If the
1247: * runq wasn't the one for the mqueue, unlock it.
1248: */
1249: wait_queue_pull_thread_locked(waitq,
1250: receiver,
1251: (waitq != &dest_mqueue->imq_wait_queue));
1252:
1253: /*
1254: * Store the kmsg and seqno where the receiver can pick it up.
1255: */
1256: receiver->ith_state = MACH_MSG_SUCCESS;
1257: receiver->ith_kmsg = kmsg;
1258: receiver->ith_seqno = dest_mqueue->imq_seqno++;
1259: ipc_mqueue_release_msgcount(dest_mqueue);
1260: imq_unlock(dest_mqueue);
1261: ip_unlock(dest_port);
1262:
1263:
1264: /*
1265: * Put sender on reply port's queue.
1266: * Also save state that the sender of
1267: * our reply message needs to determine if it
1268: * can hand off directly back to us.
1269: */
1270:
1271: self->ith_state = MACH_RCV_IN_PROGRESS;
1272: self->ith_msize = rcv_size;
1273:
1274: self->ith_option = option;
1275: self->ith_scatter_list = MACH_MSG_BODY_NULL;
1276: self->ith_scatter_list_size = scatter_list_size;
1277:
1278: waitq = &rcv_mqueue->imq_wait_queue;
1279: wait_queue_assert_wait_locked(waitq,
1280: IPC_MQUEUE_RECEIVE,
1281: THREAD_ABORTSAFE,
1282: TRUE); /* unlock? */
1283: /* rcv_mqueue is unlocked */
1284:
1285: current_task()->messages_sent++;
1286:
1287: /*
1288: * Switch directly to receiving thread, and block
1289: * this thread as though it had called ipc_mqueue_receive.
1290: */
1291: {
1292: extern unsigned sched_tick; /* kern/sched_prim.c */
1293:
1294: /* from thread_block_reason() */
1295: ast_off(AST_QUANTUM|AST_BLOCK|AST_URGENT);
1296:
1297: /*
1298: * JMM - Hacked in version of setrun scheduler op
1299: * that doesn't try to put thread on a runq. Safe
1300: * for now because we only do this for sp_mk threads.
1301: */
1302: {
1303: mk_sp_info_t sched_info;
1304:
1305: assert(receiver->sp_info != SP_INFO_NULL);
1306: sched_info = (mk_sp_info_t)receiver->sp_info;
1307: sched_info->th_state = MK_SP_RUNNABLE;
1308: }
1309:
1310: assert((receiver->state & (TH_RUN|TH_WAIT)) == TH_WAIT);
1311: assert(receiver != self);
1312:
1313: receiver->state &= ~(TH_WAIT|TH_UNINT);
1314:
1315: receiver->state |= TH_RUN;
1316: receiver->wait_result = THREAD_AWAKENED;
1317: #if NCPUS > 1 /* from thread_invoke inline */
1318: mp_disable_preemption();
1319: receiver->last_processor = current_processor();
1320: mp_enable_preemption();
1321: #endif /* NCPUS > 1 */
1322: thread_unlock(receiver);
1323: assert( receiver != self );
1324: #if THREAD_SWAPPER
1325: act_unlock(rcv_act);
1326: #endif /* THREAD_SWAPPER */
1327:
1328: /*
1329: * Prepare self (the sender) to block.
1330: */
1331: thread_lock(self);
1332:
1333: assert(!(self->state & TH_ABORT));
1334: assert(self->wait_result = -1); /* for assertions */
1335: assert(self->state & TH_RUN);
1336: self->state |= TH_WAIT;
1337: self->reason = 0; /* inline thread_invoke */
1338: thread_unlock(self);
1339:
1340: /*
1341: * Switch to the receiver now.
1342: * Inlined: thread_invoke(self, receiver, 0);
1343: */
1344: {
1345: thread_t old_thread;
1346: sched_policy_t *policy;
1347: sf_return_t sfr;
1348:
1349: mp_disable_preemption();
1350: ast_context(receiver->top_act, cpu_number());
1351: timer_switch(&receiver->system_timer);
1352:
1353: current_task()->csw++;
1354:
1355: policy = &sched_policy[self->policy];
1356: sfr = policy->sp_ops.sp_thread_done(policy, self);
1357: assert(sfr == SF_SUCCESS);
1358:
1359: old_thread = switch_context(self, 0, receiver);
1360:
1361: /* We're back! */
1362: assert(current_thread() == self);
1363: policy = &sched_policy[self->policy];
1364: sfr = policy->sp_ops.sp_thread_begin(policy, self);
1365: assert(sfr == SF_SUCCESS);
1366:
1367: assert(old_thread != self);
1368: thread_dispatch(old_thread);
1369: enable_preemption();
1370: }
1371:
1372: splx(s);
1373: }
1374:
1375: switch (self->wait_result) {
1376:
1377: case THREAD_INTERRUPTED:
1378: ipc_object_release(rcv_object);
1379: return MACH_RCV_INTERRUPTED;
1380:
1381: case THREAD_RESTART:
1382: /*
1383: * something bad happened to the port/set. If the port
1384: * is still active, it must have just changed membership.
1385: */
1386: if (!io_active(rcv_object)) {
1387: ipc_object_release(rcv_object);
1388: return MACH_RCV_PORT_DIED;
1389: } else {
1390: ipc_object_release(rcv_object);
1391: return MACH_RCV_PORT_CHANGED;
1392: }
1393:
1394: case THREAD_AWAKENED:
1395: ipc_object_release(rcv_object);
1396: break;
1397:
1398: default:
1399: panic("mmot_hotpath: bad wait result");
1400: }
1401:
1402:
1403: /*
1404: * If we allowed for just a receive too large indication
1405: * and we got one, copy out the size (stored in our kmsg
1406: * pointer) instead of the message. If we got the error,
1407: * but didn't set the flag, we just need to try the copy
1408: * out of the message and see where that takes us.
1409: */
1410: if ((option & MACH_RCV_LARGE) &&
1411: (self->ith_state == MACH_RCV_TOO_LARGE)) {
1412: mach_msg_header_t *out_msg;
1413:
1414: out_msg = (rcv_msg != MACH_MSG_NULL)?rcv_msg:msg;
1415: if (copyout((char *) &self->ith_kmsg,
1416: (char *) &msg->msgh_size,
1417: sizeof(mach_msg_size_t)))
1418: return MACH_RCV_INVALID_DATA;
1419: return MACH_RCV_TOO_LARGE;
1420: }
1421:
1422:
1423: /*
1424: * Restore state saved in sending thread's send path.
1425: */
1426: kmsg = self->ith_kmsg;
1427: assert(kmsg != IKM_NULL);
1428: dest_port = (ipc_port_t)kmsg->ikm_header.msgh_remote_port;
1429:
1430: {
1431: ip_lock(dest_port);
1432: assert(dest_port->ip_messages.imq_msgcount > 0);
1433: dest_port->ip_messages.imq_msgcount--;
1434: ip_unlock(dest_port);
1435: }
1436:
1437: hdr = &kmsg->ikm_header;
1438: send_size = hdr->msgh_size;
1439: trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1440: round_msg(send_size));
1441:
1442: if (option & MACH_RCV_TRAILER_MASK) {
1443: trailer->msgh_seqno = self->ith_seqno;
1444: trailer->msgh_trailer_size =
1445: REQUESTED_TRAILER_SIZE(option);
1446: }
1447: }
1448:
1449: fast_copyout:
1450: /*
1451: * Nothing locked and no references held, except
1452: * we have kmsg with msgh_seqno filled in. Must
1453: * still check against rcv_size and do
1454: * ipc_kmsg_copyout/ipc_kmsg_put.
1455: */
1456:
1457: reply_size = send_size + trailer->msgh_trailer_size;
1458: if (rcv_size < reply_size) {
1459: HOT(c_mmot_g_slow_copyout6++);
1460: goto slow_copyout;
1461: }
1462:
1463: /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1464:
1465: switch (hdr->msgh_bits) {
1466: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1467: MACH_MSG_TYPE_PORT_SEND_ONCE): {
1468: ipc_port_t reply_port =
1469: (ipc_port_t) hdr->msgh_local_port;
1470: mach_port_name_t dest_name, reply_name;
1471:
1472: /* receiving a request message */
1473:
1474: if (!IP_VALID(reply_port)) {
1475: HOT(c_mmot_g_slow_copyout5++);
1476: goto slow_copyout;
1477: }
1478:
1479: is_write_lock(space);
1480: assert(space->is_active);
1481:
1482: /*
1483: * To do an atomic copyout, need simultaneous
1484: * locks on both ports and the space. If
1485: * dest_port == reply_port, and simple locking is
1486: * enabled, then we will abort. Otherwise it's
1487: * OK to unlock twice.
1488: */
1489:
1490: ip_lock(dest_port);
1491: if (!ip_active(dest_port) ||
1492: !ip_lock_try(reply_port)) {
1493: HOT(c_mmot_cold_037++);
1494: goto abort_request_copyout;
1495: }
1496:
1497: if (!ip_active(reply_port)) {
1498: ip_unlock(reply_port);
1499: HOT(c_mmot_cold_038++);
1500: goto abort_request_copyout;
1501: }
1502:
1503: assert(reply_port->ip_sorights > 0);
1504: ip_unlock(reply_port);
1505:
1506: {
1507: register ipc_entry_t table;
1508: register ipc_entry_t entry;
1509: register mach_port_index_t index;
1510:
1511: /* optimized ipc_entry_get */
1512:
1513: table = space->is_table;
1514: index = table->ie_next;
1515:
1516: if (index == 0) {
1517: HOT(c_mmot_cold_039++);
1518: goto abort_request_copyout;
1519: }
1520:
1521: entry = &table[index];
1522: table->ie_next = entry->ie_next;
1523: entry->ie_request = 0;
1524:
1525: {
1526: register mach_port_gen_t gen;
1527:
1528: assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1529: gen = IE_BITS_NEW_GEN(entry->ie_bits);
1530:
1531: reply_name = MACH_PORT_MAKE(index, gen);
1532:
1533: /* optimized ipc_right_copyout */
1534:
1535: entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1536: }
1537:
1538: assert(MACH_PORT_VALID(reply_name));
1539: entry->ie_object = (ipc_object_t) reply_port;
1540: is_write_unlock(space);
1541: }
1542:
1543: /* optimized ipc_object_copyout_dest */
1544:
1545: assert(dest_port->ip_srights > 0);
1546: ip_release(dest_port);
1547:
1548: if (dest_port->ip_receiver == space)
1549: dest_name = dest_port->ip_receiver_name;
1550: else
1551: dest_name = MACH_PORT_NULL;
1552:
1553: if ((--dest_port->ip_srights == 0) &&
1554: (dest_port->ip_nsrequest != IP_NULL)) {
1555: ipc_port_t nsrequest;
1556: mach_port_mscount_t mscount;
1557:
1558: /* a rather rare case */
1559:
1560: nsrequest = dest_port->ip_nsrequest;
1561: mscount = dest_port->ip_mscount;
1562: dest_port->ip_nsrequest = IP_NULL;
1563: ip_unlock(dest_port);
1564: ipc_notify_no_senders(nsrequest, mscount);
1565: } else
1566: ip_unlock(dest_port);
1567:
1568: hdr->msgh_bits =
1569: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1570: MACH_MSG_TYPE_PORT_SEND);
1571: hdr->msgh_remote_port = (mach_port_t)reply_name;
1572: hdr->msgh_local_port = (mach_port_t)dest_name;
1573: HOT(c_mmot_hot_ok1++);
1574: goto fast_put;
1575:
1576: abort_request_copyout:
1577: ip_unlock(dest_port);
1578: is_write_unlock(space);
1579: HOT(c_mmot_g_slow_copyout4++);
1580: goto slow_copyout;
1581: }
1582:
1583: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1584: register mach_port_name_t dest_name;
1585:
1586: /* receiving a reply message */
1587:
1588: ip_lock(dest_port);
1589: if (!ip_active(dest_port)) {
1590: ip_unlock(dest_port);
1591: HOT(c_mmot_g_slow_copyout3++);
1592: goto slow_copyout;
1593: }
1594:
1595: /* optimized ipc_object_copyout_dest */
1596:
1597: assert(dest_port->ip_sorights > 0);
1598:
1599: if (dest_port->ip_receiver == space) {
1600: ip_release(dest_port);
1601: dest_port->ip_sorights--;
1602: dest_name = dest_port->ip_receiver_name;
1603: ip_unlock(dest_port);
1604: } else {
1605: ip_unlock(dest_port);
1606:
1607: ipc_notify_send_once(dest_port);
1608: dest_name = MACH_PORT_NULL;
1609: }
1610:
1611: hdr->msgh_bits = MACH_MSGH_BITS(0,
1612: MACH_MSG_TYPE_PORT_SEND_ONCE);
1613: hdr->msgh_remote_port = MACH_PORT_NULL;
1614: hdr->msgh_local_port = (ipc_port_t)dest_name;
1615: HOT(c_mmot_hot_ok2++);
1616: goto fast_put;
1617: }
1618:
1619: case MACH_MSGH_BITS_COMPLEX|
1620: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1621: register mach_port_name_t dest_name;
1622:
1623: /* receiving a complex reply message */
1624:
1625: ip_lock(dest_port);
1626: if (!ip_active(dest_port)) {
1627: ip_unlock(dest_port);
1628: HOT(c_mmot_g_slow_copyout1++);
1629: goto slow_copyout;
1630: }
1631:
1632: /* optimized ipc_object_copyout_dest */
1633:
1634: assert(dest_port->ip_sorights > 0);
1635:
1636: if (dest_port->ip_receiver == space) {
1637: ip_release(dest_port);
1638: dest_port->ip_sorights--;
1639: dest_name = dest_port->ip_receiver_name;
1640: ip_unlock(dest_port);
1641: } else {
1642: ip_unlock(dest_port);
1643:
1644: ipc_notify_send_once(dest_port);
1645: dest_name = MACH_PORT_NULL;
1646: }
1647:
1648: hdr->msgh_bits =
1649: MACH_MSGH_BITS_COMPLEX |
1650: MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1651: hdr->msgh_remote_port = MACH_PORT_NULL;
1652: hdr->msgh_local_port = (mach_port_t)dest_name;
1653:
1654: mr = ipc_kmsg_copyout_body(kmsg, space,
1655: current_map(),
1656: MACH_MSG_BODY_NULL);
1657: if (mr != MACH_MSG_SUCCESS) {
1658: if (ipc_kmsg_put(msg, kmsg, hdr->msgh_size +
1659: trailer->msgh_trailer_size) ==
1660: MACH_RCV_INVALID_DATA)
1661: return MACH_RCV_INVALID_DATA;
1662: else
1663: return mr | MACH_RCV_BODY_ERROR;
1664: }
1665: HOT(c_mmot_hot_ok3++);
1666: goto fast_put;
1667: }
1668:
1669: default:
1670: HOT(c_mmot_g_slow_copyout2++);
1671: goto slow_copyout;
1672: }
1673: /*NOTREACHED*/ panic("happy birthday rwd");
1674:
1675: fast_put:
1676: /*
1677: * We have the reply message data in kmsg,
1678: * and the reply message size (plus trailer size)
1679: * in reply_size. Just need to copy it out to the
1680: * user and free kmsg. Must check ikm_cache after
1681: * copyoutmsg. Inlined ipc_kmsg_put() here.
1682: */
1683: mr = MACH_MSG_SUCCESS;
1684: if (copyoutmsg((char *) &kmsg->ikm_header,
1685: (char *)(rcv_msg ? rcv_msg : msg),
1686: hdr->msgh_size + trailer->msgh_trailer_size))
1687: mr = MACH_RCV_INVALID_DATA;
1688: else
1689: current_task()->messages_received++;
1690:
1691: if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
1692: KMSG_IS_RT(kmsg) || !ikm_cache_put(kmsg)) {
1693: ikm_free(kmsg);
1694: }
1695:
1696: return(mr);
1697:
1698:
1699: /* BEGINNING OF WARM PATH */
1700:
1701: /*
1702: * The slow path has a few non-register temporary
1703: * variables used only for call-by-reference.
1704: */
1705:
1706: {
1707: ipc_kmsg_t temp_kmsg;
1708: mach_port_seqno_t temp_seqno;
1709: ipc_object_t temp_rcv_object;
1710: ipc_mqueue_t temp_rcv_mqueue;
1711:
1712: slow_get:
1713: /*
1714: * No locks, references, or messages held.
1715: * Still have to get the request, send it,
1716: * receive reply, etc.
1717: */
1718:
1719: mr = ipc_kmsg_get(msg, send_size, &temp_kmsg, current_space());
1720: if (mr != MACH_MSG_SUCCESS) {
1721: return(mr);
1722: /*NOTREACHED*/
1723: }
1724: kmsg = temp_kmsg;
1725: hdr = &kmsg->ikm_header;
1726: /* naturally align the message before tacking on the trailer */
1727: trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1728: send_size);
1729: /* try to get back on optimized path */
1730: HOT(c_mmot_getback_fast_copyin++);
1731: goto fast_copyin;
1732:
1733: slow_copyin:
1734: /*
1735: * We have the message data in kmsg, but
1736: * we still need to copyin, send it,
1737: * receive a reply, and do copyout.
1738: */
1739:
1740: mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1741: MACH_PORT_NULL);
1742: if (mr != MACH_MSG_SUCCESS) {
1743: if (kmsg->ikm_size != IKM_SAVED_KMSG_SIZE ||
1744: KMSG_IS_RT(kmsg) || !ikm_cache_put(kmsg))
1745: ikm_free(kmsg);
1746: return(mr);
1747: }
1748:
1749: /* try to get back on optimized path */
1750:
1751: if (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1752: HOT(c_mmot_cold_048++);
1753: goto slow_send;
1754: }
1755:
1756: dest_port = (ipc_port_t) hdr->msgh_remote_port;
1757: assert(IP_VALID(dest_port));
1758:
1759: ip_lock(dest_port);
1760: if (!ip_active(dest_port)) {
1761: ip_unlock(dest_port);
1762: goto slow_send;
1763: }
1764:
1765: if (dest_port->ip_receiver == ipc_space_kernel) {
1766: dest_port->ip_messages.imq_seqno++;
1767: ip_unlock(dest_port);
1768: goto kernel_send;
1769: }
1770:
1771: if (!imq_full(&dest_port->ip_messages) ||
1772: (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1773: MACH_MSG_TYPE_PORT_SEND_ONCE))
1774: {
1775: /*
1776: * Try an optimized ipc_mqueue_copyin.
1777: * It will work if this is a request message.
1778: */
1779:
1780: register ipc_port_t reply_port;
1781:
1782: reply_port = (ipc_port_t) hdr->msgh_local_port;
1783: if (IP_VALID(reply_port)) {
1784: if (ip_lock_try(reply_port)) {
1785: if (ip_active(reply_port) &&
1786: reply_port->ip_receiver == space &&
1787: reply_port->ip_receiver_name == rcv_name &&
1788: reply_port->ip_pset_count == 0)
1789: {
1790: /* Grab a reference to the reply port. */
1791: rcv_object = (ipc_object_t) reply_port;
1792: io_reference(rcv_object);
1793: rcv_mqueue = &reply_port->ip_messages;
1794: imq_lock(rcv_mqueue);
1795: io_unlock(rcv_object);
1796: HOT(c_mmot_getback_FastSR++);
1797: goto fast_send_receive;
1798: }
1799: ip_unlock(reply_port);
1800: }
1801: }
1802: }
1803:
1804: ip_unlock(dest_port);
1805: HOT(c_mmot_cold_050++);
1806: goto slow_send;
1807:
1808: kernel_send:
1809: /*
1810: * Special case: send message to kernel services.
1811: * The request message has been copied into the
1812: * kmsg. Nothing is locked.
1813: */
1814:
1815: {
1816: register ipc_port_t reply_port;
1817: mach_port_seqno_t local_seqno;
1818:
1819: /*
1820: * Perform the kernel function.
1821: */
1822: c_mmot_kernel_send++;
1823:
1824: current_task()->messages_sent++;
1825:
1826: kmsg = ipc_kobject_server(kmsg);
1827: if (kmsg == IKM_NULL) {
1828: /*
1829: * No reply. Take the
1830: * slow receive path.
1831: */
1832: HOT(c_mmot_cold_051++);
1833: goto slow_get_rcv_port;
1834: }
1835:
1836: /*
1837: * Check that:
1838: * the reply port is alive
1839: * we hold the receive right
1840: * the name has not changed.
1841: * the port is not in a set
1842: * If any of these are not true,
1843: * we cannot directly receive the reply
1844: * message.
1845: */
1846: hdr = &kmsg->ikm_header;
1847: send_size = hdr->msgh_size;
1848: trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1849: round_msg(send_size));
1850: reply_port = (ipc_port_t) hdr->msgh_remote_port;
1851: ip_lock(reply_port);
1852:
1853: if ((!ip_active(reply_port)) ||
1854: (reply_port->ip_receiver != space) ||
1855: (reply_port->ip_receiver_name != rcv_name) ||
1856: (reply_port->ip_pset_count != 0))
1857: {
1858: ip_unlock(reply_port);
1859: ipc_kmsg_send_always(kmsg);
1860: HOT(c_mmot_cold_052++);
1861: goto slow_get_rcv_port;
1862: }
1863:
1864: rcv_mqueue = &reply_port->ip_messages;
1865: imq_lock(rcv_mqueue);
1866: /* keep port locked, and don`t change ref count yet */
1867:
1868: /*
1869: * If there are messages on the port
1870: * or other threads waiting for a message,
1871: * we cannot directly receive the reply.
1872: */
1873: if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1874: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1875: {
1876: imq_unlock(rcv_mqueue);
1877: ip_unlock(reply_port);
1878: ipc_kmsg_send_always(kmsg);
1879: HOT(c_mmot_cold_053++);
1880: goto slow_get_rcv_port;
1881: }
1882:
1883: /*
1884: * We can directly receive this reply.
1885: * Since there were no messages queued
1886: * on the reply port, there should be
1887: * no threads blocked waiting to send.
1888: */
1889: dest_port = reply_port;
1890: local_seqno = rcv_mqueue->imq_seqno++;
1891: imq_unlock(rcv_mqueue);
1892:
1893: /*
1894: * inline ipc_object_release.
1895: * Port is still locked.
1896: * Reference count was not incremented.
1897: */
1898: ip_check_unlock(reply_port);
1899:
1900: if (option & MACH_RCV_TRAILER_MASK) {
1901: trailer->msgh_seqno = local_seqno;
1902: trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1903: }
1904: /* copy out the kernel reply */
1905: HOT(c_mmot_fastkernelreply++);
1906: goto fast_copyout;
1907: }
1908:
1909: slow_send:
1910: /*
1911: * Nothing is locked. We have acquired kmsg, but
1912: * we still need to send it and receive a reply.
1913: */
1914:
1915: mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1916: MACH_MSG_TIMEOUT_NONE);
1917: if (mr != MACH_MSG_SUCCESS) {
1918: mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1919: current_map(),
1920: MACH_MSG_BODY_NULL);
1921:
1922: (void) ipc_kmsg_put(msg, kmsg, hdr->msgh_size);
1923: return(mr);
1924: }
1925:
1926: slow_get_rcv_port:
1927: /*
1928: * We have sent the message. Copy in the receive port.
1929: */
1930: mr = ipc_mqueue_copyin(space, rcv_name,
1931: &temp_rcv_mqueue, &temp_rcv_object);
1932: if (mr != MACH_MSG_SUCCESS) {
1933: return(mr);
1934: }
1935: rcv_mqueue = temp_rcv_mqueue;
1936: rcv_object = temp_rcv_object;
1937: /* hold ref for rcv_object */
1938:
1939: slow_receive:
1940: /*
1941: * Now we have sent the request and copied in rcv_name,
1942: * so rcv_mqueue is locked and hold ref for rcv_object.
1943: * Just receive a reply and try to get back to fast path.
1944: */
1945:
1946: self->ith_option = option;
1947: self->ith_scatter_list = MACH_MSG_BODY_NULL;
1948: self->ith_scatter_list_size = scatter_list_size;
1949:
1950: mr = ipc_mqueue_receive(rcv_mqueue,
1951: MACH_MSG_OPTION_NONE,
1952: MACH_MSG_SIZE_MAX,
1953: MACH_MSG_TIMEOUT_NONE,
1954: THREAD_ABORTSAFE,
1955: &temp_kmsg, &temp_seqno);
1956: ipc_object_release(rcv_object);
1957:
1958: if (mr != MACH_MSG_SUCCESS) {
1959: return(mr);
1960: }
1961:
1962: kmsg = temp_kmsg;
1963: hdr = &kmsg->ikm_header;
1964: send_size = hdr->msgh_size;
1965: trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1966: round_msg(send_size));
1967: if (option & MACH_RCV_TRAILER_MASK) {
1968: trailer->msgh_seqno = temp_seqno;
1969: trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1970: }
1971: dest_port = (ipc_port_t) hdr->msgh_remote_port;
1972: HOT(c_mmot_cold_055++);
1973: goto fast_copyout;
1974:
1975: slow_copyout:
1976: /*
1977: * Nothing locked and no references held, except
1978: * we have kmsg with msgh_seqno filled in. Must
1979: * still check against rcv_size and do
1980: * ipc_kmsg_copyout/ipc_kmsg_put.
1981: */
1982:
1983: reply_size = send_size + trailer->msgh_trailer_size;
1984: if (rcv_size < reply_size) {
1985: if (msg_receive_error(kmsg, msg, option, temp_seqno,
1986: space) == MACH_RCV_INVALID_DATA) {
1987: mr = MACH_RCV_INVALID_DATA;
1988: return(mr);
1989: }
1990: else {
1991: mr = MACH_RCV_TOO_LARGE;
1992: return(mr);
1993: }
1994: }
1995:
1996: mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1997: MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1998: if (mr != MACH_MSG_SUCCESS) {
1999: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
2000: if (ipc_kmsg_put(msg, kmsg, reply_size) ==
2001: MACH_RCV_INVALID_DATA)
2002: mr = MACH_RCV_INVALID_DATA;
2003: }
2004: else {
2005: if (msg_receive_error(kmsg, msg, option,
2006: temp_seqno, space) == MACH_RCV_INVALID_DATA)
2007: mr = MACH_RCV_INVALID_DATA;
2008: }
2009:
2010: return(mr);
2011: }
2012:
2013: /* try to get back on optimized path */
2014: HOT(c_mmot_getback_fast_put++);
2015: goto fast_put;
2016:
2017: slow_put:
2018: mr = ipc_kmsg_put((rcv_msg != MACH_MSG_NULL)?rcv_msg:msg,
2019: kmsg,
2020: hdr->msgh_size + trailer->msgh_trailer_size);
2021: return(mr);
2022: /*NOTREACHED*/
2023: }
2024: } /* END OF HOT PATH */
2025: #endif /* ENABLE_HOTPATH */
2026:
2027: if (option & MACH_SEND_MSG) {
2028: mr = mach_msg_send(msg, option, send_size,
2029: timeout, notify);
2030: if (mr != MACH_MSG_SUCCESS) {
2031: return mr;
2032: }
2033: }
2034:
2035: if (option & MACH_RCV_MSG) {
2036: mach_msg_header_t *rcv;
2037:
2038: /*
2039: * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
2040: * and receive buffer
2041: * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
2042: * alternate receive buffer (separate send and receive buffers).
2043: */
2044: if (option & MACH_RCV_OVERWRITE)
2045: rcv = rcv_msg;
2046: else if (rcv_msg != MACH_MSG_NULL)
2047: rcv = rcv_msg;
2048: else
2049: rcv = msg;
2050: mr = mach_msg_receive(rcv, option, rcv_size, rcv_name,
2051: timeout, notify, scatter_list_size);
2052: if (mr != MACH_MSG_SUCCESS) {
2053: return mr;
2054: }
2055: }
2056:
2057: return MACH_MSG_SUCCESS;
2058: }
2059:
2060: /*
2061: * Routine: msg_receive_error [internal]
2062: * Purpose:
2063: * Builds a minimal header/trailer and copies it to
2064: * the user message buffer. Invoked when in the case of a
2065: * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
2066: * Conditions:
2067: * Nothing locked.
2068: * Returns:
2069: * MACH_MSG_SUCCESS minimal header/trailer copied
2070: * MACH_RCV_INVALID_DATA copyout to user buffer failed
2071: */
2072:
2073: mach_msg_return_t
2074: msg_receive_error(
2075: ipc_kmsg_t kmsg,
2076: mach_msg_header_t *msg,
2077: mach_msg_option_t option,
2078: mach_port_seqno_t seqno,
2079: ipc_space_t space)
2080: {
2081: mach_msg_format_0_trailer_t *trailer;
2082:
2083: /*
2084: * Copy out the destination port in the message.
2085: * Destroy all other rights and memory in the message.
2086: */
2087: ipc_kmsg_copyout_dest(kmsg, space);
2088:
2089: /*
2090: * Build a minimal message with the requested trailer.
2091: */
2092: trailer = (mach_msg_format_0_trailer_t *)
2093: ((vm_offset_t)&kmsg->ikm_header +
2094: round_msg(sizeof(mach_msg_header_t)));
2095: kmsg->ikm_header.msgh_size = sizeof(mach_msg_header_t);
2096: bcopy( (char *)&trailer_template,
2097: (char *)trailer,
2098: sizeof(trailer_template));
2099: if (option & MACH_RCV_TRAILER_MASK) {
2100: trailer->msgh_seqno = seqno;
2101: trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
2102: }
2103:
2104: /*
2105: * Copy the message to user space
2106: */
2107: if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
2108: trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
2109: return(MACH_RCV_INVALID_DATA);
2110: else
2111: return(MACH_MSG_SUCCESS);
2112: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.