![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mach_msg.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / ipc|
Return to mach_msg.c CVS log | Up to [GnuMach] / Gnu-Mach / ipc |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: */
31: /*
32: * File: ipc/mach_msg.c
33: * Author: Rich Draves
34: * Date: 1989
35: *
36: * Exported message traps. See mach/message.h.
37: */
38:
39: #include <mach/kern_return.h>
40: #include <mach/port.h>
41: #include <mach/message.h>
42: #include <kern/assert.h>
43: #include <kern/counters.h>
1.1.1.3 root 44: #include <kern/debug.h>
1.1 root 45: #include <kern/lock.h>
1.1.1.3 root 46: #include <kern/printf.h>
1.1 root 47: #include <kern/sched_prim.h>
48: #include <kern/ipc_sched.h>
1.1.1.4 root 49: #include <kern/exception.h>
1.1 root 50: #include <vm/vm_map.h>
51: #include <ipc/ipc_kmsg.h>
52: #include <ipc/ipc_marequest.h>
53: #include <ipc/ipc_mqueue.h>
54: #include <ipc/ipc_object.h>
55: #include <ipc/ipc_notify.h>
56: #include <ipc/ipc_port.h>
57: #include <ipc/ipc_pset.h>
58: #include <ipc/ipc_space.h>
59: #include <ipc/ipc_thread.h>
60: #include <ipc/ipc_entry.h>
61: #include <ipc/mach_msg.h>
1.1.1.3 root 62: #include <machine/locore.h>
63: #include <machine/pcb.h>
1.1 root 64:
65: /*
66: * Routine: mach_msg_send
67: * Purpose:
68: * Send a message.
69: * Conditions:
70: * Nothing locked.
71: * Returns:
72: * MACH_MSG_SUCCESS Sent the message.
73: * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
74: * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
75: * MACH_SEND_INVALID_DATA Couldn't copy message data.
76: * MACH_SEND_INVALID_HEADER
77: * Illegal value in the message header bits.
78: * MACH_SEND_INVALID_DEST The space is dead.
79: * MACH_SEND_INVALID_NOTIFY Bad notify port.
80: * MACH_SEND_INVALID_DEST Can't copyin destination port.
81: * MACH_SEND_INVALID_REPLY Can't copyin reply port.
82: * MACH_SEND_TIMED_OUT Timeout expired without delivery.
83: * MACH_SEND_INTERRUPTED Delivery interrupted.
84: * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
85: * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
86: * MACH_SEND_NOTIFY_IN_PROGRESS
87: * This space has already forced a message to this port.
88: */
89:
90: mach_msg_return_t
1.1.1.4 root 91: mach_msg_send(
92: mach_msg_header_t *msg,
93: mach_msg_option_t option,
94: mach_msg_size_t send_size,
95: mach_msg_timeout_t time_out,
96: mach_port_t notify)
1.1 root 97: {
98: ipc_space_t space = current_space();
99: vm_map_t map = current_map();
100: ipc_kmsg_t kmsg;
101: mach_msg_return_t mr;
102:
103: mr = ipc_kmsg_get(msg, send_size, &kmsg);
104: if (mr != MACH_MSG_SUCCESS)
105: return mr;
106:
107: if (option & MACH_SEND_CANCEL) {
108: if (notify == MACH_PORT_NULL)
109: mr = MACH_SEND_INVALID_NOTIFY;
110: else
111: mr = ipc_kmsg_copyin(kmsg, space, map, notify);
112: } else
113: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
114: if (mr != MACH_MSG_SUCCESS) {
115: ikm_free(kmsg);
116: return mr;
117: }
118:
119: if (option & MACH_SEND_NOTIFY) {
120: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT,
121: ((option & MACH_SEND_TIMEOUT) ?
122: time_out : MACH_MSG_TIMEOUT_NONE));
123: if (mr == MACH_SEND_TIMED_OUT) {
124: ipc_port_t dest = (ipc_port_t)
125: kmsg->ikm_header.msgh_remote_port;
126:
127: if (notify == MACH_PORT_NULL)
128: mr = MACH_SEND_INVALID_NOTIFY;
129: else
130: mr = ipc_marequest_create(space, dest,
131: notify, &kmsg->ikm_marequest);
132: if (mr == MACH_MSG_SUCCESS) {
133: ipc_mqueue_send_always(kmsg);
134: return MACH_SEND_WILL_NOTIFY;
135: }
136: }
137: } else
138: mr = ipc_mqueue_send(kmsg, option & MACH_SEND_TIMEOUT,
139: time_out);
140:
141: if (mr != MACH_MSG_SUCCESS) {
142: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
143:
144: assert(kmsg->ikm_marequest == IMAR_NULL);
145: (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
146: }
147:
148: return mr;
149: }
150:
151: /*
152: * Routine: mach_msg_receive
153: * Purpose:
154: * Receive a message.
155: * Conditions:
156: * Nothing locked.
157: * Returns:
158: * MACH_MSG_SUCCESS Received a message.
159: * MACH_RCV_INVALID_NAME The name doesn't denote a right,
160: * or the denoted right is not receive or port set.
161: * MACH_RCV_IN_SET Receive right is a member of a set.
162: * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
163: * MACH_RCV_TIMED_OUT Timeout expired without a message.
164: * MACH_RCV_INTERRUPTED Reception interrupted.
165: * MACH_RCV_PORT_DIED Port/set died while receiving.
166: * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
167: * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
168: * MACH_RCV_INVALID_NOTIFY Bad notify port.
169: * MACH_RCV_HEADER_ERROR
170: */
171:
172: mach_msg_return_t
1.1.1.4 root 173: mach_msg_receive(
174: mach_msg_header_t *msg,
175: mach_msg_option_t option,
176: mach_msg_size_t rcv_size,
177: mach_port_t rcv_name,
178: mach_msg_timeout_t time_out,
179: mach_port_t notify)
1.1 root 180: {
181: ipc_thread_t self = current_thread();
182: ipc_space_t space = current_space();
183: vm_map_t map = current_map();
184: ipc_object_t object;
185: ipc_mqueue_t mqueue;
186: ipc_kmsg_t kmsg;
187: mach_port_seqno_t seqno;
188: mach_msg_return_t mr;
189:
190: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
191: if (mr != MACH_MSG_SUCCESS)
192: return mr;
193: /* hold ref for object; mqueue is locked */
194:
195: /*
196: * ipc_mqueue_receive may not return, because if we block
197: * then our kernel stack may be discarded. So we save
198: * state here for mach_msg_receive_continue to pick up.
199: */
200:
201: self->ith_msg = msg;
202: self->ith_option = option;
203: self->ith_rcv_size = rcv_size;
204: self->ith_timeout = time_out;
205: self->ith_notify = notify;
206: self->ith_object = object;
207: self->ith_mqueue = mqueue;
208:
209: if (option & MACH_RCV_LARGE) {
210: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
211: rcv_size, time_out,
212: FALSE, mach_msg_receive_continue,
213: &kmsg, &seqno);
214: /* mqueue is unlocked */
215: ipc_object_release(object);
216: if (mr != MACH_MSG_SUCCESS) {
217: if (mr == MACH_RCV_TOO_LARGE) {
218: mach_msg_size_t real_size =
1.1.1.3 root 219: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 220:
221: assert(real_size > rcv_size);
222:
1.1.1.3 root 223: (void) copyout(&real_size,
224: &msg->msgh_size,
1.1 root 225: sizeof(mach_msg_size_t));
226: }
227:
228: return mr;
229: }
230:
231: kmsg->ikm_header.msgh_seqno = seqno;
232: assert(kmsg->ikm_header.msgh_size <= rcv_size);
233: } else {
234: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
235: MACH_MSG_SIZE_MAX, time_out,
236: FALSE, mach_msg_receive_continue,
237: &kmsg, &seqno);
238: /* mqueue is unlocked */
239: ipc_object_release(object);
240: if (mr != MACH_MSG_SUCCESS)
241: return mr;
242:
243: kmsg->ikm_header.msgh_seqno = seqno;
244: if (kmsg->ikm_header.msgh_size > rcv_size) {
245: ipc_kmsg_copyout_dest(kmsg, space);
246: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
247: return MACH_RCV_TOO_LARGE;
248: }
249: }
250:
251: if (option & MACH_RCV_NOTIFY) {
252: if (notify == MACH_PORT_NULL)
253: mr = MACH_RCV_INVALID_NOTIFY;
254: else
255: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
256: } else
257: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
258: if (mr != MACH_MSG_SUCCESS) {
259: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
260: (void) ipc_kmsg_put(msg, kmsg,
261: kmsg->ikm_header.msgh_size);
262: } else {
263: ipc_kmsg_copyout_dest(kmsg, space);
264: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
265: }
266:
267: return mr;
268: }
269:
270: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
271: }
272:
273: /*
274: * Routine: mach_msg_receive_continue
275: * Purpose:
276: * Continue after blocking for a message.
277: * Conditions:
278: * Nothing locked. We are running on a new kernel stack,
279: * with the receive state saved in the thread. From here
280: * control goes back to user space.
281: */
282:
283: void
1.1.1.3 root 284: mach_msg_receive_continue(void)
1.1 root 285: {
286: ipc_thread_t self = current_thread();
287: ipc_space_t space = current_space();
288: vm_map_t map = current_map();
289: mach_msg_header_t *msg = self->ith_msg;
290: mach_msg_option_t option = self->ith_option;
291: mach_msg_size_t rcv_size = self->ith_rcv_size;
292: mach_msg_timeout_t time_out = self->ith_timeout;
293: mach_port_t notify = self->ith_notify;
294: ipc_object_t object = self->ith_object;
295: ipc_mqueue_t mqueue = self->ith_mqueue;
296: ipc_kmsg_t kmsg;
297: mach_port_seqno_t seqno;
298: mach_msg_return_t mr;
299:
300: if (option & MACH_RCV_LARGE) {
301: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
302: rcv_size, time_out,
303: TRUE, mach_msg_receive_continue,
304: &kmsg, &seqno);
305: /* mqueue is unlocked */
306: ipc_object_release(object);
307: if (mr != MACH_MSG_SUCCESS) {
308: if (mr == MACH_RCV_TOO_LARGE) {
309: mach_msg_size_t real_size =
1.1.1.3 root 310: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 311:
312: assert(real_size > rcv_size);
313:
1.1.1.3 root 314: (void) copyout(&real_size,
315: &msg->msgh_size,
1.1 root 316: sizeof(mach_msg_size_t));
317: }
318:
319: thread_syscall_return(mr);
320: /*NOTREACHED*/
321: }
322:
323: kmsg->ikm_header.msgh_seqno = seqno;
324: assert(kmsg->ikm_header.msgh_size <= rcv_size);
325: } else {
326: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
327: MACH_MSG_SIZE_MAX, time_out,
328: TRUE, mach_msg_receive_continue,
329: &kmsg, &seqno);
330: /* mqueue is unlocked */
331: ipc_object_release(object);
332: if (mr != MACH_MSG_SUCCESS) {
333: thread_syscall_return(mr);
334: /*NOTREACHED*/
335: }
336:
337: kmsg->ikm_header.msgh_seqno = seqno;
338: if (kmsg->ikm_header.msgh_size > rcv_size) {
339: ipc_kmsg_copyout_dest(kmsg, space);
340: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
341: thread_syscall_return(MACH_RCV_TOO_LARGE);
342: /*NOTREACHED*/
343: }
344: }
345:
346: if (option & MACH_RCV_NOTIFY) {
347: if (notify == MACH_PORT_NULL)
348: mr = MACH_RCV_INVALID_NOTIFY;
349: else
350: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
351: } else
352: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
353: if (mr != MACH_MSG_SUCCESS) {
354: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
355: (void) ipc_kmsg_put(msg, kmsg,
356: kmsg->ikm_header.msgh_size);
357: } else {
358: ipc_kmsg_copyout_dest(kmsg, space);
359: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
360: }
361:
362: thread_syscall_return(mr);
363: /*NOTREACHED*/
364: }
365:
366: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
367: thread_syscall_return(mr);
368: /*NOTREACHED*/
369: }
370:
371: /*
372: * Routine: mach_msg_trap [mach trap]
373: * Purpose:
374: * Possibly send a message; possibly receive a message.
375: * Conditions:
376: * Nothing locked.
377: * Returns:
378: * All of mach_msg_send and mach_msg_receive error codes.
379: */
380:
381: mach_msg_return_t
1.1.1.4 root 382: mach_msg_trap(
383: mach_msg_header_t *msg,
384: mach_msg_option_t option,
385: mach_msg_size_t send_size,
386: mach_msg_size_t rcv_size,
387: mach_port_t rcv_name,
388: mach_msg_timeout_t time_out,
389: mach_port_t notify)
1.1 root 390: {
391: mach_msg_return_t mr;
392:
393: /* first check for common cases */
394:
395: if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) {
1.1.1.4 root 396: ipc_thread_t self = current_thread();
1.1 root 397: ipc_space_t space = self->task->itk_space;
1.1.1.4 root 398: ipc_kmsg_t kmsg;
399: ipc_port_t dest_port;
1.1 root 400: ipc_object_t rcv_object;
1.1.1.4 root 401: ipc_mqueue_t rcv_mqueue;
1.1 root 402: mach_msg_size_t reply_size;
403:
404: /*
405: * This case is divided into ten sections, each
406: * with a label. There are five optimized
407: * sections and six unoptimized sections, which
408: * do the same thing but handle all possible
409: * cases and are slower.
410: *
411: * The five sections for an RPC are
412: * 1) Get request message into a buffer.
413: * (fast_get or slow_get)
414: * 2) Copyin request message and rcv_name.
415: * (fast_copyin or slow_copyin)
416: * 3) Enqueue request and dequeue reply.
417: * (fast_send_receive or
418: * slow_send and slow_receive)
419: * 4) Copyout reply message.
420: * (fast_copyout or slow_copyout)
421: * 5) Put reply message to user's buffer.
422: * (fast_put or slow_put)
423: *
424: * Keep the locking hierarchy firmly in mind.
425: * (First spaces, then ports, then port sets,
426: * then message queues.) Only a non-blocking
427: * attempt can be made to acquire locks out of
428: * order, or acquire two locks on the same level.
429: * Acquiring two locks on the same level will
430: * fail if the objects are really the same,
431: * unless simple locking is disabled. This is OK,
432: * because then the extra unlock does nothing.
433: *
434: * There are two major reasons these RPCs can't use
435: * ipc_thread_switch, and use slow_send/slow_receive:
436: * 1) Kernel RPCs.
437: * 2) Servers fall behind clients, so
438: * client doesn't find a blocked server thread and
439: * server finds waiting messages and can't block.
440: */
441:
442: /*
443: fast_get:
444: */
445: /*
446: * optimized ipc_kmsg_get
447: *
448: * No locks, references, or messages held.
449: * We must clear ikm_cache before copyinmsg.
450: */
451:
452: if ((send_size > IKM_SAVED_MSG_SIZE) ||
453: (send_size < sizeof(mach_msg_header_t)) ||
454: (send_size & 3) ||
455: ((kmsg = ikm_cache()) == IKM_NULL))
456: goto slow_get;
457:
458: ikm_cache() = IKM_NULL;
459: ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
460:
1.1.1.3 root 461: if (copyinmsg(msg, &kmsg->ikm_header,
1.1 root 462: send_size)) {
463: ikm_free(kmsg);
464: goto slow_get;
465: }
466:
467: kmsg->ikm_header.msgh_size = send_size;
468:
469: fast_copyin:
470: /*
471: * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
472: *
473: * We have the request message data in kmsg.
474: * Must still do copyin, send, receive, etc.
475: *
476: * If the message isn't simple, we can't combine
477: * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
478: * because copyin of the message body might
479: * affect rcv_name.
480: */
481:
482: switch (kmsg->ikm_header.msgh_bits) {
483: case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
484: MACH_MSG_TYPE_MAKE_SEND_ONCE): {
1.1.1.4 root 485: ipc_port_t reply_port;
1.1 root 486: {
1.1.1.4 root 487: mach_port_t reply_name =
1.1 root 488: kmsg->ikm_header.msgh_local_port;
489:
490: if (reply_name != rcv_name)
491: goto slow_copyin;
492:
493: is_read_lock(space);
494: assert(space->is_active);
495:
1.1.1.4 root 496: ipc_entry_t entry;
1.1.1.5 ! root 497: entry = ipc_entry_lookup (space, reply_name);
! 498: if (entry == IE_NULL)
1.1 root 499: goto abort_request_copyin;
500: reply_port = (ipc_port_t) entry->ie_object;
501: assert(reply_port != IP_NULL);
502: }
503:
504: {
1.1.1.4 root 505: mach_port_t dest_name =
1.1 root 506: kmsg->ikm_header.msgh_remote_port;
507:
1.1.1.4 root 508: ipc_entry_t entry;
509: ipc_entry_bits_t bits;
1.1.1.5 ! root 510: entry = ipc_entry_lookup (space, dest_name);
! 511: if (entry == IE_NULL)
! 512: goto abort_request_copyin;
1.1 root 513: bits = entry->ie_bits;
514:
1.1.1.5 ! root 515: /* check type bits */
! 516: if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_SEND)
1.1 root 517: goto abort_request_copyin;
518:
519: assert(IE_BITS_UREFS(bits) > 0);
520:
521: dest_port = (ipc_port_t) entry->ie_object;
522: assert(dest_port != IP_NULL);
523: }
524:
525: /*
526: * To do an atomic copyin, need simultaneous
527: * locks on both ports and the space. If
528: * dest_port == reply_port, and simple locking is
529: * enabled, then we will abort. Otherwise it's
530: * OK to unlock twice.
531: */
532:
533: ip_lock(dest_port);
534: if (!ip_active(dest_port) ||
535: !ip_lock_try(reply_port)) {
536: ip_unlock(dest_port);
537: goto abort_request_copyin;
538: }
539: is_read_unlock(space);
540:
541: assert(dest_port->ip_srights > 0);
542: dest_port->ip_srights++;
543: ip_reference(dest_port);
544:
545: assert(ip_active(reply_port));
546: assert(reply_port->ip_receiver_name ==
547: kmsg->ikm_header.msgh_local_port);
548: assert(reply_port->ip_receiver == space);
549:
550: reply_port->ip_sorights++;
551: ip_reference(reply_port);
552:
553: kmsg->ikm_header.msgh_bits =
554: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
555: MACH_MSG_TYPE_PORT_SEND_ONCE);
556: kmsg->ikm_header.msgh_remote_port =
557: (mach_port_t) dest_port;
558: kmsg->ikm_header.msgh_local_port =
559: (mach_port_t) reply_port;
560:
561: /* make sure we can queue to the destination */
562:
563: if (dest_port->ip_receiver == ipc_space_kernel) {
564: /*
565: * The kernel server has a reference to
566: * the reply port, which it hands back
567: * to us in the reply message. We do
568: * not need to keep another reference to
569: * it.
570: */
571: ip_unlock(reply_port);
572:
573: assert(ip_active(dest_port));
574: ip_unlock(dest_port);
575: goto kernel_send;
576: }
577:
578: if (dest_port->ip_msgcount >= dest_port->ip_qlimit)
579: goto abort_request_send_receive;
580:
581: /* optimized ipc_mqueue_copyin */
582:
583: if (reply_port->ip_pset != IPS_NULL)
584: goto abort_request_send_receive;
585:
586: rcv_object = (ipc_object_t) reply_port;
587: io_reference(rcv_object);
588: rcv_mqueue = &reply_port->ip_messages;
589: imq_lock(rcv_mqueue);
590: io_unlock(rcv_object);
591: goto fast_send_receive;
592:
593: abort_request_copyin:
594: is_read_unlock(space);
595: goto slow_copyin;
596:
597: abort_request_send_receive:
598: ip_unlock(dest_port);
599: ip_unlock(reply_port);
600: goto slow_send;
601: }
602:
603: case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
604: /* sending a reply message */
605:
606: {
1.1.1.4 root 607: mach_port_t reply_name =
1.1 root 608: kmsg->ikm_header.msgh_local_port;
609:
610: if (reply_name != MACH_PORT_NULL)
611: goto slow_copyin;
612: }
613:
614: is_write_lock(space);
615: assert(space->is_active);
616:
617: {
1.1.1.4 root 618: ipc_entry_t entry;
619: mach_port_t dest_name =
1.1 root 620: kmsg->ikm_header.msgh_remote_port;
621:
1.1.1.5 ! root 622: entry = ipc_entry_lookup (space, dest_name);
! 623: if (entry == IE_NULL)
1.1 root 624: goto abort_reply_dest_copyin;
625:
1.1.1.5 ! root 626: /* check type bits */
! 627: if (IE_BITS_TYPE (entry->ie_bits) !=
! 628: MACH_PORT_TYPE_SEND_ONCE)
1.1 root 629: goto abort_reply_dest_copyin;
630:
631: /* optimized ipc_right_copyin */
632:
633: assert(IE_BITS_TYPE(entry->ie_bits) ==
634: MACH_PORT_TYPE_SEND_ONCE);
635: assert(IE_BITS_UREFS(entry->ie_bits) == 1);
636: assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0);
637:
638: if (entry->ie_request != 0)
639: goto abort_reply_dest_copyin;
640:
641: dest_port = (ipc_port_t) entry->ie_object;
642: assert(dest_port != IP_NULL);
643:
644: ip_lock(dest_port);
645: if (!ip_active(dest_port)) {
646: ip_unlock(dest_port);
647: goto abort_reply_dest_copyin;
648: }
649:
650: assert(dest_port->ip_sorights > 0);
651: entry->ie_object = IO_NULL;
1.1.1.5 ! root 652: ipc_entry_dealloc (space, dest_name, entry);
1.1 root 653: }
654:
655: kmsg->ikm_header.msgh_bits =
656: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
657: 0);
658: kmsg->ikm_header.msgh_remote_port =
659: (mach_port_t) dest_port;
660:
661: /* make sure we can queue to the destination */
662:
663: assert(dest_port->ip_receiver != ipc_space_kernel);
664:
1.1.1.5 ! root 665: /* optimized ipc_mqueue_copyin */
1.1 root 666:
667: {
1.1.1.4 root 668: ipc_entry_t entry;
669: ipc_entry_bits_t bits;
1.1.1.5 ! root 670: entry = ipc_entry_lookup (space, rcv_name);
! 671: if (entry == IE_NULL)
1.1 root 672: goto abort_reply_rcv_copyin;
673: bits = entry->ie_bits;
674:
675: /* check type bits; looking for receive or set */
676:
677: if (bits & MACH_PORT_TYPE_PORT_SET) {
1.1.1.4 root 678: ipc_pset_t rcv_pset;
1.1 root 679:
680: rcv_pset = (ipc_pset_t) entry->ie_object;
681: assert(rcv_pset != IPS_NULL);
682:
683: ips_lock(rcv_pset);
684: assert(ips_active(rcv_pset));
685:
686: rcv_object = (ipc_object_t) rcv_pset;
687: rcv_mqueue = &rcv_pset->ips_messages;
688: } else if (bits & MACH_PORT_TYPE_RECEIVE) {
1.1.1.4 root 689: ipc_port_t rcv_port;
1.1 root 690:
691: rcv_port = (ipc_port_t) entry->ie_object;
692: assert(rcv_port != IP_NULL);
693:
694: if (!ip_lock_try(rcv_port))
695: goto abort_reply_rcv_copyin;
696: assert(ip_active(rcv_port));
697:
698: if (rcv_port->ip_pset != IPS_NULL) {
699: ip_unlock(rcv_port);
700: goto abort_reply_rcv_copyin;
701: }
702:
703: rcv_object = (ipc_object_t) rcv_port;
704: rcv_mqueue = &rcv_port->ip_messages;
705: } else
706: goto abort_reply_rcv_copyin;
707: }
708:
709: is_write_unlock(space);
710: io_reference(rcv_object);
711: imq_lock(rcv_mqueue);
712: io_unlock(rcv_object);
713: goto fast_send_receive;
714:
715: abort_reply_dest_copyin:
716: is_write_unlock(space);
717: goto slow_copyin;
718:
719: abort_reply_rcv_copyin:
720: ip_unlock(dest_port);
721: is_write_unlock(space);
722: goto slow_send;
723: }
724:
725: default:
726: goto slow_copyin;
727: }
728: /*NOTREACHED*/
729:
730: fast_send_receive:
731: /*
732: * optimized ipc_mqueue_send/ipc_mqueue_receive
733: *
734: * Finished get/copyin of kmsg and copyin of rcv_name.
735: * space is unlocked, dest_port is locked,
736: * we can queue kmsg to dest_port,
737: * rcv_mqueue is locked, rcv_object holds a ref,
738: * if rcv_object is a port it isn't in a port set
739: *
740: * Note that if simple locking is turned off,
741: * then we could have dest_mqueue == rcv_mqueue
742: * and not abort when we try to lock dest_mqueue.
743: */
744:
745: assert(ip_active(dest_port));
746: assert(dest_port->ip_receiver != ipc_space_kernel);
747: assert((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
748: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
749: MACH_MSG_TYPE_PORT_SEND_ONCE));
750: assert((kmsg->ikm_header.msgh_bits &
751: MACH_MSGH_BITS_CIRCULAR) == 0);
752:
753: {
1.1.1.4 root 754: ipc_mqueue_t dest_mqueue;
755: ipc_thread_t receiver;
1.1 root 756:
757: {
1.1.1.4 root 758: ipc_pset_t dest_pset;
1.1 root 759:
760: dest_pset = dest_port->ip_pset;
761: if (dest_pset == IPS_NULL)
762: dest_mqueue = &dest_port->ip_messages;
763: else
764: dest_mqueue = &dest_pset->ips_messages;
765: }
766:
767: if (!imq_lock_try(dest_mqueue)) {
768: abort_send_receive:
769: ip_unlock(dest_port);
770: imq_unlock(rcv_mqueue);
771: ipc_object_release(rcv_object);
772: goto slow_send;
773: }
774:
775: receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
776: if ((receiver == ITH_NULL) ||
777: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
778: != IKM_NULL)) {
779: imq_unlock(dest_mqueue);
780: goto abort_send_receive;
781: }
782:
783: /*
784: * There is a receiver thread waiting, and
785: * there is no reply message for us to pick up.
786: * We have hope of hand-off, so save state.
787: */
788:
789: self->ith_msg = msg;
790: self->ith_rcv_size = rcv_size;
791: self->ith_object = rcv_object;
792: self->ith_mqueue = rcv_mqueue;
793:
794: if ((receiver->swap_func == (void (*)()) mach_msg_continue) &&
795: thread_handoff(self, mach_msg_continue, receiver)) {
796: assert(current_thread() == receiver);
797:
798: /*
799: * We can use the optimized receive code,
800: * because the receiver is using no options.
801: */
802: } else if ((receiver->swap_func ==
803: (void (*)()) exception_raise_continue) &&
804: thread_handoff(self, mach_msg_continue, receiver)) {
805: counter(c_mach_msg_trap_block_exc++);
806: assert(current_thread() == receiver);
807:
808: /*
809: * We are a reply message coming back through
810: * the optimized exception-handling path.
811: * Finish with rcv_mqueue and dest_mqueue,
812: * and then jump to exception code with
813: * dest_port still locked. We don't bother
814: * with a sequence number in this case.
815: */
816:
817: ipc_thread_enqueue_macro(
818: &rcv_mqueue->imq_threads, self);
819: self->ith_state = MACH_RCV_IN_PROGRESS;
820: self->ith_msize = MACH_MSG_SIZE_MAX;
821: imq_unlock(rcv_mqueue);
822:
823: ipc_thread_rmqueue_first_macro(
824: &dest_mqueue->imq_threads, receiver);
825: imq_unlock(dest_mqueue);
826:
827: exception_raise_continue_fast(dest_port, kmsg);
828: /*NOTREACHED*/
829: return MACH_MSG_SUCCESS;
830: } else if ((send_size <= receiver->ith_msize) &&
831: thread_handoff(self, mach_msg_continue, receiver)) {
832: assert(current_thread() == receiver);
833:
834: if ((receiver->swap_func ==
835: (void (*)()) mach_msg_receive_continue) &&
836: ((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) {
837: /*
838: * We can still use the optimized code.
839: */
840: } else {
841: counter(c_mach_msg_trap_block_slow++);
842: /*
843: * We are running as the receiver,
844: * but we can't use the optimized code.
845: * Finish send/receive processing.
846: */
847:
848: dest_port->ip_msgcount++;
849: ip_unlock(dest_port);
850:
851: ipc_thread_enqueue_macro(
852: &rcv_mqueue->imq_threads, self);
853: self->ith_state = MACH_RCV_IN_PROGRESS;
854: self->ith_msize = MACH_MSG_SIZE_MAX;
855: imq_unlock(rcv_mqueue);
856:
857: ipc_thread_rmqueue_first_macro(
858: &dest_mqueue->imq_threads, receiver);
859: receiver->ith_state = MACH_MSG_SUCCESS;
860: receiver->ith_kmsg = kmsg;
861: receiver->ith_seqno = dest_port->ip_seqno++;
862: imq_unlock(dest_mqueue);
863:
864: /*
865: * Call the receiver's continuation.
866: */
867:
868: receiver->wait_result = THREAD_AWAKENED;
869: (*receiver->swap_func)();
870: /*NOTREACHED*/
871: return MACH_MSG_SUCCESS;
872: }
873: } else {
874: /*
875: * The receiver can't accept the message,
876: * or we can't switch to the receiver.
877: */
878:
879: imq_unlock(dest_mqueue);
880: goto abort_send_receive;
881: }
882: counter(c_mach_msg_trap_block_fast++);
883:
884: /*
885: * Safe to unlock dest_port now that we are
886: * committed to this path, because we hold
887: * dest_mqueue locked. We never bother changing
888: * dest_port->ip_msgcount.
889: */
890:
891: ip_unlock(dest_port);
892:
893: /*
894: * We need to finish preparing self for its
895: * time asleep in rcv_mqueue.
896: */
897:
898: ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self);
899: self->ith_state = MACH_RCV_IN_PROGRESS;
900: self->ith_msize = MACH_MSG_SIZE_MAX;
901: imq_unlock(rcv_mqueue);
902:
903: /*
904: * Finish extracting receiver from dest_mqueue.
905: */
906:
907: ipc_thread_rmqueue_first_macro(
908: &dest_mqueue->imq_threads, receiver);
909: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
910: imq_unlock(dest_mqueue);
911:
912: /*
913: * We don't have to do any post-dequeue processing of
914: * the message. We never incremented ip_msgcount, we
915: * know it has no msg-accepted request, and blocked
916: * senders aren't a worry because we found the port
917: * with a receiver waiting.
918: */
919:
920: self = receiver;
921: space = self->task->itk_space;
922:
923: msg = self->ith_msg;
924: rcv_size = self->ith_rcv_size;
925: rcv_object = self->ith_object;
926:
927: /* inline ipc_object_release */
928: io_lock(rcv_object);
929: io_release(rcv_object);
930: io_check_unlock(rcv_object);
931: }
932:
933: fast_copyout:
934: /*
935: * Nothing locked and no references held, except
936: * we have kmsg with msgh_seqno filled in. Must
937: * still check against rcv_size and do
938: * ipc_kmsg_copyout/ipc_kmsg_put.
939: */
940:
941: assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port
942: == dest_port);
943:
944: reply_size = kmsg->ikm_header.msgh_size;
945: if (rcv_size < reply_size)
946: goto slow_copyout;
947:
948: /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
949:
950: switch (kmsg->ikm_header.msgh_bits) {
951: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
952: MACH_MSG_TYPE_PORT_SEND_ONCE): {
953: ipc_port_t reply_port =
954: (ipc_port_t) kmsg->ikm_header.msgh_local_port;
955: mach_port_t dest_name, reply_name;
1.1.1.5 ! root 956: unsigned long payload;
1.1 root 957:
958: /* receiving a request message */
959:
960: if (!IP_VALID(reply_port))
961: goto slow_copyout;
962:
963: is_write_lock(space);
964: assert(space->is_active);
965:
966: /*
967: * To do an atomic copyout, need simultaneous
968: * locks on both ports and the space. If
969: * dest_port == reply_port, and simple locking is
970: * enabled, then we will abort. Otherwise it's
971: * OK to unlock twice.
972: */
973:
974: ip_lock(dest_port);
975: if (!ip_active(dest_port) ||
976: !ip_lock_try(reply_port))
977: goto abort_request_copyout;
978:
979: if (!ip_active(reply_port)) {
980: ip_unlock(reply_port);
981: goto abort_request_copyout;
982: }
983:
984: assert(reply_port->ip_sorights > 0);
985: ip_unlock(reply_port);
986:
987: {
1.1.1.4 root 988: ipc_entry_t entry;
1.1.1.5 ! root 989: kern_return_t kr;
! 990: kr = ipc_entry_get (space, &reply_name, &entry);
! 991: if (kr)
1.1 root 992: goto abort_request_copyout;
1.1.1.5 ! root 993: assert (entry != NULL);
1.1 root 994:
995: {
1.1.1.4 root 996: mach_port_gen_t gen;
1.1 root 997:
998: assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
999: gen = entry->ie_bits + IE_BITS_GEN_ONE;
1000:
1001: /* optimized ipc_right_copyout */
1002:
1003: entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1004: }
1005:
1006: assert(MACH_PORT_VALID(reply_name));
1007: entry->ie_object = (ipc_object_t) reply_port;
1008: is_write_unlock(space);
1009: }
1010:
1011: /* optimized ipc_object_copyout_dest */
1012:
1013: assert(dest_port->ip_srights > 0);
1014: ip_release(dest_port);
1015:
1016: if (dest_port->ip_receiver == space)
1017: dest_name = dest_port->ip_receiver_name;
1018: else
1019: dest_name = MACH_PORT_NULL;
1.1.1.5 ! root 1020: payload = dest_port->ip_protected_payload;
1.1 root 1021:
1022: if ((--dest_port->ip_srights == 0) &&
1023: (dest_port->ip_nsrequest != IP_NULL)) {
1024: ipc_port_t nsrequest;
1025: mach_port_mscount_t mscount;
1026:
1027: /* a rather rare case */
1028:
1029: nsrequest = dest_port->ip_nsrequest;
1030: mscount = dest_port->ip_mscount;
1031: dest_port->ip_nsrequest = IP_NULL;
1032: ip_unlock(dest_port);
1033:
1034: ipc_notify_no_senders(nsrequest, mscount);
1035: } else
1036: ip_unlock(dest_port);
1037:
1.1.1.4 root 1038: if (! ipc_port_flag_protected_payload(dest_port)) {
1039: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
1040: MACH_MSG_TYPE_PORT_SEND_ONCE,
1041: MACH_MSG_TYPE_PORT_SEND);
1042: kmsg->ikm_header.msgh_local_port = dest_name;
1043: } else {
1044: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
1045: MACH_MSG_TYPE_PORT_SEND_ONCE,
1046: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
1047: kmsg->ikm_header.msgh_protected_payload =
1.1.1.5 ! root 1048: payload;
1.1.1.4 root 1049: }
1.1 root 1050: kmsg->ikm_header.msgh_remote_port = reply_name;
1051: goto fast_put;
1052:
1053: abort_request_copyout:
1054: ip_unlock(dest_port);
1055: is_write_unlock(space);
1056: goto slow_copyout;
1057: }
1058:
1059: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1.1.1.4 root 1060: mach_port_t dest_name;
1.1.1.5 ! root 1061: unsigned long payload;
1.1 root 1062:
1063: /* receiving a reply message */
1064:
1065: ip_lock(dest_port);
1066: if (!ip_active(dest_port))
1067: goto slow_copyout;
1068:
1069: /* optimized ipc_object_copyout_dest */
1070:
1071: assert(dest_port->ip_sorights > 0);
1072:
1.1.1.5 ! root 1073: payload = dest_port->ip_protected_payload;
! 1074:
1.1 root 1075: if (dest_port->ip_receiver == space) {
1076: ip_release(dest_port);
1077: dest_port->ip_sorights--;
1078: dest_name = dest_port->ip_receiver_name;
1079: ip_unlock(dest_port);
1080: } else {
1081: ip_unlock(dest_port);
1082:
1083: ipc_notify_send_once(dest_port);
1084: dest_name = MACH_PORT_NULL;
1085: }
1086:
1.1.1.4 root 1087: if (! ipc_port_flag_protected_payload(dest_port)) {
1088: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
1089: 0,
1090: MACH_MSG_TYPE_PORT_SEND_ONCE);
1091: kmsg->ikm_header.msgh_local_port = dest_name;
1092: } else {
1093: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
1094: 0,
1095: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
1096: kmsg->ikm_header.msgh_protected_payload =
1.1.1.5 ! root 1097: payload;
1.1.1.4 root 1098: }
1.1 root 1099: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1100: goto fast_put;
1101: }
1102:
1103: case MACH_MSGH_BITS_COMPLEX|
1104: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1.1.1.4 root 1105: mach_port_t dest_name;
1.1.1.5 ! root 1106: unsigned long payload;
1.1 root 1107:
1108: /* receiving a complex reply message */
1109:
1110: ip_lock(dest_port);
1111: if (!ip_active(dest_port))
1112: goto slow_copyout;
1113:
1114: /* optimized ipc_object_copyout_dest */
1115:
1116: assert(dest_port->ip_sorights > 0);
1117:
1.1.1.5 ! root 1118: payload = dest_port->ip_protected_payload;
! 1119:
1.1 root 1120: if (dest_port->ip_receiver == space) {
1121: ip_release(dest_port);
1122: dest_port->ip_sorights--;
1123: dest_name = dest_port->ip_receiver_name;
1124: ip_unlock(dest_port);
1125: } else {
1126: ip_unlock(dest_port);
1127:
1128: ipc_notify_send_once(dest_port);
1129: dest_name = MACH_PORT_NULL;
1130: }
1131:
1.1.1.4 root 1132: if (! ipc_port_flag_protected_payload(dest_port)) {
1133: kmsg->ikm_header.msgh_bits =
1134: MACH_MSGH_BITS_COMPLEX
1135: | MACH_MSGH_BITS(
1136: 0,
1137: MACH_MSG_TYPE_PORT_SEND_ONCE);
1138: kmsg->ikm_header.msgh_local_port = dest_name;
1139: } else {
1140: kmsg->ikm_header.msgh_bits =
1141: MACH_MSGH_BITS_COMPLEX
1142: | MACH_MSGH_BITS(
1143: 0,
1144: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
1145: kmsg->ikm_header.msgh_protected_payload =
1.1.1.5 ! root 1146: payload;
1.1.1.4 root 1147: }
1.1 root 1148: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1149:
1150: mr = ipc_kmsg_copyout_body(
1151: (vm_offset_t) (&kmsg->ikm_header + 1),
1152: (vm_offset_t) &kmsg->ikm_header
1153: + kmsg->ikm_header.msgh_size,
1154: space,
1155: current_map());
1156:
1157: if (mr != MACH_MSG_SUCCESS) {
1158: (void) ipc_kmsg_put(msg, kmsg,
1159: kmsg->ikm_header.msgh_size);
1160: return mr | MACH_RCV_BODY_ERROR;
1161: }
1162: goto fast_put;
1163: }
1164:
1165: default:
1166: goto slow_copyout;
1167: }
1168: /*NOTREACHED*/
1169:
1170: fast_put:
1171: /*
1172: * We have the reply message data in kmsg,
1173: * and the reply message size in reply_size.
1174: * Just need to copy it out to the user and free kmsg.
1175: * We must check ikm_cache after copyoutmsg.
1176: */
1177:
1178: ikm_check_initialized(kmsg, kmsg->ikm_size);
1179:
1180: if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
1.1.1.3 root 1181: copyoutmsg(&kmsg->ikm_header, msg,
1.1 root 1182: reply_size) ||
1183: (ikm_cache() != IKM_NULL))
1184: goto slow_put;
1185:
1186: ikm_cache() = kmsg;
1187: thread_syscall_return(MACH_MSG_SUCCESS);
1188: /*NOTREACHED*/
1189: return MACH_MSG_SUCCESS; /* help for the compiler */
1190:
1191: /*
1192: * The slow path has a few non-register temporary
1193: * variables used only for call-by-reference.
1194: */
1195:
1196: {
1197: ipc_kmsg_t temp_kmsg;
1198: mach_port_seqno_t temp_seqno;
1199: ipc_object_t temp_rcv_object;
1200: ipc_mqueue_t temp_rcv_mqueue;
1201:
1202: slow_get:
1203: /*
1204: * No locks, references, or messages held.
1205: * Still have to get the request, send it,
1206: * receive reply, etc.
1207: */
1208:
1209: mr = ipc_kmsg_get(msg, send_size, &temp_kmsg);
1210: if (mr != MACH_MSG_SUCCESS) {
1211: thread_syscall_return(mr);
1212: /*NOTREACHED*/
1213: }
1214: kmsg = temp_kmsg;
1215:
1216: /* try to get back on optimized path */
1217: goto fast_copyin;
1218:
1219: slow_copyin:
1220: /*
1221: * We have the message data in kmsg, but
1222: * we still need to copyin, send it,
1223: * receive a reply, and do copyout.
1224: */
1225:
1226: mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1227: MACH_PORT_NULL);
1228: if (mr != MACH_MSG_SUCCESS) {
1229: ikm_free(kmsg);
1230: thread_syscall_return(mr);
1231: /*NOTREACHED*/
1232: }
1233:
1234: /* try to get back on optimized path */
1235:
1236: if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR)
1237: goto slow_send;
1238:
1239: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1240: assert(IP_VALID(dest_port));
1241:
1242: ip_lock(dest_port);
1243: if (dest_port->ip_receiver == ipc_space_kernel) {
1244: assert(ip_active(dest_port));
1245: ip_unlock(dest_port);
1246: goto kernel_send;
1247: }
1248:
1249: if (ip_active(dest_port) &&
1250: ((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
1251: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
1252: MACH_MSG_TYPE_PORT_SEND_ONCE)))
1253: {
1254: /*
1255: * Try an optimized ipc_mqueue_copyin.
1256: * It will work if this is a request message.
1257: */
1258:
1.1.1.4 root 1259: ipc_port_t reply_port;
1.1 root 1260:
1261: reply_port = (ipc_port_t)
1262: kmsg->ikm_header.msgh_local_port;
1263: if (IP_VALID(reply_port)) {
1264: if (ip_lock_try(reply_port)) {
1265: if (ip_active(reply_port) &&
1266: reply_port->ip_receiver == space &&
1267: reply_port->ip_receiver_name == rcv_name &&
1268: reply_port->ip_pset == IPS_NULL)
1269: {
1270: /* Grab a reference to the reply port. */
1271: rcv_object = (ipc_object_t) reply_port;
1272: io_reference(rcv_object);
1273: rcv_mqueue = &reply_port->ip_messages;
1274: imq_lock(rcv_mqueue);
1275: io_unlock(rcv_object);
1276: goto fast_send_receive;
1277: }
1278: ip_unlock(reply_port);
1279: }
1280: }
1281: }
1282:
1283: ip_unlock(dest_port);
1284: goto slow_send;
1285:
1286: kernel_send:
1287: /*
1288: * Special case: send message to kernel services.
1289: * The request message has been copied into the
1290: * kmsg. Nothing is locked.
1291: */
1292:
1293: {
1.1.1.4 root 1294: ipc_port_t reply_port;
1.1 root 1295:
1296: /*
1297: * Perform the kernel function.
1298: */
1299:
1300: kmsg = ipc_kobject_server(kmsg);
1301: if (kmsg == IKM_NULL) {
1302: /*
1303: * No reply. Take the
1304: * slow receive path.
1305: */
1306: goto slow_get_rcv_port;
1307: }
1308:
1309: /*
1310: * Check that:
1311: * the reply port is alive
1312: * we hold the receive right
1313: * the name has not changed.
1314: * the port is not in a set
1315: * If any of these are not true,
1316: * we cannot directly receive the reply
1317: * message.
1318: */
1319: reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1320: ip_lock(reply_port);
1321:
1322: if ((!ip_active(reply_port)) ||
1323: (reply_port->ip_receiver != space) ||
1324: (reply_port->ip_receiver_name != rcv_name) ||
1325: (reply_port->ip_pset != IPS_NULL))
1326: {
1327: ip_unlock(reply_port);
1328: ipc_mqueue_send_always(kmsg);
1329: goto slow_get_rcv_port;
1330: }
1331:
1332: rcv_mqueue = &reply_port->ip_messages;
1333: imq_lock(rcv_mqueue);
1334: /* keep port locked, and don`t change ref count yet */
1335:
1336: /*
1337: * If there are messages on the port
1338: * or other threads waiting for a message,
1339: * we cannot directly receive the reply.
1340: */
1341: if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads)
1342: != ITH_NULL) ||
1343: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
1344: != IKM_NULL))
1345: {
1346: imq_unlock(rcv_mqueue);
1347: ip_unlock(reply_port);
1348: ipc_mqueue_send_always(kmsg);
1349: goto slow_get_rcv_port;
1350: }
1351:
1352: /*
1353: * We can directly receive this reply.
1354: * Since the kernel reply never blocks,
1355: * it holds no message_accepted request.
1356: * Since there were no messages queued
1357: * on the reply port, there should be
1358: * no threads blocked waiting to send.
1359: */
1360:
1361: assert(kmsg->ikm_marequest == IMAR_NULL);
1362: assert(ipc_thread_queue_first(&reply_port->ip_blocked)
1363: == ITH_NULL);
1364:
1365: dest_port = reply_port;
1366: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
1367: imq_unlock(rcv_mqueue);
1368:
1369: /*
1370: * inline ipc_object_release.
1371: * Port is still locked.
1372: * Reference count was not incremented.
1373: */
1374: ip_check_unlock(reply_port);
1375:
1376: /* copy out the kernel reply */
1377: goto fast_copyout;
1378: }
1379:
1380: slow_send:
1381: /*
1382: * Nothing is locked. We have acquired kmsg, but
1383: * we still need to send it and receive a reply.
1384: */
1385:
1386: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1387: MACH_MSG_TIMEOUT_NONE);
1388: if (mr != MACH_MSG_SUCCESS) {
1389: mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1390: current_map());
1391:
1392: assert(kmsg->ikm_marequest == IMAR_NULL);
1393: (void) ipc_kmsg_put(msg, kmsg,
1394: kmsg->ikm_header.msgh_size);
1395: thread_syscall_return(mr);
1396: /*NOTREACHED*/
1397: }
1398:
1399: slow_get_rcv_port:
1400: /*
1401: * We have sent the message. Copy in the receive port.
1402: */
1403: mr = ipc_mqueue_copyin(space, rcv_name,
1404: &temp_rcv_mqueue, &temp_rcv_object);
1405: if (mr != MACH_MSG_SUCCESS) {
1406: thread_syscall_return(mr);
1407: /*NOTREACHED*/
1408: }
1409: rcv_mqueue = temp_rcv_mqueue;
1410: rcv_object = temp_rcv_object;
1411: /* hold ref for rcv_object; rcv_mqueue is locked */
1412:
1413: /*
1414: slow_receive:
1415: */
1416: /*
1417: * Now we have sent the request and copied in rcv_name,
1418: * so rcv_mqueue is locked and hold ref for rcv_object.
1419: * Just receive a reply and try to get back to fast path.
1420: *
1421: * ipc_mqueue_receive may not return, because if we block
1422: * then our kernel stack may be discarded. So we save
1423: * state here for mach_msg_continue to pick up.
1424: */
1425:
1426: self->ith_msg = msg;
1427: self->ith_rcv_size = rcv_size;
1428: self->ith_object = rcv_object;
1429: self->ith_mqueue = rcv_mqueue;
1430:
1431: mr = ipc_mqueue_receive(rcv_mqueue,
1432: MACH_MSG_OPTION_NONE,
1433: MACH_MSG_SIZE_MAX,
1434: MACH_MSG_TIMEOUT_NONE,
1435: FALSE, mach_msg_continue,
1436: &temp_kmsg, &temp_seqno);
1437: /* rcv_mqueue is unlocked */
1438: ipc_object_release(rcv_object);
1439: if (mr != MACH_MSG_SUCCESS) {
1440: thread_syscall_return(mr);
1441: /*NOTREACHED*/
1442: }
1443:
1444: (kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno;
1445: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1446: goto fast_copyout;
1447:
1448: slow_copyout:
1449: /*
1450: * Nothing locked and no references held, except
1451: * we have kmsg with msgh_seqno filled in. Must
1452: * still check against rcv_size and do
1453: * ipc_kmsg_copyout/ipc_kmsg_put.
1454: */
1455:
1456: reply_size = kmsg->ikm_header.msgh_size;
1457: if (rcv_size < reply_size) {
1458: ipc_kmsg_copyout_dest(kmsg, space);
1459: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1460: thread_syscall_return(MACH_RCV_TOO_LARGE);
1461: /*NOTREACHED*/
1462: }
1463:
1464: mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1465: MACH_PORT_NULL);
1466: if (mr != MACH_MSG_SUCCESS) {
1467: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1468: (void) ipc_kmsg_put(msg, kmsg,
1469: kmsg->ikm_header.msgh_size);
1470: } else {
1471: ipc_kmsg_copyout_dest(kmsg, space);
1472: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1473: }
1474:
1475: thread_syscall_return(mr);
1476: /*NOTREACHED*/
1477: }
1478:
1479: /* try to get back on optimized path */
1480:
1481: goto fast_put;
1482:
1483: slow_put:
1484: mr = ipc_kmsg_put(msg, kmsg, reply_size);
1485: thread_syscall_return(mr);
1486: /*NOTREACHED*/
1487: }
1488: } else if (option == MACH_SEND_MSG) {
1489: ipc_space_t space = current_space();
1490: vm_map_t map = current_map();
1491: ipc_kmsg_t kmsg;
1492:
1493: mr = ipc_kmsg_get(msg, send_size, &kmsg);
1494: if (mr != MACH_MSG_SUCCESS)
1495: return mr;
1496:
1497: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1498: if (mr != MACH_MSG_SUCCESS) {
1499: ikm_free(kmsg);
1500: return mr;
1501: }
1502:
1503: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1504: MACH_MSG_TIMEOUT_NONE);
1505: if (mr != MACH_MSG_SUCCESS) {
1506: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
1507:
1508: assert(kmsg->ikm_marequest == IMAR_NULL);
1509: (void) ipc_kmsg_put(msg, kmsg,
1510: kmsg->ikm_header.msgh_size);
1511: }
1512:
1513: return mr;
1514: } else if (option == MACH_RCV_MSG) {
1515: ipc_thread_t self = current_thread();
1516: ipc_space_t space = current_space();
1517: vm_map_t map = current_map();
1518: ipc_object_t object;
1519: ipc_mqueue_t mqueue;
1520: ipc_kmsg_t kmsg;
1521: mach_port_seqno_t seqno;
1522:
1523: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1524: if (mr != MACH_MSG_SUCCESS)
1525: return mr;
1526: /* hold ref for object; mqueue is locked */
1527:
1528: /*
1529: * ipc_mqueue_receive may not return, because if we block
1530: * then our kernel stack may be discarded. So we save
1531: * state here for mach_msg_continue to pick up.
1532: */
1533:
1534: self->ith_msg = msg;
1535: self->ith_rcv_size = rcv_size;
1536: self->ith_object = object;
1537: self->ith_mqueue = mqueue;
1538:
1539: mr = ipc_mqueue_receive(mqueue,
1540: MACH_MSG_OPTION_NONE,
1541: MACH_MSG_SIZE_MAX,
1542: MACH_MSG_TIMEOUT_NONE,
1543: FALSE, mach_msg_continue,
1544: &kmsg, &seqno);
1545: /* mqueue is unlocked */
1546: ipc_object_release(object);
1547: if (mr != MACH_MSG_SUCCESS)
1548: return mr;
1549:
1550: kmsg->ikm_header.msgh_seqno = seqno;
1551: if (rcv_size < kmsg->ikm_header.msgh_size) {
1552: ipc_kmsg_copyout_dest(kmsg, space);
1553: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1554: return MACH_RCV_TOO_LARGE;
1555: }
1556:
1557: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1558: if (mr != MACH_MSG_SUCCESS) {
1559: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1560: (void) ipc_kmsg_put(msg, kmsg,
1561: kmsg->ikm_header.msgh_size);
1562: } else {
1563: ipc_kmsg_copyout_dest(kmsg, space);
1564: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1565: }
1566:
1567: return mr;
1568: }
1569:
1570: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1571: } else if (option == MACH_MSG_OPTION_NONE) {
1572: /*
1573: * We can measure the "null mach_msg_trap"
1574: * (syscall entry and thread_syscall_return exit)
1575: * with this path.
1576: */
1577:
1578: thread_syscall_return(MACH_MSG_SUCCESS);
1579: /*NOTREACHED*/
1580: }
1581:
1582: if (option & MACH_SEND_MSG) {
1583: mr = mach_msg_send(msg, option, send_size,
1584: time_out, notify);
1585: if (mr != MACH_MSG_SUCCESS)
1586: return mr;
1587: }
1588:
1589: if (option & MACH_RCV_MSG) {
1590: mr = mach_msg_receive(msg, option, rcv_size, rcv_name,
1591: time_out, notify);
1592: if (mr != MACH_MSG_SUCCESS)
1593: return mr;
1594: }
1595:
1596: return MACH_MSG_SUCCESS;
1597: }
1598:
1599: /*
1600: * Routine: mach_msg_continue
1601: * Purpose:
1602: * Continue after blocking for a message.
1603: * Conditions:
1604: * Nothing locked. We are running on a new kernel stack,
1605: * with the receive state saved in the thread. From here
1606: * control goes back to user space.
1607: */
1608:
1609: void
1.1.1.3 root 1610: mach_msg_continue(void)
1.1 root 1611: {
1612: ipc_thread_t thread = current_thread();
1613: task_t task = thread->task;
1614: ipc_space_t space = task->itk_space;
1615: vm_map_t map = task->map;
1616: mach_msg_header_t *msg = thread->ith_msg;
1617: mach_msg_size_t rcv_size = thread->ith_rcv_size;
1618: ipc_object_t object = thread->ith_object;
1619: ipc_mqueue_t mqueue = thread->ith_mqueue;
1620: ipc_kmsg_t kmsg;
1621: mach_port_seqno_t seqno;
1622: mach_msg_return_t mr;
1623:
1624: mr = ipc_mqueue_receive(mqueue, MACH_MSG_OPTION_NONE,
1625: MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE,
1626: TRUE, mach_msg_continue, &kmsg, &seqno);
1627: /* mqueue is unlocked */
1628: ipc_object_release(object);
1629: if (mr != MACH_MSG_SUCCESS) {
1630: thread_syscall_return(mr);
1631: /*NOTREACHED*/
1632: }
1633:
1634: kmsg->ikm_header.msgh_seqno = seqno;
1635: if (kmsg->ikm_header.msgh_size > rcv_size) {
1636: ipc_kmsg_copyout_dest(kmsg, space);
1637: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1638: thread_syscall_return(MACH_RCV_TOO_LARGE);
1639: /*NOTREACHED*/
1640: }
1641:
1642: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1643: if (mr != MACH_MSG_SUCCESS) {
1644: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1645: (void) ipc_kmsg_put(msg, kmsg,
1646: kmsg->ikm_header.msgh_size);
1647: } else {
1648: ipc_kmsg_copyout_dest(kmsg, space);
1649: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1650: }
1651:
1652: thread_syscall_return(mr);
1653: /*NOTREACHED*/
1654: }
1655:
1656: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1657: thread_syscall_return(mr);
1658: /*NOTREACHED*/
1659: }
1660:
1661: /*
1662: * Routine: mach_msg_interrupt
1663: * Purpose:
1664: * Attempts to force a thread waiting at mach_msg_continue or
1665: * mach_msg_receive_continue into a clean point. Returns TRUE
1666: * if this was possible.
1667: * Conditions:
1668: * Nothing locked. The thread must NOT be runnable.
1669: */
1670:
1671: boolean_t
1.1.1.4 root 1672: mach_msg_interrupt(thread_t thread)
1.1 root 1673: {
1674: ipc_mqueue_t mqueue;
1675:
1676: assert((thread->swap_func == (void (*)()) mach_msg_continue) ||
1677: (thread->swap_func == (void (*)()) mach_msg_receive_continue));
1678:
1679: mqueue = thread->ith_mqueue;
1680: imq_lock(mqueue);
1681: if (thread->ith_state != MACH_RCV_IN_PROGRESS) {
1682: /*
1683: * The thread is no longer waiting for a message.
1684: * It may have a message sitting in ith_kmsg.
1685: * We can't clean this up.
1686: */
1687:
1688: imq_unlock(mqueue);
1689: return FALSE;
1690: }
1691: ipc_thread_rmqueue(&mqueue->imq_threads, thread);
1692: imq_unlock(mqueue);
1693:
1694: ipc_object_release(thread->ith_object);
1695:
1696: thread_set_syscall_return(thread, MACH_RCV_INTERRUPTED);
1697: thread->swap_func = thread_exception_return;
1698: return TRUE;
1699: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.