![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mach_msg.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / ipc|
Return to mach_msg.c CVS log | Up to [GnuMach] / Gnu-Mach / ipc |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: */
31: /*
32: * File: ipc/mach_msg.c
33: * Author: Rich Draves
34: * Date: 1989
35: *
36: * Exported message traps. See mach/message.h.
37: */
38:
39: #include <mach/kern_return.h>
40: #include <mach/port.h>
41: #include <mach/message.h>
42: #include <kern/assert.h>
43: #include <kern/counters.h>
1.1.1.3 root 44: #include <kern/debug.h>
1.1 root 45: #include <kern/lock.h>
1.1.1.3 root 46: #include <kern/printf.h>
1.1 root 47: #include <kern/sched_prim.h>
48: #include <kern/ipc_sched.h>
1.1.1.4 ! root 49: #include <kern/exception.h>
1.1 root 50: #include <vm/vm_map.h>
51: #include <ipc/ipc_kmsg.h>
52: #include <ipc/ipc_marequest.h>
53: #include <ipc/ipc_mqueue.h>
54: #include <ipc/ipc_object.h>
55: #include <ipc/ipc_notify.h>
56: #include <ipc/ipc_port.h>
57: #include <ipc/ipc_pset.h>
58: #include <ipc/ipc_space.h>
59: #include <ipc/ipc_thread.h>
60: #include <ipc/ipc_entry.h>
61: #include <ipc/mach_msg.h>
1.1.1.3 root 62: #include <machine/locore.h>
63: #include <machine/pcb.h>
1.1 root 64:
65: /*
66: * Routine: mach_msg_send
67: * Purpose:
68: * Send a message.
69: * Conditions:
70: * Nothing locked.
71: * Returns:
72: * MACH_MSG_SUCCESS Sent the message.
73: * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
74: * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
75: * MACH_SEND_INVALID_DATA Couldn't copy message data.
76: * MACH_SEND_INVALID_HEADER
77: * Illegal value in the message header bits.
78: * MACH_SEND_INVALID_DEST The space is dead.
79: * MACH_SEND_INVALID_NOTIFY Bad notify port.
80: * MACH_SEND_INVALID_DEST Can't copyin destination port.
81: * MACH_SEND_INVALID_REPLY Can't copyin reply port.
82: * MACH_SEND_TIMED_OUT Timeout expired without delivery.
83: * MACH_SEND_INTERRUPTED Delivery interrupted.
84: * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
85: * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
86: * MACH_SEND_NOTIFY_IN_PROGRESS
87: * This space has already forced a message to this port.
88: */
89:
90: mach_msg_return_t
1.1.1.4 ! root 91: mach_msg_send(
! 92: mach_msg_header_t *msg,
! 93: mach_msg_option_t option,
! 94: mach_msg_size_t send_size,
! 95: mach_msg_timeout_t time_out,
! 96: mach_port_t notify)
1.1 root 97: {
98: ipc_space_t space = current_space();
99: vm_map_t map = current_map();
100: ipc_kmsg_t kmsg;
101: mach_msg_return_t mr;
102:
103: mr = ipc_kmsg_get(msg, send_size, &kmsg);
104: if (mr != MACH_MSG_SUCCESS)
105: return mr;
106:
107: if (option & MACH_SEND_CANCEL) {
108: if (notify == MACH_PORT_NULL)
109: mr = MACH_SEND_INVALID_NOTIFY;
110: else
111: mr = ipc_kmsg_copyin(kmsg, space, map, notify);
112: } else
113: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
114: if (mr != MACH_MSG_SUCCESS) {
115: ikm_free(kmsg);
116: return mr;
117: }
118:
119: if (option & MACH_SEND_NOTIFY) {
120: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT,
121: ((option & MACH_SEND_TIMEOUT) ?
122: time_out : MACH_MSG_TIMEOUT_NONE));
123: if (mr == MACH_SEND_TIMED_OUT) {
124: ipc_port_t dest = (ipc_port_t)
125: kmsg->ikm_header.msgh_remote_port;
126:
127: if (notify == MACH_PORT_NULL)
128: mr = MACH_SEND_INVALID_NOTIFY;
129: else
130: mr = ipc_marequest_create(space, dest,
131: notify, &kmsg->ikm_marequest);
132: if (mr == MACH_MSG_SUCCESS) {
133: ipc_mqueue_send_always(kmsg);
134: return MACH_SEND_WILL_NOTIFY;
135: }
136: }
137: } else
138: mr = ipc_mqueue_send(kmsg, option & MACH_SEND_TIMEOUT,
139: time_out);
140:
141: if (mr != MACH_MSG_SUCCESS) {
142: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
143:
144: assert(kmsg->ikm_marequest == IMAR_NULL);
145: (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
146: }
147:
148: return mr;
149: }
150:
151: /*
152: * Routine: mach_msg_receive
153: * Purpose:
154: * Receive a message.
155: * Conditions:
156: * Nothing locked.
157: * Returns:
158: * MACH_MSG_SUCCESS Received a message.
159: * MACH_RCV_INVALID_NAME The name doesn't denote a right,
160: * or the denoted right is not receive or port set.
161: * MACH_RCV_IN_SET Receive right is a member of a set.
162: * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
163: * MACH_RCV_TIMED_OUT Timeout expired without a message.
164: * MACH_RCV_INTERRUPTED Reception interrupted.
165: * MACH_RCV_PORT_DIED Port/set died while receiving.
166: * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
167: * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
168: * MACH_RCV_INVALID_NOTIFY Bad notify port.
169: * MACH_RCV_HEADER_ERROR
170: */
171:
172: mach_msg_return_t
1.1.1.4 ! root 173: mach_msg_receive(
! 174: mach_msg_header_t *msg,
! 175: mach_msg_option_t option,
! 176: mach_msg_size_t rcv_size,
! 177: mach_port_t rcv_name,
! 178: mach_msg_timeout_t time_out,
! 179: mach_port_t notify)
1.1 root 180: {
181: ipc_thread_t self = current_thread();
182: ipc_space_t space = current_space();
183: vm_map_t map = current_map();
184: ipc_object_t object;
185: ipc_mqueue_t mqueue;
186: ipc_kmsg_t kmsg;
187: mach_port_seqno_t seqno;
188: mach_msg_return_t mr;
189:
190: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
191: if (mr != MACH_MSG_SUCCESS)
192: return mr;
193: /* hold ref for object; mqueue is locked */
194:
195: /*
196: * ipc_mqueue_receive may not return, because if we block
197: * then our kernel stack may be discarded. So we save
198: * state here for mach_msg_receive_continue to pick up.
199: */
200:
201: self->ith_msg = msg;
202: self->ith_option = option;
203: self->ith_rcv_size = rcv_size;
204: self->ith_timeout = time_out;
205: self->ith_notify = notify;
206: self->ith_object = object;
207: self->ith_mqueue = mqueue;
208:
209: if (option & MACH_RCV_LARGE) {
210: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
211: rcv_size, time_out,
212: FALSE, mach_msg_receive_continue,
213: &kmsg, &seqno);
214: /* mqueue is unlocked */
215: ipc_object_release(object);
216: if (mr != MACH_MSG_SUCCESS) {
217: if (mr == MACH_RCV_TOO_LARGE) {
218: mach_msg_size_t real_size =
1.1.1.3 root 219: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 220:
221: assert(real_size > rcv_size);
222:
1.1.1.3 root 223: (void) copyout(&real_size,
224: &msg->msgh_size,
1.1 root 225: sizeof(mach_msg_size_t));
226: }
227:
228: return mr;
229: }
230:
231: kmsg->ikm_header.msgh_seqno = seqno;
232: assert(kmsg->ikm_header.msgh_size <= rcv_size);
233: } else {
234: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
235: MACH_MSG_SIZE_MAX, time_out,
236: FALSE, mach_msg_receive_continue,
237: &kmsg, &seqno);
238: /* mqueue is unlocked */
239: ipc_object_release(object);
240: if (mr != MACH_MSG_SUCCESS)
241: return mr;
242:
243: kmsg->ikm_header.msgh_seqno = seqno;
244: if (kmsg->ikm_header.msgh_size > rcv_size) {
245: ipc_kmsg_copyout_dest(kmsg, space);
246: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
247: return MACH_RCV_TOO_LARGE;
248: }
249: }
250:
251: if (option & MACH_RCV_NOTIFY) {
252: if (notify == MACH_PORT_NULL)
253: mr = MACH_RCV_INVALID_NOTIFY;
254: else
255: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
256: } else
257: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
258: if (mr != MACH_MSG_SUCCESS) {
259: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
260: (void) ipc_kmsg_put(msg, kmsg,
261: kmsg->ikm_header.msgh_size);
262: } else {
263: ipc_kmsg_copyout_dest(kmsg, space);
264: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
265: }
266:
267: return mr;
268: }
269:
270: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
271: }
272:
273: /*
274: * Routine: mach_msg_receive_continue
275: * Purpose:
276: * Continue after blocking for a message.
277: * Conditions:
278: * Nothing locked. We are running on a new kernel stack,
279: * with the receive state saved in the thread. From here
280: * control goes back to user space.
281: */
282:
283: void
1.1.1.3 root 284: mach_msg_receive_continue(void)
1.1 root 285: {
286: ipc_thread_t self = current_thread();
287: ipc_space_t space = current_space();
288: vm_map_t map = current_map();
289: mach_msg_header_t *msg = self->ith_msg;
290: mach_msg_option_t option = self->ith_option;
291: mach_msg_size_t rcv_size = self->ith_rcv_size;
292: mach_msg_timeout_t time_out = self->ith_timeout;
293: mach_port_t notify = self->ith_notify;
294: ipc_object_t object = self->ith_object;
295: ipc_mqueue_t mqueue = self->ith_mqueue;
296: ipc_kmsg_t kmsg;
297: mach_port_seqno_t seqno;
298: mach_msg_return_t mr;
299:
300: if (option & MACH_RCV_LARGE) {
301: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
302: rcv_size, time_out,
303: TRUE, mach_msg_receive_continue,
304: &kmsg, &seqno);
305: /* mqueue is unlocked */
306: ipc_object_release(object);
307: if (mr != MACH_MSG_SUCCESS) {
308: if (mr == MACH_RCV_TOO_LARGE) {
309: mach_msg_size_t real_size =
1.1.1.3 root 310: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 311:
312: assert(real_size > rcv_size);
313:
1.1.1.3 root 314: (void) copyout(&real_size,
315: &msg->msgh_size,
1.1 root 316: sizeof(mach_msg_size_t));
317: }
318:
319: thread_syscall_return(mr);
320: /*NOTREACHED*/
321: }
322:
323: kmsg->ikm_header.msgh_seqno = seqno;
324: assert(kmsg->ikm_header.msgh_size <= rcv_size);
325: } else {
326: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
327: MACH_MSG_SIZE_MAX, time_out,
328: TRUE, mach_msg_receive_continue,
329: &kmsg, &seqno);
330: /* mqueue is unlocked */
331: ipc_object_release(object);
332: if (mr != MACH_MSG_SUCCESS) {
333: thread_syscall_return(mr);
334: /*NOTREACHED*/
335: }
336:
337: kmsg->ikm_header.msgh_seqno = seqno;
338: if (kmsg->ikm_header.msgh_size > rcv_size) {
339: ipc_kmsg_copyout_dest(kmsg, space);
340: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
341: thread_syscall_return(MACH_RCV_TOO_LARGE);
342: /*NOTREACHED*/
343: }
344: }
345:
346: if (option & MACH_RCV_NOTIFY) {
347: if (notify == MACH_PORT_NULL)
348: mr = MACH_RCV_INVALID_NOTIFY;
349: else
350: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
351: } else
352: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
353: if (mr != MACH_MSG_SUCCESS) {
354: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
355: (void) ipc_kmsg_put(msg, kmsg,
356: kmsg->ikm_header.msgh_size);
357: } else {
358: ipc_kmsg_copyout_dest(kmsg, space);
359: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
360: }
361:
362: thread_syscall_return(mr);
363: /*NOTREACHED*/
364: }
365:
366: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
367: thread_syscall_return(mr);
368: /*NOTREACHED*/
369: }
370:
371: /*
372: * Routine: mach_msg_trap [mach trap]
373: * Purpose:
374: * Possibly send a message; possibly receive a message.
375: * Conditions:
376: * Nothing locked.
377: * Returns:
378: * All of mach_msg_send and mach_msg_receive error codes.
379: */
380:
381: mach_msg_return_t
1.1.1.4 ! root 382: mach_msg_trap(
! 383: mach_msg_header_t *msg,
! 384: mach_msg_option_t option,
! 385: mach_msg_size_t send_size,
! 386: mach_msg_size_t rcv_size,
! 387: mach_port_t rcv_name,
! 388: mach_msg_timeout_t time_out,
! 389: mach_port_t notify)
1.1 root 390: {
391: mach_msg_return_t mr;
392:
393: /* first check for common cases */
394:
395: if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) {
1.1.1.4 ! root 396: ipc_thread_t self = current_thread();
1.1 root 397: ipc_space_t space = self->task->itk_space;
1.1.1.4 ! root 398: ipc_kmsg_t kmsg;
! 399: ipc_port_t dest_port;
1.1 root 400: ipc_object_t rcv_object;
1.1.1.4 ! root 401: ipc_mqueue_t rcv_mqueue;
1.1 root 402: mach_msg_size_t reply_size;
403:
404: /*
405: * This case is divided into ten sections, each
406: * with a label. There are five optimized
407: * sections and six unoptimized sections, which
408: * do the same thing but handle all possible
409: * cases and are slower.
410: *
411: * The five sections for an RPC are
412: * 1) Get request message into a buffer.
413: * (fast_get or slow_get)
414: * 2) Copyin request message and rcv_name.
415: * (fast_copyin or slow_copyin)
416: * 3) Enqueue request and dequeue reply.
417: * (fast_send_receive or
418: * slow_send and slow_receive)
419: * 4) Copyout reply message.
420: * (fast_copyout or slow_copyout)
421: * 5) Put reply message to user's buffer.
422: * (fast_put or slow_put)
423: *
424: * Keep the locking hierarchy firmly in mind.
425: * (First spaces, then ports, then port sets,
426: * then message queues.) Only a non-blocking
427: * attempt can be made to acquire locks out of
428: * order, or acquire two locks on the same level.
429: * Acquiring two locks on the same level will
430: * fail if the objects are really the same,
431: * unless simple locking is disabled. This is OK,
432: * because then the extra unlock does nothing.
433: *
434: * There are two major reasons these RPCs can't use
435: * ipc_thread_switch, and use slow_send/slow_receive:
436: * 1) Kernel RPCs.
437: * 2) Servers fall behind clients, so
438: * client doesn't find a blocked server thread and
439: * server finds waiting messages and can't block.
440: */
441:
442: /*
443: fast_get:
444: */
445: /*
446: * optimized ipc_kmsg_get
447: *
448: * No locks, references, or messages held.
449: * We must clear ikm_cache before copyinmsg.
450: */
451:
452: if ((send_size > IKM_SAVED_MSG_SIZE) ||
453: (send_size < sizeof(mach_msg_header_t)) ||
454: (send_size & 3) ||
455: ((kmsg = ikm_cache()) == IKM_NULL))
456: goto slow_get;
457:
458: ikm_cache() = IKM_NULL;
459: ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
460:
1.1.1.3 root 461: if (copyinmsg(msg, &kmsg->ikm_header,
1.1 root 462: send_size)) {
463: ikm_free(kmsg);
464: goto slow_get;
465: }
466:
467: kmsg->ikm_header.msgh_size = send_size;
468:
469: fast_copyin:
470: /*
471: * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
472: *
473: * We have the request message data in kmsg.
474: * Must still do copyin, send, receive, etc.
475: *
476: * If the message isn't simple, we can't combine
477: * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
478: * because copyin of the message body might
479: * affect rcv_name.
480: */
481:
482: switch (kmsg->ikm_header.msgh_bits) {
483: case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
484: MACH_MSG_TYPE_MAKE_SEND_ONCE): {
1.1.1.4 ! root 485: ipc_entry_t table;
! 486: ipc_entry_num_t size;
! 487: ipc_port_t reply_port;
1.1 root 488:
489: /* sending a request message */
490:
491: {
1.1.1.4 ! root 492: mach_port_index_t index;
! 493: mach_port_gen_t gen;
1.1 root 494:
495: {
1.1.1.4 ! root 496: mach_port_t reply_name =
1.1 root 497: kmsg->ikm_header.msgh_local_port;
498:
499: if (reply_name != rcv_name)
500: goto slow_copyin;
501:
502: /* optimized ipc_entry_lookup of reply_name */
503:
504: index = MACH_PORT_INDEX(reply_name);
505: gen = MACH_PORT_GEN(reply_name);
506: }
507:
508: is_read_lock(space);
509: assert(space->is_active);
510:
511: size = space->is_table_size;
512: table = space->is_table;
513:
514: if (index >= size)
515: goto abort_request_copyin;
516:
517: {
1.1.1.4 ! root 518: ipc_entry_t entry;
! 519: ipc_entry_bits_t bits;
1.1 root 520:
521: entry = &table[index];
522: bits = entry->ie_bits;
523:
524: /* check generation number and type bit */
525:
526: if ((bits & (IE_BITS_GEN_MASK|
527: MACH_PORT_TYPE_RECEIVE)) !=
528: (gen | MACH_PORT_TYPE_RECEIVE))
529: goto abort_request_copyin;
530:
531: reply_port = (ipc_port_t) entry->ie_object;
532: assert(reply_port != IP_NULL);
533: }
534: }
535:
536: /* optimized ipc_entry_lookup of dest_name */
537:
538: {
1.1.1.4 ! root 539: mach_port_index_t index;
! 540: mach_port_gen_t gen;
1.1 root 541:
542: {
1.1.1.4 ! root 543: mach_port_t dest_name =
1.1 root 544: kmsg->ikm_header.msgh_remote_port;
545:
546: index = MACH_PORT_INDEX(dest_name);
547: gen = MACH_PORT_GEN(dest_name);
548: }
549:
550: if (index >= size)
551: goto abort_request_copyin;
552:
553: {
1.1.1.4 ! root 554: ipc_entry_t entry;
! 555: ipc_entry_bits_t bits;
1.1 root 556:
557: entry = &table[index];
558: bits = entry->ie_bits;
559:
560: /* check generation number and type bit */
561:
562: if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
563: (gen | MACH_PORT_TYPE_SEND))
564: goto abort_request_copyin;
565:
566: assert(IE_BITS_UREFS(bits) > 0);
567:
568: dest_port = (ipc_port_t) entry->ie_object;
569: assert(dest_port != IP_NULL);
570: }
571: }
572:
573: /*
574: * To do an atomic copyin, need simultaneous
575: * locks on both ports and the space. If
576: * dest_port == reply_port, and simple locking is
577: * enabled, then we will abort. Otherwise it's
578: * OK to unlock twice.
579: */
580:
581: ip_lock(dest_port);
582: if (!ip_active(dest_port) ||
583: !ip_lock_try(reply_port)) {
584: ip_unlock(dest_port);
585: goto abort_request_copyin;
586: }
587: is_read_unlock(space);
588:
589: assert(dest_port->ip_srights > 0);
590: dest_port->ip_srights++;
591: ip_reference(dest_port);
592:
593: assert(ip_active(reply_port));
594: assert(reply_port->ip_receiver_name ==
595: kmsg->ikm_header.msgh_local_port);
596: assert(reply_port->ip_receiver == space);
597:
598: reply_port->ip_sorights++;
599: ip_reference(reply_port);
600:
601: kmsg->ikm_header.msgh_bits =
602: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
603: MACH_MSG_TYPE_PORT_SEND_ONCE);
604: kmsg->ikm_header.msgh_remote_port =
605: (mach_port_t) dest_port;
606: kmsg->ikm_header.msgh_local_port =
607: (mach_port_t) reply_port;
608:
609: /* make sure we can queue to the destination */
610:
611: if (dest_port->ip_receiver == ipc_space_kernel) {
612: /*
613: * The kernel server has a reference to
614: * the reply port, which it hands back
615: * to us in the reply message. We do
616: * not need to keep another reference to
617: * it.
618: */
619: ip_unlock(reply_port);
620:
621: assert(ip_active(dest_port));
622: ip_unlock(dest_port);
623: goto kernel_send;
624: }
625:
626: if (dest_port->ip_msgcount >= dest_port->ip_qlimit)
627: goto abort_request_send_receive;
628:
629: /* optimized ipc_mqueue_copyin */
630:
631: if (reply_port->ip_pset != IPS_NULL)
632: goto abort_request_send_receive;
633:
634: rcv_object = (ipc_object_t) reply_port;
635: io_reference(rcv_object);
636: rcv_mqueue = &reply_port->ip_messages;
637: imq_lock(rcv_mqueue);
638: io_unlock(rcv_object);
639: goto fast_send_receive;
640:
641: abort_request_copyin:
642: is_read_unlock(space);
643: goto slow_copyin;
644:
645: abort_request_send_receive:
646: ip_unlock(dest_port);
647: ip_unlock(reply_port);
648: goto slow_send;
649: }
650:
651: case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
1.1.1.4 ! root 652: ipc_entry_num_t size;
! 653: ipc_entry_t table;
1.1 root 654:
655: /* sending a reply message */
656:
657: {
1.1.1.4 ! root 658: mach_port_t reply_name =
1.1 root 659: kmsg->ikm_header.msgh_local_port;
660:
661: if (reply_name != MACH_PORT_NULL)
662: goto slow_copyin;
663: }
664:
665: is_write_lock(space);
666: assert(space->is_active);
667:
668: /* optimized ipc_entry_lookup */
669:
670: size = space->is_table_size;
671: table = space->is_table;
672:
673: {
1.1.1.4 ! root 674: ipc_entry_t entry;
! 675: mach_port_gen_t gen;
! 676: mach_port_index_t index;
1.1 root 677:
678: {
1.1.1.4 ! root 679: mach_port_t dest_name =
1.1 root 680: kmsg->ikm_header.msgh_remote_port;
681:
682: index = MACH_PORT_INDEX(dest_name);
683: gen = MACH_PORT_GEN(dest_name);
684: }
685:
686: if (index >= size)
687: goto abort_reply_dest_copyin;
688:
689: entry = &table[index];
690:
691: /* check generation, collision bit, and type bit */
692:
693: if ((entry->ie_bits & (IE_BITS_GEN_MASK|
694: IE_BITS_COLLISION|
695: MACH_PORT_TYPE_SEND_ONCE)) !=
696: (gen | MACH_PORT_TYPE_SEND_ONCE))
697: goto abort_reply_dest_copyin;
698:
699: /* optimized ipc_right_copyin */
700:
701: assert(IE_BITS_TYPE(entry->ie_bits) ==
702: MACH_PORT_TYPE_SEND_ONCE);
703: assert(IE_BITS_UREFS(entry->ie_bits) == 1);
704: assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0);
705:
706: if (entry->ie_request != 0)
707: goto abort_reply_dest_copyin;
708:
709: dest_port = (ipc_port_t) entry->ie_object;
710: assert(dest_port != IP_NULL);
711:
712: ip_lock(dest_port);
713: if (!ip_active(dest_port)) {
714: ip_unlock(dest_port);
715: goto abort_reply_dest_copyin;
716: }
717:
718: assert(dest_port->ip_sorights > 0);
719:
720: /* optimized ipc_entry_dealloc */
721:
722: entry->ie_next = table->ie_next;
723: table->ie_next = index;
724: entry->ie_bits = gen;
725: entry->ie_object = IO_NULL;
726: }
727:
728: kmsg->ikm_header.msgh_bits =
729: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
730: 0);
731: kmsg->ikm_header.msgh_remote_port =
732: (mach_port_t) dest_port;
733:
734: /* make sure we can queue to the destination */
735:
736: assert(dest_port->ip_receiver != ipc_space_kernel);
737:
738: /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
739:
740: {
1.1.1.4 ! root 741: ipc_entry_t entry;
! 742: ipc_entry_bits_t bits;
1.1 root 743:
744: {
1.1.1.4 ! root 745: mach_port_index_t index;
! 746: mach_port_gen_t gen;
1.1 root 747:
748: index = MACH_PORT_INDEX(rcv_name);
749: gen = MACH_PORT_GEN(rcv_name);
750:
751: if (index >= size)
752: goto abort_reply_rcv_copyin;
753:
754: entry = &table[index];
755: bits = entry->ie_bits;
756:
757: /* check generation number */
758:
759: if ((bits & IE_BITS_GEN_MASK) != gen)
760: goto abort_reply_rcv_copyin;
761: }
762:
763: /* check type bits; looking for receive or set */
764:
765: if (bits & MACH_PORT_TYPE_PORT_SET) {
1.1.1.4 ! root 766: ipc_pset_t rcv_pset;
1.1 root 767:
768: rcv_pset = (ipc_pset_t) entry->ie_object;
769: assert(rcv_pset != IPS_NULL);
770:
771: ips_lock(rcv_pset);
772: assert(ips_active(rcv_pset));
773:
774: rcv_object = (ipc_object_t) rcv_pset;
775: rcv_mqueue = &rcv_pset->ips_messages;
776: } else if (bits & MACH_PORT_TYPE_RECEIVE) {
1.1.1.4 ! root 777: ipc_port_t rcv_port;
1.1 root 778:
779: rcv_port = (ipc_port_t) entry->ie_object;
780: assert(rcv_port != IP_NULL);
781:
782: if (!ip_lock_try(rcv_port))
783: goto abort_reply_rcv_copyin;
784: assert(ip_active(rcv_port));
785:
786: if (rcv_port->ip_pset != IPS_NULL) {
787: ip_unlock(rcv_port);
788: goto abort_reply_rcv_copyin;
789: }
790:
791: rcv_object = (ipc_object_t) rcv_port;
792: rcv_mqueue = &rcv_port->ip_messages;
793: } else
794: goto abort_reply_rcv_copyin;
795: }
796:
797: is_write_unlock(space);
798: io_reference(rcv_object);
799: imq_lock(rcv_mqueue);
800: io_unlock(rcv_object);
801: goto fast_send_receive;
802:
803: abort_reply_dest_copyin:
804: is_write_unlock(space);
805: goto slow_copyin;
806:
807: abort_reply_rcv_copyin:
808: ip_unlock(dest_port);
809: is_write_unlock(space);
810: goto slow_send;
811: }
812:
813: default:
814: goto slow_copyin;
815: }
816: /*NOTREACHED*/
817:
818: fast_send_receive:
819: /*
820: * optimized ipc_mqueue_send/ipc_mqueue_receive
821: *
822: * Finished get/copyin of kmsg and copyin of rcv_name.
823: * space is unlocked, dest_port is locked,
824: * we can queue kmsg to dest_port,
825: * rcv_mqueue is locked, rcv_object holds a ref,
826: * if rcv_object is a port it isn't in a port set
827: *
828: * Note that if simple locking is turned off,
829: * then we could have dest_mqueue == rcv_mqueue
830: * and not abort when we try to lock dest_mqueue.
831: */
832:
833: assert(ip_active(dest_port));
834: assert(dest_port->ip_receiver != ipc_space_kernel);
835: assert((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
836: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
837: MACH_MSG_TYPE_PORT_SEND_ONCE));
838: assert((kmsg->ikm_header.msgh_bits &
839: MACH_MSGH_BITS_CIRCULAR) == 0);
840:
841: {
1.1.1.4 ! root 842: ipc_mqueue_t dest_mqueue;
! 843: ipc_thread_t receiver;
1.1 root 844:
845: {
1.1.1.4 ! root 846: ipc_pset_t dest_pset;
1.1 root 847:
848: dest_pset = dest_port->ip_pset;
849: if (dest_pset == IPS_NULL)
850: dest_mqueue = &dest_port->ip_messages;
851: else
852: dest_mqueue = &dest_pset->ips_messages;
853: }
854:
855: if (!imq_lock_try(dest_mqueue)) {
856: abort_send_receive:
857: ip_unlock(dest_port);
858: imq_unlock(rcv_mqueue);
859: ipc_object_release(rcv_object);
860: goto slow_send;
861: }
862:
863: receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
864: if ((receiver == ITH_NULL) ||
865: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
866: != IKM_NULL)) {
867: imq_unlock(dest_mqueue);
868: goto abort_send_receive;
869: }
870:
871: /*
872: * There is a receiver thread waiting, and
873: * there is no reply message for us to pick up.
874: * We have hope of hand-off, so save state.
875: */
876:
877: self->ith_msg = msg;
878: self->ith_rcv_size = rcv_size;
879: self->ith_object = rcv_object;
880: self->ith_mqueue = rcv_mqueue;
881:
882: if ((receiver->swap_func == (void (*)()) mach_msg_continue) &&
883: thread_handoff(self, mach_msg_continue, receiver)) {
884: assert(current_thread() == receiver);
885:
886: /*
887: * We can use the optimized receive code,
888: * because the receiver is using no options.
889: */
890: } else if ((receiver->swap_func ==
891: (void (*)()) exception_raise_continue) &&
892: thread_handoff(self, mach_msg_continue, receiver)) {
893: counter(c_mach_msg_trap_block_exc++);
894: assert(current_thread() == receiver);
895:
896: /*
897: * We are a reply message coming back through
898: * the optimized exception-handling path.
899: * Finish with rcv_mqueue and dest_mqueue,
900: * and then jump to exception code with
901: * dest_port still locked. We don't bother
902: * with a sequence number in this case.
903: */
904:
905: ipc_thread_enqueue_macro(
906: &rcv_mqueue->imq_threads, self);
907: self->ith_state = MACH_RCV_IN_PROGRESS;
908: self->ith_msize = MACH_MSG_SIZE_MAX;
909: imq_unlock(rcv_mqueue);
910:
911: ipc_thread_rmqueue_first_macro(
912: &dest_mqueue->imq_threads, receiver);
913: imq_unlock(dest_mqueue);
914:
915: exception_raise_continue_fast(dest_port, kmsg);
916: /*NOTREACHED*/
917: return MACH_MSG_SUCCESS;
918: } else if ((send_size <= receiver->ith_msize) &&
919: thread_handoff(self, mach_msg_continue, receiver)) {
920: assert(current_thread() == receiver);
921:
922: if ((receiver->swap_func ==
923: (void (*)()) mach_msg_receive_continue) &&
924: ((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) {
925: /*
926: * We can still use the optimized code.
927: */
928: } else {
929: counter(c_mach_msg_trap_block_slow++);
930: /*
931: * We are running as the receiver,
932: * but we can't use the optimized code.
933: * Finish send/receive processing.
934: */
935:
936: dest_port->ip_msgcount++;
937: ip_unlock(dest_port);
938:
939: ipc_thread_enqueue_macro(
940: &rcv_mqueue->imq_threads, self);
941: self->ith_state = MACH_RCV_IN_PROGRESS;
942: self->ith_msize = MACH_MSG_SIZE_MAX;
943: imq_unlock(rcv_mqueue);
944:
945: ipc_thread_rmqueue_first_macro(
946: &dest_mqueue->imq_threads, receiver);
947: receiver->ith_state = MACH_MSG_SUCCESS;
948: receiver->ith_kmsg = kmsg;
949: receiver->ith_seqno = dest_port->ip_seqno++;
950: imq_unlock(dest_mqueue);
951:
952: /*
953: * Call the receiver's continuation.
954: */
955:
956: receiver->wait_result = THREAD_AWAKENED;
957: (*receiver->swap_func)();
958: /*NOTREACHED*/
959: return MACH_MSG_SUCCESS;
960: }
961: } else {
962: /*
963: * The receiver can't accept the message,
964: * or we can't switch to the receiver.
965: */
966:
967: imq_unlock(dest_mqueue);
968: goto abort_send_receive;
969: }
970: counter(c_mach_msg_trap_block_fast++);
971:
972: /*
973: * Safe to unlock dest_port now that we are
974: * committed to this path, because we hold
975: * dest_mqueue locked. We never bother changing
976: * dest_port->ip_msgcount.
977: */
978:
979: ip_unlock(dest_port);
980:
981: /*
982: * We need to finish preparing self for its
983: * time asleep in rcv_mqueue.
984: */
985:
986: ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self);
987: self->ith_state = MACH_RCV_IN_PROGRESS;
988: self->ith_msize = MACH_MSG_SIZE_MAX;
989: imq_unlock(rcv_mqueue);
990:
991: /*
992: * Finish extracting receiver from dest_mqueue.
993: */
994:
995: ipc_thread_rmqueue_first_macro(
996: &dest_mqueue->imq_threads, receiver);
997: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
998: imq_unlock(dest_mqueue);
999:
1000: /*
1001: * We don't have to do any post-dequeue processing of
1002: * the message. We never incremented ip_msgcount, we
1003: * know it has no msg-accepted request, and blocked
1004: * senders aren't a worry because we found the port
1005: * with a receiver waiting.
1006: */
1007:
1008: self = receiver;
1009: space = self->task->itk_space;
1010:
1011: msg = self->ith_msg;
1012: rcv_size = self->ith_rcv_size;
1013: rcv_object = self->ith_object;
1014:
1015: /* inline ipc_object_release */
1016: io_lock(rcv_object);
1017: io_release(rcv_object);
1018: io_check_unlock(rcv_object);
1019: }
1020:
1021: fast_copyout:
1022: /*
1023: * Nothing locked and no references held, except
1024: * we have kmsg with msgh_seqno filled in. Must
1025: * still check against rcv_size and do
1026: * ipc_kmsg_copyout/ipc_kmsg_put.
1027: */
1028:
1029: assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port
1030: == dest_port);
1031:
1032: reply_size = kmsg->ikm_header.msgh_size;
1033: if (rcv_size < reply_size)
1034: goto slow_copyout;
1035:
1036: /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1037:
1038: switch (kmsg->ikm_header.msgh_bits) {
1039: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1040: MACH_MSG_TYPE_PORT_SEND_ONCE): {
1041: ipc_port_t reply_port =
1042: (ipc_port_t) kmsg->ikm_header.msgh_local_port;
1043: mach_port_t dest_name, reply_name;
1044:
1045: /* receiving a request message */
1046:
1047: if (!IP_VALID(reply_port))
1048: goto slow_copyout;
1049:
1050: is_write_lock(space);
1051: assert(space->is_active);
1052:
1053: /*
1054: * To do an atomic copyout, need simultaneous
1055: * locks on both ports and the space. If
1056: * dest_port == reply_port, and simple locking is
1057: * enabled, then we will abort. Otherwise it's
1058: * OK to unlock twice.
1059: */
1060:
1061: ip_lock(dest_port);
1062: if (!ip_active(dest_port) ||
1063: !ip_lock_try(reply_port))
1064: goto abort_request_copyout;
1065:
1066: if (!ip_active(reply_port)) {
1067: ip_unlock(reply_port);
1068: goto abort_request_copyout;
1069: }
1070:
1071: assert(reply_port->ip_sorights > 0);
1072: ip_unlock(reply_port);
1073:
1074: {
1.1.1.4 ! root 1075: ipc_entry_t table;
! 1076: ipc_entry_t entry;
! 1077: mach_port_index_t index;
1.1 root 1078:
1079: /* optimized ipc_entry_get */
1080:
1081: table = space->is_table;
1082: index = table->ie_next;
1083:
1084: if (index == 0)
1085: goto abort_request_copyout;
1086:
1087: entry = &table[index];
1088: table->ie_next = entry->ie_next;
1089: entry->ie_request = 0;
1090:
1091: {
1.1.1.4 ! root 1092: mach_port_gen_t gen;
1.1 root 1093:
1094: assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1095: gen = entry->ie_bits + IE_BITS_GEN_ONE;
1096:
1097: reply_name = MACH_PORT_MAKE(index, gen);
1098:
1099: /* optimized ipc_right_copyout */
1100:
1101: entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1102: }
1103:
1104: assert(MACH_PORT_VALID(reply_name));
1105: entry->ie_object = (ipc_object_t) reply_port;
1106: is_write_unlock(space);
1107: }
1108:
1109: /* optimized ipc_object_copyout_dest */
1110:
1111: assert(dest_port->ip_srights > 0);
1112: ip_release(dest_port);
1113:
1114: if (dest_port->ip_receiver == space)
1115: dest_name = dest_port->ip_receiver_name;
1116: else
1117: dest_name = MACH_PORT_NULL;
1118:
1119: if ((--dest_port->ip_srights == 0) &&
1120: (dest_port->ip_nsrequest != IP_NULL)) {
1121: ipc_port_t nsrequest;
1122: mach_port_mscount_t mscount;
1123:
1124: /* a rather rare case */
1125:
1126: nsrequest = dest_port->ip_nsrequest;
1127: mscount = dest_port->ip_mscount;
1128: dest_port->ip_nsrequest = IP_NULL;
1129: ip_unlock(dest_port);
1130:
1131: ipc_notify_no_senders(nsrequest, mscount);
1132: } else
1133: ip_unlock(dest_port);
1134:
1.1.1.4 ! root 1135: if (! ipc_port_flag_protected_payload(dest_port)) {
! 1136: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
! 1137: MACH_MSG_TYPE_PORT_SEND_ONCE,
! 1138: MACH_MSG_TYPE_PORT_SEND);
! 1139: kmsg->ikm_header.msgh_local_port = dest_name;
! 1140: } else {
! 1141: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
! 1142: MACH_MSG_TYPE_PORT_SEND_ONCE,
! 1143: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
! 1144: kmsg->ikm_header.msgh_protected_payload =
! 1145: dest_port->ip_protected_payload;
! 1146: }
1.1 root 1147: kmsg->ikm_header.msgh_remote_port = reply_name;
1148: goto fast_put;
1149:
1150: abort_request_copyout:
1151: ip_unlock(dest_port);
1152: is_write_unlock(space);
1153: goto slow_copyout;
1154: }
1155:
1156: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1.1.1.4 ! root 1157: mach_port_t dest_name;
1.1 root 1158:
1159: /* receiving a reply message */
1160:
1161: ip_lock(dest_port);
1162: if (!ip_active(dest_port))
1163: goto slow_copyout;
1164:
1165: /* optimized ipc_object_copyout_dest */
1166:
1167: assert(dest_port->ip_sorights > 0);
1168:
1169: if (dest_port->ip_receiver == space) {
1170: ip_release(dest_port);
1171: dest_port->ip_sorights--;
1172: dest_name = dest_port->ip_receiver_name;
1173: ip_unlock(dest_port);
1174: } else {
1175: ip_unlock(dest_port);
1176:
1177: ipc_notify_send_once(dest_port);
1178: dest_name = MACH_PORT_NULL;
1179: }
1180:
1.1.1.4 ! root 1181: if (! ipc_port_flag_protected_payload(dest_port)) {
! 1182: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
! 1183: 0,
! 1184: MACH_MSG_TYPE_PORT_SEND_ONCE);
! 1185: kmsg->ikm_header.msgh_local_port = dest_name;
! 1186: } else {
! 1187: kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(
! 1188: 0,
! 1189: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
! 1190: kmsg->ikm_header.msgh_protected_payload =
! 1191: dest_port->ip_protected_payload;
! 1192: }
1.1 root 1193: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1194: goto fast_put;
1195: }
1196:
1197: case MACH_MSGH_BITS_COMPLEX|
1198: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1.1.1.4 ! root 1199: mach_port_t dest_name;
1.1 root 1200:
1201: /* receiving a complex reply message */
1202:
1203: ip_lock(dest_port);
1204: if (!ip_active(dest_port))
1205: goto slow_copyout;
1206:
1207: /* optimized ipc_object_copyout_dest */
1208:
1209: assert(dest_port->ip_sorights > 0);
1210:
1211: if (dest_port->ip_receiver == space) {
1212: ip_release(dest_port);
1213: dest_port->ip_sorights--;
1214: dest_name = dest_port->ip_receiver_name;
1215: ip_unlock(dest_port);
1216: } else {
1217: ip_unlock(dest_port);
1218:
1219: ipc_notify_send_once(dest_port);
1220: dest_name = MACH_PORT_NULL;
1221: }
1222:
1.1.1.4 ! root 1223: if (! ipc_port_flag_protected_payload(dest_port)) {
! 1224: kmsg->ikm_header.msgh_bits =
! 1225: MACH_MSGH_BITS_COMPLEX
! 1226: | MACH_MSGH_BITS(
! 1227: 0,
! 1228: MACH_MSG_TYPE_PORT_SEND_ONCE);
! 1229: kmsg->ikm_header.msgh_local_port = dest_name;
! 1230: } else {
! 1231: kmsg->ikm_header.msgh_bits =
! 1232: MACH_MSGH_BITS_COMPLEX
! 1233: | MACH_MSGH_BITS(
! 1234: 0,
! 1235: MACH_MSG_TYPE_PROTECTED_PAYLOAD);
! 1236: kmsg->ikm_header.msgh_protected_payload =
! 1237: dest_port->ip_protected_payload;
! 1238: }
1.1 root 1239: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1240:
1241: mr = ipc_kmsg_copyout_body(
1242: (vm_offset_t) (&kmsg->ikm_header + 1),
1243: (vm_offset_t) &kmsg->ikm_header
1244: + kmsg->ikm_header.msgh_size,
1245: space,
1246: current_map());
1247:
1248: if (mr != MACH_MSG_SUCCESS) {
1249: (void) ipc_kmsg_put(msg, kmsg,
1250: kmsg->ikm_header.msgh_size);
1251: return mr | MACH_RCV_BODY_ERROR;
1252: }
1253: goto fast_put;
1254: }
1255:
1256: default:
1257: goto slow_copyout;
1258: }
1259: /*NOTREACHED*/
1260:
1261: fast_put:
1262: /*
1263: * We have the reply message data in kmsg,
1264: * and the reply message size in reply_size.
1265: * Just need to copy it out to the user and free kmsg.
1266: * We must check ikm_cache after copyoutmsg.
1267: */
1268:
1269: ikm_check_initialized(kmsg, kmsg->ikm_size);
1270:
1271: if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
1.1.1.3 root 1272: copyoutmsg(&kmsg->ikm_header, msg,
1.1 root 1273: reply_size) ||
1274: (ikm_cache() != IKM_NULL))
1275: goto slow_put;
1276:
1277: ikm_cache() = kmsg;
1278: thread_syscall_return(MACH_MSG_SUCCESS);
1279: /*NOTREACHED*/
1280: return MACH_MSG_SUCCESS; /* help for the compiler */
1281:
1282: /*
1283: * The slow path has a few non-register temporary
1284: * variables used only for call-by-reference.
1285: */
1286:
1287: {
1288: ipc_kmsg_t temp_kmsg;
1289: mach_port_seqno_t temp_seqno;
1290: ipc_object_t temp_rcv_object;
1291: ipc_mqueue_t temp_rcv_mqueue;
1292:
1293: slow_get:
1294: /*
1295: * No locks, references, or messages held.
1296: * Still have to get the request, send it,
1297: * receive reply, etc.
1298: */
1299:
1300: mr = ipc_kmsg_get(msg, send_size, &temp_kmsg);
1301: if (mr != MACH_MSG_SUCCESS) {
1302: thread_syscall_return(mr);
1303: /*NOTREACHED*/
1304: }
1305: kmsg = temp_kmsg;
1306:
1307: /* try to get back on optimized path */
1308: goto fast_copyin;
1309:
1310: slow_copyin:
1311: /*
1312: * We have the message data in kmsg, but
1313: * we still need to copyin, send it,
1314: * receive a reply, and do copyout.
1315: */
1316:
1317: mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1318: MACH_PORT_NULL);
1319: if (mr != MACH_MSG_SUCCESS) {
1320: ikm_free(kmsg);
1321: thread_syscall_return(mr);
1322: /*NOTREACHED*/
1323: }
1324:
1325: /* try to get back on optimized path */
1326:
1327: if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR)
1328: goto slow_send;
1329:
1330: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1331: assert(IP_VALID(dest_port));
1332:
1333: ip_lock(dest_port);
1334: if (dest_port->ip_receiver == ipc_space_kernel) {
1335: assert(ip_active(dest_port));
1336: ip_unlock(dest_port);
1337: goto kernel_send;
1338: }
1339:
1340: if (ip_active(dest_port) &&
1341: ((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
1342: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
1343: MACH_MSG_TYPE_PORT_SEND_ONCE)))
1344: {
1345: /*
1346: * Try an optimized ipc_mqueue_copyin.
1347: * It will work if this is a request message.
1348: */
1349:
1.1.1.4 ! root 1350: ipc_port_t reply_port;
1.1 root 1351:
1352: reply_port = (ipc_port_t)
1353: kmsg->ikm_header.msgh_local_port;
1354: if (IP_VALID(reply_port)) {
1355: if (ip_lock_try(reply_port)) {
1356: if (ip_active(reply_port) &&
1357: reply_port->ip_receiver == space &&
1358: reply_port->ip_receiver_name == rcv_name &&
1359: reply_port->ip_pset == IPS_NULL)
1360: {
1361: /* Grab a reference to the reply port. */
1362: rcv_object = (ipc_object_t) reply_port;
1363: io_reference(rcv_object);
1364: rcv_mqueue = &reply_port->ip_messages;
1365: imq_lock(rcv_mqueue);
1366: io_unlock(rcv_object);
1367: goto fast_send_receive;
1368: }
1369: ip_unlock(reply_port);
1370: }
1371: }
1372: }
1373:
1374: ip_unlock(dest_port);
1375: goto slow_send;
1376:
1377: kernel_send:
1378: /*
1379: * Special case: send message to kernel services.
1380: * The request message has been copied into the
1381: * kmsg. Nothing is locked.
1382: */
1383:
1384: {
1.1.1.4 ! root 1385: ipc_port_t reply_port;
1.1 root 1386:
1387: /*
1388: * Perform the kernel function.
1389: */
1390:
1391: kmsg = ipc_kobject_server(kmsg);
1392: if (kmsg == IKM_NULL) {
1393: /*
1394: * No reply. Take the
1395: * slow receive path.
1396: */
1397: goto slow_get_rcv_port;
1398: }
1399:
1400: /*
1401: * Check that:
1402: * the reply port is alive
1403: * we hold the receive right
1404: * the name has not changed.
1405: * the port is not in a set
1406: * If any of these are not true,
1407: * we cannot directly receive the reply
1408: * message.
1409: */
1410: reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1411: ip_lock(reply_port);
1412:
1413: if ((!ip_active(reply_port)) ||
1414: (reply_port->ip_receiver != space) ||
1415: (reply_port->ip_receiver_name != rcv_name) ||
1416: (reply_port->ip_pset != IPS_NULL))
1417: {
1418: ip_unlock(reply_port);
1419: ipc_mqueue_send_always(kmsg);
1420: goto slow_get_rcv_port;
1421: }
1422:
1423: rcv_mqueue = &reply_port->ip_messages;
1424: imq_lock(rcv_mqueue);
1425: /* keep port locked, and don`t change ref count yet */
1426:
1427: /*
1428: * If there are messages on the port
1429: * or other threads waiting for a message,
1430: * we cannot directly receive the reply.
1431: */
1432: if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads)
1433: != ITH_NULL) ||
1434: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
1435: != IKM_NULL))
1436: {
1437: imq_unlock(rcv_mqueue);
1438: ip_unlock(reply_port);
1439: ipc_mqueue_send_always(kmsg);
1440: goto slow_get_rcv_port;
1441: }
1442:
1443: /*
1444: * We can directly receive this reply.
1445: * Since the kernel reply never blocks,
1446: * it holds no message_accepted request.
1447: * Since there were no messages queued
1448: * on the reply port, there should be
1449: * no threads blocked waiting to send.
1450: */
1451:
1452: assert(kmsg->ikm_marequest == IMAR_NULL);
1453: assert(ipc_thread_queue_first(&reply_port->ip_blocked)
1454: == ITH_NULL);
1455:
1456: dest_port = reply_port;
1457: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
1458: imq_unlock(rcv_mqueue);
1459:
1460: /*
1461: * inline ipc_object_release.
1462: * Port is still locked.
1463: * Reference count was not incremented.
1464: */
1465: ip_check_unlock(reply_port);
1466:
1467: /* copy out the kernel reply */
1468: goto fast_copyout;
1469: }
1470:
1471: slow_send:
1472: /*
1473: * Nothing is locked. We have acquired kmsg, but
1474: * we still need to send it and receive a reply.
1475: */
1476:
1477: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1478: MACH_MSG_TIMEOUT_NONE);
1479: if (mr != MACH_MSG_SUCCESS) {
1480: mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1481: current_map());
1482:
1483: assert(kmsg->ikm_marequest == IMAR_NULL);
1484: (void) ipc_kmsg_put(msg, kmsg,
1485: kmsg->ikm_header.msgh_size);
1486: thread_syscall_return(mr);
1487: /*NOTREACHED*/
1488: }
1489:
1490: slow_get_rcv_port:
1491: /*
1492: * We have sent the message. Copy in the receive port.
1493: */
1494: mr = ipc_mqueue_copyin(space, rcv_name,
1495: &temp_rcv_mqueue, &temp_rcv_object);
1496: if (mr != MACH_MSG_SUCCESS) {
1497: thread_syscall_return(mr);
1498: /*NOTREACHED*/
1499: }
1500: rcv_mqueue = temp_rcv_mqueue;
1501: rcv_object = temp_rcv_object;
1502: /* hold ref for rcv_object; rcv_mqueue is locked */
1503:
1504: /*
1505: slow_receive:
1506: */
1507: /*
1508: * Now we have sent the request and copied in rcv_name,
1509: * so rcv_mqueue is locked and hold ref for rcv_object.
1510: * Just receive a reply and try to get back to fast path.
1511: *
1512: * ipc_mqueue_receive may not return, because if we block
1513: * then our kernel stack may be discarded. So we save
1514: * state here for mach_msg_continue to pick up.
1515: */
1516:
1517: self->ith_msg = msg;
1518: self->ith_rcv_size = rcv_size;
1519: self->ith_object = rcv_object;
1520: self->ith_mqueue = rcv_mqueue;
1521:
1522: mr = ipc_mqueue_receive(rcv_mqueue,
1523: MACH_MSG_OPTION_NONE,
1524: MACH_MSG_SIZE_MAX,
1525: MACH_MSG_TIMEOUT_NONE,
1526: FALSE, mach_msg_continue,
1527: &temp_kmsg, &temp_seqno);
1528: /* rcv_mqueue is unlocked */
1529: ipc_object_release(rcv_object);
1530: if (mr != MACH_MSG_SUCCESS) {
1531: thread_syscall_return(mr);
1532: /*NOTREACHED*/
1533: }
1534:
1535: (kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno;
1536: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1537: goto fast_copyout;
1538:
1539: slow_copyout:
1540: /*
1541: * Nothing locked and no references held, except
1542: * we have kmsg with msgh_seqno filled in. Must
1543: * still check against rcv_size and do
1544: * ipc_kmsg_copyout/ipc_kmsg_put.
1545: */
1546:
1547: reply_size = kmsg->ikm_header.msgh_size;
1548: if (rcv_size < reply_size) {
1549: ipc_kmsg_copyout_dest(kmsg, space);
1550: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1551: thread_syscall_return(MACH_RCV_TOO_LARGE);
1552: /*NOTREACHED*/
1553: }
1554:
1555: mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1556: MACH_PORT_NULL);
1557: if (mr != MACH_MSG_SUCCESS) {
1558: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1559: (void) ipc_kmsg_put(msg, kmsg,
1560: kmsg->ikm_header.msgh_size);
1561: } else {
1562: ipc_kmsg_copyout_dest(kmsg, space);
1563: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1564: }
1565:
1566: thread_syscall_return(mr);
1567: /*NOTREACHED*/
1568: }
1569:
1570: /* try to get back on optimized path */
1571:
1572: goto fast_put;
1573:
1574: slow_put:
1575: mr = ipc_kmsg_put(msg, kmsg, reply_size);
1576: thread_syscall_return(mr);
1577: /*NOTREACHED*/
1578: }
1579: } else if (option == MACH_SEND_MSG) {
1580: ipc_space_t space = current_space();
1581: vm_map_t map = current_map();
1582: ipc_kmsg_t kmsg;
1583:
1584: mr = ipc_kmsg_get(msg, send_size, &kmsg);
1585: if (mr != MACH_MSG_SUCCESS)
1586: return mr;
1587:
1588: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1589: if (mr != MACH_MSG_SUCCESS) {
1590: ikm_free(kmsg);
1591: return mr;
1592: }
1593:
1594: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1595: MACH_MSG_TIMEOUT_NONE);
1596: if (mr != MACH_MSG_SUCCESS) {
1597: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
1598:
1599: assert(kmsg->ikm_marequest == IMAR_NULL);
1600: (void) ipc_kmsg_put(msg, kmsg,
1601: kmsg->ikm_header.msgh_size);
1602: }
1603:
1604: return mr;
1605: } else if (option == MACH_RCV_MSG) {
1606: ipc_thread_t self = current_thread();
1607: ipc_space_t space = current_space();
1608: vm_map_t map = current_map();
1609: ipc_object_t object;
1610: ipc_mqueue_t mqueue;
1611: ipc_kmsg_t kmsg;
1612: mach_port_seqno_t seqno;
1613:
1614: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1615: if (mr != MACH_MSG_SUCCESS)
1616: return mr;
1617: /* hold ref for object; mqueue is locked */
1618:
1619: /*
1620: * ipc_mqueue_receive may not return, because if we block
1621: * then our kernel stack may be discarded. So we save
1622: * state here for mach_msg_continue to pick up.
1623: */
1624:
1625: self->ith_msg = msg;
1626: self->ith_rcv_size = rcv_size;
1627: self->ith_object = object;
1628: self->ith_mqueue = mqueue;
1629:
1630: mr = ipc_mqueue_receive(mqueue,
1631: MACH_MSG_OPTION_NONE,
1632: MACH_MSG_SIZE_MAX,
1633: MACH_MSG_TIMEOUT_NONE,
1634: FALSE, mach_msg_continue,
1635: &kmsg, &seqno);
1636: /* mqueue is unlocked */
1637: ipc_object_release(object);
1638: if (mr != MACH_MSG_SUCCESS)
1639: return mr;
1640:
1641: kmsg->ikm_header.msgh_seqno = seqno;
1642: if (rcv_size < kmsg->ikm_header.msgh_size) {
1643: ipc_kmsg_copyout_dest(kmsg, space);
1644: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1645: return MACH_RCV_TOO_LARGE;
1646: }
1647:
1648: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1649: if (mr != MACH_MSG_SUCCESS) {
1650: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1651: (void) ipc_kmsg_put(msg, kmsg,
1652: kmsg->ikm_header.msgh_size);
1653: } else {
1654: ipc_kmsg_copyout_dest(kmsg, space);
1655: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1656: }
1657:
1658: return mr;
1659: }
1660:
1661: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1662: } else if (option == MACH_MSG_OPTION_NONE) {
1663: /*
1664: * We can measure the "null mach_msg_trap"
1665: * (syscall entry and thread_syscall_return exit)
1666: * with this path.
1667: */
1668:
1669: thread_syscall_return(MACH_MSG_SUCCESS);
1670: /*NOTREACHED*/
1671: }
1672:
1673: if (option & MACH_SEND_MSG) {
1674: mr = mach_msg_send(msg, option, send_size,
1675: time_out, notify);
1676: if (mr != MACH_MSG_SUCCESS)
1677: return mr;
1678: }
1679:
1680: if (option & MACH_RCV_MSG) {
1681: mr = mach_msg_receive(msg, option, rcv_size, rcv_name,
1682: time_out, notify);
1683: if (mr != MACH_MSG_SUCCESS)
1684: return mr;
1685: }
1686:
1687: return MACH_MSG_SUCCESS;
1688: }
1689:
1690: /*
1691: * Routine: mach_msg_continue
1692: * Purpose:
1693: * Continue after blocking for a message.
1694: * Conditions:
1695: * Nothing locked. We are running on a new kernel stack,
1696: * with the receive state saved in the thread. From here
1697: * control goes back to user space.
1698: */
1699:
1700: void
1.1.1.3 root 1701: mach_msg_continue(void)
1.1 root 1702: {
1703: ipc_thread_t thread = current_thread();
1704: task_t task = thread->task;
1705: ipc_space_t space = task->itk_space;
1706: vm_map_t map = task->map;
1707: mach_msg_header_t *msg = thread->ith_msg;
1708: mach_msg_size_t rcv_size = thread->ith_rcv_size;
1709: ipc_object_t object = thread->ith_object;
1710: ipc_mqueue_t mqueue = thread->ith_mqueue;
1711: ipc_kmsg_t kmsg;
1712: mach_port_seqno_t seqno;
1713: mach_msg_return_t mr;
1714:
1715: mr = ipc_mqueue_receive(mqueue, MACH_MSG_OPTION_NONE,
1716: MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE,
1717: TRUE, mach_msg_continue, &kmsg, &seqno);
1718: /* mqueue is unlocked */
1719: ipc_object_release(object);
1720: if (mr != MACH_MSG_SUCCESS) {
1721: thread_syscall_return(mr);
1722: /*NOTREACHED*/
1723: }
1724:
1725: kmsg->ikm_header.msgh_seqno = seqno;
1726: if (kmsg->ikm_header.msgh_size > rcv_size) {
1727: ipc_kmsg_copyout_dest(kmsg, space);
1728: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1729: thread_syscall_return(MACH_RCV_TOO_LARGE);
1730: /*NOTREACHED*/
1731: }
1732:
1733: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1734: if (mr != MACH_MSG_SUCCESS) {
1735: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1736: (void) ipc_kmsg_put(msg, kmsg,
1737: kmsg->ikm_header.msgh_size);
1738: } else {
1739: ipc_kmsg_copyout_dest(kmsg, space);
1740: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1741: }
1742:
1743: thread_syscall_return(mr);
1744: /*NOTREACHED*/
1745: }
1746:
1747: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1748: thread_syscall_return(mr);
1749: /*NOTREACHED*/
1750: }
1751:
1752: /*
1753: * Routine: mach_msg_interrupt
1754: * Purpose:
1755: * Attempts to force a thread waiting at mach_msg_continue or
1756: * mach_msg_receive_continue into a clean point. Returns TRUE
1757: * if this was possible.
1758: * Conditions:
1759: * Nothing locked. The thread must NOT be runnable.
1760: */
1761:
1762: boolean_t
1.1.1.4 ! root 1763: mach_msg_interrupt(thread_t thread)
1.1 root 1764: {
1765: ipc_mqueue_t mqueue;
1766:
1767: assert((thread->swap_func == (void (*)()) mach_msg_continue) ||
1768: (thread->swap_func == (void (*)()) mach_msg_receive_continue));
1769:
1770: mqueue = thread->ith_mqueue;
1771: imq_lock(mqueue);
1772: if (thread->ith_state != MACH_RCV_IN_PROGRESS) {
1773: /*
1774: * The thread is no longer waiting for a message.
1775: * It may have a message sitting in ith_kmsg.
1776: * We can't clean this up.
1777: */
1778:
1779: imq_unlock(mqueue);
1780: return FALSE;
1781: }
1782: ipc_thread_rmqueue(&mqueue->imq_threads, thread);
1783: imq_unlock(mqueue);
1784:
1785: ipc_object_release(thread->ith_object);
1786:
1787: thread_set_syscall_return(thread, MACH_RCV_INTERRUPTED);
1788: thread->swap_func = thread_exception_return;
1789: return TRUE;
1790: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.