![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mach_msg.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / ipc|
Return to mach_msg.c CVS log | Up to [GnuMach] / Gnu-Mach / ipc |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: */
31: /*
32: * File: ipc/mach_msg.c
33: * Author: Rich Draves
34: * Date: 1989
35: *
36: * Exported message traps. See mach/message.h.
37: */
38:
39: #include <mach/kern_return.h>
40: #include <mach/port.h>
41: #include <mach/message.h>
42: #include <kern/assert.h>
43: #include <kern/counters.h>
1.1.1.3 ! root 44: #include <kern/debug.h>
1.1 root 45: #include <kern/lock.h>
1.1.1.3 ! root 46: #include <kern/printf.h>
1.1 root 47: #include <kern/sched_prim.h>
48: #include <kern/ipc_sched.h>
49: #include <vm/vm_map.h>
50: #include <ipc/ipc_kmsg.h>
51: #include <ipc/ipc_marequest.h>
52: #include <ipc/ipc_mqueue.h>
53: #include <ipc/ipc_object.h>
54: #include <ipc/ipc_notify.h>
55: #include <ipc/ipc_port.h>
56: #include <ipc/ipc_pset.h>
57: #include <ipc/ipc_space.h>
58: #include <ipc/ipc_thread.h>
59: #include <ipc/ipc_entry.h>
60: #include <ipc/mach_msg.h>
1.1.1.3 ! root 61: #include <machine/locore.h>
! 62: #include <machine/pcb.h>
1.1 root 63:
64: extern void exception_raise_continue();
65: extern void exception_raise_continue_fast();
66:
67: /*
68: * Routine: mach_msg_send
69: * Purpose:
70: * Send a message.
71: * Conditions:
72: * Nothing locked.
73: * Returns:
74: * MACH_MSG_SUCCESS Sent the message.
75: * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
76: * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
77: * MACH_SEND_INVALID_DATA Couldn't copy message data.
78: * MACH_SEND_INVALID_HEADER
79: * Illegal value in the message header bits.
80: * MACH_SEND_INVALID_DEST The space is dead.
81: * MACH_SEND_INVALID_NOTIFY Bad notify port.
82: * MACH_SEND_INVALID_DEST Can't copyin destination port.
83: * MACH_SEND_INVALID_REPLY Can't copyin reply port.
84: * MACH_SEND_TIMED_OUT Timeout expired without delivery.
85: * MACH_SEND_INTERRUPTED Delivery interrupted.
86: * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
87: * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
88: * MACH_SEND_NOTIFY_IN_PROGRESS
89: * This space has already forced a message to this port.
90: */
91:
92: mach_msg_return_t
93: mach_msg_send(msg, option, send_size, time_out, notify)
94: mach_msg_header_t *msg;
95: mach_msg_option_t option;
96: mach_msg_size_t send_size;
97: mach_msg_timeout_t time_out;
98: mach_port_t notify;
99: {
100: ipc_space_t space = current_space();
101: vm_map_t map = current_map();
102: ipc_kmsg_t kmsg;
103: mach_msg_return_t mr;
104:
105: mr = ipc_kmsg_get(msg, send_size, &kmsg);
106: if (mr != MACH_MSG_SUCCESS)
107: return mr;
108:
109: if (option & MACH_SEND_CANCEL) {
110: if (notify == MACH_PORT_NULL)
111: mr = MACH_SEND_INVALID_NOTIFY;
112: else
113: mr = ipc_kmsg_copyin(kmsg, space, map, notify);
114: } else
115: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
116: if (mr != MACH_MSG_SUCCESS) {
117: ikm_free(kmsg);
118: return mr;
119: }
120:
121: if (option & MACH_SEND_NOTIFY) {
122: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT,
123: ((option & MACH_SEND_TIMEOUT) ?
124: time_out : MACH_MSG_TIMEOUT_NONE));
125: if (mr == MACH_SEND_TIMED_OUT) {
126: ipc_port_t dest = (ipc_port_t)
127: kmsg->ikm_header.msgh_remote_port;
128:
129: if (notify == MACH_PORT_NULL)
130: mr = MACH_SEND_INVALID_NOTIFY;
131: else
132: mr = ipc_marequest_create(space, dest,
133: notify, &kmsg->ikm_marequest);
134: if (mr == MACH_MSG_SUCCESS) {
135: ipc_mqueue_send_always(kmsg);
136: return MACH_SEND_WILL_NOTIFY;
137: }
138: }
139: } else
140: mr = ipc_mqueue_send(kmsg, option & MACH_SEND_TIMEOUT,
141: time_out);
142:
143: if (mr != MACH_MSG_SUCCESS) {
144: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
145:
146: assert(kmsg->ikm_marequest == IMAR_NULL);
147: (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
148: }
149:
150: return mr;
151: }
152:
153: /*
154: * Routine: mach_msg_receive
155: * Purpose:
156: * Receive a message.
157: * Conditions:
158: * Nothing locked.
159: * Returns:
160: * MACH_MSG_SUCCESS Received a message.
161: * MACH_RCV_INVALID_NAME The name doesn't denote a right,
162: * or the denoted right is not receive or port set.
163: * MACH_RCV_IN_SET Receive right is a member of a set.
164: * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
165: * MACH_RCV_TIMED_OUT Timeout expired without a message.
166: * MACH_RCV_INTERRUPTED Reception interrupted.
167: * MACH_RCV_PORT_DIED Port/set died while receiving.
168: * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
169: * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
170: * MACH_RCV_INVALID_NOTIFY Bad notify port.
171: * MACH_RCV_HEADER_ERROR
172: */
173:
174: mach_msg_return_t
175: mach_msg_receive(msg, option, rcv_size, rcv_name, time_out, notify)
176: mach_msg_header_t *msg;
177: mach_msg_option_t option;
178: mach_msg_size_t rcv_size;
179: mach_port_t rcv_name;
180: mach_msg_timeout_t time_out;
181: mach_port_t notify;
182: {
183: ipc_thread_t self = current_thread();
184: ipc_space_t space = current_space();
185: vm_map_t map = current_map();
186: ipc_object_t object;
187: ipc_mqueue_t mqueue;
188: ipc_kmsg_t kmsg;
189: mach_port_seqno_t seqno;
190: mach_msg_return_t mr;
191:
192: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
193: if (mr != MACH_MSG_SUCCESS)
194: return mr;
195: /* hold ref for object; mqueue is locked */
196:
197: /*
198: * ipc_mqueue_receive may not return, because if we block
199: * then our kernel stack may be discarded. So we save
200: * state here for mach_msg_receive_continue to pick up.
201: */
202:
203: self->ith_msg = msg;
204: self->ith_option = option;
205: self->ith_rcv_size = rcv_size;
206: self->ith_timeout = time_out;
207: self->ith_notify = notify;
208: self->ith_object = object;
209: self->ith_mqueue = mqueue;
210:
211: if (option & MACH_RCV_LARGE) {
212: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
213: rcv_size, time_out,
214: FALSE, mach_msg_receive_continue,
215: &kmsg, &seqno);
216: /* mqueue is unlocked */
217: ipc_object_release(object);
218: if (mr != MACH_MSG_SUCCESS) {
219: if (mr == MACH_RCV_TOO_LARGE) {
220: mach_msg_size_t real_size =
1.1.1.3 ! root 221: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 222:
223: assert(real_size > rcv_size);
224:
1.1.1.3 ! root 225: (void) copyout(&real_size,
! 226: &msg->msgh_size,
1.1 root 227: sizeof(mach_msg_size_t));
228: }
229:
230: return mr;
231: }
232:
233: kmsg->ikm_header.msgh_seqno = seqno;
234: assert(kmsg->ikm_header.msgh_size <= rcv_size);
235: } else {
236: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
237: MACH_MSG_SIZE_MAX, time_out,
238: FALSE, mach_msg_receive_continue,
239: &kmsg, &seqno);
240: /* mqueue is unlocked */
241: ipc_object_release(object);
242: if (mr != MACH_MSG_SUCCESS)
243: return mr;
244:
245: kmsg->ikm_header.msgh_seqno = seqno;
246: if (kmsg->ikm_header.msgh_size > rcv_size) {
247: ipc_kmsg_copyout_dest(kmsg, space);
248: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
249: return MACH_RCV_TOO_LARGE;
250: }
251: }
252:
253: if (option & MACH_RCV_NOTIFY) {
254: if (notify == MACH_PORT_NULL)
255: mr = MACH_RCV_INVALID_NOTIFY;
256: else
257: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
258: } else
259: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
260: if (mr != MACH_MSG_SUCCESS) {
261: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
262: (void) ipc_kmsg_put(msg, kmsg,
263: kmsg->ikm_header.msgh_size);
264: } else {
265: ipc_kmsg_copyout_dest(kmsg, space);
266: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
267: }
268:
269: return mr;
270: }
271:
272: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
273: }
274:
275: /*
276: * Routine: mach_msg_receive_continue
277: * Purpose:
278: * Continue after blocking for a message.
279: * Conditions:
280: * Nothing locked. We are running on a new kernel stack,
281: * with the receive state saved in the thread. From here
282: * control goes back to user space.
283: */
284:
285: void
1.1.1.3 ! root 286: mach_msg_receive_continue(void)
1.1 root 287: {
288: ipc_thread_t self = current_thread();
289: ipc_space_t space = current_space();
290: vm_map_t map = current_map();
291: mach_msg_header_t *msg = self->ith_msg;
292: mach_msg_option_t option = self->ith_option;
293: mach_msg_size_t rcv_size = self->ith_rcv_size;
294: mach_msg_timeout_t time_out = self->ith_timeout;
295: mach_port_t notify = self->ith_notify;
296: ipc_object_t object = self->ith_object;
297: ipc_mqueue_t mqueue = self->ith_mqueue;
298: ipc_kmsg_t kmsg;
299: mach_port_seqno_t seqno;
300: mach_msg_return_t mr;
301:
302: if (option & MACH_RCV_LARGE) {
303: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
304: rcv_size, time_out,
305: TRUE, mach_msg_receive_continue,
306: &kmsg, &seqno);
307: /* mqueue is unlocked */
308: ipc_object_release(object);
309: if (mr != MACH_MSG_SUCCESS) {
310: if (mr == MACH_RCV_TOO_LARGE) {
311: mach_msg_size_t real_size =
1.1.1.3 ! root 312: (mach_msg_size_t) (vm_offset_t) kmsg;
1.1 root 313:
314: assert(real_size > rcv_size);
315:
1.1.1.3 ! root 316: (void) copyout(&real_size,
! 317: &msg->msgh_size,
1.1 root 318: sizeof(mach_msg_size_t));
319: }
320:
321: thread_syscall_return(mr);
322: /*NOTREACHED*/
323: }
324:
325: kmsg->ikm_header.msgh_seqno = seqno;
326: assert(kmsg->ikm_header.msgh_size <= rcv_size);
327: } else {
328: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT,
329: MACH_MSG_SIZE_MAX, time_out,
330: TRUE, mach_msg_receive_continue,
331: &kmsg, &seqno);
332: /* mqueue is unlocked */
333: ipc_object_release(object);
334: if (mr != MACH_MSG_SUCCESS) {
335: thread_syscall_return(mr);
336: /*NOTREACHED*/
337: }
338:
339: kmsg->ikm_header.msgh_seqno = seqno;
340: if (kmsg->ikm_header.msgh_size > rcv_size) {
341: ipc_kmsg_copyout_dest(kmsg, space);
342: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
343: thread_syscall_return(MACH_RCV_TOO_LARGE);
344: /*NOTREACHED*/
345: }
346: }
347:
348: if (option & MACH_RCV_NOTIFY) {
349: if (notify == MACH_PORT_NULL)
350: mr = MACH_RCV_INVALID_NOTIFY;
351: else
352: mr = ipc_kmsg_copyout(kmsg, space, map, notify);
353: } else
354: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
355: if (mr != MACH_MSG_SUCCESS) {
356: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
357: (void) ipc_kmsg_put(msg, kmsg,
358: kmsg->ikm_header.msgh_size);
359: } else {
360: ipc_kmsg_copyout_dest(kmsg, space);
361: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
362: }
363:
364: thread_syscall_return(mr);
365: /*NOTREACHED*/
366: }
367:
368: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
369: thread_syscall_return(mr);
370: /*NOTREACHED*/
371: }
372:
373: /*
374: * Routine: mach_msg_trap [mach trap]
375: * Purpose:
376: * Possibly send a message; possibly receive a message.
377: * Conditions:
378: * Nothing locked.
379: * Returns:
380: * All of mach_msg_send and mach_msg_receive error codes.
381: */
382:
383: mach_msg_return_t
384: mach_msg_trap(msg, option, send_size, rcv_size, rcv_name, time_out, notify)
385: mach_msg_header_t *msg;
386: mach_msg_option_t option;
387: mach_msg_size_t send_size;
388: mach_msg_size_t rcv_size;
389: mach_port_t rcv_name;
390: mach_msg_timeout_t time_out;
391: mach_port_t notify;
392: {
393: mach_msg_return_t mr;
394:
395: /* first check for common cases */
396:
397: if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) {
398: register ipc_thread_t self = current_thread();
399: ipc_space_t space = self->task->itk_space;
400: register ipc_kmsg_t kmsg;
401: register ipc_port_t dest_port;
402: ipc_object_t rcv_object;
403: register ipc_mqueue_t rcv_mqueue;
404: mach_msg_size_t reply_size;
405:
406: /*
407: * This case is divided into ten sections, each
408: * with a label. There are five optimized
409: * sections and six unoptimized sections, which
410: * do the same thing but handle all possible
411: * cases and are slower.
412: *
413: * The five sections for an RPC are
414: * 1) Get request message into a buffer.
415: * (fast_get or slow_get)
416: * 2) Copyin request message and rcv_name.
417: * (fast_copyin or slow_copyin)
418: * 3) Enqueue request and dequeue reply.
419: * (fast_send_receive or
420: * slow_send and slow_receive)
421: * 4) Copyout reply message.
422: * (fast_copyout or slow_copyout)
423: * 5) Put reply message to user's buffer.
424: * (fast_put or slow_put)
425: *
426: * Keep the locking hierarchy firmly in mind.
427: * (First spaces, then ports, then port sets,
428: * then message queues.) Only a non-blocking
429: * attempt can be made to acquire locks out of
430: * order, or acquire two locks on the same level.
431: * Acquiring two locks on the same level will
432: * fail if the objects are really the same,
433: * unless simple locking is disabled. This is OK,
434: * because then the extra unlock does nothing.
435: *
436: * There are two major reasons these RPCs can't use
437: * ipc_thread_switch, and use slow_send/slow_receive:
438: * 1) Kernel RPCs.
439: * 2) Servers fall behind clients, so
440: * client doesn't find a blocked server thread and
441: * server finds waiting messages and can't block.
442: */
443:
444: /*
445: fast_get:
446: */
447: /*
448: * optimized ipc_kmsg_get
449: *
450: * No locks, references, or messages held.
451: * We must clear ikm_cache before copyinmsg.
452: */
453:
454: if ((send_size > IKM_SAVED_MSG_SIZE) ||
455: (send_size < sizeof(mach_msg_header_t)) ||
456: (send_size & 3) ||
457: ((kmsg = ikm_cache()) == IKM_NULL))
458: goto slow_get;
459:
460: ikm_cache() = IKM_NULL;
461: ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
462:
1.1.1.3 ! root 463: if (copyinmsg(msg, &kmsg->ikm_header,
1.1 root 464: send_size)) {
465: ikm_free(kmsg);
466: goto slow_get;
467: }
468:
469: kmsg->ikm_header.msgh_size = send_size;
470:
471: fast_copyin:
472: /*
473: * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
474: *
475: * We have the request message data in kmsg.
476: * Must still do copyin, send, receive, etc.
477: *
478: * If the message isn't simple, we can't combine
479: * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
480: * because copyin of the message body might
481: * affect rcv_name.
482: */
483:
484: switch (kmsg->ikm_header.msgh_bits) {
485: case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
486: MACH_MSG_TYPE_MAKE_SEND_ONCE): {
487: register ipc_entry_t table;
488: register ipc_entry_num_t size;
489: register ipc_port_t reply_port;
490:
491: /* sending a request message */
492:
493: {
494: register mach_port_index_t index;
495: register mach_port_gen_t gen;
496:
497: {
498: register mach_port_t reply_name =
499: kmsg->ikm_header.msgh_local_port;
500:
501: if (reply_name != rcv_name)
502: goto slow_copyin;
503:
504: /* optimized ipc_entry_lookup of reply_name */
505:
506: index = MACH_PORT_INDEX(reply_name);
507: gen = MACH_PORT_GEN(reply_name);
508: }
509:
510: is_read_lock(space);
511: assert(space->is_active);
512:
513: size = space->is_table_size;
514: table = space->is_table;
515:
516: if (index >= size)
517: goto abort_request_copyin;
518:
519: {
520: register ipc_entry_t entry;
521: register ipc_entry_bits_t bits;
522:
523: entry = &table[index];
524: bits = entry->ie_bits;
525:
526: /* check generation number and type bit */
527:
528: if ((bits & (IE_BITS_GEN_MASK|
529: MACH_PORT_TYPE_RECEIVE)) !=
530: (gen | MACH_PORT_TYPE_RECEIVE))
531: goto abort_request_copyin;
532:
533: reply_port = (ipc_port_t) entry->ie_object;
534: assert(reply_port != IP_NULL);
535: }
536: }
537:
538: /* optimized ipc_entry_lookup of dest_name */
539:
540: {
541: register mach_port_index_t index;
542: register mach_port_gen_t gen;
543:
544: {
545: register mach_port_t dest_name =
546: kmsg->ikm_header.msgh_remote_port;
547:
548: index = MACH_PORT_INDEX(dest_name);
549: gen = MACH_PORT_GEN(dest_name);
550: }
551:
552: if (index >= size)
553: goto abort_request_copyin;
554:
555: {
556: register ipc_entry_t entry;
557: register ipc_entry_bits_t bits;
558:
559: entry = &table[index];
560: bits = entry->ie_bits;
561:
562: /* check generation number and type bit */
563:
564: if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
565: (gen | MACH_PORT_TYPE_SEND))
566: goto abort_request_copyin;
567:
568: assert(IE_BITS_UREFS(bits) > 0);
569:
570: dest_port = (ipc_port_t) entry->ie_object;
571: assert(dest_port != IP_NULL);
572: }
573: }
574:
575: /*
576: * To do an atomic copyin, need simultaneous
577: * locks on both ports and the space. If
578: * dest_port == reply_port, and simple locking is
579: * enabled, then we will abort. Otherwise it's
580: * OK to unlock twice.
581: */
582:
583: ip_lock(dest_port);
584: if (!ip_active(dest_port) ||
585: !ip_lock_try(reply_port)) {
586: ip_unlock(dest_port);
587: goto abort_request_copyin;
588: }
589: is_read_unlock(space);
590:
591: assert(dest_port->ip_srights > 0);
592: dest_port->ip_srights++;
593: ip_reference(dest_port);
594:
595: assert(ip_active(reply_port));
596: assert(reply_port->ip_receiver_name ==
597: kmsg->ikm_header.msgh_local_port);
598: assert(reply_port->ip_receiver == space);
599:
600: reply_port->ip_sorights++;
601: ip_reference(reply_port);
602:
603: kmsg->ikm_header.msgh_bits =
604: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
605: MACH_MSG_TYPE_PORT_SEND_ONCE);
606: kmsg->ikm_header.msgh_remote_port =
607: (mach_port_t) dest_port;
608: kmsg->ikm_header.msgh_local_port =
609: (mach_port_t) reply_port;
610:
611: /* make sure we can queue to the destination */
612:
613: if (dest_port->ip_receiver == ipc_space_kernel) {
614: /*
615: * The kernel server has a reference to
616: * the reply port, which it hands back
617: * to us in the reply message. We do
618: * not need to keep another reference to
619: * it.
620: */
621: ip_unlock(reply_port);
622:
623: assert(ip_active(dest_port));
624: ip_unlock(dest_port);
625: goto kernel_send;
626: }
627:
628: if (dest_port->ip_msgcount >= dest_port->ip_qlimit)
629: goto abort_request_send_receive;
630:
631: /* optimized ipc_mqueue_copyin */
632:
633: if (reply_port->ip_pset != IPS_NULL)
634: goto abort_request_send_receive;
635:
636: rcv_object = (ipc_object_t) reply_port;
637: io_reference(rcv_object);
638: rcv_mqueue = &reply_port->ip_messages;
639: imq_lock(rcv_mqueue);
640: io_unlock(rcv_object);
641: goto fast_send_receive;
642:
643: abort_request_copyin:
644: is_read_unlock(space);
645: goto slow_copyin;
646:
647: abort_request_send_receive:
648: ip_unlock(dest_port);
649: ip_unlock(reply_port);
650: goto slow_send;
651: }
652:
653: case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
654: register ipc_entry_num_t size;
655: register ipc_entry_t table;
656:
657: /* sending a reply message */
658:
659: {
660: register mach_port_t reply_name =
661: kmsg->ikm_header.msgh_local_port;
662:
663: if (reply_name != MACH_PORT_NULL)
664: goto slow_copyin;
665: }
666:
667: is_write_lock(space);
668: assert(space->is_active);
669:
670: /* optimized ipc_entry_lookup */
671:
672: size = space->is_table_size;
673: table = space->is_table;
674:
675: {
676: register ipc_entry_t entry;
677: register mach_port_gen_t gen;
678: register mach_port_index_t index;
679:
680: {
681: register mach_port_t dest_name =
682: kmsg->ikm_header.msgh_remote_port;
683:
684: index = MACH_PORT_INDEX(dest_name);
685: gen = MACH_PORT_GEN(dest_name);
686: }
687:
688: if (index >= size)
689: goto abort_reply_dest_copyin;
690:
691: entry = &table[index];
692:
693: /* check generation, collision bit, and type bit */
694:
695: if ((entry->ie_bits & (IE_BITS_GEN_MASK|
696: IE_BITS_COLLISION|
697: MACH_PORT_TYPE_SEND_ONCE)) !=
698: (gen | MACH_PORT_TYPE_SEND_ONCE))
699: goto abort_reply_dest_copyin;
700:
701: /* optimized ipc_right_copyin */
702:
703: assert(IE_BITS_TYPE(entry->ie_bits) ==
704: MACH_PORT_TYPE_SEND_ONCE);
705: assert(IE_BITS_UREFS(entry->ie_bits) == 1);
706: assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0);
707:
708: if (entry->ie_request != 0)
709: goto abort_reply_dest_copyin;
710:
711: dest_port = (ipc_port_t) entry->ie_object;
712: assert(dest_port != IP_NULL);
713:
714: ip_lock(dest_port);
715: if (!ip_active(dest_port)) {
716: ip_unlock(dest_port);
717: goto abort_reply_dest_copyin;
718: }
719:
720: assert(dest_port->ip_sorights > 0);
721:
722: /* optimized ipc_entry_dealloc */
723:
724: entry->ie_next = table->ie_next;
725: table->ie_next = index;
726: entry->ie_bits = gen;
727: entry->ie_object = IO_NULL;
728: }
729:
730: kmsg->ikm_header.msgh_bits =
731: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
732: 0);
733: kmsg->ikm_header.msgh_remote_port =
734: (mach_port_t) dest_port;
735:
736: /* make sure we can queue to the destination */
737:
738: assert(dest_port->ip_receiver != ipc_space_kernel);
739:
740: /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
741:
742: {
743: register ipc_entry_t entry;
744: register ipc_entry_bits_t bits;
745:
746: {
747: register mach_port_index_t index;
748: register mach_port_gen_t gen;
749:
750: index = MACH_PORT_INDEX(rcv_name);
751: gen = MACH_PORT_GEN(rcv_name);
752:
753: if (index >= size)
754: goto abort_reply_rcv_copyin;
755:
756: entry = &table[index];
757: bits = entry->ie_bits;
758:
759: /* check generation number */
760:
761: if ((bits & IE_BITS_GEN_MASK) != gen)
762: goto abort_reply_rcv_copyin;
763: }
764:
765: /* check type bits; looking for receive or set */
766:
767: if (bits & MACH_PORT_TYPE_PORT_SET) {
768: register ipc_pset_t rcv_pset;
769:
770: rcv_pset = (ipc_pset_t) entry->ie_object;
771: assert(rcv_pset != IPS_NULL);
772:
773: ips_lock(rcv_pset);
774: assert(ips_active(rcv_pset));
775:
776: rcv_object = (ipc_object_t) rcv_pset;
777: rcv_mqueue = &rcv_pset->ips_messages;
778: } else if (bits & MACH_PORT_TYPE_RECEIVE) {
779: register ipc_port_t rcv_port;
780:
781: rcv_port = (ipc_port_t) entry->ie_object;
782: assert(rcv_port != IP_NULL);
783:
784: if (!ip_lock_try(rcv_port))
785: goto abort_reply_rcv_copyin;
786: assert(ip_active(rcv_port));
787:
788: if (rcv_port->ip_pset != IPS_NULL) {
789: ip_unlock(rcv_port);
790: goto abort_reply_rcv_copyin;
791: }
792:
793: rcv_object = (ipc_object_t) rcv_port;
794: rcv_mqueue = &rcv_port->ip_messages;
795: } else
796: goto abort_reply_rcv_copyin;
797: }
798:
799: is_write_unlock(space);
800: io_reference(rcv_object);
801: imq_lock(rcv_mqueue);
802: io_unlock(rcv_object);
803: goto fast_send_receive;
804:
805: abort_reply_dest_copyin:
806: is_write_unlock(space);
807: goto slow_copyin;
808:
809: abort_reply_rcv_copyin:
810: ip_unlock(dest_port);
811: is_write_unlock(space);
812: goto slow_send;
813: }
814:
815: default:
816: goto slow_copyin;
817: }
818: /*NOTREACHED*/
819:
820: fast_send_receive:
821: /*
822: * optimized ipc_mqueue_send/ipc_mqueue_receive
823: *
824: * Finished get/copyin of kmsg and copyin of rcv_name.
825: * space is unlocked, dest_port is locked,
826: * we can queue kmsg to dest_port,
827: * rcv_mqueue is locked, rcv_object holds a ref,
828: * if rcv_object is a port it isn't in a port set
829: *
830: * Note that if simple locking is turned off,
831: * then we could have dest_mqueue == rcv_mqueue
832: * and not abort when we try to lock dest_mqueue.
833: */
834:
835: assert(ip_active(dest_port));
836: assert(dest_port->ip_receiver != ipc_space_kernel);
837: assert((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
838: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
839: MACH_MSG_TYPE_PORT_SEND_ONCE));
840: assert((kmsg->ikm_header.msgh_bits &
841: MACH_MSGH_BITS_CIRCULAR) == 0);
842:
843: {
844: register ipc_mqueue_t dest_mqueue;
845: register ipc_thread_t receiver;
846:
847: {
848: register ipc_pset_t dest_pset;
849:
850: dest_pset = dest_port->ip_pset;
851: if (dest_pset == IPS_NULL)
852: dest_mqueue = &dest_port->ip_messages;
853: else
854: dest_mqueue = &dest_pset->ips_messages;
855: }
856:
857: if (!imq_lock_try(dest_mqueue)) {
858: abort_send_receive:
859: ip_unlock(dest_port);
860: imq_unlock(rcv_mqueue);
861: ipc_object_release(rcv_object);
862: goto slow_send;
863: }
864:
865: receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
866: if ((receiver == ITH_NULL) ||
867: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
868: != IKM_NULL)) {
869: imq_unlock(dest_mqueue);
870: goto abort_send_receive;
871: }
872:
873: /*
874: * There is a receiver thread waiting, and
875: * there is no reply message for us to pick up.
876: * We have hope of hand-off, so save state.
877: */
878:
879: self->ith_msg = msg;
880: self->ith_rcv_size = rcv_size;
881: self->ith_object = rcv_object;
882: self->ith_mqueue = rcv_mqueue;
883:
884: if ((receiver->swap_func == (void (*)()) mach_msg_continue) &&
885: thread_handoff(self, mach_msg_continue, receiver)) {
886: assert(current_thread() == receiver);
887:
888: /*
889: * We can use the optimized receive code,
890: * because the receiver is using no options.
891: */
892: } else if ((receiver->swap_func ==
893: (void (*)()) exception_raise_continue) &&
894: thread_handoff(self, mach_msg_continue, receiver)) {
895: counter(c_mach_msg_trap_block_exc++);
896: assert(current_thread() == receiver);
897:
898: /*
899: * We are a reply message coming back through
900: * the optimized exception-handling path.
901: * Finish with rcv_mqueue and dest_mqueue,
902: * and then jump to exception code with
903: * dest_port still locked. We don't bother
904: * with a sequence number in this case.
905: */
906:
907: ipc_thread_enqueue_macro(
908: &rcv_mqueue->imq_threads, self);
909: self->ith_state = MACH_RCV_IN_PROGRESS;
910: self->ith_msize = MACH_MSG_SIZE_MAX;
911: imq_unlock(rcv_mqueue);
912:
913: ipc_thread_rmqueue_first_macro(
914: &dest_mqueue->imq_threads, receiver);
915: imq_unlock(dest_mqueue);
916:
917: exception_raise_continue_fast(dest_port, kmsg);
918: /*NOTREACHED*/
919: return MACH_MSG_SUCCESS;
920: } else if ((send_size <= receiver->ith_msize) &&
921: thread_handoff(self, mach_msg_continue, receiver)) {
922: assert(current_thread() == receiver);
923:
924: if ((receiver->swap_func ==
925: (void (*)()) mach_msg_receive_continue) &&
926: ((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) {
927: /*
928: * We can still use the optimized code.
929: */
930: } else {
931: counter(c_mach_msg_trap_block_slow++);
932: /*
933: * We are running as the receiver,
934: * but we can't use the optimized code.
935: * Finish send/receive processing.
936: */
937:
938: dest_port->ip_msgcount++;
939: ip_unlock(dest_port);
940:
941: ipc_thread_enqueue_macro(
942: &rcv_mqueue->imq_threads, self);
943: self->ith_state = MACH_RCV_IN_PROGRESS;
944: self->ith_msize = MACH_MSG_SIZE_MAX;
945: imq_unlock(rcv_mqueue);
946:
947: ipc_thread_rmqueue_first_macro(
948: &dest_mqueue->imq_threads, receiver);
949: receiver->ith_state = MACH_MSG_SUCCESS;
950: receiver->ith_kmsg = kmsg;
951: receiver->ith_seqno = dest_port->ip_seqno++;
952: imq_unlock(dest_mqueue);
953:
954: /*
955: * Call the receiver's continuation.
956: */
957:
958: receiver->wait_result = THREAD_AWAKENED;
959: (*receiver->swap_func)();
960: /*NOTREACHED*/
961: return MACH_MSG_SUCCESS;
962: }
963: } else {
964: /*
965: * The receiver can't accept the message,
966: * or we can't switch to the receiver.
967: */
968:
969: imq_unlock(dest_mqueue);
970: goto abort_send_receive;
971: }
972: counter(c_mach_msg_trap_block_fast++);
973:
974: /*
975: * Safe to unlock dest_port now that we are
976: * committed to this path, because we hold
977: * dest_mqueue locked. We never bother changing
978: * dest_port->ip_msgcount.
979: */
980:
981: ip_unlock(dest_port);
982:
983: /*
984: * We need to finish preparing self for its
985: * time asleep in rcv_mqueue.
986: */
987:
988: ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self);
989: self->ith_state = MACH_RCV_IN_PROGRESS;
990: self->ith_msize = MACH_MSG_SIZE_MAX;
991: imq_unlock(rcv_mqueue);
992:
993: /*
994: * Finish extracting receiver from dest_mqueue.
995: */
996:
997: ipc_thread_rmqueue_first_macro(
998: &dest_mqueue->imq_threads, receiver);
999: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
1000: imq_unlock(dest_mqueue);
1001:
1002: /*
1003: * We don't have to do any post-dequeue processing of
1004: * the message. We never incremented ip_msgcount, we
1005: * know it has no msg-accepted request, and blocked
1006: * senders aren't a worry because we found the port
1007: * with a receiver waiting.
1008: */
1009:
1010: self = receiver;
1011: space = self->task->itk_space;
1012:
1013: msg = self->ith_msg;
1014: rcv_size = self->ith_rcv_size;
1015: rcv_object = self->ith_object;
1016:
1017: /* inline ipc_object_release */
1018: io_lock(rcv_object);
1019: io_release(rcv_object);
1020: io_check_unlock(rcv_object);
1021: }
1022:
1023: fast_copyout:
1024: /*
1025: * Nothing locked and no references held, except
1026: * we have kmsg with msgh_seqno filled in. Must
1027: * still check against rcv_size and do
1028: * ipc_kmsg_copyout/ipc_kmsg_put.
1029: */
1030:
1031: assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port
1032: == dest_port);
1033:
1034: reply_size = kmsg->ikm_header.msgh_size;
1035: if (rcv_size < reply_size)
1036: goto slow_copyout;
1037:
1038: /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1039:
1040: switch (kmsg->ikm_header.msgh_bits) {
1041: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1042: MACH_MSG_TYPE_PORT_SEND_ONCE): {
1043: ipc_port_t reply_port =
1044: (ipc_port_t) kmsg->ikm_header.msgh_local_port;
1045: mach_port_t dest_name, reply_name;
1046:
1047: /* receiving a request message */
1048:
1049: if (!IP_VALID(reply_port))
1050: goto slow_copyout;
1051:
1052: is_write_lock(space);
1053: assert(space->is_active);
1054:
1055: /*
1056: * To do an atomic copyout, need simultaneous
1057: * locks on both ports and the space. If
1058: * dest_port == reply_port, and simple locking is
1059: * enabled, then we will abort. Otherwise it's
1060: * OK to unlock twice.
1061: */
1062:
1063: ip_lock(dest_port);
1064: if (!ip_active(dest_port) ||
1065: !ip_lock_try(reply_port))
1066: goto abort_request_copyout;
1067:
1068: if (!ip_active(reply_port)) {
1069: ip_unlock(reply_port);
1070: goto abort_request_copyout;
1071: }
1072:
1073: assert(reply_port->ip_sorights > 0);
1074: ip_unlock(reply_port);
1075:
1076: {
1077: register ipc_entry_t table;
1078: register ipc_entry_t entry;
1079: register mach_port_index_t index;
1080:
1081: /* optimized ipc_entry_get */
1082:
1083: table = space->is_table;
1084: index = table->ie_next;
1085:
1086: if (index == 0)
1087: goto abort_request_copyout;
1088:
1089: entry = &table[index];
1090: table->ie_next = entry->ie_next;
1091: entry->ie_request = 0;
1092:
1093: {
1094: register mach_port_gen_t gen;
1095:
1096: assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1097: gen = entry->ie_bits + IE_BITS_GEN_ONE;
1098:
1099: reply_name = MACH_PORT_MAKE(index, gen);
1100:
1101: /* optimized ipc_right_copyout */
1102:
1103: entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1104: }
1105:
1106: assert(MACH_PORT_VALID(reply_name));
1107: entry->ie_object = (ipc_object_t) reply_port;
1108: is_write_unlock(space);
1109: }
1110:
1111: /* optimized ipc_object_copyout_dest */
1112:
1113: assert(dest_port->ip_srights > 0);
1114: ip_release(dest_port);
1115:
1116: if (dest_port->ip_receiver == space)
1117: dest_name = dest_port->ip_receiver_name;
1118: else
1119: dest_name = MACH_PORT_NULL;
1120:
1121: if ((--dest_port->ip_srights == 0) &&
1122: (dest_port->ip_nsrequest != IP_NULL)) {
1123: ipc_port_t nsrequest;
1124: mach_port_mscount_t mscount;
1125:
1126: /* a rather rare case */
1127:
1128: nsrequest = dest_port->ip_nsrequest;
1129: mscount = dest_port->ip_mscount;
1130: dest_port->ip_nsrequest = IP_NULL;
1131: ip_unlock(dest_port);
1132:
1133: ipc_notify_no_senders(nsrequest, mscount);
1134: } else
1135: ip_unlock(dest_port);
1136:
1137: kmsg->ikm_header.msgh_bits =
1138: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1139: MACH_MSG_TYPE_PORT_SEND);
1140: kmsg->ikm_header.msgh_remote_port = reply_name;
1141: kmsg->ikm_header.msgh_local_port = dest_name;
1142: goto fast_put;
1143:
1144: abort_request_copyout:
1145: ip_unlock(dest_port);
1146: is_write_unlock(space);
1147: goto slow_copyout;
1148: }
1149:
1150: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1151: register mach_port_t dest_name;
1152:
1153: /* receiving a reply message */
1154:
1155: ip_lock(dest_port);
1156: if (!ip_active(dest_port))
1157: goto slow_copyout;
1158:
1159: /* optimized ipc_object_copyout_dest */
1160:
1161: assert(dest_port->ip_sorights > 0);
1162:
1163: if (dest_port->ip_receiver == space) {
1164: ip_release(dest_port);
1165: dest_port->ip_sorights--;
1166: dest_name = dest_port->ip_receiver_name;
1167: ip_unlock(dest_port);
1168: } else {
1169: ip_unlock(dest_port);
1170:
1171: ipc_notify_send_once(dest_port);
1172: dest_name = MACH_PORT_NULL;
1173: }
1174:
1175: kmsg->ikm_header.msgh_bits =
1176: MACH_MSGH_BITS(0,
1177: MACH_MSG_TYPE_PORT_SEND_ONCE);
1178: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1179: kmsg->ikm_header.msgh_local_port = dest_name;
1180: goto fast_put;
1181: }
1182:
1183: case MACH_MSGH_BITS_COMPLEX|
1184: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1185: register mach_port_t dest_name;
1186:
1187: /* receiving a complex reply message */
1188:
1189: ip_lock(dest_port);
1190: if (!ip_active(dest_port))
1191: goto slow_copyout;
1192:
1193: /* optimized ipc_object_copyout_dest */
1194:
1195: assert(dest_port->ip_sorights > 0);
1196:
1197: if (dest_port->ip_receiver == space) {
1198: ip_release(dest_port);
1199: dest_port->ip_sorights--;
1200: dest_name = dest_port->ip_receiver_name;
1201: ip_unlock(dest_port);
1202: } else {
1203: ip_unlock(dest_port);
1204:
1205: ipc_notify_send_once(dest_port);
1206: dest_name = MACH_PORT_NULL;
1207: }
1208:
1209: kmsg->ikm_header.msgh_bits =
1210: MACH_MSGH_BITS_COMPLEX |
1211: MACH_MSGH_BITS(0,
1212: MACH_MSG_TYPE_PORT_SEND_ONCE);
1213: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
1214: kmsg->ikm_header.msgh_local_port = dest_name;
1215:
1216: mr = ipc_kmsg_copyout_body(
1217: (vm_offset_t) (&kmsg->ikm_header + 1),
1218: (vm_offset_t) &kmsg->ikm_header
1219: + kmsg->ikm_header.msgh_size,
1220: space,
1221: current_map());
1222:
1223: if (mr != MACH_MSG_SUCCESS) {
1224: (void) ipc_kmsg_put(msg, kmsg,
1225: kmsg->ikm_header.msgh_size);
1226: return mr | MACH_RCV_BODY_ERROR;
1227: }
1228: goto fast_put;
1229: }
1230:
1231: default:
1232: goto slow_copyout;
1233: }
1234: /*NOTREACHED*/
1235:
1236: fast_put:
1237: /*
1238: * We have the reply message data in kmsg,
1239: * and the reply message size in reply_size.
1240: * Just need to copy it out to the user and free kmsg.
1241: * We must check ikm_cache after copyoutmsg.
1242: */
1243:
1244: ikm_check_initialized(kmsg, kmsg->ikm_size);
1245:
1246: if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
1.1.1.3 ! root 1247: copyoutmsg(&kmsg->ikm_header, msg,
1.1 root 1248: reply_size) ||
1249: (ikm_cache() != IKM_NULL))
1250: goto slow_put;
1251:
1252: ikm_cache() = kmsg;
1253: thread_syscall_return(MACH_MSG_SUCCESS);
1254: /*NOTREACHED*/
1255: return MACH_MSG_SUCCESS; /* help for the compiler */
1256:
1257: /*
1258: * The slow path has a few non-register temporary
1259: * variables used only for call-by-reference.
1260: */
1261:
1262: {
1263: ipc_kmsg_t temp_kmsg;
1264: mach_port_seqno_t temp_seqno;
1265: ipc_object_t temp_rcv_object;
1266: ipc_mqueue_t temp_rcv_mqueue;
1267:
1268: slow_get:
1269: /*
1270: * No locks, references, or messages held.
1271: * Still have to get the request, send it,
1272: * receive reply, etc.
1273: */
1274:
1275: mr = ipc_kmsg_get(msg, send_size, &temp_kmsg);
1276: if (mr != MACH_MSG_SUCCESS) {
1277: thread_syscall_return(mr);
1278: /*NOTREACHED*/
1279: }
1280: kmsg = temp_kmsg;
1281:
1282: /* try to get back on optimized path */
1283: goto fast_copyin;
1284:
1285: slow_copyin:
1286: /*
1287: * We have the message data in kmsg, but
1288: * we still need to copyin, send it,
1289: * receive a reply, and do copyout.
1290: */
1291:
1292: mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1293: MACH_PORT_NULL);
1294: if (mr != MACH_MSG_SUCCESS) {
1295: ikm_free(kmsg);
1296: thread_syscall_return(mr);
1297: /*NOTREACHED*/
1298: }
1299:
1300: /* try to get back on optimized path */
1301:
1302: if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR)
1303: goto slow_send;
1304:
1305: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1306: assert(IP_VALID(dest_port));
1307:
1308: ip_lock(dest_port);
1309: if (dest_port->ip_receiver == ipc_space_kernel) {
1310: assert(ip_active(dest_port));
1311: ip_unlock(dest_port);
1312: goto kernel_send;
1313: }
1314:
1315: if (ip_active(dest_port) &&
1316: ((dest_port->ip_msgcount < dest_port->ip_qlimit) ||
1317: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
1318: MACH_MSG_TYPE_PORT_SEND_ONCE)))
1319: {
1320: /*
1321: * Try an optimized ipc_mqueue_copyin.
1322: * It will work if this is a request message.
1323: */
1324:
1325: register ipc_port_t reply_port;
1326:
1327: reply_port = (ipc_port_t)
1328: kmsg->ikm_header.msgh_local_port;
1329: if (IP_VALID(reply_port)) {
1330: if (ip_lock_try(reply_port)) {
1331: if (ip_active(reply_port) &&
1332: reply_port->ip_receiver == space &&
1333: reply_port->ip_receiver_name == rcv_name &&
1334: reply_port->ip_pset == IPS_NULL)
1335: {
1336: /* Grab a reference to the reply port. */
1337: rcv_object = (ipc_object_t) reply_port;
1338: io_reference(rcv_object);
1339: rcv_mqueue = &reply_port->ip_messages;
1340: imq_lock(rcv_mqueue);
1341: io_unlock(rcv_object);
1342: goto fast_send_receive;
1343: }
1344: ip_unlock(reply_port);
1345: }
1346: }
1347: }
1348:
1349: ip_unlock(dest_port);
1350: goto slow_send;
1351:
1352: kernel_send:
1353: /*
1354: * Special case: send message to kernel services.
1355: * The request message has been copied into the
1356: * kmsg. Nothing is locked.
1357: */
1358:
1359: {
1360: register ipc_port_t reply_port;
1361:
1362: /*
1363: * Perform the kernel function.
1364: */
1365:
1366: kmsg = ipc_kobject_server(kmsg);
1367: if (kmsg == IKM_NULL) {
1368: /*
1369: * No reply. Take the
1370: * slow receive path.
1371: */
1372: goto slow_get_rcv_port;
1373: }
1374:
1375: /*
1376: * Check that:
1377: * the reply port is alive
1378: * we hold the receive right
1379: * the name has not changed.
1380: * the port is not in a set
1381: * If any of these are not true,
1382: * we cannot directly receive the reply
1383: * message.
1384: */
1385: reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1386: ip_lock(reply_port);
1387:
1388: if ((!ip_active(reply_port)) ||
1389: (reply_port->ip_receiver != space) ||
1390: (reply_port->ip_receiver_name != rcv_name) ||
1391: (reply_port->ip_pset != IPS_NULL))
1392: {
1393: ip_unlock(reply_port);
1394: ipc_mqueue_send_always(kmsg);
1395: goto slow_get_rcv_port;
1396: }
1397:
1398: rcv_mqueue = &reply_port->ip_messages;
1399: imq_lock(rcv_mqueue);
1400: /* keep port locked, and don`t change ref count yet */
1401:
1402: /*
1403: * If there are messages on the port
1404: * or other threads waiting for a message,
1405: * we cannot directly receive the reply.
1406: */
1407: if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads)
1408: != ITH_NULL) ||
1409: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages)
1410: != IKM_NULL))
1411: {
1412: imq_unlock(rcv_mqueue);
1413: ip_unlock(reply_port);
1414: ipc_mqueue_send_always(kmsg);
1415: goto slow_get_rcv_port;
1416: }
1417:
1418: /*
1419: * We can directly receive this reply.
1420: * Since the kernel reply never blocks,
1421: * it holds no message_accepted request.
1422: * Since there were no messages queued
1423: * on the reply port, there should be
1424: * no threads blocked waiting to send.
1425: */
1426:
1427: assert(kmsg->ikm_marequest == IMAR_NULL);
1428: assert(ipc_thread_queue_first(&reply_port->ip_blocked)
1429: == ITH_NULL);
1430:
1431: dest_port = reply_port;
1432: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++;
1433: imq_unlock(rcv_mqueue);
1434:
1435: /*
1436: * inline ipc_object_release.
1437: * Port is still locked.
1438: * Reference count was not incremented.
1439: */
1440: ip_check_unlock(reply_port);
1441:
1442: /* copy out the kernel reply */
1443: goto fast_copyout;
1444: }
1445:
1446: slow_send:
1447: /*
1448: * Nothing is locked. We have acquired kmsg, but
1449: * we still need to send it and receive a reply.
1450: */
1451:
1452: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1453: MACH_MSG_TIMEOUT_NONE);
1454: if (mr != MACH_MSG_SUCCESS) {
1455: mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1456: current_map());
1457:
1458: assert(kmsg->ikm_marequest == IMAR_NULL);
1459: (void) ipc_kmsg_put(msg, kmsg,
1460: kmsg->ikm_header.msgh_size);
1461: thread_syscall_return(mr);
1462: /*NOTREACHED*/
1463: }
1464:
1465: slow_get_rcv_port:
1466: /*
1467: * We have sent the message. Copy in the receive port.
1468: */
1469: mr = ipc_mqueue_copyin(space, rcv_name,
1470: &temp_rcv_mqueue, &temp_rcv_object);
1471: if (mr != MACH_MSG_SUCCESS) {
1472: thread_syscall_return(mr);
1473: /*NOTREACHED*/
1474: }
1475: rcv_mqueue = temp_rcv_mqueue;
1476: rcv_object = temp_rcv_object;
1477: /* hold ref for rcv_object; rcv_mqueue is locked */
1478:
1479: /*
1480: slow_receive:
1481: */
1482: /*
1483: * Now we have sent the request and copied in rcv_name,
1484: * so rcv_mqueue is locked and hold ref for rcv_object.
1485: * Just receive a reply and try to get back to fast path.
1486: *
1487: * ipc_mqueue_receive may not return, because if we block
1488: * then our kernel stack may be discarded. So we save
1489: * state here for mach_msg_continue to pick up.
1490: */
1491:
1492: self->ith_msg = msg;
1493: self->ith_rcv_size = rcv_size;
1494: self->ith_object = rcv_object;
1495: self->ith_mqueue = rcv_mqueue;
1496:
1497: mr = ipc_mqueue_receive(rcv_mqueue,
1498: MACH_MSG_OPTION_NONE,
1499: MACH_MSG_SIZE_MAX,
1500: MACH_MSG_TIMEOUT_NONE,
1501: FALSE, mach_msg_continue,
1502: &temp_kmsg, &temp_seqno);
1503: /* rcv_mqueue is unlocked */
1504: ipc_object_release(rcv_object);
1505: if (mr != MACH_MSG_SUCCESS) {
1506: thread_syscall_return(mr);
1507: /*NOTREACHED*/
1508: }
1509:
1510: (kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno;
1511: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port;
1512: goto fast_copyout;
1513:
1514: slow_copyout:
1515: /*
1516: * Nothing locked and no references held, except
1517: * we have kmsg with msgh_seqno filled in. Must
1518: * still check against rcv_size and do
1519: * ipc_kmsg_copyout/ipc_kmsg_put.
1520: */
1521:
1522: reply_size = kmsg->ikm_header.msgh_size;
1523: if (rcv_size < reply_size) {
1524: ipc_kmsg_copyout_dest(kmsg, space);
1525: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1526: thread_syscall_return(MACH_RCV_TOO_LARGE);
1527: /*NOTREACHED*/
1528: }
1529:
1530: mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1531: MACH_PORT_NULL);
1532: if (mr != MACH_MSG_SUCCESS) {
1533: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1534: (void) ipc_kmsg_put(msg, kmsg,
1535: kmsg->ikm_header.msgh_size);
1536: } else {
1537: ipc_kmsg_copyout_dest(kmsg, space);
1538: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1539: }
1540:
1541: thread_syscall_return(mr);
1542: /*NOTREACHED*/
1543: }
1544:
1545: /* try to get back on optimized path */
1546:
1547: goto fast_put;
1548:
1549: slow_put:
1550: mr = ipc_kmsg_put(msg, kmsg, reply_size);
1551: thread_syscall_return(mr);
1552: /*NOTREACHED*/
1553: }
1554: } else if (option == MACH_SEND_MSG) {
1555: ipc_space_t space = current_space();
1556: vm_map_t map = current_map();
1557: ipc_kmsg_t kmsg;
1558:
1559: mr = ipc_kmsg_get(msg, send_size, &kmsg);
1560: if (mr != MACH_MSG_SUCCESS)
1561: return mr;
1562:
1563: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1564: if (mr != MACH_MSG_SUCCESS) {
1565: ikm_free(kmsg);
1566: return mr;
1567: }
1568:
1569: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE,
1570: MACH_MSG_TIMEOUT_NONE);
1571: if (mr != MACH_MSG_SUCCESS) {
1572: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map);
1573:
1574: assert(kmsg->ikm_marequest == IMAR_NULL);
1575: (void) ipc_kmsg_put(msg, kmsg,
1576: kmsg->ikm_header.msgh_size);
1577: }
1578:
1579: return mr;
1580: } else if (option == MACH_RCV_MSG) {
1581: ipc_thread_t self = current_thread();
1582: ipc_space_t space = current_space();
1583: vm_map_t map = current_map();
1584: ipc_object_t object;
1585: ipc_mqueue_t mqueue;
1586: ipc_kmsg_t kmsg;
1587: mach_port_seqno_t seqno;
1588:
1589: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1590: if (mr != MACH_MSG_SUCCESS)
1591: return mr;
1592: /* hold ref for object; mqueue is locked */
1593:
1594: /*
1595: * ipc_mqueue_receive may not return, because if we block
1596: * then our kernel stack may be discarded. So we save
1597: * state here for mach_msg_continue to pick up.
1598: */
1599:
1600: self->ith_msg = msg;
1601: self->ith_rcv_size = rcv_size;
1602: self->ith_object = object;
1603: self->ith_mqueue = mqueue;
1604:
1605: mr = ipc_mqueue_receive(mqueue,
1606: MACH_MSG_OPTION_NONE,
1607: MACH_MSG_SIZE_MAX,
1608: MACH_MSG_TIMEOUT_NONE,
1609: FALSE, mach_msg_continue,
1610: &kmsg, &seqno);
1611: /* mqueue is unlocked */
1612: ipc_object_release(object);
1613: if (mr != MACH_MSG_SUCCESS)
1614: return mr;
1615:
1616: kmsg->ikm_header.msgh_seqno = seqno;
1617: if (rcv_size < kmsg->ikm_header.msgh_size) {
1618: ipc_kmsg_copyout_dest(kmsg, space);
1619: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1620: return MACH_RCV_TOO_LARGE;
1621: }
1622:
1623: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1624: if (mr != MACH_MSG_SUCCESS) {
1625: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1626: (void) ipc_kmsg_put(msg, kmsg,
1627: kmsg->ikm_header.msgh_size);
1628: } else {
1629: ipc_kmsg_copyout_dest(kmsg, space);
1630: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1631: }
1632:
1633: return mr;
1634: }
1635:
1636: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1637: } else if (option == MACH_MSG_OPTION_NONE) {
1638: /*
1639: * We can measure the "null mach_msg_trap"
1640: * (syscall entry and thread_syscall_return exit)
1641: * with this path.
1642: */
1643:
1644: thread_syscall_return(MACH_MSG_SUCCESS);
1645: /*NOTREACHED*/
1646: }
1647:
1648: if (option & MACH_SEND_MSG) {
1649: mr = mach_msg_send(msg, option, send_size,
1650: time_out, notify);
1651: if (mr != MACH_MSG_SUCCESS)
1652: return mr;
1653: }
1654:
1655: if (option & MACH_RCV_MSG) {
1656: mr = mach_msg_receive(msg, option, rcv_size, rcv_name,
1657: time_out, notify);
1658: if (mr != MACH_MSG_SUCCESS)
1659: return mr;
1660: }
1661:
1662: return MACH_MSG_SUCCESS;
1663: }
1664:
1665: /*
1666: * Routine: mach_msg_continue
1667: * Purpose:
1668: * Continue after blocking for a message.
1669: * Conditions:
1670: * Nothing locked. We are running on a new kernel stack,
1671: * with the receive state saved in the thread. From here
1672: * control goes back to user space.
1673: */
1674:
1675: void
1.1.1.3 ! root 1676: mach_msg_continue(void)
1.1 root 1677: {
1678: ipc_thread_t thread = current_thread();
1679: task_t task = thread->task;
1680: ipc_space_t space = task->itk_space;
1681: vm_map_t map = task->map;
1682: mach_msg_header_t *msg = thread->ith_msg;
1683: mach_msg_size_t rcv_size = thread->ith_rcv_size;
1684: ipc_object_t object = thread->ith_object;
1685: ipc_mqueue_t mqueue = thread->ith_mqueue;
1686: ipc_kmsg_t kmsg;
1687: mach_port_seqno_t seqno;
1688: mach_msg_return_t mr;
1689:
1690: mr = ipc_mqueue_receive(mqueue, MACH_MSG_OPTION_NONE,
1691: MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE,
1692: TRUE, mach_msg_continue, &kmsg, &seqno);
1693: /* mqueue is unlocked */
1694: ipc_object_release(object);
1695: if (mr != MACH_MSG_SUCCESS) {
1696: thread_syscall_return(mr);
1697: /*NOTREACHED*/
1698: }
1699:
1700: kmsg->ikm_header.msgh_seqno = seqno;
1701: if (kmsg->ikm_header.msgh_size > rcv_size) {
1702: ipc_kmsg_copyout_dest(kmsg, space);
1703: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1704: thread_syscall_return(MACH_RCV_TOO_LARGE);
1705: /*NOTREACHED*/
1706: }
1707:
1708: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL);
1709: if (mr != MACH_MSG_SUCCESS) {
1710: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1711: (void) ipc_kmsg_put(msg, kmsg,
1712: kmsg->ikm_header.msgh_size);
1713: } else {
1714: ipc_kmsg_copyout_dest(kmsg, space);
1715: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg);
1716: }
1717:
1718: thread_syscall_return(mr);
1719: /*NOTREACHED*/
1720: }
1721:
1722: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
1723: thread_syscall_return(mr);
1724: /*NOTREACHED*/
1725: }
1726:
1727: /*
1728: * Routine: mach_msg_interrupt
1729: * Purpose:
1730: * Attempts to force a thread waiting at mach_msg_continue or
1731: * mach_msg_receive_continue into a clean point. Returns TRUE
1732: * if this was possible.
1733: * Conditions:
1734: * Nothing locked. The thread must NOT be runnable.
1735: */
1736:
1737: boolean_t
1738: mach_msg_interrupt(thread)
1739: thread_t thread;
1740: {
1741: ipc_mqueue_t mqueue;
1742:
1743: assert((thread->swap_func == (void (*)()) mach_msg_continue) ||
1744: (thread->swap_func == (void (*)()) mach_msg_receive_continue));
1745:
1746: mqueue = thread->ith_mqueue;
1747: imq_lock(mqueue);
1748: if (thread->ith_state != MACH_RCV_IN_PROGRESS) {
1749: /*
1750: * The thread is no longer waiting for a message.
1751: * It may have a message sitting in ith_kmsg.
1752: * We can't clean this up.
1753: */
1754:
1755: imq_unlock(mqueue);
1756: return FALSE;
1757: }
1758: ipc_thread_rmqueue(&mqueue->imq_threads, thread);
1759: imq_unlock(mqueue);
1760:
1761: ipc_object_release(thread->ith_object);
1762:
1763: thread_set_syscall_return(thread, MACH_RCV_INTERRUPTED);
1764: thread->swap_func = thread_exception_return;
1765: return TRUE;
1766: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.