![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mach_msg.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / ipc|
Return to mach_msg.c CVS log | Up to [GnuMach] / OSKit-Mach / ipc |
1.1 ! root 1: /* ! 2: * Mach Operating System ! 3: * Copyright (c) 1991,1990,1989 Carnegie Mellon University. ! 4: * Copyright (c) 1993,1994 The University of Utah and ! 5: * the Computer Systems Laboratory (CSL). ! 6: * All rights reserved. ! 7: * ! 8: * Permission to use, copy, modify and distribute this software and its ! 9: * documentation is hereby granted, provided that both the copyright ! 10: * notice and this permission notice appear in all copies of the ! 11: * software, derivative works or modified versions, and any portions ! 12: * thereof, and that both notices appear in supporting documentation. ! 13: * ! 14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF ! 15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY ! 16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF ! 17: * THIS SOFTWARE. ! 18: * ! 19: * Carnegie Mellon requests users of this software to return to ! 20: * ! 21: * Software Distribution Coordinator or [email protected] ! 22: * School of Computer Science ! 23: * Carnegie Mellon University ! 24: * Pittsburgh PA 15213-3890 ! 25: * ! 26: * any improvements or extensions that they make and grant Carnegie Mellon ! 27: * the rights to redistribute these changes. ! 28: */ ! 29: /* ! 30: */ ! 31: /* ! 32: * File: ipc/mach_msg.c ! 33: * Author: Rich Draves ! 34: * Date: 1989 ! 35: * ! 36: * Exported message traps. See mach/message.h. ! 37: */ ! 38: ! 39: #include <mach_ipc_compat.h> ! 40: #include <norma_ipc.h> ! 41: ! 42: #include <mach/kern_return.h> ! 43: #include <mach/port.h> ! 44: #include <mach/message.h> ! 45: #include <kern/assert.h> ! 46: #include <kern/counters.h> ! 47: #include <kern/lock.h> ! 48: #include <kern/sched_prim.h> ! 49: #include <kern/ipc_sched.h> ! 50: #include <vm/vm_map.h> ! 51: #include <ipc/ipc_kmsg.h> ! 52: #include <ipc/ipc_marequest.h> ! 53: #include <ipc/ipc_mqueue.h> ! 54: #include <ipc/ipc_object.h> ! 55: #include <ipc/ipc_notify.h> ! 56: #include <ipc/ipc_port.h> ! 57: #include <ipc/ipc_pset.h> ! 58: #include <ipc/ipc_space.h> ! 59: #include <ipc/ipc_thread.h> ! 60: #include <ipc/ipc_entry.h> ! 61: #include <ipc/mach_msg.h> ! 62: ! 63: ! 64: ! 65: extern void exception_raise_continue(); ! 66: extern void exception_raise_continue_fast(); ! 67: #ifndef CONTINUATIONS ! 68: #define mach_msg_receive_continue 0 ! 69: #define msg_receive_continue 0 ! 70: #endif ! 71: ! 72: /* ! 73: * Routine: mach_msg_send ! 74: * Purpose: ! 75: * Send a message. ! 76: * Conditions: ! 77: * Nothing locked. ! 78: * Returns: ! 79: * MACH_MSG_SUCCESS Sent the message. ! 80: * MACH_SEND_MSG_TOO_SMALL Message smaller than a header. ! 81: * MACH_SEND_NO_BUFFER Couldn't allocate buffer. ! 82: * MACH_SEND_INVALID_DATA Couldn't copy message data. ! 83: * MACH_SEND_INVALID_HEADER ! 84: * Illegal value in the message header bits. ! 85: * MACH_SEND_INVALID_DEST The space is dead. ! 86: * MACH_SEND_INVALID_NOTIFY Bad notify port. ! 87: * MACH_SEND_INVALID_DEST Can't copyin destination port. ! 88: * MACH_SEND_INVALID_REPLY Can't copyin reply port. ! 89: * MACH_SEND_TIMED_OUT Timeout expired without delivery. ! 90: * MACH_SEND_INTERRUPTED Delivery interrupted. ! 91: * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request. ! 92: * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested. ! 93: * MACH_SEND_NOTIFY_IN_PROGRESS ! 94: * This space has already forced a message to this port. ! 95: */ ! 96: ! 97: mach_msg_return_t ! 98: mach_msg_send(msg, option, send_size, time_out, notify) ! 99: mach_msg_header_t *msg; ! 100: mach_msg_option_t option; ! 101: mach_msg_size_t send_size; ! 102: mach_msg_timeout_t time_out; ! 103: mach_port_t notify; ! 104: { ! 105: ipc_space_t space = current_space(); ! 106: vm_map_t map = current_map(); ! 107: ipc_kmsg_t kmsg; ! 108: mach_msg_return_t mr; ! 109: ! 110: mr = ipc_kmsg_get(msg, send_size, &kmsg); ! 111: if (mr != MACH_MSG_SUCCESS) ! 112: return mr; ! 113: ! 114: if (option & MACH_SEND_CANCEL) { ! 115: if (notify == MACH_PORT_NULL) ! 116: mr = MACH_SEND_INVALID_NOTIFY; ! 117: else ! 118: mr = ipc_kmsg_copyin(kmsg, space, map, notify); ! 119: } else ! 120: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL); ! 121: if (mr != MACH_MSG_SUCCESS) { ! 122: ikm_free(kmsg); ! 123: return mr; ! 124: } ! 125: ! 126: if (option & MACH_SEND_NOTIFY) { ! 127: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT, ! 128: ((option & MACH_SEND_TIMEOUT) ? ! 129: time_out : MACH_MSG_TIMEOUT_NONE)); ! 130: if (mr == MACH_SEND_TIMED_OUT) { ! 131: ipc_port_t dest = (ipc_port_t) ! 132: kmsg->ikm_header.msgh_remote_port; ! 133: ! 134: if (notify == MACH_PORT_NULL) ! 135: mr = MACH_SEND_INVALID_NOTIFY; ! 136: else ! 137: mr = ipc_marequest_create(space, dest, ! 138: notify, &kmsg->ikm_marequest); ! 139: if (mr == MACH_MSG_SUCCESS) { ! 140: ipc_mqueue_send_always(kmsg); ! 141: return MACH_SEND_WILL_NOTIFY; ! 142: } ! 143: } ! 144: } else ! 145: mr = ipc_mqueue_send(kmsg, option & MACH_SEND_TIMEOUT, ! 146: time_out); ! 147: ! 148: if (mr != MACH_MSG_SUCCESS) { ! 149: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map); ! 150: ! 151: assert(kmsg->ikm_marequest == IMAR_NULL); ! 152: (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); ! 153: } ! 154: ! 155: return mr; ! 156: } ! 157: ! 158: /* ! 159: * Routine: mach_msg_receive ! 160: * Purpose: ! 161: * Receive a message. ! 162: * Conditions: ! 163: * Nothing locked. ! 164: * Returns: ! 165: * MACH_MSG_SUCCESS Received a message. ! 166: * MACH_RCV_INVALID_NAME The name doesn't denote a right, ! 167: * or the denoted right is not receive or port set. ! 168: * MACH_RCV_IN_SET Receive right is a member of a set. ! 169: * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer. ! 170: * MACH_RCV_TIMED_OUT Timeout expired without a message. ! 171: * MACH_RCV_INTERRUPTED Reception interrupted. ! 172: * MACH_RCV_PORT_DIED Port/set died while receiving. ! 173: * MACH_RCV_PORT_CHANGED Port moved into set while receiving. ! 174: * MACH_RCV_INVALID_DATA Couldn't copy to user buffer. ! 175: * MACH_RCV_INVALID_NOTIFY Bad notify port. ! 176: * MACH_RCV_HEADER_ERROR ! 177: */ ! 178: ! 179: mach_msg_return_t ! 180: mach_msg_receive(msg, option, rcv_size, rcv_name, time_out, notify) ! 181: mach_msg_header_t *msg; ! 182: mach_msg_option_t option; ! 183: mach_msg_size_t rcv_size; ! 184: mach_port_t rcv_name; ! 185: mach_msg_timeout_t time_out; ! 186: mach_port_t notify; ! 187: { ! 188: ipc_thread_t self = current_thread(); ! 189: ipc_space_t space = current_space(); ! 190: vm_map_t map = current_map(); ! 191: ipc_object_t object; ! 192: ipc_mqueue_t mqueue; ! 193: ipc_kmsg_t kmsg; ! 194: mach_port_seqno_t seqno; ! 195: mach_msg_return_t mr; ! 196: ! 197: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object); ! 198: if (mr != MACH_MSG_SUCCESS) ! 199: return mr; ! 200: /* hold ref for object; mqueue is locked */ ! 201: ! 202: #ifdef CONTINUATIONS ! 203: /* ! 204: * ipc_mqueue_receive may not return, because if we block ! 205: * then our kernel stack may be discarded. So we save ! 206: * state here for mach_msg_receive_continue to pick up. ! 207: */ ! 208: ! 209: self->ith_msg = msg; ! 210: self->ith_option = option; ! 211: self->ith_rcv_size = rcv_size; ! 212: self->ith_timeout = time_out; ! 213: self->ith_notify = notify; ! 214: self->ith_object = object; ! 215: self->ith_mqueue = mqueue; ! 216: #endif ! 217: ! 218: if (option & MACH_RCV_LARGE) { ! 219: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT, ! 220: rcv_size, time_out, ! 221: FALSE, mach_msg_receive_continue, ! 222: &kmsg, &seqno); ! 223: /* mqueue is unlocked */ ! 224: ipc_object_release(object); ! 225: if (mr != MACH_MSG_SUCCESS) { ! 226: if (mr == MACH_RCV_TOO_LARGE) { ! 227: mach_msg_size_t real_size = ! 228: (mach_msg_size_t) (natural_t) kmsg; ! 229: ! 230: assert(real_size > rcv_size); ! 231: ! 232: (void) copyout((vm_offset_t) &real_size, ! 233: (vm_offset_t) &msg->msgh_size, ! 234: sizeof(mach_msg_size_t)); ! 235: } ! 236: ! 237: return mr; ! 238: } ! 239: ! 240: kmsg->ikm_header.msgh_seqno = seqno; ! 241: assert(kmsg->ikm_header.msgh_size <= rcv_size); ! 242: } else { ! 243: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT, ! 244: MACH_MSG_SIZE_MAX, time_out, ! 245: FALSE, mach_msg_receive_continue, ! 246: &kmsg, &seqno); ! 247: /* mqueue is unlocked */ ! 248: ipc_object_release(object); ! 249: if (mr != MACH_MSG_SUCCESS) ! 250: return mr; ! 251: ! 252: kmsg->ikm_header.msgh_seqno = seqno; ! 253: if (kmsg->ikm_header.msgh_size > rcv_size) { ! 254: ipc_kmsg_copyout_dest(kmsg, space); ! 255: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 256: return MACH_RCV_TOO_LARGE; ! 257: } ! 258: } ! 259: ! 260: if (option & MACH_RCV_NOTIFY) { ! 261: if (notify == MACH_PORT_NULL) ! 262: mr = MACH_RCV_INVALID_NOTIFY; ! 263: else ! 264: mr = ipc_kmsg_copyout(kmsg, space, map, notify); ! 265: } else ! 266: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL); ! 267: if (mr != MACH_MSG_SUCCESS) { ! 268: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { ! 269: (void) ipc_kmsg_put(msg, kmsg, ! 270: kmsg->ikm_header.msgh_size); ! 271: } else { ! 272: ipc_kmsg_copyout_dest(kmsg, space); ! 273: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 274: } ! 275: ! 276: return mr; ! 277: } ! 278: ! 279: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); ! 280: } ! 281: ! 282: #ifdef CONTINUATIONS ! 283: /* ! 284: * Routine: mach_msg_receive_continue ! 285: * Purpose: ! 286: * Continue after blocking for a message. ! 287: * Conditions: ! 288: * Nothing locked. We are running on a new kernel stack, ! 289: * with the receive state saved in the thread. From here ! 290: * control goes back to user space. ! 291: */ ! 292: ! 293: void ! 294: mach_msg_receive_continue() ! 295: { ! 296: ipc_thread_t self = current_thread(); ! 297: ipc_space_t space = current_space(); ! 298: vm_map_t map = current_map(); ! 299: mach_msg_header_t *msg = self->ith_msg; ! 300: mach_msg_option_t option = self->ith_option; ! 301: mach_msg_size_t rcv_size = self->ith_rcv_size; ! 302: mach_msg_timeout_t time_out = self->ith_timeout; ! 303: mach_port_t notify = self->ith_notify; ! 304: ipc_object_t object = self->ith_object; ! 305: ipc_mqueue_t mqueue = self->ith_mqueue; ! 306: ipc_kmsg_t kmsg; ! 307: mach_port_seqno_t seqno; ! 308: mach_msg_return_t mr; ! 309: ! 310: if (option & MACH_RCV_LARGE) { ! 311: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT, ! 312: rcv_size, time_out, ! 313: TRUE, mach_msg_receive_continue, ! 314: &kmsg, &seqno); ! 315: /* mqueue is unlocked */ ! 316: ipc_object_release(object); ! 317: if (mr != MACH_MSG_SUCCESS) { ! 318: if (mr == MACH_RCV_TOO_LARGE) { ! 319: mach_msg_size_t real_size = ! 320: (mach_msg_size_t) (natural_t) kmsg; ! 321: ! 322: assert(real_size > rcv_size); ! 323: ! 324: (void) copyout((vm_offset_t) &real_size, ! 325: (vm_offset_t) &msg->msgh_size, ! 326: sizeof(mach_msg_size_t)); ! 327: } ! 328: ! 329: thread_syscall_return(mr); ! 330: /*NOTREACHED*/ ! 331: } ! 332: ! 333: kmsg->ikm_header.msgh_seqno = seqno; ! 334: assert(kmsg->ikm_header.msgh_size <= rcv_size); ! 335: } else { ! 336: mr = ipc_mqueue_receive(mqueue, option & MACH_RCV_TIMEOUT, ! 337: MACH_MSG_SIZE_MAX, time_out, ! 338: TRUE, mach_msg_receive_continue, ! 339: &kmsg, &seqno); ! 340: /* mqueue is unlocked */ ! 341: ipc_object_release(object); ! 342: if (mr != MACH_MSG_SUCCESS) { ! 343: thread_syscall_return(mr); ! 344: /*NOTREACHED*/ ! 345: } ! 346: ! 347: kmsg->ikm_header.msgh_seqno = seqno; ! 348: if (kmsg->ikm_header.msgh_size > rcv_size) { ! 349: ipc_kmsg_copyout_dest(kmsg, space); ! 350: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 351: thread_syscall_return(MACH_RCV_TOO_LARGE); ! 352: /*NOTREACHED*/ ! 353: } ! 354: } ! 355: ! 356: if (option & MACH_RCV_NOTIFY) { ! 357: if (notify == MACH_PORT_NULL) ! 358: mr = MACH_RCV_INVALID_NOTIFY; ! 359: else ! 360: mr = ipc_kmsg_copyout(kmsg, space, map, notify); ! 361: } else ! 362: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL); ! 363: if (mr != MACH_MSG_SUCCESS) { ! 364: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { ! 365: (void) ipc_kmsg_put(msg, kmsg, ! 366: kmsg->ikm_header.msgh_size); ! 367: } else { ! 368: ipc_kmsg_copyout_dest(kmsg, space); ! 369: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 370: } ! 371: ! 372: thread_syscall_return(mr); ! 373: /*NOTREACHED*/ ! 374: } ! 375: ! 376: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); ! 377: thread_syscall_return(mr); ! 378: /*NOTREACHED*/ ! 379: } ! 380: #endif /* CONTINUATIONS */ ! 381: ! 382: /* ! 383: * Routine: mach_msg_trap [mach trap] ! 384: * Purpose: ! 385: * Possibly send a message; possibly receive a message. ! 386: * Conditions: ! 387: * Nothing locked. ! 388: * Returns: ! 389: * All of mach_msg_send and mach_msg_receive error codes. ! 390: */ ! 391: ! 392: mach_msg_return_t ! 393: mach_msg_trap(msg, option, send_size, rcv_size, rcv_name, time_out, notify) ! 394: mach_msg_header_t *msg; ! 395: mach_msg_option_t option; ! 396: mach_msg_size_t send_size; ! 397: mach_msg_size_t rcv_size; ! 398: mach_port_t rcv_name; ! 399: mach_msg_timeout_t time_out; ! 400: mach_port_t notify; ! 401: { ! 402: mach_msg_return_t mr; ! 403: ! 404: #ifdef CONTINUATIONS ! 405: /* first check for common cases */ ! 406: ! 407: if (option == (MACH_SEND_MSG|MACH_RCV_MSG)) { ! 408: register ipc_thread_t self = current_thread(); ! 409: ipc_space_t space = self->task->itk_space; ! 410: register ipc_kmsg_t kmsg; ! 411: register ipc_port_t dest_port; ! 412: ipc_object_t rcv_object; ! 413: register ipc_mqueue_t rcv_mqueue; ! 414: mach_msg_size_t reply_size; ! 415: ! 416: /* ! 417: * This case is divided into ten sections, each ! 418: * with a label. There are five optimized ! 419: * sections and six unoptimized sections, which ! 420: * do the same thing but handle all possible ! 421: * cases and are slower. ! 422: * ! 423: * The five sections for an RPC are ! 424: * 1) Get request message into a buffer. ! 425: * (fast_get or slow_get) ! 426: * 2) Copyin request message and rcv_name. ! 427: * (fast_copyin or slow_copyin) ! 428: * 3) Enqueue request and dequeue reply. ! 429: * (fast_send_receive or ! 430: * slow_send and slow_receive) ! 431: * 4) Copyout reply message. ! 432: * (fast_copyout or slow_copyout) ! 433: * 5) Put reply message to user's buffer. ! 434: * (fast_put or slow_put) ! 435: * ! 436: * Keep the locking hierarchy firmly in mind. ! 437: * (First spaces, then ports, then port sets, ! 438: * then message queues.) Only a non-blocking ! 439: * attempt can be made to acquire locks out of ! 440: * order, or acquire two locks on the same level. ! 441: * Acquiring two locks on the same level will ! 442: * fail if the objects are really the same, ! 443: * unless simple locking is disabled. This is OK, ! 444: * because then the extra unlock does nothing. ! 445: * ! 446: * There are two major reasons these RPCs can't use ! 447: * ipc_thread_switch, and use slow_send/slow_receive: ! 448: * 1) Kernel RPCs. ! 449: * 2) Servers fall behind clients, so ! 450: * client doesn't find a blocked server thread and ! 451: * server finds waiting messages and can't block. ! 452: */ ! 453: ! 454: /* ! 455: fast_get: ! 456: */ ! 457: /* ! 458: * optimized ipc_kmsg_get ! 459: * ! 460: * No locks, references, or messages held. ! 461: * We must clear ikm_cache before copyinmsg. ! 462: */ ! 463: ! 464: if ((send_size > IKM_SAVED_MSG_SIZE) || ! 465: (send_size < sizeof(mach_msg_header_t)) || ! 466: (send_size & 3) || ! 467: ((kmsg = ikm_cache()) == IKM_NULL)) ! 468: goto slow_get; ! 469: ! 470: ikm_cache() = IKM_NULL; ! 471: ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE); ! 472: ! 473: if (copyinmsg((vm_offset_t) msg, (vm_offset_t) &kmsg->ikm_header, ! 474: send_size)) { ! 475: ikm_free(kmsg); ! 476: goto slow_get; ! 477: } ! 478: ! 479: kmsg->ikm_header.msgh_size = send_size; ! 480: ! 481: fast_copyin: ! 482: /* ! 483: * optimized ipc_kmsg_copyin/ipc_mqueue_copyin ! 484: * ! 485: * We have the request message data in kmsg. ! 486: * Must still do copyin, send, receive, etc. ! 487: * ! 488: * If the message isn't simple, we can't combine ! 489: * ipc_kmsg_copyin_header and ipc_mqueue_copyin, ! 490: * because copyin of the message body might ! 491: * affect rcv_name. ! 492: */ ! 493: ! 494: switch (kmsg->ikm_header.msgh_bits) { ! 495: case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, ! 496: MACH_MSG_TYPE_MAKE_SEND_ONCE): { ! 497: register ipc_entry_t table; ! 498: register ipc_entry_num_t size; ! 499: register ipc_port_t reply_port; ! 500: ! 501: /* sending a request message */ ! 502: ! 503: { ! 504: register mach_port_index_t index; ! 505: register mach_port_gen_t gen; ! 506: ! 507: { ! 508: register mach_port_t reply_name = ! 509: kmsg->ikm_header.msgh_local_port; ! 510: ! 511: if (reply_name != rcv_name) ! 512: goto slow_copyin; ! 513: ! 514: /* optimized ipc_entry_lookup of reply_name */ ! 515: ! 516: index = MACH_PORT_INDEX(reply_name); ! 517: gen = MACH_PORT_GEN(reply_name); ! 518: } ! 519: ! 520: is_read_lock(space); ! 521: assert(space->is_active); ! 522: ! 523: size = space->is_table_size; ! 524: table = space->is_table; ! 525: ! 526: if (index >= size) ! 527: goto abort_request_copyin; ! 528: ! 529: { ! 530: register ipc_entry_t entry; ! 531: register ipc_entry_bits_t bits; ! 532: ! 533: entry = &table[index]; ! 534: bits = entry->ie_bits; ! 535: ! 536: /* check generation number and type bit */ ! 537: ! 538: if ((bits & (IE_BITS_GEN_MASK| ! 539: MACH_PORT_TYPE_RECEIVE)) != ! 540: (gen | MACH_PORT_TYPE_RECEIVE)) ! 541: goto abort_request_copyin; ! 542: ! 543: reply_port = (ipc_port_t) entry->ie_object; ! 544: assert(reply_port != IP_NULL); ! 545: } ! 546: } ! 547: ! 548: /* optimized ipc_entry_lookup of dest_name */ ! 549: ! 550: { ! 551: register mach_port_index_t index; ! 552: register mach_port_gen_t gen; ! 553: ! 554: { ! 555: register mach_port_t dest_name = ! 556: kmsg->ikm_header.msgh_remote_port; ! 557: ! 558: index = MACH_PORT_INDEX(dest_name); ! 559: gen = MACH_PORT_GEN(dest_name); ! 560: } ! 561: ! 562: if (index >= size) ! 563: goto abort_request_copyin; ! 564: ! 565: { ! 566: register ipc_entry_t entry; ! 567: register ipc_entry_bits_t bits; ! 568: ! 569: entry = &table[index]; ! 570: bits = entry->ie_bits; ! 571: ! 572: /* check generation number and type bit */ ! 573: ! 574: if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) != ! 575: (gen | MACH_PORT_TYPE_SEND)) ! 576: goto abort_request_copyin; ! 577: ! 578: assert(IE_BITS_UREFS(bits) > 0); ! 579: ! 580: dest_port = (ipc_port_t) entry->ie_object; ! 581: assert(dest_port != IP_NULL); ! 582: } ! 583: } ! 584: ! 585: /* ! 586: * To do an atomic copyin, need simultaneous ! 587: * locks on both ports and the space. If ! 588: * dest_port == reply_port, and simple locking is ! 589: * enabled, then we will abort. Otherwise it's ! 590: * OK to unlock twice. ! 591: */ ! 592: ! 593: ip_lock(dest_port); ! 594: if (!ip_active(dest_port) || ! 595: !ip_lock_try(reply_port)) { ! 596: ip_unlock(dest_port); ! 597: goto abort_request_copyin; ! 598: } ! 599: is_read_unlock(space); ! 600: ! 601: assert(dest_port->ip_srights > 0); ! 602: dest_port->ip_srights++; ! 603: ip_reference(dest_port); ! 604: ! 605: assert(ip_active(reply_port)); ! 606: assert(reply_port->ip_receiver_name == ! 607: kmsg->ikm_header.msgh_local_port); ! 608: assert(reply_port->ip_receiver == space); ! 609: ! 610: reply_port->ip_sorights++; ! 611: ip_reference(reply_port); ! 612: ! 613: kmsg->ikm_header.msgh_bits = ! 614: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, ! 615: MACH_MSG_TYPE_PORT_SEND_ONCE); ! 616: kmsg->ikm_header.msgh_remote_port = ! 617: (mach_port_t) dest_port; ! 618: kmsg->ikm_header.msgh_local_port = ! 619: (mach_port_t) reply_port; ! 620: ! 621: /* make sure we can queue to the destination */ ! 622: ! 623: if (dest_port->ip_receiver == ipc_space_kernel) { ! 624: /* ! 625: * The kernel server has a reference to ! 626: * the reply port, which it hands back ! 627: * to us in the reply message. We do ! 628: * not need to keep another reference to ! 629: * it. ! 630: */ ! 631: ip_unlock(reply_port); ! 632: ! 633: assert(ip_active(dest_port)); ! 634: ip_unlock(dest_port); ! 635: goto kernel_send; ! 636: } ! 637: ! 638: #if NORMA_IPC ! 639: if (IP_NORMA_IS_PROXY(dest_port)) { ! 640: ip_unlock(dest_port); ! 641: ip_unlock(reply_port); ! 642: goto norma_send; ! 643: } ! 644: #endif /* NORMA_IPC */ ! 645: ! 646: if (dest_port->ip_msgcount >= dest_port->ip_qlimit) ! 647: goto abort_request_send_receive; ! 648: ! 649: /* optimized ipc_mqueue_copyin */ ! 650: ! 651: if (reply_port->ip_pset != IPS_NULL) ! 652: goto abort_request_send_receive; ! 653: ! 654: rcv_object = (ipc_object_t) reply_port; ! 655: io_reference(rcv_object); ! 656: rcv_mqueue = &reply_port->ip_messages; ! 657: imq_lock(rcv_mqueue); ! 658: io_unlock(rcv_object); ! 659: goto fast_send_receive; ! 660: ! 661: abort_request_copyin: ! 662: is_read_unlock(space); ! 663: goto slow_copyin; ! 664: ! 665: abort_request_send_receive: ! 666: ip_unlock(dest_port); ! 667: ip_unlock(reply_port); ! 668: goto slow_send; ! 669: } ! 670: ! 671: case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): { ! 672: register ipc_entry_num_t size; ! 673: register ipc_entry_t table; ! 674: ! 675: /* sending a reply message */ ! 676: ! 677: { ! 678: register mach_port_t reply_name = ! 679: kmsg->ikm_header.msgh_local_port; ! 680: ! 681: if (reply_name != MACH_PORT_NULL) ! 682: goto slow_copyin; ! 683: } ! 684: ! 685: is_write_lock(space); ! 686: assert(space->is_active); ! 687: ! 688: /* optimized ipc_entry_lookup */ ! 689: ! 690: size = space->is_table_size; ! 691: table = space->is_table; ! 692: ! 693: { ! 694: register ipc_entry_t entry; ! 695: register mach_port_gen_t gen; ! 696: register mach_port_index_t index; ! 697: ! 698: { ! 699: register mach_port_t dest_name = ! 700: kmsg->ikm_header.msgh_remote_port; ! 701: ! 702: index = MACH_PORT_INDEX(dest_name); ! 703: gen = MACH_PORT_GEN(dest_name); ! 704: } ! 705: ! 706: if (index >= size) ! 707: goto abort_reply_dest_copyin; ! 708: ! 709: entry = &table[index]; ! 710: ! 711: /* check generation, collision bit, and type bit */ ! 712: ! 713: if ((entry->ie_bits & (IE_BITS_GEN_MASK| ! 714: IE_BITS_COLLISION| ! 715: MACH_PORT_TYPE_SEND_ONCE)) != ! 716: (gen | MACH_PORT_TYPE_SEND_ONCE)) ! 717: goto abort_reply_dest_copyin; ! 718: ! 719: /* optimized ipc_right_copyin */ ! 720: ! 721: assert(IE_BITS_TYPE(entry->ie_bits) == ! 722: MACH_PORT_TYPE_SEND_ONCE); ! 723: assert(IE_BITS_UREFS(entry->ie_bits) == 1); ! 724: assert((entry->ie_bits & IE_BITS_MAREQUEST) == 0); ! 725: ! 726: if (entry->ie_request != 0) ! 727: goto abort_reply_dest_copyin; ! 728: ! 729: dest_port = (ipc_port_t) entry->ie_object; ! 730: assert(dest_port != IP_NULL); ! 731: ! 732: ip_lock(dest_port); ! 733: if (!ip_active(dest_port)) { ! 734: ip_unlock(dest_port); ! 735: goto abort_reply_dest_copyin; ! 736: } ! 737: ! 738: assert(dest_port->ip_sorights > 0); ! 739: ! 740: /* optimized ipc_entry_dealloc */ ! 741: ! 742: entry->ie_next = table->ie_next; ! 743: table->ie_next = index; ! 744: entry->ie_bits = gen; ! 745: entry->ie_object = IO_NULL; ! 746: } ! 747: ! 748: kmsg->ikm_header.msgh_bits = ! 749: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, ! 750: 0); ! 751: kmsg->ikm_header.msgh_remote_port = ! 752: (mach_port_t) dest_port; ! 753: ! 754: /* make sure we can queue to the destination */ ! 755: ! 756: assert(dest_port->ip_receiver != ipc_space_kernel); ! 757: #if NORMA_IPC ! 758: if (IP_NORMA_IS_PROXY(dest_port)) { ! 759: is_write_unlock(space); ! 760: ip_unlock(dest_port); ! 761: goto norma_send; ! 762: } ! 763: #endif /* NORMA_IPC */ ! 764: ! 765: /* optimized ipc_entry_lookup/ipc_mqueue_copyin */ ! 766: ! 767: { ! 768: register ipc_entry_t entry; ! 769: register ipc_entry_bits_t bits; ! 770: ! 771: { ! 772: register mach_port_index_t index; ! 773: register mach_port_gen_t gen; ! 774: ! 775: index = MACH_PORT_INDEX(rcv_name); ! 776: gen = MACH_PORT_GEN(rcv_name); ! 777: ! 778: if (index >= size) ! 779: goto abort_reply_rcv_copyin; ! 780: ! 781: entry = &table[index]; ! 782: bits = entry->ie_bits; ! 783: ! 784: /* check generation number */ ! 785: ! 786: if ((bits & IE_BITS_GEN_MASK) != gen) ! 787: goto abort_reply_rcv_copyin; ! 788: } ! 789: ! 790: /* check type bits; looking for receive or set */ ! 791: ! 792: if (bits & MACH_PORT_TYPE_PORT_SET) { ! 793: register ipc_pset_t rcv_pset; ! 794: ! 795: rcv_pset = (ipc_pset_t) entry->ie_object; ! 796: assert(rcv_pset != IPS_NULL); ! 797: ! 798: ips_lock(rcv_pset); ! 799: assert(ips_active(rcv_pset)); ! 800: ! 801: rcv_object = (ipc_object_t) rcv_pset; ! 802: rcv_mqueue = &rcv_pset->ips_messages; ! 803: } else if (bits & MACH_PORT_TYPE_RECEIVE) { ! 804: register ipc_port_t rcv_port; ! 805: ! 806: rcv_port = (ipc_port_t) entry->ie_object; ! 807: assert(rcv_port != IP_NULL); ! 808: ! 809: if (!ip_lock_try(rcv_port)) ! 810: goto abort_reply_rcv_copyin; ! 811: assert(ip_active(rcv_port)); ! 812: ! 813: if (rcv_port->ip_pset != IPS_NULL) { ! 814: ip_unlock(rcv_port); ! 815: goto abort_reply_rcv_copyin; ! 816: } ! 817: ! 818: rcv_object = (ipc_object_t) rcv_port; ! 819: rcv_mqueue = &rcv_port->ip_messages; ! 820: } else ! 821: goto abort_reply_rcv_copyin; ! 822: } ! 823: ! 824: is_write_unlock(space); ! 825: io_reference(rcv_object); ! 826: imq_lock(rcv_mqueue); ! 827: io_unlock(rcv_object); ! 828: goto fast_send_receive; ! 829: ! 830: abort_reply_dest_copyin: ! 831: is_write_unlock(space); ! 832: goto slow_copyin; ! 833: ! 834: abort_reply_rcv_copyin: ! 835: ip_unlock(dest_port); ! 836: is_write_unlock(space); ! 837: goto slow_send; ! 838: } ! 839: ! 840: default: ! 841: goto slow_copyin; ! 842: } ! 843: /*NOTREACHED*/ ! 844: ! 845: fast_send_receive: ! 846: /* ! 847: * optimized ipc_mqueue_send/ipc_mqueue_receive ! 848: * ! 849: * Finished get/copyin of kmsg and copyin of rcv_name. ! 850: * space is unlocked, dest_port is locked, ! 851: * we can queue kmsg to dest_port, ! 852: * rcv_mqueue is locked, rcv_object holds a ref, ! 853: * if rcv_object is a port it isn't in a port set ! 854: * ! 855: * Note that if simple locking is turned off, ! 856: * then we could have dest_mqueue == rcv_mqueue ! 857: * and not abort when we try to lock dest_mqueue. ! 858: */ ! 859: ! 860: assert(ip_active(dest_port)); ! 861: assert(dest_port->ip_receiver != ipc_space_kernel); ! 862: #if NORMA_IPC ! 863: assert(! IP_NORMA_IS_PROXY(dest_port)); ! 864: #endif /* NORMA_IPC */ ! 865: assert((dest_port->ip_msgcount < dest_port->ip_qlimit) || ! 866: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) == ! 867: MACH_MSG_TYPE_PORT_SEND_ONCE)); ! 868: assert((kmsg->ikm_header.msgh_bits & ! 869: MACH_MSGH_BITS_CIRCULAR) == 0); ! 870: ! 871: { ! 872: register ipc_mqueue_t dest_mqueue; ! 873: register ipc_thread_t receiver; ! 874: ! 875: { ! 876: register ipc_pset_t dest_pset; ! 877: ! 878: dest_pset = dest_port->ip_pset; ! 879: if (dest_pset == IPS_NULL) ! 880: dest_mqueue = &dest_port->ip_messages; ! 881: else ! 882: dest_mqueue = &dest_pset->ips_messages; ! 883: } ! 884: ! 885: if (!imq_lock_try(dest_mqueue)) { ! 886: abort_send_receive: ! 887: ip_unlock(dest_port); ! 888: imq_unlock(rcv_mqueue); ! 889: ipc_object_release(rcv_object); ! 890: goto slow_send; ! 891: } ! 892: ! 893: receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads); ! 894: if ((receiver == ITH_NULL) || ! 895: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) ! 896: != IKM_NULL)) { ! 897: imq_unlock(dest_mqueue); ! 898: goto abort_send_receive; ! 899: } ! 900: ! 901: /* ! 902: * There is a receiver thread waiting, and ! 903: * there is no reply message for us to pick up. ! 904: * We have hope of hand-off, so save state. ! 905: */ ! 906: ! 907: self->ith_msg = msg; ! 908: self->ith_rcv_size = rcv_size; ! 909: self->ith_object = rcv_object; ! 910: self->ith_mqueue = rcv_mqueue; ! 911: ! 912: if ((receiver->swap_func == (void (*)()) mach_msg_continue) && ! 913: thread_handoff(self, mach_msg_continue, receiver)) { ! 914: assert(current_thread() == receiver); ! 915: ! 916: /* ! 917: * We can use the optimized receive code, ! 918: * because the receiver is using no options. ! 919: */ ! 920: } else if ((receiver->swap_func == ! 921: (void (*)()) exception_raise_continue) && ! 922: thread_handoff(self, mach_msg_continue, receiver)) { ! 923: counter(c_mach_msg_trap_block_exc++); ! 924: assert(current_thread() == receiver); ! 925: ! 926: /* ! 927: * We are a reply message coming back through ! 928: * the optimized exception-handling path. ! 929: * Finish with rcv_mqueue and dest_mqueue, ! 930: * and then jump to exception code with ! 931: * dest_port still locked. We don't bother ! 932: * with a sequence number in this case. ! 933: */ ! 934: ! 935: ipc_thread_enqueue_macro( ! 936: &rcv_mqueue->imq_threads, self); ! 937: self->ith_state = MACH_RCV_IN_PROGRESS; ! 938: self->ith_msize = MACH_MSG_SIZE_MAX; ! 939: imq_unlock(rcv_mqueue); ! 940: ! 941: ipc_thread_rmqueue_first_macro( ! 942: &dest_mqueue->imq_threads, receiver); ! 943: imq_unlock(dest_mqueue); ! 944: ! 945: exception_raise_continue_fast(dest_port, kmsg); ! 946: /*NOTREACHED*/ ! 947: return MACH_MSG_SUCCESS; ! 948: } else if ((send_size <= receiver->ith_msize) && ! 949: thread_handoff(self, mach_msg_continue, receiver)) { ! 950: assert(current_thread() == receiver); ! 951: ! 952: if ((receiver->swap_func == ! 953: (void (*)()) mach_msg_receive_continue) && ! 954: ((receiver->ith_option & MACH_RCV_NOTIFY) == 0)) { ! 955: /* ! 956: * We can still use the optimized code. ! 957: */ ! 958: } else { ! 959: counter(c_mach_msg_trap_block_slow++); ! 960: /* ! 961: * We are running as the receiver, ! 962: * but we can't use the optimized code. ! 963: * Finish send/receive processing. ! 964: */ ! 965: ! 966: dest_port->ip_msgcount++; ! 967: ip_unlock(dest_port); ! 968: ! 969: ipc_thread_enqueue_macro( ! 970: &rcv_mqueue->imq_threads, self); ! 971: self->ith_state = MACH_RCV_IN_PROGRESS; ! 972: self->ith_msize = MACH_MSG_SIZE_MAX; ! 973: imq_unlock(rcv_mqueue); ! 974: ! 975: ipc_thread_rmqueue_first_macro( ! 976: &dest_mqueue->imq_threads, receiver); ! 977: receiver->ith_state = MACH_MSG_SUCCESS; ! 978: receiver->ith_kmsg = kmsg; ! 979: receiver->ith_seqno = dest_port->ip_seqno++; ! 980: imq_unlock(dest_mqueue); ! 981: ! 982: /* ! 983: * Call the receiver's continuation. ! 984: */ ! 985: ! 986: receiver->wait_result = THREAD_AWAKENED; ! 987: (*receiver->swap_func)(); ! 988: /*NOTREACHED*/ ! 989: return MACH_MSG_SUCCESS; ! 990: } ! 991: } else { ! 992: /* ! 993: * The receiver can't accept the message, ! 994: * or we can't switch to the receiver. ! 995: */ ! 996: ! 997: imq_unlock(dest_mqueue); ! 998: goto abort_send_receive; ! 999: } ! 1000: counter(c_mach_msg_trap_block_fast++); ! 1001: ! 1002: /* ! 1003: * Safe to unlock dest_port now that we are ! 1004: * committed to this path, because we hold ! 1005: * dest_mqueue locked. We never bother changing ! 1006: * dest_port->ip_msgcount. ! 1007: */ ! 1008: ! 1009: ip_unlock(dest_port); ! 1010: ! 1011: /* ! 1012: * We need to finish preparing self for its ! 1013: * time asleep in rcv_mqueue. ! 1014: */ ! 1015: ! 1016: ipc_thread_enqueue_macro(&rcv_mqueue->imq_threads, self); ! 1017: self->ith_state = MACH_RCV_IN_PROGRESS; ! 1018: self->ith_msize = MACH_MSG_SIZE_MAX; ! 1019: imq_unlock(rcv_mqueue); ! 1020: ! 1021: /* ! 1022: * Finish extracting receiver from dest_mqueue. ! 1023: */ ! 1024: ! 1025: ipc_thread_rmqueue_first_macro( ! 1026: &dest_mqueue->imq_threads, receiver); ! 1027: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++; ! 1028: imq_unlock(dest_mqueue); ! 1029: ! 1030: /* ! 1031: * We don't have to do any post-dequeue processing of ! 1032: * the message. We never incremented ip_msgcount, we ! 1033: * know it has no msg-accepted request, and blocked ! 1034: * senders aren't a worry because we found the port ! 1035: * with a receiver waiting. ! 1036: */ ! 1037: ! 1038: self = receiver; ! 1039: space = self->task->itk_space; ! 1040: ! 1041: msg = self->ith_msg; ! 1042: rcv_size = self->ith_rcv_size; ! 1043: rcv_object = self->ith_object; ! 1044: ! 1045: /* inline ipc_object_release */ ! 1046: io_lock(rcv_object); ! 1047: io_release(rcv_object); ! 1048: io_check_unlock(rcv_object); ! 1049: } ! 1050: ! 1051: fast_copyout: ! 1052: /* ! 1053: * Nothing locked and no references held, except ! 1054: * we have kmsg with msgh_seqno filled in. Must ! 1055: * still check against rcv_size and do ! 1056: * ipc_kmsg_copyout/ipc_kmsg_put. ! 1057: */ ! 1058: ! 1059: assert((ipc_port_t) kmsg->ikm_header.msgh_remote_port ! 1060: == dest_port); ! 1061: ! 1062: reply_size = kmsg->ikm_header.msgh_size; ! 1063: if (rcv_size < reply_size) ! 1064: goto slow_copyout; ! 1065: ! 1066: /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */ ! 1067: ! 1068: switch (kmsg->ikm_header.msgh_bits) { ! 1069: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, ! 1070: MACH_MSG_TYPE_PORT_SEND_ONCE): { ! 1071: ipc_port_t reply_port = ! 1072: (ipc_port_t) kmsg->ikm_header.msgh_local_port; ! 1073: mach_port_t dest_name, reply_name; ! 1074: ! 1075: /* receiving a request message */ ! 1076: ! 1077: if (!IP_VALID(reply_port)) ! 1078: goto slow_copyout; ! 1079: ! 1080: is_write_lock(space); ! 1081: assert(space->is_active); ! 1082: ! 1083: /* ! 1084: * To do an atomic copyout, need simultaneous ! 1085: * locks on both ports and the space. If ! 1086: * dest_port == reply_port, and simple locking is ! 1087: * enabled, then we will abort. Otherwise it's ! 1088: * OK to unlock twice. ! 1089: */ ! 1090: ! 1091: ip_lock(dest_port); ! 1092: if (!ip_active(dest_port) || ! 1093: !ip_lock_try(reply_port)) ! 1094: goto abort_request_copyout; ! 1095: ! 1096: if (!ip_active(reply_port)) { ! 1097: ip_unlock(reply_port); ! 1098: goto abort_request_copyout; ! 1099: } ! 1100: ! 1101: assert(reply_port->ip_sorights > 0); ! 1102: ip_unlock(reply_port); ! 1103: ! 1104: { ! 1105: register ipc_entry_t table; ! 1106: register ipc_entry_t entry; ! 1107: register mach_port_index_t index; ! 1108: ! 1109: /* optimized ipc_entry_get */ ! 1110: ! 1111: table = space->is_table; ! 1112: index = table->ie_next; ! 1113: ! 1114: if (index == 0) ! 1115: goto abort_request_copyout; ! 1116: ! 1117: entry = &table[index]; ! 1118: table->ie_next = entry->ie_next; ! 1119: entry->ie_request = 0; ! 1120: ! 1121: { ! 1122: register mach_port_gen_t gen; ! 1123: ! 1124: assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0); ! 1125: gen = entry->ie_bits + IE_BITS_GEN_ONE; ! 1126: ! 1127: reply_name = MACH_PORT_MAKE(index, gen); ! 1128: ! 1129: /* optimized ipc_right_copyout */ ! 1130: ! 1131: entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1); ! 1132: } ! 1133: ! 1134: assert(MACH_PORT_VALID(reply_name)); ! 1135: entry->ie_object = (ipc_object_t) reply_port; ! 1136: is_write_unlock(space); ! 1137: } ! 1138: ! 1139: /* optimized ipc_object_copyout_dest */ ! 1140: ! 1141: assert(dest_port->ip_srights > 0); ! 1142: ip_release(dest_port); ! 1143: ! 1144: if (dest_port->ip_receiver == space) ! 1145: dest_name = dest_port->ip_receiver_name; ! 1146: else ! 1147: dest_name = MACH_PORT_NULL; ! 1148: ! 1149: if ((--dest_port->ip_srights == 0) && ! 1150: (dest_port->ip_nsrequest != IP_NULL)) { ! 1151: ipc_port_t nsrequest; ! 1152: mach_port_mscount_t mscount; ! 1153: ! 1154: /* a rather rare case */ ! 1155: ! 1156: nsrequest = dest_port->ip_nsrequest; ! 1157: mscount = dest_port->ip_mscount; ! 1158: dest_port->ip_nsrequest = IP_NULL; ! 1159: ip_unlock(dest_port); ! 1160: ! 1161: ipc_notify_no_senders(nsrequest, mscount); ! 1162: } else ! 1163: ip_unlock(dest_port); ! 1164: ! 1165: kmsg->ikm_header.msgh_bits = ! 1166: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, ! 1167: MACH_MSG_TYPE_PORT_SEND); ! 1168: kmsg->ikm_header.msgh_remote_port = reply_name; ! 1169: kmsg->ikm_header.msgh_local_port = dest_name; ! 1170: goto fast_put; ! 1171: ! 1172: abort_request_copyout: ! 1173: ip_unlock(dest_port); ! 1174: is_write_unlock(space); ! 1175: goto slow_copyout; ! 1176: } ! 1177: ! 1178: case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { ! 1179: register mach_port_t dest_name; ! 1180: ! 1181: /* receiving a reply message */ ! 1182: ! 1183: ip_lock(dest_port); ! 1184: if (!ip_active(dest_port)) ! 1185: goto slow_copyout; ! 1186: ! 1187: /* optimized ipc_object_copyout_dest */ ! 1188: ! 1189: assert(dest_port->ip_sorights > 0); ! 1190: ! 1191: if (dest_port->ip_receiver == space) { ! 1192: ip_release(dest_port); ! 1193: dest_port->ip_sorights--; ! 1194: dest_name = dest_port->ip_receiver_name; ! 1195: ip_unlock(dest_port); ! 1196: } else { ! 1197: ip_unlock(dest_port); ! 1198: ! 1199: ipc_notify_send_once(dest_port); ! 1200: dest_name = MACH_PORT_NULL; ! 1201: } ! 1202: ! 1203: kmsg->ikm_header.msgh_bits = ! 1204: MACH_MSGH_BITS(0, ! 1205: MACH_MSG_TYPE_PORT_SEND_ONCE); ! 1206: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL; ! 1207: kmsg->ikm_header.msgh_local_port = dest_name; ! 1208: goto fast_put; ! 1209: } ! 1210: ! 1211: case MACH_MSGH_BITS_COMPLEX| ! 1212: MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { ! 1213: register mach_port_t dest_name; ! 1214: ! 1215: /* receiving a complex reply message */ ! 1216: ! 1217: ip_lock(dest_port); ! 1218: if (!ip_active(dest_port)) ! 1219: goto slow_copyout; ! 1220: ! 1221: /* optimized ipc_object_copyout_dest */ ! 1222: ! 1223: assert(dest_port->ip_sorights > 0); ! 1224: ! 1225: if (dest_port->ip_receiver == space) { ! 1226: ip_release(dest_port); ! 1227: dest_port->ip_sorights--; ! 1228: dest_name = dest_port->ip_receiver_name; ! 1229: ip_unlock(dest_port); ! 1230: } else { ! 1231: ip_unlock(dest_port); ! 1232: ! 1233: ipc_notify_send_once(dest_port); ! 1234: dest_name = MACH_PORT_NULL; ! 1235: } ! 1236: ! 1237: kmsg->ikm_header.msgh_bits = ! 1238: MACH_MSGH_BITS_COMPLEX | ! 1239: MACH_MSGH_BITS(0, ! 1240: MACH_MSG_TYPE_PORT_SEND_ONCE); ! 1241: kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL; ! 1242: kmsg->ikm_header.msgh_local_port = dest_name; ! 1243: ! 1244: mr = ipc_kmsg_copyout_body( ! 1245: (vm_offset_t) (&kmsg->ikm_header + 1), ! 1246: (vm_offset_t) &kmsg->ikm_header ! 1247: + kmsg->ikm_header.msgh_size, ! 1248: space, ! 1249: current_map()); ! 1250: ! 1251: if (mr != MACH_MSG_SUCCESS) { ! 1252: (void) ipc_kmsg_put(msg, kmsg, ! 1253: kmsg->ikm_header.msgh_size); ! 1254: return mr | MACH_RCV_BODY_ERROR; ! 1255: } ! 1256: goto fast_put; ! 1257: } ! 1258: ! 1259: default: ! 1260: goto slow_copyout; ! 1261: } ! 1262: /*NOTREACHED*/ ! 1263: ! 1264: fast_put: ! 1265: /* ! 1266: * We have the reply message data in kmsg, ! 1267: * and the reply message size in reply_size. ! 1268: * Just need to copy it out to the user and free kmsg. ! 1269: * We must check ikm_cache after copyoutmsg. ! 1270: */ ! 1271: ! 1272: ikm_check_initialized(kmsg, kmsg->ikm_size); ! 1273: ! 1274: if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) || ! 1275: copyoutmsg((vm_offset_t) &kmsg->ikm_header, (vm_offset_t) msg, ! 1276: reply_size) || ! 1277: (ikm_cache() != IKM_NULL)) ! 1278: goto slow_put; ! 1279: ! 1280: ikm_cache() = kmsg; ! 1281: thread_syscall_return(MACH_MSG_SUCCESS); ! 1282: /*NOTREACHED*/ ! 1283: return MACH_MSG_SUCCESS; /* help for the compiler */ ! 1284: ! 1285: /* ! 1286: * The slow path has a few non-register temporary ! 1287: * variables used only for call-by-reference. ! 1288: */ ! 1289: ! 1290: { ! 1291: ipc_kmsg_t temp_kmsg; ! 1292: mach_port_seqno_t temp_seqno; ! 1293: ipc_object_t temp_rcv_object; ! 1294: ipc_mqueue_t temp_rcv_mqueue; ! 1295: ! 1296: slow_get: ! 1297: /* ! 1298: * No locks, references, or messages held. ! 1299: * Still have to get the request, send it, ! 1300: * receive reply, etc. ! 1301: */ ! 1302: ! 1303: mr = ipc_kmsg_get(msg, send_size, &temp_kmsg); ! 1304: if (mr != MACH_MSG_SUCCESS) { ! 1305: thread_syscall_return(mr); ! 1306: /*NOTREACHED*/ ! 1307: } ! 1308: kmsg = temp_kmsg; ! 1309: ! 1310: /* try to get back on optimized path */ ! 1311: goto fast_copyin; ! 1312: ! 1313: slow_copyin: ! 1314: /* ! 1315: * We have the message data in kmsg, but ! 1316: * we still need to copyin, send it, ! 1317: * receive a reply, and do copyout. ! 1318: */ ! 1319: ! 1320: mr = ipc_kmsg_copyin(kmsg, space, current_map(), ! 1321: MACH_PORT_NULL); ! 1322: if (mr != MACH_MSG_SUCCESS) { ! 1323: ikm_free(kmsg); ! 1324: thread_syscall_return(mr); ! 1325: /*NOTREACHED*/ ! 1326: } ! 1327: ! 1328: /* try to get back on optimized path */ ! 1329: ! 1330: if (kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_CIRCULAR) ! 1331: goto slow_send; ! 1332: ! 1333: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; ! 1334: assert(IP_VALID(dest_port)); ! 1335: ! 1336: ip_lock(dest_port); ! 1337: if (dest_port->ip_receiver == ipc_space_kernel) { ! 1338: assert(ip_active(dest_port)); ! 1339: ip_unlock(dest_port); ! 1340: goto kernel_send; ! 1341: } ! 1342: ! 1343: if (ip_active(dest_port) && ! 1344: #if NORMA_IPC ! 1345: (! IP_NORMA_IS_PROXY(dest_port)) && ! 1346: #endif /* NORMA_IPC */ ! 1347: ((dest_port->ip_msgcount < dest_port->ip_qlimit) || ! 1348: (MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) == ! 1349: MACH_MSG_TYPE_PORT_SEND_ONCE))) ! 1350: { ! 1351: /* ! 1352: * Try an optimized ipc_mqueue_copyin. ! 1353: * It will work if this is a request message. ! 1354: */ ! 1355: ! 1356: register ipc_port_t reply_port; ! 1357: ! 1358: reply_port = (ipc_port_t) ! 1359: kmsg->ikm_header.msgh_local_port; ! 1360: if (IP_VALID(reply_port)) { ! 1361: if (ip_lock_try(reply_port)) { ! 1362: if (ip_active(reply_port) && ! 1363: reply_port->ip_receiver == space && ! 1364: reply_port->ip_receiver_name == rcv_name && ! 1365: reply_port->ip_pset == IPS_NULL) ! 1366: { ! 1367: /* Grab a reference to the reply port. */ ! 1368: rcv_object = (ipc_object_t) reply_port; ! 1369: io_reference(rcv_object); ! 1370: rcv_mqueue = &reply_port->ip_messages; ! 1371: imq_lock(rcv_mqueue); ! 1372: io_unlock(rcv_object); ! 1373: goto fast_send_receive; ! 1374: } ! 1375: ip_unlock(reply_port); ! 1376: } ! 1377: } ! 1378: } ! 1379: ! 1380: ip_unlock(dest_port); ! 1381: goto slow_send; ! 1382: ! 1383: #if NORMA_IPC ! 1384: norma_send: ! 1385: /* ! 1386: * Nothing is locked. We have acquired kmsg, but ! 1387: * we still need to send it and receive a reply. ! 1388: */ ! 1389: ! 1390: mr = norma_ipc_send(kmsg); ! 1391: if (mr != MACH_MSG_SUCCESS) { ! 1392: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, ! 1393: current_map()); ! 1394: ! 1395: assert(kmsg->ikm_marequest == IMAR_NULL); ! 1396: (void) ipc_kmsg_put(msg, kmsg, ! 1397: kmsg->ikm_header.msgh_size); ! 1398: thread_syscall_return(mr); ! 1399: /*NOTREACHED*/ ! 1400: } ! 1401: ! 1402: goto slow_get_rcv_port; ! 1403: #endif /* NORMA_IPC */ ! 1404: ! 1405: kernel_send: ! 1406: /* ! 1407: * Special case: send message to kernel services. ! 1408: * The request message has been copied into the ! 1409: * kmsg. Nothing is locked. ! 1410: */ ! 1411: ! 1412: { ! 1413: register ipc_port_t reply_port; ! 1414: ! 1415: /* ! 1416: * Perform the kernel function. ! 1417: */ ! 1418: ! 1419: kmsg = ipc_kobject_server(kmsg); ! 1420: if (kmsg == IKM_NULL) { ! 1421: /* ! 1422: * No reply. Take the ! 1423: * slow receive path. ! 1424: */ ! 1425: goto slow_get_rcv_port; ! 1426: } ! 1427: ! 1428: /* ! 1429: * Check that: ! 1430: * the reply port is alive ! 1431: * we hold the receive right ! 1432: * the name has not changed. ! 1433: * the port is not in a set ! 1434: * If any of these are not true, ! 1435: * we cannot directly receive the reply ! 1436: * message. ! 1437: */ ! 1438: reply_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; ! 1439: ip_lock(reply_port); ! 1440: ! 1441: if ((!ip_active(reply_port)) || ! 1442: (reply_port->ip_receiver != space) || ! 1443: (reply_port->ip_receiver_name != rcv_name) || ! 1444: (reply_port->ip_pset != IPS_NULL)) ! 1445: { ! 1446: ip_unlock(reply_port); ! 1447: ipc_mqueue_send_always(kmsg); ! 1448: goto slow_get_rcv_port; ! 1449: } ! 1450: ! 1451: rcv_mqueue = &reply_port->ip_messages; ! 1452: imq_lock(rcv_mqueue); ! 1453: /* keep port locked, and don`t change ref count yet */ ! 1454: ! 1455: /* ! 1456: * If there are messages on the port ! 1457: * or other threads waiting for a message, ! 1458: * we cannot directly receive the reply. ! 1459: */ ! 1460: if ((ipc_thread_queue_first(&rcv_mqueue->imq_threads) ! 1461: != ITH_NULL) || ! 1462: (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) ! 1463: != IKM_NULL)) ! 1464: { ! 1465: imq_unlock(rcv_mqueue); ! 1466: ip_unlock(reply_port); ! 1467: ipc_mqueue_send_always(kmsg); ! 1468: goto slow_get_rcv_port; ! 1469: } ! 1470: ! 1471: /* ! 1472: * We can directly receive this reply. ! 1473: * Since the kernel reply never blocks, ! 1474: * it holds no message_accepted request. ! 1475: * Since there were no messages queued ! 1476: * on the reply port, there should be ! 1477: * no threads blocked waiting to send. ! 1478: */ ! 1479: ! 1480: assert(kmsg->ikm_marequest == IMAR_NULL); ! 1481: assert(ipc_thread_queue_first(&reply_port->ip_blocked) ! 1482: == ITH_NULL); ! 1483: ! 1484: dest_port = reply_port; ! 1485: kmsg->ikm_header.msgh_seqno = dest_port->ip_seqno++; ! 1486: imq_unlock(rcv_mqueue); ! 1487: ! 1488: /* ! 1489: * inline ipc_object_release. ! 1490: * Port is still locked. ! 1491: * Reference count was not incremented. ! 1492: */ ! 1493: ip_check_unlock(reply_port); ! 1494: ! 1495: /* copy out the kernel reply */ ! 1496: goto fast_copyout; ! 1497: } ! 1498: ! 1499: slow_send: ! 1500: /* ! 1501: * Nothing is locked. We have acquired kmsg, but ! 1502: * we still need to send it and receive a reply. ! 1503: */ ! 1504: ! 1505: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE, ! 1506: MACH_MSG_TIMEOUT_NONE); ! 1507: if (mr != MACH_MSG_SUCCESS) { ! 1508: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, ! 1509: current_map()); ! 1510: ! 1511: assert(kmsg->ikm_marequest == IMAR_NULL); ! 1512: (void) ipc_kmsg_put(msg, kmsg, ! 1513: kmsg->ikm_header.msgh_size); ! 1514: thread_syscall_return(mr); ! 1515: /*NOTREACHED*/ ! 1516: } ! 1517: ! 1518: slow_get_rcv_port: ! 1519: /* ! 1520: * We have sent the message. Copy in the receive port. ! 1521: */ ! 1522: mr = ipc_mqueue_copyin(space, rcv_name, ! 1523: &temp_rcv_mqueue, &temp_rcv_object); ! 1524: if (mr != MACH_MSG_SUCCESS) { ! 1525: thread_syscall_return(mr); ! 1526: /*NOTREACHED*/ ! 1527: } ! 1528: rcv_mqueue = temp_rcv_mqueue; ! 1529: rcv_object = temp_rcv_object; ! 1530: /* hold ref for rcv_object; rcv_mqueue is locked */ ! 1531: ! 1532: /* ! 1533: slow_receive: ! 1534: */ ! 1535: /* ! 1536: * Now we have sent the request and copied in rcv_name, ! 1537: * so rcv_mqueue is locked and hold ref for rcv_object. ! 1538: * Just receive a reply and try to get back to fast path. ! 1539: * ! 1540: * ipc_mqueue_receive may not return, because if we block ! 1541: * then our kernel stack may be discarded. So we save ! 1542: * state here for mach_msg_continue to pick up. ! 1543: */ ! 1544: ! 1545: self->ith_msg = msg; ! 1546: self->ith_rcv_size = rcv_size; ! 1547: self->ith_object = rcv_object; ! 1548: self->ith_mqueue = rcv_mqueue; ! 1549: ! 1550: mr = ipc_mqueue_receive(rcv_mqueue, ! 1551: MACH_MSG_OPTION_NONE, ! 1552: MACH_MSG_SIZE_MAX, ! 1553: MACH_MSG_TIMEOUT_NONE, ! 1554: FALSE, mach_msg_continue, ! 1555: &temp_kmsg, &temp_seqno); ! 1556: /* rcv_mqueue is unlocked */ ! 1557: ipc_object_release(rcv_object); ! 1558: if (mr != MACH_MSG_SUCCESS) { ! 1559: thread_syscall_return(mr); ! 1560: /*NOTREACHED*/ ! 1561: } ! 1562: ! 1563: (kmsg = temp_kmsg)->ikm_header.msgh_seqno = temp_seqno; ! 1564: dest_port = (ipc_port_t) kmsg->ikm_header.msgh_remote_port; ! 1565: goto fast_copyout; ! 1566: ! 1567: slow_copyout: ! 1568: /* ! 1569: * Nothing locked and no references held, except ! 1570: * we have kmsg with msgh_seqno filled in. Must ! 1571: * still check against rcv_size and do ! 1572: * ipc_kmsg_copyout/ipc_kmsg_put. ! 1573: */ ! 1574: ! 1575: reply_size = kmsg->ikm_header.msgh_size; ! 1576: if (rcv_size < reply_size) { ! 1577: ipc_kmsg_copyout_dest(kmsg, space); ! 1578: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1579: thread_syscall_return(MACH_RCV_TOO_LARGE); ! 1580: /*NOTREACHED*/ ! 1581: } ! 1582: ! 1583: mr = ipc_kmsg_copyout(kmsg, space, current_map(), ! 1584: MACH_PORT_NULL); ! 1585: if (mr != MACH_MSG_SUCCESS) { ! 1586: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { ! 1587: (void) ipc_kmsg_put(msg, kmsg, ! 1588: kmsg->ikm_header.msgh_size); ! 1589: } else { ! 1590: ipc_kmsg_copyout_dest(kmsg, space); ! 1591: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1592: } ! 1593: ! 1594: thread_syscall_return(mr); ! 1595: /*NOTREACHED*/ ! 1596: } ! 1597: ! 1598: /* try to get back on optimized path */ ! 1599: ! 1600: goto fast_put; ! 1601: ! 1602: slow_put: ! 1603: mr = ipc_kmsg_put(msg, kmsg, reply_size); ! 1604: thread_syscall_return(mr); ! 1605: /*NOTREACHED*/ ! 1606: } ! 1607: } else if (option == MACH_SEND_MSG) { ! 1608: ipc_space_t space = current_space(); ! 1609: vm_map_t map = current_map(); ! 1610: ipc_kmsg_t kmsg; ! 1611: ! 1612: mr = ipc_kmsg_get(msg, send_size, &kmsg); ! 1613: if (mr != MACH_MSG_SUCCESS) ! 1614: return mr; ! 1615: ! 1616: mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL); ! 1617: if (mr != MACH_MSG_SUCCESS) { ! 1618: ikm_free(kmsg); ! 1619: return mr; ! 1620: } ! 1621: ! 1622: mr = ipc_mqueue_send(kmsg, MACH_MSG_OPTION_NONE, ! 1623: MACH_MSG_TIMEOUT_NONE); ! 1624: if (mr != MACH_MSG_SUCCESS) { ! 1625: mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map); ! 1626: ! 1627: assert(kmsg->ikm_marequest == IMAR_NULL); ! 1628: (void) ipc_kmsg_put(msg, kmsg, ! 1629: kmsg->ikm_header.msgh_size); ! 1630: } ! 1631: ! 1632: return mr; ! 1633: } else if (option == MACH_RCV_MSG) { ! 1634: ipc_thread_t self = current_thread(); ! 1635: ipc_space_t space = current_space(); ! 1636: vm_map_t map = current_map(); ! 1637: ipc_object_t object; ! 1638: ipc_mqueue_t mqueue; ! 1639: ipc_kmsg_t kmsg; ! 1640: mach_port_seqno_t seqno; ! 1641: ! 1642: mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object); ! 1643: if (mr != MACH_MSG_SUCCESS) ! 1644: return mr; ! 1645: /* hold ref for object; mqueue is locked */ ! 1646: ! 1647: /* ! 1648: * ipc_mqueue_receive may not return, because if we block ! 1649: * then our kernel stack may be discarded. So we save ! 1650: * state here for mach_msg_continue to pick up. ! 1651: */ ! 1652: ! 1653: self->ith_msg = msg; ! 1654: self->ith_rcv_size = rcv_size; ! 1655: self->ith_object = object; ! 1656: self->ith_mqueue = mqueue; ! 1657: ! 1658: mr = ipc_mqueue_receive(mqueue, ! 1659: MACH_MSG_OPTION_NONE, ! 1660: MACH_MSG_SIZE_MAX, ! 1661: MACH_MSG_TIMEOUT_NONE, ! 1662: FALSE, mach_msg_continue, ! 1663: &kmsg, &seqno); ! 1664: /* mqueue is unlocked */ ! 1665: ipc_object_release(object); ! 1666: if (mr != MACH_MSG_SUCCESS) ! 1667: return mr; ! 1668: ! 1669: kmsg->ikm_header.msgh_seqno = seqno; ! 1670: if (rcv_size < kmsg->ikm_header.msgh_size) { ! 1671: ipc_kmsg_copyout_dest(kmsg, space); ! 1672: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1673: return MACH_RCV_TOO_LARGE; ! 1674: } ! 1675: ! 1676: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL); ! 1677: if (mr != MACH_MSG_SUCCESS) { ! 1678: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { ! 1679: (void) ipc_kmsg_put(msg, kmsg, ! 1680: kmsg->ikm_header.msgh_size); ! 1681: } else { ! 1682: ipc_kmsg_copyout_dest(kmsg, space); ! 1683: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1684: } ! 1685: ! 1686: return mr; ! 1687: } ! 1688: ! 1689: return ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); ! 1690: } else if (option == MACH_MSG_OPTION_NONE) { ! 1691: /* ! 1692: * We can measure the "null mach_msg_trap" ! 1693: * (syscall entry and thread_syscall_return exit) ! 1694: * with this path. ! 1695: */ ! 1696: ! 1697: thread_syscall_return(MACH_MSG_SUCCESS); ! 1698: /*NOTREACHED*/ ! 1699: } ! 1700: #endif /* CONTINUATIONS */ ! 1701: ! 1702: if (option & MACH_SEND_MSG) { ! 1703: mr = mach_msg_send(msg, option, send_size, ! 1704: time_out, notify); ! 1705: if (mr != MACH_MSG_SUCCESS) ! 1706: return mr; ! 1707: } ! 1708: ! 1709: if (option & MACH_RCV_MSG) { ! 1710: mr = mach_msg_receive(msg, option, rcv_size, rcv_name, ! 1711: time_out, notify); ! 1712: if (mr != MACH_MSG_SUCCESS) ! 1713: return mr; ! 1714: } ! 1715: ! 1716: return MACH_MSG_SUCCESS; ! 1717: } ! 1718: ! 1719: #ifdef CONTINUATIONS ! 1720: /* ! 1721: * Routine: mach_msg_continue ! 1722: * Purpose: ! 1723: * Continue after blocking for a message. ! 1724: * Conditions: ! 1725: * Nothing locked. We are running on a new kernel stack, ! 1726: * with the receive state saved in the thread. From here ! 1727: * control goes back to user space. ! 1728: */ ! 1729: ! 1730: void ! 1731: mach_msg_continue() ! 1732: { ! 1733: ipc_thread_t thread = current_thread(); ! 1734: task_t task = thread->task; ! 1735: ipc_space_t space = task->itk_space; ! 1736: vm_map_t map = task->map; ! 1737: mach_msg_header_t *msg = thread->ith_msg; ! 1738: mach_msg_size_t rcv_size = thread->ith_rcv_size; ! 1739: ipc_object_t object = thread->ith_object; ! 1740: ipc_mqueue_t mqueue = thread->ith_mqueue; ! 1741: ipc_kmsg_t kmsg; ! 1742: mach_port_seqno_t seqno; ! 1743: mach_msg_return_t mr; ! 1744: ! 1745: mr = ipc_mqueue_receive(mqueue, MACH_MSG_OPTION_NONE, ! 1746: MACH_MSG_SIZE_MAX, MACH_MSG_TIMEOUT_NONE, ! 1747: TRUE, mach_msg_continue, &kmsg, &seqno); ! 1748: /* mqueue is unlocked */ ! 1749: ipc_object_release(object); ! 1750: if (mr != MACH_MSG_SUCCESS) { ! 1751: thread_syscall_return(mr); ! 1752: /*NOTREACHED*/ ! 1753: } ! 1754: ! 1755: kmsg->ikm_header.msgh_seqno = seqno; ! 1756: if (kmsg->ikm_header.msgh_size > rcv_size) { ! 1757: ipc_kmsg_copyout_dest(kmsg, space); ! 1758: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1759: thread_syscall_return(MACH_RCV_TOO_LARGE); ! 1760: /*NOTREACHED*/ ! 1761: } ! 1762: ! 1763: mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL); ! 1764: if (mr != MACH_MSG_SUCCESS) { ! 1765: if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { ! 1766: (void) ipc_kmsg_put(msg, kmsg, ! 1767: kmsg->ikm_header.msgh_size); ! 1768: } else { ! 1769: ipc_kmsg_copyout_dest(kmsg, space); ! 1770: (void) ipc_kmsg_put(msg, kmsg, sizeof *msg); ! 1771: } ! 1772: ! 1773: thread_syscall_return(mr); ! 1774: /*NOTREACHED*/ ! 1775: } ! 1776: ! 1777: mr = ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size); ! 1778: thread_syscall_return(mr); ! 1779: /*NOTREACHED*/ ! 1780: } ! 1781: ! 1782: /* ! 1783: * Routine: mach_msg_interrupt ! 1784: * Purpose: ! 1785: * Attempts to force a thread waiting at mach_msg_continue or ! 1786: * mach_msg_receive_continue into a clean point. Returns TRUE ! 1787: * if this was possible. ! 1788: * Conditions: ! 1789: * Nothing locked. The thread must NOT be runnable. ! 1790: */ ! 1791: ! 1792: boolean_t ! 1793: mach_msg_interrupt(thread) ! 1794: thread_t thread; ! 1795: { ! 1796: ipc_mqueue_t mqueue; ! 1797: ! 1798: assert((thread->swap_func == (void (*)()) mach_msg_continue) || ! 1799: (thread->swap_func == (void (*)()) mach_msg_receive_continue)); ! 1800: ! 1801: mqueue = thread->ith_mqueue; ! 1802: imq_lock(mqueue); ! 1803: if (thread->ith_state != MACH_RCV_IN_PROGRESS) { ! 1804: /* ! 1805: * The thread is no longer waiting for a message. ! 1806: * It may have a message sitting in ith_kmsg. ! 1807: * We can't clean this up. ! 1808: */ ! 1809: ! 1810: imq_unlock(mqueue); ! 1811: return FALSE; ! 1812: } ! 1813: ipc_thread_rmqueue(&mqueue->imq_threads, thread); ! 1814: imq_unlock(mqueue); ! 1815: ! 1816: ipc_object_release(thread->ith_object); ! 1817: ! 1818: thread_set_syscall_return(thread, MACH_RCV_INTERRUPTED); ! 1819: thread->swap_func = thread_exception_return; ! 1820: return TRUE; ! 1821: } ! 1822: #endif /* CONTINUATIONS */ ! 1823: ! 1824: #if MACH_IPC_COMPAT ! 1825: ! 1826: /* ! 1827: * Routine: msg_return_translate ! 1828: * Purpose: ! 1829: * Translate from new error code to old error code. ! 1830: */ ! 1831: ! 1832: msg_return_t ! 1833: msg_return_translate(mr) ! 1834: mach_msg_return_t mr; ! 1835: { ! 1836: switch (mr &~ MACH_MSG_MASK) { ! 1837: case MACH_MSG_SUCCESS: ! 1838: return 0; /* SEND_SUCCESS/RCV_SUCCESS/RPC_SUCCESS */ ! 1839: ! 1840: case MACH_SEND_NO_BUFFER: ! 1841: case MACH_SEND_NO_NOTIFY: ! 1842: printf("msg_return_translate: %x -> interrupted\n", mr); ! 1843: return SEND_INTERRUPTED; ! 1844: ! 1845: case MACH_SEND_MSG_TOO_SMALL: ! 1846: return SEND_MSG_TOO_SMALL; ! 1847: case MACH_SEND_INVALID_DATA: ! 1848: case MACH_SEND_INVALID_MEMORY: ! 1849: return SEND_INVALID_MEMORY; ! 1850: case MACH_SEND_TIMED_OUT: ! 1851: return SEND_TIMED_OUT; ! 1852: case MACH_SEND_INTERRUPTED: ! 1853: return SEND_INTERRUPTED; ! 1854: case MACH_SEND_INVALID_DEST: ! 1855: case MACH_SEND_INVALID_REPLY: ! 1856: case MACH_SEND_INVALID_RIGHT: ! 1857: case MACH_SEND_INVALID_TYPE: ! 1858: return SEND_INVALID_PORT; ! 1859: case MACH_SEND_WILL_NOTIFY: ! 1860: return SEND_WILL_NOTIFY; ! 1861: case MACH_SEND_NOTIFY_IN_PROGRESS: ! 1862: return SEND_NOTIFY_IN_PROGRESS; ! 1863: ! 1864: case MACH_RCV_INVALID_NAME: ! 1865: case MACH_RCV_IN_SET: ! 1866: case MACH_RCV_PORT_DIED: ! 1867: return RCV_INVALID_PORT; ! 1868: case MACH_RCV_TOO_LARGE: ! 1869: return RCV_TOO_LARGE; ! 1870: case MACH_RCV_TIMED_OUT: ! 1871: return RCV_TIMED_OUT; ! 1872: case MACH_RCV_INTERRUPTED: ! 1873: return RCV_INTERRUPTED; ! 1874: case MACH_RCV_PORT_CHANGED: ! 1875: return RCV_PORT_CHANGE; ! 1876: case MACH_RCV_INVALID_DATA: ! 1877: return RCV_INVALID_MEMORY; ! 1878: ! 1879: case MACH_SEND_IN_PROGRESS: ! 1880: case MACH_SEND_INVALID_NOTIFY: ! 1881: case MACH_SEND_INVALID_HEADER: ! 1882: case MACH_RCV_IN_PROGRESS: ! 1883: case MACH_RCV_INVALID_NOTIFY: ! 1884: case MACH_RCV_HEADER_ERROR: ! 1885: case MACH_RCV_BODY_ERROR: ! 1886: default: ! 1887: #if MACH_ASSERT ! 1888: assert(!"msg_return_translate"); ! 1889: #else ! 1890: panic("msg_return_translate"); ! 1891: #endif ! 1892: } ! 1893: } ! 1894: ! 1895: /* ! 1896: * Routine: msg_send_trap [mach trap] ! 1897: * Purpose: ! 1898: * Send a message. ! 1899: * Conditions: ! 1900: * Nothing locked. ! 1901: * Returns: ! 1902: */ ! 1903: ! 1904: msg_return_t ! 1905: msg_send_trap(msg, option, send_size, time_out) ! 1906: msg_header_t *msg; ! 1907: msg_option_t option; ! 1908: msg_size_t send_size; ! 1909: msg_timeout_t time_out; ! 1910: { ! 1911: ipc_space_t space = current_space(); ! 1912: vm_map_t map = current_map(); ! 1913: ipc_kmsg_t kmsg; ! 1914: mach_msg_return_t mr; ! 1915: ! 1916: send_size = (send_size + 3) & ~3; /* round up */ ! 1917: ! 1918: if (send_size > MSG_SIZE_MAX) ! 1919: return SEND_MSG_TOO_LARGE; ! 1920: ! 1921: mr = ipc_kmsg_get((mach_msg_header_t *) msg, ! 1922: (mach_msg_size_t) send_size, ! 1923: &kmsg); ! 1924: if (mr != MACH_MSG_SUCCESS) ! 1925: return msg_return_translate(mr); ! 1926: ! 1927: mr = ipc_kmsg_copyin_compat(kmsg, space, map); ! 1928: if (mr != MACH_MSG_SUCCESS) { ! 1929: ikm_free(kmsg); ! 1930: return msg_return_translate(mr); ! 1931: } ! 1932: ! 1933: if (option & SEND_NOTIFY) { ! 1934: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT, ! 1935: ((option & SEND_TIMEOUT) ? ! 1936: (mach_msg_timeout_t) time_out : ! 1937: MACH_MSG_TIMEOUT_NONE)); ! 1938: if (mr == MACH_SEND_TIMED_OUT) { ! 1939: ipc_port_t dest = (ipc_port_t) ! 1940: kmsg->ikm_header.msgh_remote_port; ! 1941: ! 1942: mr = ipc_marequest_create(space, dest, MACH_PORT_NULL, ! 1943: &kmsg->ikm_marequest); ! 1944: if (mr == MACH_MSG_SUCCESS) { ! 1945: ipc_mqueue_send_always(kmsg); ! 1946: return SEND_WILL_NOTIFY; ! 1947: } ! 1948: } ! 1949: } else ! 1950: mr = ipc_mqueue_send(kmsg, ! 1951: ((option & SEND_TIMEOUT) ? ! 1952: MACH_SEND_TIMEOUT : ! 1953: MACH_MSG_OPTION_NONE), ! 1954: (mach_msg_timeout_t) time_out); ! 1955: ! 1956: if (mr != MACH_MSG_SUCCESS) ! 1957: ipc_kmsg_destroy(kmsg); ! 1958: ! 1959: return msg_return_translate(mr); ! 1960: } ! 1961: ! 1962: /* ! 1963: * Routine: msg_receive_trap [mach trap] ! 1964: * Purpose: ! 1965: * Receive a message. ! 1966: * Conditions: ! 1967: * Nothing locked. ! 1968: * Returns: ! 1969: */ ! 1970: ! 1971: msg_return_t ! 1972: msg_receive_trap(msg, option, rcv_size, rcv_name, time_out) ! 1973: msg_header_t *msg; ! 1974: msg_option_t option; ! 1975: msg_size_t rcv_size; ! 1976: port_name_t rcv_name; ! 1977: msg_timeout_t time_out; ! 1978: { ! 1979: ipc_thread_t self; ! 1980: ipc_space_t space = current_space(); ! 1981: vm_map_t map = current_map(); ! 1982: ipc_object_t object; ! 1983: ipc_mqueue_t mqueue; ! 1984: ipc_kmsg_t kmsg; ! 1985: mach_port_seqno_t seqno; ! 1986: mach_msg_return_t mr; ! 1987: ! 1988: mr = ipc_mqueue_copyin(space, (mach_port_t) rcv_name, ! 1989: &mqueue, &object); ! 1990: if (mr != MACH_MSG_SUCCESS) ! 1991: return msg_return_translate(mr); ! 1992: /* hold ref for object; mqueue is locked */ ! 1993: ! 1994: #ifdef CONTINUATIONS ! 1995: /* ! 1996: * ipc_mqueue_receive may not return, because if we block ! 1997: * then our kernel stack may be discarded. So we save ! 1998: * state here for msg_receive_continue to pick up. ! 1999: */ ! 2000: ! 2001: self = current_thread(); ! 2002: self->ith_msg = (mach_msg_header_t *) msg; ! 2003: self->ith_option = (mach_msg_option_t) option; ! 2004: self->ith_rcv_size = (mach_msg_size_t) rcv_size; ! 2005: self->ith_timeout = (mach_msg_timeout_t) time_out; ! 2006: self->ith_object = object; ! 2007: self->ith_mqueue = mqueue; ! 2008: #endif /* CONTINUATIONS */ ! 2009: ! 2010: mr = ipc_mqueue_receive(mqueue, ! 2011: (option & RCV_TIMEOUT) ? ! 2012: MACH_RCV_TIMEOUT : MACH_MSG_OPTION_NONE, ! 2013: (mach_msg_size_t) rcv_size, ! 2014: (mach_msg_timeout_t) time_out, ! 2015: FALSE, msg_receive_continue, ! 2016: &kmsg, &seqno); ! 2017: /* mqueue is unlocked */ ! 2018: ipc_object_release(object); ! 2019: if (mr != MACH_MSG_SUCCESS) { ! 2020: if (mr == MACH_RCV_TOO_LARGE) { ! 2021: msg_size_t real_size = ! 2022: (msg_size_t) (mach_msg_size_t) kmsg; ! 2023: ! 2024: assert(real_size > rcv_size); ! 2025: ! 2026: (void) copyout((vm_offset_t) &real_size, ! 2027: (vm_offset_t) &msg->msg_size, ! 2028: sizeof(msg_size_t)); ! 2029: } ! 2030: ! 2031: return msg_return_translate(mr); ! 2032: } ! 2033: ! 2034: assert(kmsg->ikm_header.msgh_size <= (mach_msg_size_t) rcv_size); ! 2035: ! 2036: mr = ipc_kmsg_copyout_compat(kmsg, space, map); ! 2037: assert(mr == MACH_MSG_SUCCESS); ! 2038: ! 2039: mr = ipc_kmsg_put((mach_msg_header_t *) msg, kmsg, ! 2040: kmsg->ikm_header.msgh_size); ! 2041: return msg_return_translate(mr); ! 2042: } ! 2043: ! 2044: /* ! 2045: * Routine: msg_rpc_trap [mach trap] ! 2046: * Purpose: ! 2047: * Send and receive a message. ! 2048: * Conditions: ! 2049: * Nothing locked. ! 2050: * Returns: ! 2051: */ ! 2052: ! 2053: msg_return_t ! 2054: msg_rpc_trap(msg, option, send_size, rcv_size, send_timeout, rcv_timeout) ! 2055: msg_header_t *msg; ! 2056: msg_option_t option; ! 2057: msg_size_t send_size; ! 2058: msg_size_t rcv_size; ! 2059: msg_timeout_t send_timeout; ! 2060: msg_timeout_t rcv_timeout; ! 2061: { ! 2062: ipc_thread_t self; ! 2063: ipc_space_t space = current_space(); ! 2064: vm_map_t map = current_map(); ! 2065: ipc_port_t reply; ! 2066: ipc_pset_t pset; ! 2067: ipc_mqueue_t mqueue; ! 2068: ipc_kmsg_t kmsg; ! 2069: mach_port_seqno_t seqno; ! 2070: mach_msg_return_t mr; ! 2071: ! 2072: /* ! 2073: * Instead of using msg_send_trap and msg_receive_trap, ! 2074: * we implement msg_rpc_trap directly. The difference ! 2075: * is how the reply port is handled. Instead of using ! 2076: * ipc_mqueue_copyin, we save a reference for the reply ! 2077: * port carried in the sent message. For example, ! 2078: * consider a rename kernel call which changes the name ! 2079: * of the call's own reply port. This is the behaviour ! 2080: * of the Mach 2.5 msg_rpc_trap. ! 2081: */ ! 2082: ! 2083: send_size = (send_size + 3) & ~3; /* round up */ ! 2084: ! 2085: if (send_size > MSG_SIZE_MAX) ! 2086: return SEND_MSG_TOO_LARGE; ! 2087: ! 2088: mr = ipc_kmsg_get((mach_msg_header_t *) msg, ! 2089: (mach_msg_size_t) send_size, ! 2090: &kmsg); ! 2091: if (mr != MACH_MSG_SUCCESS) ! 2092: return msg_return_translate(mr); ! 2093: ! 2094: mr = ipc_kmsg_copyin_compat(kmsg, space, map); ! 2095: if (mr != MACH_MSG_SUCCESS) { ! 2096: ikm_free(kmsg); ! 2097: return msg_return_translate(mr); ! 2098: } ! 2099: ! 2100: reply = (ipc_port_t) kmsg->ikm_header.msgh_local_port; ! 2101: if (IP_VALID(reply)) ! 2102: ipc_port_reference(reply); ! 2103: ! 2104: if (option & SEND_NOTIFY) { ! 2105: mr = ipc_mqueue_send(kmsg, MACH_SEND_TIMEOUT, ! 2106: ((option & SEND_TIMEOUT) ? ! 2107: (mach_msg_timeout_t) send_timeout : ! 2108: MACH_MSG_TIMEOUT_NONE)); ! 2109: if (mr == MACH_SEND_TIMED_OUT) { ! 2110: ipc_port_t dest = (ipc_port_t) ! 2111: kmsg->ikm_header.msgh_remote_port; ! 2112: ! 2113: mr = ipc_marequest_create(space, dest, MACH_PORT_NULL, ! 2114: &kmsg->ikm_marequest); ! 2115: if (mr == MACH_MSG_SUCCESS) { ! 2116: ipc_mqueue_send_always(kmsg); ! 2117: if (IP_VALID(reply)) ! 2118: ipc_port_release(reply); ! 2119: return SEND_WILL_NOTIFY; ! 2120: } ! 2121: } ! 2122: } else ! 2123: mr = ipc_mqueue_send(kmsg, ! 2124: ((option & SEND_TIMEOUT) ? ! 2125: MACH_SEND_TIMEOUT : ! 2126: MACH_MSG_OPTION_NONE), ! 2127: (mach_msg_timeout_t) send_timeout); ! 2128: ! 2129: if (mr != MACH_MSG_SUCCESS) { ! 2130: ipc_kmsg_destroy(kmsg); ! 2131: if (IP_VALID(reply)) ! 2132: ipc_port_release(reply); ! 2133: return msg_return_translate(mr); ! 2134: } ! 2135: ! 2136: if (!IP_VALID(reply)) ! 2137: return RCV_INVALID_PORT; ! 2138: ! 2139: ip_lock(reply); ! 2140: if (reply->ip_receiver != space) { ! 2141: ip_release(reply); ! 2142: ip_check_unlock(reply); ! 2143: return RCV_INVALID_PORT; ! 2144: } ! 2145: ! 2146: assert(ip_active(reply)); ! 2147: pset = reply->ip_pset; ! 2148: ! 2149: if (pset != IPS_NULL) { ! 2150: ips_lock(pset); ! 2151: if (ips_active(pset)) { ! 2152: ips_unlock(pset); ! 2153: ip_release(reply); ! 2154: ip_unlock(reply); ! 2155: return RCV_INVALID_PORT; ! 2156: } ! 2157: ! 2158: ipc_pset_remove(pset, reply); ! 2159: ips_check_unlock(pset); ! 2160: assert(reply->ip_pset == IPS_NULL); ! 2161: } ! 2162: ! 2163: mqueue = &reply->ip_messages; ! 2164: imq_lock(mqueue); ! 2165: ip_unlock(reply); ! 2166: ! 2167: #ifdef CONTINUATIONS ! 2168: /* ! 2169: * ipc_mqueue_receive may not return, because if we block ! 2170: * then our kernel stack may be discarded. So we save ! 2171: * state here for msg_receive_continue to pick up. ! 2172: */ ! 2173: ! 2174: self = current_thread(); ! 2175: self->ith_msg = (mach_msg_header_t *) msg; ! 2176: self->ith_option = (mach_msg_option_t) option; ! 2177: self->ith_rcv_size = (mach_msg_size_t) rcv_size; ! 2178: self->ith_timeout = (mach_msg_timeout_t) rcv_timeout; ! 2179: self->ith_object = (ipc_object_t) reply; ! 2180: self->ith_mqueue = mqueue; ! 2181: #endif /* CONTINUATIONS */ ! 2182: ! 2183: mr = ipc_mqueue_receive(mqueue, ! 2184: (option & RCV_TIMEOUT) ? ! 2185: MACH_RCV_TIMEOUT : MACH_MSG_OPTION_NONE, ! 2186: (mach_msg_size_t) rcv_size, ! 2187: (mach_msg_timeout_t) rcv_timeout, ! 2188: FALSE, msg_receive_continue, ! 2189: &kmsg, &seqno); ! 2190: /* mqueue is unlocked */ ! 2191: ipc_port_release(reply); ! 2192: if (mr != MACH_MSG_SUCCESS) { ! 2193: if (mr == MACH_RCV_TOO_LARGE) { ! 2194: msg_size_t real_size = ! 2195: (msg_size_t) (mach_msg_size_t) kmsg; ! 2196: ! 2197: assert(real_size > rcv_size); ! 2198: ! 2199: (void) copyout((vm_offset_t) &real_size, ! 2200: (vm_offset_t) &msg->msg_size, ! 2201: sizeof(msg_size_t)); ! 2202: } ! 2203: ! 2204: return msg_return_translate(mr); ! 2205: } ! 2206: ! 2207: assert(kmsg->ikm_header.msgh_size <= (mach_msg_size_t) rcv_size); ! 2208: ! 2209: mr = ipc_kmsg_copyout_compat(kmsg, space, map); ! 2210: assert(mr == MACH_MSG_SUCCESS); ! 2211: ! 2212: mr = ipc_kmsg_put((mach_msg_header_t *) msg, ! 2213: kmsg, kmsg->ikm_header.msgh_size); ! 2214: return msg_return_translate(mr); ! 2215: } ! 2216: ! 2217: #ifdef CONTINUATIONS ! 2218: /* ! 2219: * Routine: msg_receive_continue ! 2220: * Purpose: ! 2221: * Continue after blocking for a message. ! 2222: * Conditions: ! 2223: * Nothing locked. We are running on a new kernel stack, ! 2224: * with the receive state saved in the thread. From here ! 2225: * control goes back to user space. ! 2226: */ ! 2227: ! 2228: void ! 2229: msg_receive_continue() ! 2230: { ! 2231: ipc_thread_t self = current_thread(); ! 2232: msg_header_t *msg = (msg_header_t *) self->ith_msg; ! 2233: msg_option_t option = (msg_option_t) self->ith_option; ! 2234: msg_size_t rcv_size = (msg_size_t) self->ith_rcv_size; ! 2235: msg_timeout_t time_out = (msg_timeout_t) self->ith_timeout; ! 2236: ipc_object_t object = self->ith_object; ! 2237: ipc_mqueue_t mqueue = self->ith_mqueue; ! 2238: ipc_kmsg_t kmsg; ! 2239: mach_port_seqno_t seqno; ! 2240: mach_msg_return_t mr; ! 2241: ! 2242: mr = ipc_mqueue_receive(mqueue, ! 2243: (option & RCV_TIMEOUT) ? ! 2244: MACH_RCV_TIMEOUT : MACH_MSG_OPTION_NONE, ! 2245: (mach_msg_size_t) rcv_size, ! 2246: (mach_msg_timeout_t) time_out, ! 2247: TRUE, msg_receive_continue, ! 2248: &kmsg, &seqno); ! 2249: /* mqueue is unlocked */ ! 2250: ipc_object_release(object); ! 2251: if (mr != MACH_MSG_SUCCESS) { ! 2252: if (mr == MACH_RCV_TOO_LARGE) { ! 2253: msg_size_t real_size = ! 2254: (msg_size_t) (mach_msg_size_t) kmsg; ! 2255: ! 2256: assert(real_size > rcv_size); ! 2257: ! 2258: (void) copyout((vm_offset_t) &real_size, ! 2259: (vm_offset_t) &msg->msg_size, ! 2260: sizeof(msg_size_t)); ! 2261: } ! 2262: ! 2263: thread_syscall_return(msg_return_translate(mr)); ! 2264: /*NOTREACHED*/ ! 2265: } ! 2266: ! 2267: assert(kmsg->ikm_header.msgh_size <= (mach_msg_size_t) rcv_size); ! 2268: ! 2269: mr = ipc_kmsg_copyout_compat(kmsg, current_space(), current_map()); ! 2270: assert(mr == MACH_MSG_SUCCESS); ! 2271: ! 2272: mr = ipc_kmsg_put((mach_msg_header_t *) msg, kmsg, ! 2273: kmsg->ikm_header.msgh_size); ! 2274: thread_syscall_return(msg_return_translate(mr)); ! 2275: /*NOTREACHED*/ ! 2276: } ! 2277: #endif /* CONTINUATIONS */ ! 2278: ! 2279: #endif /* MACH_IPC_COMPAT */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.