![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to ib_cmrc.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / net / infiniband|
Return to ib_cmrc.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / net / infiniband |
1.1 root 1: /*
2: * Copyright (C) 2009 Fen Systems Ltd <[email protected]>.
3: * All rights reserved.
4: *
5: * Redistribution and use in source and binary forms, with or without
6: * modification, are permitted provided that the following conditions
7: * are met:
8: *
9: * Redistributions of source code must retain the above copyright
10: * notice, this list of conditions and the following disclaimer.
11: *
12: * Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in
14: * the documentation and/or other materials provided with the
15: * distribution.
16: *
17: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18: * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19: * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
20: * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
21: * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
22: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
28: * OF THE POSSIBILITY OF SUCH DAMAGE.
29: */
30:
31: FILE_LICENCE ( BSD2 );
32:
33: #include <stdlib.h>
34: #include <string.h>
35: #include <errno.h>
36: #include <ipxe/iobuf.h>
37: #include <ipxe/xfer.h>
38: #include <ipxe/process.h>
39: #include <ipxe/infiniband.h>
40: #include <ipxe/ib_cm.h>
41: #include <ipxe/ib_cmrc.h>
42:
43: /**
44: * @file
45: *
46: * Infiniband Communication-managed Reliable Connections
47: *
48: */
49:
50: /** CMRC number of send WQEs
51: *
52: * This is a policy decision.
53: */
54: #define IB_CMRC_NUM_SEND_WQES 4
55:
56: /** CMRC number of receive WQEs
57: *
58: * This is a policy decision.
59: */
60: #define IB_CMRC_NUM_RECV_WQES 2
61:
62: /** CMRC number of completion queue entries
63: *
64: * This is a policy decision
65: */
66: #define IB_CMRC_NUM_CQES 8
67:
68: /** An Infiniband Communication-Managed Reliable Connection */
69: struct ib_cmrc_connection {
70: /** Reference count */
71: struct refcnt refcnt;
72: /** Data transfer interface */
73: struct interface xfer;
74: /** Infiniband device */
75: struct ib_device *ibdev;
76: /** Completion queue */
77: struct ib_completion_queue *cq;
78: /** Queue pair */
79: struct ib_queue_pair *qp;
80: /** Connection */
81: struct ib_connection *conn;
82: /** Destination GID */
83: union ib_gid dgid;
84: /** Service ID */
85: union ib_guid service_id;
86: /** QP is connected */
87: int connected;
88: /** Shutdown process */
89: struct process shutdown;
90: };
91:
92: /**
93: * Shut down CMRC connection gracefully
94: *
95: * @v process Process
96: *
97: * The Infiniband data structures are not reference-counted or
98: * guarded. It is therefore unsafe to shut them down while we may be
99: * in the middle of a callback from the Infiniband stack (e.g. in a
100: * receive completion handler).
101: *
102: * This shutdown process will run some time after the call to
103: * ib_cmrc_close(), after control has returned out of the Infiniband
104: * core, and will shut down the Infiniband interfaces cleanly.
105: *
106: * The shutdown process holds an implicit reference on the CMRC
107: * connection, ensuring that the structure is not freed before the
108: * shutdown process has run.
109: */
110: static void ib_cmrc_shutdown ( struct process *process ) {
111: struct ib_cmrc_connection *cmrc =
112: container_of ( process, struct ib_cmrc_connection, shutdown );
113:
114: DBGC ( cmrc, "CMRC %p shutting down\n", cmrc );
115:
116: /* Shut down Infiniband interface */
117: ib_destroy_conn ( cmrc->ibdev, cmrc->qp, cmrc->conn );
118: ib_destroy_qp ( cmrc->ibdev, cmrc->qp );
119: ib_destroy_cq ( cmrc->ibdev, cmrc->cq );
120: ib_close ( cmrc->ibdev );
121:
122: /* Remove process from run queue */
123: process_del ( &cmrc->shutdown );
124:
125: /* Drop the remaining reference */
126: ref_put ( &cmrc->refcnt );
127: }
128:
129: /**
130: * Close CMRC connection
131: *
132: * @v cmrc Communication-Managed Reliable Connection
133: * @v rc Reason for close
134: */
135: static void ib_cmrc_close ( struct ib_cmrc_connection *cmrc, int rc ) {
136:
137: /* Close data transfer interface */
138: intf_shutdown ( &cmrc->xfer, rc );
139:
140: /* Schedule shutdown process */
141: process_add ( &cmrc->shutdown );
142: }
143:
144: /**
145: * Handle change of CMRC connection status
146: *
147: * @v ibdev Infiniband device
148: * @v qp Queue pair
149: * @v conn Connection
150: * @v rc_cm Connection status code
151: * @v private_data Private data, if available
152: * @v private_data_len Length of private data
153: */
154: static void ib_cmrc_changed ( struct ib_device *ibdev __unused,
155: struct ib_queue_pair *qp,
156: struct ib_connection *conn __unused, int rc_cm,
157: void *private_data, size_t private_data_len ) {
158: struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
159: int rc_xfer;
160:
161: /* Record connection status */
162: if ( rc_cm == 0 ) {
163: DBGC ( cmrc, "CMRC %p connected\n", cmrc );
164: cmrc->connected = 1;
165: } else {
166: DBGC ( cmrc, "CMRC %p disconnected: %s\n",
167: cmrc, strerror ( rc_cm ) );
168: cmrc->connected = 0;
169: }
170:
171: /* Pass up any private data */
172: DBGC2 ( cmrc, "CMRC %p received private data:\n", cmrc );
173: DBGC2_HDA ( cmrc, 0, private_data, private_data_len );
174: if ( private_data &&
175: ( rc_xfer = xfer_deliver_raw ( &cmrc->xfer, private_data,
176: private_data_len ) ) != 0 ) {
177: DBGC ( cmrc, "CMRC %p could not deliver private data: %s\n",
178: cmrc, strerror ( rc_xfer ) );
179: ib_cmrc_close ( cmrc, rc_xfer );
180: return;
181: }
182:
183: /* If we are disconnected, close the upper connection */
184: if ( rc_cm != 0 ) {
185: ib_cmrc_close ( cmrc, rc_cm );
186: return;
187: }
188: }
189:
190: /** CMRC connection operations */
191: static struct ib_connection_operations ib_cmrc_conn_op = {
192: .changed = ib_cmrc_changed,
193: };
194:
195: /**
196: * Handle CMRC send completion
197: *
198: * @v ibdev Infiniband device
199: * @v qp Queue pair
200: * @v iobuf I/O buffer
201: * @v rc Completion status code
202: */
203: static void ib_cmrc_complete_send ( struct ib_device *ibdev __unused,
204: struct ib_queue_pair *qp,
205: struct io_buffer *iobuf, int rc ) {
206: struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
207:
208: /* Free the completed I/O buffer */
209: free_iob ( iobuf );
210:
211: /* Close the connection on any send errors */
212: if ( rc != 0 ) {
213: DBGC ( cmrc, "CMRC %p send error: %s\n",
214: cmrc, strerror ( rc ) );
215: ib_cmrc_close ( cmrc, rc );
216: return;
217: }
218: }
219:
220: /**
221: * Handle CMRC receive completion
222: *
223: * @v ibdev Infiniband device
224: * @v qp Queue pair
225: * @v av Address vector, or NULL
226: * @v iobuf I/O buffer
227: * @v rc Completion status code
228: */
229: static void ib_cmrc_complete_recv ( struct ib_device *ibdev __unused,
230: struct ib_queue_pair *qp,
231: struct ib_address_vector *av __unused,
232: struct io_buffer *iobuf, int rc ) {
233: struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
234:
235: /* Close the connection on any receive errors */
236: if ( rc != 0 ) {
237: DBGC ( cmrc, "CMRC %p receive error: %s\n",
238: cmrc, strerror ( rc ) );
239: free_iob ( iobuf );
240: ib_cmrc_close ( cmrc, rc );
241: return;
242: }
243:
244: DBGC2 ( cmrc, "CMRC %p received:\n", cmrc );
245: DBGC2_HDA ( cmrc, 0, iobuf->data, iob_len ( iobuf ) );
246:
247: /* Pass up data */
248: if ( ( rc = xfer_deliver_iob ( &cmrc->xfer, iobuf ) ) != 0 ) {
249: DBGC ( cmrc, "CMRC %p could not deliver data: %s\n",
250: cmrc, strerror ( rc ) );
251: ib_cmrc_close ( cmrc, rc );
252: return;
253: }
254: }
255:
256: /** Infiniband CMRC completion operations */
257: static struct ib_completion_queue_operations ib_cmrc_completion_ops = {
258: .complete_send = ib_cmrc_complete_send,
259: .complete_recv = ib_cmrc_complete_recv,
260: };
261:
262: /**
263: * Send data via CMRC
264: *
265: * @v cmrc CMRC connection
266: * @v iobuf Datagram I/O buffer
267: * @v meta Data transfer metadata
268: * @ret rc Return status code
269: */
270: static int ib_cmrc_xfer_deliver ( struct ib_cmrc_connection *cmrc,
271: struct io_buffer *iobuf,
272: struct xfer_metadata *meta __unused ) {
273: int rc;
274:
275: /* If no connection has yet been attempted, send this datagram
276: * as the CM REQ private data. Otherwise, send it via the QP.
277: */
278: if ( ! cmrc->connected ) {
279:
280: /* Abort if we have already sent a CM connection request */
281: if ( cmrc->conn ) {
282: DBGC ( cmrc, "CMRC %p attempt to send before "
283: "connection is complete\n", cmrc );
284: rc = -EIO;
285: goto out;
286: }
287:
288: /* Send via CM connection request */
289: cmrc->conn = ib_create_conn ( cmrc->ibdev, cmrc->qp,
290: &cmrc->dgid, &cmrc->service_id,
291: iobuf->data, iob_len ( iobuf ),
292: &ib_cmrc_conn_op );
293: if ( ! cmrc->conn ) {
294: DBGC ( cmrc, "CMRC %p could not connect\n", cmrc );
295: rc = -ENOMEM;
296: goto out;
297: }
298:
299: } else {
300:
301: /* Send via QP */
302: if ( ( rc = ib_post_send ( cmrc->ibdev, cmrc->qp, NULL,
303: iob_disown ( iobuf ) ) ) != 0 ) {
304: DBGC ( cmrc, "CMRC %p could not send: %s\n",
305: cmrc, strerror ( rc ) );
306: goto out;
307: }
308:
309: }
310: return 0;
311:
312: out:
313: /* Free the I/O buffer if necessary */
314: free_iob ( iobuf );
315:
316: /* Close the connection on any errors */
317: if ( rc != 0 )
318: ib_cmrc_close ( cmrc, rc );
319:
320: return rc;
321: }
322:
323: /**
324: * Check CMRC flow control window
325: *
326: * @v cmrc CMRC connection
327: * @ret len Length of window
328: */
329: static size_t ib_cmrc_xfer_window ( struct ib_cmrc_connection *cmrc ) {
330:
331: /* We indicate a window only when we are successfully
332: * connected.
333: */
334: return ( cmrc->connected ? IB_MAX_PAYLOAD_SIZE : 0 );
335: }
336:
337: /**
338: * Identify device underlying CMRC connection
339: *
340: * @v cmrc CMRC connection
341: * @ret device Underlying device
342: */
343: static struct device *
344: ib_cmrc_identify_device ( struct ib_cmrc_connection *cmrc ) {
345: return cmrc->ibdev->dev;
346: }
347:
348: /** CMRC data transfer interface operations */
349: static struct interface_operation ib_cmrc_xfer_operations[] = {
350: INTF_OP ( xfer_deliver, struct ib_cmrc_connection *,
351: ib_cmrc_xfer_deliver ),
352: INTF_OP ( xfer_window, struct ib_cmrc_connection *,
353: ib_cmrc_xfer_window ),
354: INTF_OP ( intf_close, struct ib_cmrc_connection *, ib_cmrc_close ),
355: INTF_OP ( identify_device, struct ib_cmrc_connection *,
356: ib_cmrc_identify_device ),
357: };
358:
359: /** CMRC data transfer interface descriptor */
360: static struct interface_descriptor ib_cmrc_xfer_desc =
361: INTF_DESC ( struct ib_cmrc_connection, xfer, ib_cmrc_xfer_operations );
362:
363: /**
364: * Open CMRC connection
365: *
366: * @v xfer Data transfer interface
367: * @v ibdev Infiniband device
368: * @v dgid Destination GID
369: * @v service_id Service ID
370: * @ret rc Returns status code
371: */
372: int ib_cmrc_open ( struct interface *xfer, struct ib_device *ibdev,
373: union ib_gid *dgid, union ib_guid *service_id ) {
374: struct ib_cmrc_connection *cmrc;
375: int rc;
376:
377: /* Allocate and initialise structure */
378: cmrc = zalloc ( sizeof ( *cmrc ) );
379: if ( ! cmrc ) {
380: rc = -ENOMEM;
381: goto err_alloc;
382: }
383: ref_init ( &cmrc->refcnt, NULL );
384: intf_init ( &cmrc->xfer, &ib_cmrc_xfer_desc, &cmrc->refcnt );
385: cmrc->ibdev = ibdev;
386: memcpy ( &cmrc->dgid, dgid, sizeof ( cmrc->dgid ) );
387: memcpy ( &cmrc->service_id, service_id, sizeof ( cmrc->service_id ) );
388: process_init_stopped ( &cmrc->shutdown, ib_cmrc_shutdown,
389: &cmrc->refcnt );
390:
391: /* Open Infiniband device */
392: if ( ( rc = ib_open ( ibdev ) ) != 0 ) {
393: DBGC ( cmrc, "CMRC %p could not open device: %s\n",
394: cmrc, strerror ( rc ) );
395: goto err_open;
396: }
397:
398: /* Create completion queue */
399: cmrc->cq = ib_create_cq ( ibdev, IB_CMRC_NUM_CQES,
400: &ib_cmrc_completion_ops );
401: if ( ! cmrc->cq ) {
402: DBGC ( cmrc, "CMRC %p could not create completion queue\n",
403: cmrc );
404: rc = -ENOMEM;
405: goto err_create_cq;
406: }
407:
408: /* Create queue pair */
409: cmrc->qp = ib_create_qp ( ibdev, IB_QPT_RC, IB_CMRC_NUM_SEND_WQES,
410: cmrc->cq, IB_CMRC_NUM_RECV_WQES, cmrc->cq );
411: if ( ! cmrc->qp ) {
412: DBGC ( cmrc, "CMRC %p could not create queue pair\n", cmrc );
413: rc = -ENOMEM;
414: goto err_create_qp;
415: }
416: ib_qp_set_ownerdata ( cmrc->qp, cmrc );
417: DBGC ( cmrc, "CMRC %p using QPN %lx\n", cmrc, cmrc->qp->qpn );
418:
419: /* Attach to parent interface, transfer reference (implicitly)
420: * to our shutdown process, and return.
421: */
422: intf_plug_plug ( &cmrc->xfer, xfer );
423: return 0;
424:
425: ib_destroy_qp ( ibdev, cmrc->qp );
426: err_create_qp:
427: ib_destroy_cq ( ibdev, cmrc->cq );
428: err_create_cq:
429: ib_close ( ibdev );
430: err_open:
431: ref_put ( &cmrc->refcnt );
432: err_alloc:
433: return rc;
434: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.