![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to ib_mi.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / net / infiniband|
Return to ib_mi.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / net / infiniband |
1.1 root 1: /*
2: * Copyright (C) 2009 Michael Brown <[email protected]>.
3: *
4: * This program is free software; you can redistribute it and/or
5: * modify it under the terms of the GNU General Public License as
6: * published by the Free Software Foundation; either version 2 of the
7: * License, or any later version.
8: *
9: * This program is distributed in the hope that it will be useful, but
10: * WITHOUT ANY WARRANTY; without even the implied warranty of
11: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12: * General Public License for more details.
13: *
14: * You should have received a copy of the GNU General Public License
15: * along with this program; if not, write to the Free Software
16: * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17: */
18:
19: FILE_LICENCE ( GPL2_OR_LATER );
20:
21: #include <stdint.h>
22: #include <stdlib.h>
23: #include <string.h>
24: #include <errno.h>
25: #include <stdio.h>
26: #include <unistd.h>
27: #include <byteswap.h>
28: #include <ipxe/infiniband.h>
29: #include <ipxe/iobuf.h>
30: #include <ipxe/ib_mi.h>
31:
32: /**
33: * @file
34: *
35: * Infiniband management interfaces
36: *
37: */
38:
39: /** Management interface number of send WQEs
40: *
41: * This is a policy decision.
42: */
43: #define IB_MI_NUM_SEND_WQES 4
44:
45: /** Management interface number of receive WQEs
46: *
47: * This is a policy decision.
48: */
49: #define IB_MI_NUM_RECV_WQES 2
50:
51: /** Management interface number of completion queue entries
52: *
53: * This is a policy decision
54: */
55: #define IB_MI_NUM_CQES 8
56:
57: /** TID magic signature */
58: #define IB_MI_TID_MAGIC ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
59:
60: /** TID to use for next MAD */
61: static unsigned int next_tid;
62:
63: /**
64: * Handle received MAD
65: *
66: * @v ibdev Infiniband device
67: * @v mi Management interface
68: * @v mad Received MAD
69: * @v av Source address vector
70: * @ret rc Return status code
71: */
72: static int ib_mi_handle ( struct ib_device *ibdev,
73: struct ib_mad_interface *mi,
74: union ib_mad *mad,
75: struct ib_address_vector *av ) {
76: struct ib_mad_hdr *hdr = &mad->hdr;
77: struct ib_mad_transaction *madx;
78: struct ib_mad_agent *agent;
79:
80: /* Look for a matching transaction by TID */
81: list_for_each_entry ( madx, &mi->madx, list ) {
82: if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
83: sizeof ( hdr->tid ) ) != 0 )
84: continue;
85: /* Found a matching transaction */
86: madx->op->complete ( ibdev, mi, madx, 0, mad, av );
87: return 0;
88: }
89:
90: /* If there is no matching transaction, look for a listening agent */
91: for_each_table_entry ( agent, IB_MAD_AGENTS ) {
92: if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
93: ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
94: ( agent->class_version != hdr->class_version ) ||
95: ( agent->attr_id != hdr->attr_id ) )
96: continue;
97: /* Found a matching agent */
98: agent->handle ( ibdev, mi, mad, av );
99: return 0;
100: }
101:
102: /* Otherwise, ignore it */
103: DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
104: mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
105: return -ENOTSUP;
106: }
107:
108: /**
109: * Complete receive via management interface
110: *
111: *
112: * @v ibdev Infiniband device
113: * @v qp Queue pair
114: * @v av Address vector
115: * @v iobuf I/O buffer
116: * @v rc Completion status code
117: */
118: static void ib_mi_complete_recv ( struct ib_device *ibdev,
119: struct ib_queue_pair *qp,
120: struct ib_address_vector *av,
121: struct io_buffer *iobuf, int rc ) {
122: struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
123: union ib_mad *mad;
124: struct ib_mad_hdr *hdr;
125:
126: /* Ignore errors */
127: if ( rc != 0 ) {
128: DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
129: goto out;
130: }
131:
132: /* Sanity checks */
133: if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
134: DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
135: mi, iob_len ( iobuf ) );
136: DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
137: goto out;
138: }
139: mad = iobuf->data;
140: hdr = &mad->hdr;
141: if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
142: DBGC ( mi, "MI %p RX unsupported base version %x\n",
143: mi, hdr->base_version );
144: DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
145: goto out;
146: }
147: DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
148: "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
149: hdr->mgmt_class, hdr->class_version, hdr->method,
150: ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
151: DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
152:
153: /* Handle MAD */
154: if ( ( rc = ib_mi_handle ( ibdev, mi, mad, av ) ) != 0 )
155: goto out;
156:
157: out:
158: free_iob ( iobuf );
159: }
160:
161: /** Management interface completion operations */
162: static struct ib_completion_queue_operations ib_mi_completion_ops = {
163: .complete_recv = ib_mi_complete_recv,
164: };
165:
166: /**
167: * Transmit MAD
168: *
169: * @v ibdev Infiniband device
170: * @v mi Management interface
171: * @v mad MAD
172: * @v av Destination address vector
173: * @ret rc Return status code
174: */
175: int ib_mi_send ( struct ib_device *ibdev, struct ib_mad_interface *mi,
176: union ib_mad *mad, struct ib_address_vector *av ) {
177: struct ib_mad_hdr *hdr = &mad->hdr;
178: struct io_buffer *iobuf;
179: int rc;
180:
181: /* Set common fields */
182: hdr->base_version = IB_MGMT_BASE_VERSION;
183: if ( ( hdr->tid[0] == 0 ) && ( hdr->tid[1] == 0 ) ) {
184: hdr->tid[0] = htonl ( IB_MI_TID_MAGIC );
185: hdr->tid[1] = htonl ( ++next_tid );
186: }
187: DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
188: "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
189: hdr->mgmt_class, hdr->class_version, hdr->method,
190: ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
191: DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
192:
193: /* Construct directed route portion of response, if necessary */
194: if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
195: struct ib_mad_smp *smp = &mad->smp;
196: unsigned int hop_pointer;
197: unsigned int hop_count;
198:
199: smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
200: hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
201: hop_count = smp->mad_hdr.class_specific.smp.hop_count;
202: assert ( hop_count == hop_pointer );
203: if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
204: sizeof ( smp->return_path.hops[0] ) ) ) {
205: smp->return_path.hops[hop_pointer] = ibdev->port;
206: } else {
207: DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
208: "%d\n", mi, ntohl ( hdr->tid[0] ),
209: ntohl ( hdr->tid[1] ), hop_pointer );
210: return -EINVAL;
211: }
212: }
213:
214: /* Construct I/O buffer */
215: iobuf = alloc_iob ( sizeof ( *mad ) );
216: if ( ! iobuf ) {
217: DBGC ( mi, "MI %p could not allocate buffer for TID "
218: "%08x%08x\n",
219: mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
220: return -ENOMEM;
221: }
222: memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );
223:
224: /* Send I/O buffer */
225: if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
226: DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
227: mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
228: strerror ( rc ) );
229: free_iob ( iobuf );
230: return rc;
231: }
232:
233: return 0;
234: }
235:
236: /**
237: * Handle management transaction timer expiry
238: *
239: * @v timer Retry timer
240: * @v expired Failure indicator
241: */
242: static void ib_mi_timer_expired ( struct retry_timer *timer, int expired ) {
243: struct ib_mad_transaction *madx =
244: container_of ( timer, struct ib_mad_transaction, timer );
245: struct ib_mad_interface *mi = madx->mi;
246: struct ib_device *ibdev = mi->ibdev;
247: struct ib_mad_hdr *hdr = &madx->mad.hdr;
248:
249: /* Abandon transaction if we have tried too many times */
250: if ( expired ) {
251: DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
252: mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
253: madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
254: return;
255: }
256:
257: /* Restart retransmission timer */
258: start_timer ( timer );
259:
260: /* Resend MAD */
261: ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
262: }
263:
264: /**
265: * Create management transaction
266: *
267: * @v ibdev Infiniband device
268: * @v mi Management interface
269: * @v mad MAD to send
270: * @v av Destination address, or NULL to use SM's GSI
271: * @v op Management transaction operations
272: * @ret madx Management transaction, or NULL
273: */
274: struct ib_mad_transaction *
275: ib_create_madx ( struct ib_device *ibdev, struct ib_mad_interface *mi,
276: union ib_mad *mad, struct ib_address_vector *av,
277: struct ib_mad_transaction_operations *op ) {
278: struct ib_mad_transaction *madx;
279:
280: /* Allocate and initialise structure */
281: madx = zalloc ( sizeof ( *madx ) );
282: if ( ! madx )
283: return NULL;
284: timer_init ( &madx->timer, ib_mi_timer_expired, NULL );
285: madx->mi = mi;
286: madx->op = op;
287:
288: /* Determine address vector */
289: if ( av ) {
290: memcpy ( &madx->av, av, sizeof ( madx->av ) );
291: } else {
292: madx->av.lid = ibdev->sm_lid;
293: madx->av.sl = ibdev->sm_sl;
294: madx->av.qpn = IB_QPN_GSI;
295: madx->av.qkey = IB_QKEY_GSI;
296: }
297:
298: /* Copy MAD */
299: memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );
300:
301: /* Add to list and start timer to send initial MAD */
302: list_add ( &madx->list, &mi->madx );
303: start_timer_nodelay ( &madx->timer );
304:
305: return madx;
306: }
307:
308: /**
309: * Destroy management transaction
310: *
311: * @v ibdev Infiniband device
312: * @v mi Management interface
313: * @v madx Management transaction
314: */
315: void ib_destroy_madx ( struct ib_device *ibdev __unused,
316: struct ib_mad_interface *mi __unused,
317: struct ib_mad_transaction *madx ) {
318:
319: /* Stop timer and remove from list */
320: stop_timer ( &madx->timer );
321: list_del ( &madx->list );
322:
323: /* Free transaction */
324: free ( madx );
325: }
326:
327: /**
328: * Create management interface
329: *
330: * @v ibdev Infiniband device
331: * @v type Queue pair type
332: * @ret mi Management agent, or NULL
333: */
334: struct ib_mad_interface * ib_create_mi ( struct ib_device *ibdev,
335: enum ib_queue_pair_type type ) {
336: struct ib_mad_interface *mi;
337: int rc;
338:
339: /* Allocate and initialise fields */
340: mi = zalloc ( sizeof ( *mi ) );
341: if ( ! mi )
342: goto err_alloc;
343: mi->ibdev = ibdev;
344: INIT_LIST_HEAD ( &mi->madx );
345:
346: /* Create completion queue */
347: mi->cq = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops );
348: if ( ! mi->cq ) {
349: DBGC ( mi, "MI %p could not allocate completion queue\n", mi );
350: goto err_create_cq;
351: }
352:
353: /* Create queue pair */
354: mi->qp = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
355: IB_MI_NUM_RECV_WQES, mi->cq );
356: if ( ! mi->qp ) {
357: DBGC ( mi, "MI %p could not allocate queue pair\n", mi );
358: goto err_create_qp;
359: }
360: ib_qp_set_ownerdata ( mi->qp, mi );
361: DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
362: mi, ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" ), mi->qp->qpn );
363:
364: /* Set queue key */
365: mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
366: if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
367: DBGC ( mi, "MI %p could not set queue key: %s\n",
368: mi, strerror ( rc ) );
369: goto err_modify_qp;
370: }
371:
372: /* Fill receive ring */
373: ib_refill_recv ( ibdev, mi->qp );
374: return mi;
375:
376: err_modify_qp:
377: ib_destroy_qp ( ibdev, mi->qp );
378: err_create_qp:
379: ib_destroy_cq ( ibdev, mi->cq );
380: err_create_cq:
381: free ( mi );
382: err_alloc:
383: return NULL;
384: }
385:
386: /**
387: * Destroy management interface
388: *
389: * @v mi Management interface
390: */
391: void ib_destroy_mi ( struct ib_device *ibdev, struct ib_mad_interface *mi ) {
392: struct ib_mad_transaction *madx;
393: struct ib_mad_transaction *tmp;
394:
395: /* Flush any outstanding requests */
396: list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
397: DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
398: mi, ntohl ( madx->mad.hdr.tid[0] ),
399: ntohl ( madx->mad.hdr.tid[1] ) );
400: madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
401: }
402:
403: ib_destroy_qp ( ibdev, mi->qp );
404: ib_destroy_cq ( ibdev, mi->cq );
405: free ( mi );
406: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.