![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to hermon.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / drivers / infiniband|
Return to hermon.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / drivers / infiniband |
1.1 root 1: /*
2: * Copyright (C) 2008 Michael Brown <[email protected]>.
3: * Copyright (C) 2008 Mellanox Technologies Ltd.
4: *
5: * This program is free software; you can redistribute it and/or
6: * modify it under the terms of the GNU General Public License as
7: * published by the Free Software Foundation; either version 2 of the
8: * License, or any later version.
9: *
10: * This program is distributed in the hope that it will be useful, but
11: * WITHOUT ANY WARRANTY; without even the implied warranty of
12: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13: * General Public License for more details.
14: *
15: * You should have received a copy of the GNU General Public License
16: * along with this program; if not, write to the Free Software
17: * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18: */
19:
20: FILE_LICENCE ( GPL2_OR_LATER );
21:
22: #include <stdint.h>
23: #include <stdlib.h>
24: #include <stdio.h>
25: #include <string.h>
26: #include <strings.h>
27: #include <unistd.h>
28: #include <errno.h>
29: #include <byteswap.h>
30: #include <ipxe/io.h>
31: #include <ipxe/pci.h>
32: #include <ipxe/pcibackup.h>
33: #include <ipxe/malloc.h>
34: #include <ipxe/umalloc.h>
35: #include <ipxe/iobuf.h>
36: #include <ipxe/netdevice.h>
37: #include <ipxe/infiniband.h>
38: #include <ipxe/ib_smc.h>
39: #include <ipxe/if_ether.h>
40: #include <ipxe/ethernet.h>
41: #include <ipxe/fcoe.h>
42: #include <ipxe/vlan.h>
43: #include <ipxe/bofm.h>
44: #include "hermon.h"
45:
46: /**
47: * @file
48: *
49: * Mellanox Hermon Infiniband HCA
50: *
51: */
52:
53: /***************************************************************************
54: *
55: * Queue number allocation
56: *
57: ***************************************************************************
58: */
59:
60: /**
61: * Allocate offsets within usage bitmask
62: *
63: * @v bits Usage bitmask
64: * @v bits_len Length of usage bitmask
65: * @v num_bits Number of contiguous bits to allocate within bitmask
66: * @ret bit First free bit within bitmask, or negative error
67: */
68: static int hermon_bitmask_alloc ( hermon_bitmask_t *bits,
69: unsigned int bits_len,
70: unsigned int num_bits ) {
71: unsigned int bit = 0;
72: hermon_bitmask_t mask = 1;
73: unsigned int found = 0;
74:
75: /* Search bits for num_bits contiguous free bits */
76: while ( bit < bits_len ) {
77: if ( ( mask & *bits ) == 0 ) {
78: if ( ++found == num_bits )
79: goto found;
80: } else {
81: found = 0;
82: }
83: bit++;
84: mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) );
85: if ( mask == 1 )
86: bits++;
87: }
88: return -ENFILE;
89:
90: found:
91: /* Mark bits as in-use */
92: do {
93: *bits |= mask;
94: if ( mask == 1 )
95: bits--;
96: mask = ( mask >> 1 ) | ( mask << ( 8 * sizeof ( mask ) - 1 ) );
97: } while ( --found );
98:
99: return ( bit - num_bits + 1 );
100: }
101:
102: /**
103: * Free offsets within usage bitmask
104: *
105: * @v bits Usage bitmask
106: * @v bit Starting bit within bitmask
107: * @v num_bits Number of contiguous bits to free within bitmask
108: */
109: static void hermon_bitmask_free ( hermon_bitmask_t *bits,
110: int bit, unsigned int num_bits ) {
111: hermon_bitmask_t mask;
112:
113: for ( ; num_bits ; bit++, num_bits-- ) {
114: mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) );
115: bits[ ( bit / ( 8 * sizeof ( mask ) ) ) ] &= ~mask;
116: }
117: }
118:
119: /***************************************************************************
120: *
121: * HCA commands
122: *
123: ***************************************************************************
124: */
125:
126: /**
127: * Wait for Hermon command completion
128: *
129: * @v hermon Hermon device
130: * @v hcr HCA command registers
131: * @ret rc Return status code
132: */
133: static int hermon_cmd_wait ( struct hermon *hermon,
134: struct hermonprm_hca_command_register *hcr ) {
135: unsigned int wait;
136:
137: for ( wait = HERMON_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
138: hcr->u.dwords[6] =
139: readl ( hermon->config + HERMON_HCR_REG ( 6 ) );
140: if ( ( MLX_GET ( hcr, go ) == 0 ) &&
141: ( MLX_GET ( hcr, t ) == hermon->toggle ) )
142: return 0;
143: mdelay ( 1 );
144: }
145: return -EBUSY;
146: }
147:
148: /**
149: * Issue HCA command
150: *
151: * @v hermon Hermon device
152: * @v command Command opcode, flags and input/output lengths
153: * @v op_mod Opcode modifier (0 if no modifier applicable)
154: * @v in Input parameters
155: * @v in_mod Input modifier (0 if no modifier applicable)
156: * @v out Output parameters
157: * @ret rc Return status code
158: */
159: static int hermon_cmd ( struct hermon *hermon, unsigned long command,
160: unsigned int op_mod, const void *in,
161: unsigned int in_mod, void *out ) {
162: struct hermonprm_hca_command_register hcr;
163: unsigned int opcode = HERMON_HCR_OPCODE ( command );
164: size_t in_len = HERMON_HCR_IN_LEN ( command );
165: size_t out_len = HERMON_HCR_OUT_LEN ( command );
166: void *in_buffer;
167: void *out_buffer;
168: unsigned int status;
169: unsigned int i;
170: int rc;
171:
172: assert ( in_len <= HERMON_MBOX_SIZE );
173: assert ( out_len <= HERMON_MBOX_SIZE );
174:
175: DBGC2 ( hermon, "Hermon %p command %02x in %zx%s out %zx%s\n",
176: hermon, opcode, in_len,
177: ( ( command & HERMON_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
178: ( ( command & HERMON_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
179:
180: /* Check that HCR is free */
181: if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
182: DBGC ( hermon, "Hermon %p command interface locked\n",
183: hermon );
184: return rc;
185: }
186:
187: /* Flip HCR toggle */
188: hermon->toggle = ( 1 - hermon->toggle );
189:
190: /* Prepare HCR */
191: memset ( &hcr, 0, sizeof ( hcr ) );
192: in_buffer = &hcr.u.dwords[0];
193: if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
194: memset ( hermon->mailbox_in, 0, HERMON_MBOX_SIZE );
195: in_buffer = hermon->mailbox_in;
196: MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) );
197: MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
198: }
199: memcpy ( in_buffer, in, in_len );
200: MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
201: out_buffer = &hcr.u.dwords[3];
202: if ( out_len && ( command & HERMON_HCR_OUT_MBOX ) ) {
203: out_buffer = hermon->mailbox_out;
204: MLX_FILL_H ( &hcr, 3, out_param_h,
205: virt_to_bus ( out_buffer ) );
206: MLX_FILL_1 ( &hcr, 4, out_param_l,
207: virt_to_bus ( out_buffer ) );
208: }
209: MLX_FILL_4 ( &hcr, 6,
210: opcode, opcode,
211: opcode_modifier, op_mod,
212: go, 1,
213: t, hermon->toggle );
214: DBGC ( hermon, "Hermon %p issuing command %04x\n",
215: hermon, opcode );
216: DBGC2_HDA ( hermon, virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
217: &hcr, sizeof ( hcr ) );
218: if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
219: DBGC2 ( hermon, "Input mailbox:\n" );
220: DBGC2_HDA ( hermon, virt_to_phys ( in_buffer ), in_buffer,
221: ( ( in_len < 512 ) ? in_len : 512 ) );
222: }
223:
224: /* Issue command */
225: for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
226: i++ ) {
227: writel ( hcr.u.dwords[i],
228: hermon->config + HERMON_HCR_REG ( i ) );
229: barrier();
230: }
231:
232: /* Wait for command completion */
233: if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
234: DBGC ( hermon, "Hermon %p timed out waiting for command:\n",
235: hermon );
236: DBGC_HDA ( hermon,
237: virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
238: &hcr, sizeof ( hcr ) );
239: return rc;
240: }
241:
242: /* Check command status */
243: status = MLX_GET ( &hcr, status );
244: if ( status != 0 ) {
245: DBGC ( hermon, "Hermon %p command failed with status %02x:\n",
246: hermon, status );
247: DBGC_HDA ( hermon,
248: virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
249: &hcr, sizeof ( hcr ) );
250: return -EIO;
251: }
252:
253: /* Read output parameters, if any */
254: hcr.u.dwords[3] = readl ( hermon->config + HERMON_HCR_REG ( 3 ) );
255: hcr.u.dwords[4] = readl ( hermon->config + HERMON_HCR_REG ( 4 ) );
256: memcpy ( out, out_buffer, out_len );
257: if ( out_len ) {
258: DBGC2 ( hermon, "Output%s:\n",
259: ( command & HERMON_HCR_OUT_MBOX ) ? " mailbox" : "" );
260: DBGC2_HDA ( hermon, virt_to_phys ( out_buffer ), out_buffer,
261: ( ( out_len < 512 ) ? out_len : 512 ) );
262: }
263:
264: return 0;
265: }
266:
267: static inline int
268: hermon_cmd_query_dev_cap ( struct hermon *hermon,
269: struct hermonprm_query_dev_cap *dev_cap ) {
270: return hermon_cmd ( hermon,
271: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_DEV_CAP,
272: 1, sizeof ( *dev_cap ) ),
273: 0, NULL, 0, dev_cap );
274: }
275:
276: static inline int
277: hermon_cmd_query_fw ( struct hermon *hermon, struct hermonprm_query_fw *fw ) {
278: return hermon_cmd ( hermon,
279: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_FW,
280: 1, sizeof ( *fw ) ),
281: 0, NULL, 0, fw );
282: }
283:
284: static inline int
285: hermon_cmd_init_hca ( struct hermon *hermon,
286: const struct hermonprm_init_hca *init_hca ) {
287: return hermon_cmd ( hermon,
288: HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_HCA,
289: 1, sizeof ( *init_hca ) ),
290: 0, init_hca, 0, NULL );
291: }
292:
293: static inline int
294: hermon_cmd_close_hca ( struct hermon *hermon ) {
295: return hermon_cmd ( hermon,
296: HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_HCA ),
297: 0, NULL, 0, NULL );
298: }
299:
300: static inline int
301: hermon_cmd_init_port ( struct hermon *hermon, unsigned int port ) {
302: return hermon_cmd ( hermon,
303: HERMON_HCR_VOID_CMD ( HERMON_HCR_INIT_PORT ),
304: 0, NULL, port, NULL );
305: }
306:
307: static inline int
308: hermon_cmd_close_port ( struct hermon *hermon, unsigned int port ) {
309: return hermon_cmd ( hermon,
310: HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_PORT ),
311: 0, NULL, port, NULL );
312: }
313:
314: static inline int
315: hermon_cmd_set_port ( struct hermon *hermon, int is_ethernet,
316: unsigned int port_selector,
317: const union hermonprm_set_port *set_port ) {
318: return hermon_cmd ( hermon,
319: HERMON_HCR_IN_CMD ( HERMON_HCR_SET_PORT,
320: 1, sizeof ( *set_port ) ),
321: is_ethernet, set_port, port_selector, NULL );
322: }
323:
324: static inline int
325: hermon_cmd_sw2hw_mpt ( struct hermon *hermon, unsigned int index,
326: const struct hermonprm_mpt *mpt ) {
327: return hermon_cmd ( hermon,
328: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_MPT,
329: 1, sizeof ( *mpt ) ),
330: 0, mpt, index, NULL );
331: }
332:
333: static inline int
334: hermon_cmd_write_mtt ( struct hermon *hermon,
335: const struct hermonprm_write_mtt *write_mtt ) {
336: return hermon_cmd ( hermon,
337: HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MTT,
338: 1, sizeof ( *write_mtt ) ),
339: 0, write_mtt, 1, NULL );
340: }
341:
342: static inline int
343: hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map,
344: const struct hermonprm_event_mask *mask ) {
345: return hermon_cmd ( hermon,
346: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ,
347: 0, sizeof ( *mask ) ),
348: 0, mask, index_map, NULL );
349: }
350:
351: static inline int
352: hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index,
353: const struct hermonprm_eqc *eqctx ) {
354: return hermon_cmd ( hermon,
355: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ,
356: 1, sizeof ( *eqctx ) ),
357: 0, eqctx, index, NULL );
358: }
359:
360: static inline int
361: hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index,
362: struct hermonprm_eqc *eqctx ) {
363: return hermon_cmd ( hermon,
364: HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ,
365: 1, sizeof ( *eqctx ) ),
366: 1, NULL, index, eqctx );
367: }
368:
369: static inline int
370: hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index,
371: struct hermonprm_eqc *eqctx ) {
372: return hermon_cmd ( hermon,
373: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ,
374: 1, sizeof ( *eqctx ) ),
375: 0, NULL, index, eqctx );
376: }
377:
378: static inline int
379: hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn,
380: const struct hermonprm_completion_queue_context *cqctx ){
381: return hermon_cmd ( hermon,
382: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_CQ,
383: 1, sizeof ( *cqctx ) ),
384: 0, cqctx, cqn, NULL );
385: }
386:
387: static inline int
388: hermon_cmd_hw2sw_cq ( struct hermon *hermon, unsigned long cqn,
389: struct hermonprm_completion_queue_context *cqctx ) {
390: return hermon_cmd ( hermon,
391: HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_CQ,
392: 1, sizeof ( *cqctx ) ),
393: 0, NULL, cqn, cqctx );
394: }
395:
396: static inline int
397: hermon_cmd_query_cq ( struct hermon *hermon, unsigned long cqn,
398: struct hermonprm_completion_queue_context *cqctx ) {
399: return hermon_cmd ( hermon,
400: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_CQ,
401: 1, sizeof ( *cqctx ) ),
402: 0, NULL, cqn, cqctx );
403: }
404:
405: static inline int
406: hermon_cmd_rst2init_qp ( struct hermon *hermon, unsigned long qpn,
407: const struct hermonprm_qp_ee_state_transitions *ctx ){
408: return hermon_cmd ( hermon,
409: HERMON_HCR_IN_CMD ( HERMON_HCR_RST2INIT_QP,
410: 1, sizeof ( *ctx ) ),
411: 0, ctx, qpn, NULL );
412: }
413:
414: static inline int
415: hermon_cmd_init2rtr_qp ( struct hermon *hermon, unsigned long qpn,
416: const struct hermonprm_qp_ee_state_transitions *ctx ){
417: return hermon_cmd ( hermon,
418: HERMON_HCR_IN_CMD ( HERMON_HCR_INIT2RTR_QP,
419: 1, sizeof ( *ctx ) ),
420: 0, ctx, qpn, NULL );
421: }
422:
423: static inline int
424: hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn,
425: const struct hermonprm_qp_ee_state_transitions *ctx ) {
426: return hermon_cmd ( hermon,
427: HERMON_HCR_IN_CMD ( HERMON_HCR_RTR2RTS_QP,
428: 1, sizeof ( *ctx ) ),
429: 0, ctx, qpn, NULL );
430: }
431:
432: static inline int
433: hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn,
434: const struct hermonprm_qp_ee_state_transitions *ctx ) {
435: return hermon_cmd ( hermon,
436: HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP,
437: 1, sizeof ( *ctx ) ),
438: 0, ctx, qpn, NULL );
439: }
440:
441: static inline int
442: hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) {
443: return hermon_cmd ( hermon,
444: HERMON_HCR_VOID_CMD ( HERMON_HCR_2RST_QP ),
445: 0x03, NULL, qpn, NULL );
446: }
447:
448: static inline int
449: hermon_cmd_query_qp ( struct hermon *hermon, unsigned long qpn,
450: struct hermonprm_qp_ee_state_transitions *ctx ) {
451: return hermon_cmd ( hermon,
452: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_QP,
453: 1, sizeof ( *ctx ) ),
454: 0, NULL, qpn, ctx );
455: }
456:
457: static inline int
458: hermon_cmd_conf_special_qp ( struct hermon *hermon, unsigned int internal_qps,
459: unsigned long base_qpn ) {
460: return hermon_cmd ( hermon,
461: HERMON_HCR_VOID_CMD ( HERMON_HCR_CONF_SPECIAL_QP ),
462: internal_qps, NULL, base_qpn, NULL );
463: }
464:
465: static inline int
466: hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port,
467: union hermonprm_mad *mad ) {
468: return hermon_cmd ( hermon,
469: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC,
470: 1, sizeof ( *mad ),
471: 1, sizeof ( *mad ) ),
472: 0x03, mad, port, mad );
473: }
474:
475: static inline int
476: hermon_cmd_read_mcg ( struct hermon *hermon, unsigned int index,
477: struct hermonprm_mcg_entry *mcg ) {
478: return hermon_cmd ( hermon,
479: HERMON_HCR_OUT_CMD ( HERMON_HCR_READ_MCG,
480: 1, sizeof ( *mcg ) ),
481: 0, NULL, index, mcg );
482: }
483:
484: static inline int
485: hermon_cmd_write_mcg ( struct hermon *hermon, unsigned int index,
486: const struct hermonprm_mcg_entry *mcg ) {
487: return hermon_cmd ( hermon,
488: HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MCG,
489: 1, sizeof ( *mcg ) ),
490: 0, mcg, index, NULL );
491: }
492:
493: static inline int
494: hermon_cmd_mgid_hash ( struct hermon *hermon, const union ib_gid *gid,
495: struct hermonprm_mgm_hash *hash ) {
496: return hermon_cmd ( hermon,
497: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MGID_HASH,
498: 1, sizeof ( *gid ),
499: 0, sizeof ( *hash ) ),
500: 0, gid, 0, hash );
501: }
502:
503: static inline int
504: hermon_cmd_mod_stat_cfg ( struct hermon *hermon, unsigned int mode,
505: unsigned int input_mod,
506: struct hermonprm_scalar_parameter *portion ) {
507: return hermon_cmd ( hermon,
508: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MOD_STAT_CFG,
509: 0, sizeof ( *portion ),
510: 0, sizeof ( *portion ) ),
511: mode, portion, input_mod, portion );
512: }
513:
514: static inline int
515: hermon_cmd_query_port ( struct hermon *hermon, unsigned int port,
516: struct hermonprm_query_port_cap *query_port ) {
517: return hermon_cmd ( hermon,
518: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_PORT,
519: 1, sizeof ( *query_port ) ),
520: 0, NULL, port, query_port );
521: }
522:
523: static inline int
524: hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
525: struct hermonprm_sense_port *port_type ) {
526: return hermon_cmd ( hermon,
527: HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
528: 0, sizeof ( *port_type ) ),
529: 0, NULL, port, port_type );
530: }
531:
532: static inline int
533: hermon_cmd_run_fw ( struct hermon *hermon ) {
534: return hermon_cmd ( hermon,
535: HERMON_HCR_VOID_CMD ( HERMON_HCR_RUN_FW ),
536: 0, NULL, 0, NULL );
537: }
538:
539: static inline int
540: hermon_cmd_unmap_icm ( struct hermon *hermon, unsigned int page_count,
541: const struct hermonprm_scalar_parameter *offset ) {
542: return hermon_cmd ( hermon,
543: HERMON_HCR_IN_CMD ( HERMON_HCR_UNMAP_ICM,
544: 0, sizeof ( *offset ) ),
545: 0, offset, page_count, NULL );
546: }
547:
548: static inline int
549: hermon_cmd_map_icm ( struct hermon *hermon,
550: const struct hermonprm_virtual_physical_mapping *map ) {
551: return hermon_cmd ( hermon,
552: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM,
553: 1, sizeof ( *map ) ),
554: 0, map, 1, NULL );
555: }
556:
557: static inline int
558: hermon_cmd_unmap_icm_aux ( struct hermon *hermon ) {
559: return hermon_cmd ( hermon,
560: HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_ICM_AUX ),
561: 0, NULL, 0, NULL );
562: }
563:
564: static inline int
565: hermon_cmd_map_icm_aux ( struct hermon *hermon,
566: const struct hermonprm_virtual_physical_mapping *map ) {
567: return hermon_cmd ( hermon,
568: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM_AUX,
569: 1, sizeof ( *map ) ),
570: 0, map, 1, NULL );
571: }
572:
573: static inline int
574: hermon_cmd_set_icm_size ( struct hermon *hermon,
575: const struct hermonprm_scalar_parameter *icm_size,
576: struct hermonprm_scalar_parameter *icm_aux_size ) {
577: return hermon_cmd ( hermon,
578: HERMON_HCR_INOUT_CMD ( HERMON_HCR_SET_ICM_SIZE,
579: 0, sizeof ( *icm_size ),
580: 0, sizeof (*icm_aux_size) ),
581: 0, icm_size, 0, icm_aux_size );
582: }
583:
584: static inline int
585: hermon_cmd_unmap_fa ( struct hermon *hermon ) {
586: return hermon_cmd ( hermon,
587: HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_FA ),
588: 0, NULL, 0, NULL );
589: }
590:
591: static inline int
592: hermon_cmd_map_fa ( struct hermon *hermon,
593: const struct hermonprm_virtual_physical_mapping *map ) {
594: return hermon_cmd ( hermon,
595: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_FA,
596: 1, sizeof ( *map ) ),
597: 0, map, 1, NULL );
598: }
599:
600: /***************************************************************************
601: *
602: * Memory translation table operations
603: *
604: ***************************************************************************
605: */
606:
607: /**
608: * Allocate MTT entries
609: *
610: * @v hermon Hermon device
611: * @v memory Memory to map into MTT
612: * @v len Length of memory to map
613: * @v mtt MTT descriptor to fill in
614: * @ret rc Return status code
615: */
616: static int hermon_alloc_mtt ( struct hermon *hermon,
617: const void *memory, size_t len,
618: struct hermon_mtt *mtt ) {
619: struct hermonprm_write_mtt write_mtt;
620: physaddr_t start;
621: physaddr_t addr;
622: unsigned int page_offset;
623: unsigned int num_pages;
624: int mtt_offset;
625: unsigned int mtt_base_addr;
626: unsigned int i;
627: int rc;
628:
629: /* Find available MTT entries */
630: start = virt_to_phys ( memory );
631: page_offset = ( start & ( HERMON_PAGE_SIZE - 1 ) );
632: start -= page_offset;
633: len += page_offset;
634: num_pages = ( ( len + HERMON_PAGE_SIZE - 1 ) / HERMON_PAGE_SIZE );
635: mtt_offset = hermon_bitmask_alloc ( hermon->mtt_inuse, HERMON_MAX_MTTS,
636: num_pages );
637: if ( mtt_offset < 0 ) {
638: DBGC ( hermon, "Hermon %p could not allocate %d MTT entries\n",
639: hermon, num_pages );
640: rc = mtt_offset;
641: goto err_mtt_offset;
642: }
643: mtt_base_addr = ( ( hermon->cap.reserved_mtts + mtt_offset ) *
644: hermon->cap.mtt_entry_size );
645: addr = start;
646:
647: /* Fill in MTT structure */
648: mtt->mtt_offset = mtt_offset;
649: mtt->num_pages = num_pages;
650: mtt->mtt_base_addr = mtt_base_addr;
651: mtt->page_offset = page_offset;
652:
653: /* Construct and issue WRITE_MTT commands */
654: for ( i = 0 ; i < num_pages ; i++ ) {
655: memset ( &write_mtt, 0, sizeof ( write_mtt ) );
656: MLX_FILL_1 ( &write_mtt.mtt_base_addr, 1,
657: value, mtt_base_addr );
658: MLX_FILL_H ( &write_mtt.mtt, 0, ptag_h, addr );
659: MLX_FILL_2 ( &write_mtt.mtt, 1,
660: p, 1,
661: ptag_l, ( addr >> 3 ) );
662: if ( ( rc = hermon_cmd_write_mtt ( hermon,
663: &write_mtt ) ) != 0 ) {
664: DBGC ( hermon, "Hermon %p could not write MTT at %x\n",
665: hermon, mtt_base_addr );
666: goto err_write_mtt;
667: }
668: addr += HERMON_PAGE_SIZE;
669: mtt_base_addr += hermon->cap.mtt_entry_size;
670: }
671:
672: DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] for "
673: "[%08lx,%08lx,%08lx,%08lx)\n", hermon, mtt->mtt_offset,
674: ( mtt->mtt_offset + mtt->num_pages - 1 ), start,
675: ( start + page_offset ), ( start + len ), addr );
676:
677: return 0;
678:
679: err_write_mtt:
680: hermon_bitmask_free ( hermon->mtt_inuse, mtt_offset, num_pages );
681: err_mtt_offset:
682: return rc;
683: }
684:
685: /**
686: * Free MTT entries
687: *
688: * @v hermon Hermon device
689: * @v mtt MTT descriptor
690: */
691: static void hermon_free_mtt ( struct hermon *hermon,
692: struct hermon_mtt *mtt ) {
693:
694: DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] freed\n",
695: hermon, mtt->mtt_offset,
696: ( mtt->mtt_offset + mtt->num_pages - 1 ) );
697: hermon_bitmask_free ( hermon->mtt_inuse, mtt->mtt_offset,
698: mtt->num_pages );
699: }
700:
701: /***************************************************************************
702: *
703: * Static configuration operations
704: *
705: ***************************************************************************
706: */
707:
708: /**
709: * Calculate offset within static configuration
710: *
711: * @v field Field
712: * @ret offset Offset
713: */
714: #define HERMON_MOD_STAT_CFG_OFFSET( field ) \
715: ( ( MLX_BIT_OFFSET ( struct hermonprm_mod_stat_cfg_st, field ) / 8 ) \
716: & ~( sizeof ( struct hermonprm_scalar_parameter ) - 1 ) )
717:
718: /**
719: * Query or modify static configuration
720: *
721: * @v hermon Hermon device
722: * @v port Port
723: * @v mode Command mode
724: * @v offset Offset within static configuration
725: * @v stat_cfg Static configuration
726: * @ret rc Return status code
727: */
728: static int hermon_mod_stat_cfg ( struct hermon *hermon, unsigned int port,
729: unsigned int mode, unsigned int offset,
730: struct hermonprm_mod_stat_cfg *stat_cfg ) {
731: struct hermonprm_scalar_parameter *portion =
732: ( ( void * ) &stat_cfg->u.bytes[offset] );
733: struct hermonprm_mod_stat_cfg_input_mod mod;
734: int rc;
735:
736: /* Sanity check */
737: assert ( ( offset % sizeof ( *portion ) ) == 0 );
738:
739: /* Construct input modifier */
740: memset ( &mod, 0, sizeof ( mod ) );
741: MLX_FILL_2 ( &mod, 0,
742: portnum, port,
743: offset, offset );
744:
745: /* Issue command */
746: if ( ( rc = hermon_cmd_mod_stat_cfg ( hermon, mode,
747: be32_to_cpu ( mod.u.dwords[0] ),
748: portion ) ) != 0 )
749: return rc;
750:
751: return 0;
752: }
753:
754: /***************************************************************************
755: *
756: * MAD operations
757: *
758: ***************************************************************************
759: */
760:
761: /**
762: * Issue management datagram
763: *
764: * @v ibdev Infiniband device
765: * @v mad Management datagram
766: * @ret rc Return status code
767: */
768: static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
769: struct hermon *hermon = ib_get_drvdata ( ibdev );
770: union hermonprm_mad mad_ifc;
771: int rc;
772:
773: linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
774: mad_size_mismatch );
775:
776: /* Copy in request packet */
777: memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
778:
779: /* Issue MAD */
780: if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
781: &mad_ifc ) ) != 0 ) {
782: DBGC ( hermon, "Hermon %p port %d could not issue MAD IFC: "
783: "%s\n", hermon, ibdev->port, strerror ( rc ) );
784: return rc;
785: }
786:
787: /* Copy out reply packet */
788: memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
789:
790: if ( mad->hdr.status != 0 ) {
791: DBGC ( hermon, "Hermon %p port %d MAD IFC status %04x\n",
792: hermon, ibdev->port, ntohs ( mad->hdr.status ) );
793: return -EIO;
794: }
795: return 0;
796: }
797:
798: /***************************************************************************
799: *
800: * Completion queue operations
801: *
802: ***************************************************************************
803: */
804:
805: /**
806: * Dump completion queue context (for debugging only)
807: *
808: * @v hermon Hermon device
809: * @v cq Completion queue
810: * @ret rc Return status code
811: */
812: static __attribute__ (( unused )) int
813: hermon_dump_cqctx ( struct hermon *hermon, struct ib_completion_queue *cq ) {
814: struct hermonprm_completion_queue_context cqctx;
815: int rc;
816:
817: memset ( &cqctx, 0, sizeof ( cqctx ) );
818: if ( ( rc = hermon_cmd_query_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
819: DBGC ( hermon, "Hermon %p CQN %#lx QUERY_CQ failed: %s\n",
820: hermon, cq->cqn, strerror ( rc ) );
821: return rc;
822: }
823: DBGC ( hermon, "Hermon %p CQN %#lx context:\n", hermon, cq->cqn );
824: DBGC_HDA ( hermon, 0, &cqctx, sizeof ( cqctx ) );
825:
826: return 0;
827: }
828:
829: /**
830: * Create completion queue
831: *
832: * @v ibdev Infiniband device
833: * @v cq Completion queue
834: * @ret rc Return status code
835: */
836: static int hermon_create_cq ( struct ib_device *ibdev,
837: struct ib_completion_queue *cq ) {
838: struct hermon *hermon = ib_get_drvdata ( ibdev );
839: struct hermon_completion_queue *hermon_cq;
840: struct hermonprm_completion_queue_context cqctx;
841: int cqn_offset;
842: unsigned int i;
843: int rc;
844:
845: /* Find a free completion queue number */
846: cqn_offset = hermon_bitmask_alloc ( hermon->cq_inuse,
847: HERMON_MAX_CQS, 1 );
848: if ( cqn_offset < 0 ) {
849: DBGC ( hermon, "Hermon %p out of completion queues\n",
850: hermon );
851: rc = cqn_offset;
852: goto err_cqn_offset;
853: }
854: cq->cqn = ( hermon->cap.reserved_cqs + cqn_offset );
855:
856: /* Allocate control structures */
857: hermon_cq = zalloc ( sizeof ( *hermon_cq ) );
858: if ( ! hermon_cq ) {
859: rc = -ENOMEM;
860: goto err_hermon_cq;
861: }
862:
863: /* Allocate doorbell */
864: hermon_cq->doorbell = malloc_dma ( sizeof ( hermon_cq->doorbell[0] ),
865: sizeof ( hermon_cq->doorbell[0] ) );
866: if ( ! hermon_cq->doorbell ) {
867: rc = -ENOMEM;
868: goto err_doorbell;
869: }
870: memset ( hermon_cq->doorbell, 0, sizeof ( hermon_cq->doorbell[0] ) );
871:
872: /* Allocate completion queue itself */
873: hermon_cq->cqe_size = ( cq->num_cqes * sizeof ( hermon_cq->cqe[0] ) );
874: hermon_cq->cqe = malloc_dma ( hermon_cq->cqe_size,
875: sizeof ( hermon_cq->cqe[0] ) );
876: if ( ! hermon_cq->cqe ) {
877: rc = -ENOMEM;
878: goto err_cqe;
879: }
880: memset ( hermon_cq->cqe, 0, hermon_cq->cqe_size );
881: for ( i = 0 ; i < cq->num_cqes ; i++ ) {
882: MLX_FILL_1 ( &hermon_cq->cqe[i].normal, 7, owner, 1 );
883: }
884: barrier();
885:
886: /* Allocate MTT entries */
887: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_cq->cqe,
888: hermon_cq->cqe_size,
889: &hermon_cq->mtt ) ) != 0 )
890: goto err_alloc_mtt;
891:
892: /* Hand queue over to hardware */
893: memset ( &cqctx, 0, sizeof ( cqctx ) );
894: MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
895: MLX_FILL_1 ( &cqctx, 2,
896: page_offset, ( hermon_cq->mtt.page_offset >> 5 ) );
897: MLX_FILL_2 ( &cqctx, 3,
898: usr_page, HERMON_UAR_NON_EQ_PAGE,
899: log_cq_size, fls ( cq->num_cqes - 1 ) );
900: MLX_FILL_1 ( &cqctx, 5, c_eqn, hermon->eq.eqn );
901: MLX_FILL_H ( &cqctx, 6, mtt_base_addr_h,
902: hermon_cq->mtt.mtt_base_addr );
903: MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l,
904: ( hermon_cq->mtt.mtt_base_addr >> 3 ) );
905: MLX_FILL_H ( &cqctx, 14, db_record_addr_h,
906: virt_to_phys ( hermon_cq->doorbell ) );
907: MLX_FILL_1 ( &cqctx, 15, db_record_addr_l,
908: ( virt_to_phys ( hermon_cq->doorbell ) >> 3 ) );
909: if ( ( rc = hermon_cmd_sw2hw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
910: DBGC ( hermon, "Hermon %p CQN %#lx SW2HW_CQ failed: %s\n",
911: hermon, cq->cqn, strerror ( rc ) );
912: goto err_sw2hw_cq;
913: }
914:
915: DBGC ( hermon, "Hermon %p CQN %#lx ring [%08lx,%08lx), doorbell "
916: "%08lx\n", hermon, cq->cqn, virt_to_phys ( hermon_cq->cqe ),
917: ( virt_to_phys ( hermon_cq->cqe ) + hermon_cq->cqe_size ),
918: virt_to_phys ( hermon_cq->doorbell ) );
919: ib_cq_set_drvdata ( cq, hermon_cq );
920: return 0;
921:
922: err_sw2hw_cq:
923: hermon_free_mtt ( hermon, &hermon_cq->mtt );
924: err_alloc_mtt:
925: free_dma ( hermon_cq->cqe, hermon_cq->cqe_size );
926: err_cqe:
927: free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
928: err_doorbell:
929: free ( hermon_cq );
930: err_hermon_cq:
931: hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
932: err_cqn_offset:
933: return rc;
934: }
935:
936: /**
937: * Destroy completion queue
938: *
939: * @v ibdev Infiniband device
940: * @v cq Completion queue
941: */
942: static void hermon_destroy_cq ( struct ib_device *ibdev,
943: struct ib_completion_queue *cq ) {
944: struct hermon *hermon = ib_get_drvdata ( ibdev );
945: struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
946: struct hermonprm_completion_queue_context cqctx;
947: int cqn_offset;
948: int rc;
949:
950: /* Take ownership back from hardware */
951: if ( ( rc = hermon_cmd_hw2sw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
952: DBGC ( hermon, "Hermon %p CQN %#lx FATAL HW2SW_CQ failed: "
953: "%s\n", hermon, cq->cqn, strerror ( rc ) );
954: /* Leak memory and return; at least we avoid corruption */
955: return;
956: }
957:
958: /* Free MTT entries */
959: hermon_free_mtt ( hermon, &hermon_cq->mtt );
960:
961: /* Free memory */
962: free_dma ( hermon_cq->cqe, hermon_cq->cqe_size );
963: free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
964: free ( hermon_cq );
965:
966: /* Mark queue number as free */
967: cqn_offset = ( cq->cqn - hermon->cap.reserved_cqs );
968: hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
969:
970: ib_cq_set_drvdata ( cq, NULL );
971: }
972:
973: /***************************************************************************
974: *
975: * Queue pair operations
976: *
977: ***************************************************************************
978: */
979:
980: /**
981: * Assign queue pair number
982: *
983: * @v ibdev Infiniband device
984: * @v qp Queue pair
985: * @ret rc Return status code
986: */
987: static int hermon_alloc_qpn ( struct ib_device *ibdev,
988: struct ib_queue_pair *qp ) {
989: struct hermon *hermon = ib_get_drvdata ( ibdev );
990: unsigned int port_offset;
991: int qpn_offset;
992:
993: /* Calculate queue pair number */
994: port_offset = ( ibdev->port - HERMON_PORT_BASE );
995:
996: switch ( qp->type ) {
997: case IB_QPT_SMI:
998: qp->qpn = ( hermon->special_qpn_base + port_offset );
999: return 0;
1000: case IB_QPT_GSI:
1001: qp->qpn = ( hermon->special_qpn_base + 2 + port_offset );
1002: return 0;
1003: case IB_QPT_UD:
1004: case IB_QPT_RC:
1005: case IB_QPT_ETH:
1006: /* Find a free queue pair number */
1007: qpn_offset = hermon_bitmask_alloc ( hermon->qp_inuse,
1008: HERMON_MAX_QPS, 1 );
1009: if ( qpn_offset < 0 ) {
1010: DBGC ( hermon, "Hermon %p out of queue pairs\n",
1011: hermon );
1012: return qpn_offset;
1013: }
1014: qp->qpn = ( ( random() & HERMON_QPN_RANDOM_MASK ) |
1015: ( hermon->qpn_base + qpn_offset ) );
1016: return 0;
1017: default:
1018: DBGC ( hermon, "Hermon %p unsupported QP type %d\n",
1019: hermon, qp->type );
1020: return -ENOTSUP;
1021: }
1022: }
1023:
1024: /**
1025: * Free queue pair number
1026: *
1027: * @v ibdev Infiniband device
1028: * @v qp Queue pair
1029: */
1030: static void hermon_free_qpn ( struct ib_device *ibdev,
1031: struct ib_queue_pair *qp ) {
1032: struct hermon *hermon = ib_get_drvdata ( ibdev );
1033: int qpn_offset;
1034:
1035: qpn_offset = ( ( qp->qpn & ~HERMON_QPN_RANDOM_MASK )
1036: - hermon->qpn_base );
1037: if ( qpn_offset >= 0 )
1038: hermon_bitmask_free ( hermon->qp_inuse, qpn_offset, 1 );
1039: }
1040:
1041: /**
1042: * Calculate transmission rate
1043: *
1044: * @v av Address vector
1045: * @ret hermon_rate Hermon rate
1046: */
1047: static unsigned int hermon_rate ( struct ib_address_vector *av ) {
1048: return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) )
1049: ? ( av->rate + 5 ) : 0 );
1050: }
1051:
1052: /**
1053: * Calculate schedule queue
1054: *
1055: * @v ibdev Infiniband device
1056: * @v qp Queue pair
1057: * @ret sched_queue Schedule queue
1058: */
1059: static unsigned int hermon_sched_queue ( struct ib_device *ibdev,
1060: struct ib_queue_pair *qp ) {
1061: return ( ( ( qp->type == IB_QPT_SMI ) ?
1062: HERMON_SCHED_QP0 : HERMON_SCHED_DEFAULT ) |
1063: ( ( ibdev->port - 1 ) << 6 ) );
1064: }
1065:
1066: /** Queue pair transport service type map */
1067: static uint8_t hermon_qp_st[] = {
1068: [IB_QPT_SMI] = HERMON_ST_MLX,
1069: [IB_QPT_GSI] = HERMON_ST_MLX,
1070: [IB_QPT_UD] = HERMON_ST_UD,
1071: [IB_QPT_RC] = HERMON_ST_RC,
1072: [IB_QPT_ETH] = HERMON_ST_MLX,
1073: };
1074:
1075: /**
1076: * Dump queue pair context (for debugging only)
1077: *
1078: * @v hermon Hermon device
1079: * @v qp Queue pair
1080: * @ret rc Return status code
1081: */
1082: static __attribute__ (( unused )) int
1083: hermon_dump_qpctx ( struct hermon *hermon, struct ib_queue_pair *qp ) {
1084: struct hermonprm_qp_ee_state_transitions qpctx;
1085: int rc;
1086:
1087: memset ( &qpctx, 0, sizeof ( qpctx ) );
1088: if ( ( rc = hermon_cmd_query_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ) {
1089: DBGC ( hermon, "Hermon %p QPN %#lx QUERY_QP failed: %s\n",
1090: hermon, qp->qpn, strerror ( rc ) );
1091: return rc;
1092: }
1093: DBGC ( hermon, "Hermon %p QPN %#lx context:\n", hermon, qp->qpn );
1094: DBGC_HDA ( hermon, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) );
1095:
1096: return 0;
1097: }
1098:
1099: /**
1100: * Create queue pair
1101: *
1102: * @v ibdev Infiniband device
1103: * @v qp Queue pair
1104: * @ret rc Return status code
1105: */
1106: static int hermon_create_qp ( struct ib_device *ibdev,
1107: struct ib_queue_pair *qp ) {
1108: struct hermon *hermon = ib_get_drvdata ( ibdev );
1109: struct hermon_queue_pair *hermon_qp;
1110: struct hermonprm_qp_ee_state_transitions qpctx;
1111: int rc;
1112:
1113: /* Calculate queue pair number */
1114: if ( ( rc = hermon_alloc_qpn ( ibdev, qp ) ) != 0 )
1115: goto err_alloc_qpn;
1116:
1117: /* Allocate control structures */
1118: hermon_qp = zalloc ( sizeof ( *hermon_qp ) );
1119: if ( ! hermon_qp ) {
1120: rc = -ENOMEM;
1121: goto err_hermon_qp;
1122: }
1123:
1124: /* Allocate doorbells */
1125: hermon_qp->recv.doorbell =
1126: malloc_dma ( sizeof ( hermon_qp->recv.doorbell[0] ),
1127: sizeof ( hermon_qp->recv.doorbell[0] ) );
1128: if ( ! hermon_qp->recv.doorbell ) {
1129: rc = -ENOMEM;
1130: goto err_recv_doorbell;
1131: }
1132: memset ( hermon_qp->recv.doorbell, 0,
1133: sizeof ( hermon_qp->recv.doorbell[0] ) );
1134: hermon_qp->send.doorbell =
1135: ( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE +
1136: HERMON_DB_POST_SND_OFFSET );
1137:
1138: /* Allocate work queue buffer */
1139: hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 +
1140: ( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) );
1141: hermon_qp->send.num_wqes =
1142: ( 1 << fls ( hermon_qp->send.num_wqes - 1 ) ); /* round up */
1143: hermon_qp->send.wqe_size = ( hermon_qp->send.num_wqes *
1144: sizeof ( hermon_qp->send.wqe[0] ) );
1145: hermon_qp->recv.wqe_size = ( qp->recv.num_wqes *
1146: sizeof ( hermon_qp->recv.wqe[0] ) );
1147: hermon_qp->wqe_size = ( hermon_qp->send.wqe_size +
1148: hermon_qp->recv.wqe_size );
1149: hermon_qp->wqe = malloc_dma ( hermon_qp->wqe_size,
1150: sizeof ( hermon_qp->send.wqe[0] ) );
1151: if ( ! hermon_qp->wqe ) {
1152: rc = -ENOMEM;
1153: goto err_alloc_wqe;
1154: }
1155: hermon_qp->send.wqe = hermon_qp->wqe;
1156: memset ( hermon_qp->send.wqe, 0xff, hermon_qp->send.wqe_size );
1157: hermon_qp->recv.wqe = ( hermon_qp->wqe + hermon_qp->send.wqe_size );
1158: memset ( hermon_qp->recv.wqe, 0, hermon_qp->recv.wqe_size );
1159:
1160: /* Allocate MTT entries */
1161: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_qp->wqe,
1162: hermon_qp->wqe_size,
1163: &hermon_qp->mtt ) ) != 0 ) {
1164: goto err_alloc_mtt;
1165: }
1166:
1167: /* Transition queue to INIT state */
1168: memset ( &qpctx, 0, sizeof ( qpctx ) );
1169: MLX_FILL_2 ( &qpctx, 2,
1170: qpc_eec_data.pm_state, HERMON_PM_STATE_MIGRATED,
1171: qpc_eec_data.st, hermon_qp_st[qp->type] );
1172: MLX_FILL_1 ( &qpctx, 3, qpc_eec_data.pd, HERMON_GLOBAL_PD );
1173: MLX_FILL_4 ( &qpctx, 4,
1174: qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
1175: qpc_eec_data.log_rq_stride,
1176: ( fls ( sizeof ( hermon_qp->recv.wqe[0] ) - 1 ) - 4 ),
1177: qpc_eec_data.log_sq_size,
1178: fls ( hermon_qp->send.num_wqes - 1 ),
1179: qpc_eec_data.log_sq_stride,
1180: ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) );
1181: MLX_FILL_1 ( &qpctx, 5,
1182: qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE );
1183: MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
1184: MLX_FILL_4 ( &qpctx, 38,
1185: qpc_eec_data.rre, 1,
1186: qpc_eec_data.rwe, 1,
1187: qpc_eec_data.rae, 1,
1188: qpc_eec_data.page_offset,
1189: ( hermon_qp->mtt.page_offset >> 6 ) );
1190: MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
1191: MLX_FILL_H ( &qpctx, 42, qpc_eec_data.db_record_addr_h,
1192: virt_to_phys ( hermon_qp->recv.doorbell ) );
1193: MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l,
1194: ( virt_to_phys ( hermon_qp->recv.doorbell ) >> 2 ) );
1195: MLX_FILL_H ( &qpctx, 52, qpc_eec_data.mtt_base_addr_h,
1196: hermon_qp->mtt.mtt_base_addr );
1197: MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l,
1198: ( hermon_qp->mtt.mtt_base_addr >> 3 ) );
1199: if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn,
1200: &qpctx ) ) != 0 ) {
1201: DBGC ( hermon, "Hermon %p QPN %#lx RST2INIT_QP failed: %s\n",
1202: hermon, qp->qpn, strerror ( rc ) );
1203: goto err_rst2init_qp;
1204: }
1205: hermon_qp->state = HERMON_QP_ST_INIT;
1206:
1207: DBGC ( hermon, "Hermon %p QPN %#lx send ring [%08lx,%08lx), doorbell "
1208: "%08lx\n", hermon, qp->qpn,
1209: virt_to_phys ( hermon_qp->send.wqe ),
1210: ( virt_to_phys ( hermon_qp->send.wqe ) +
1211: hermon_qp->send.wqe_size ),
1212: virt_to_phys ( hermon_qp->send.doorbell ) );
1213: DBGC ( hermon, "Hermon %p QPN %#lx receive ring [%08lx,%08lx), "
1214: "doorbell %08lx\n", hermon, qp->qpn,
1215: virt_to_phys ( hermon_qp->recv.wqe ),
1216: ( virt_to_phys ( hermon_qp->recv.wqe ) +
1217: hermon_qp->recv.wqe_size ),
1218: virt_to_phys ( hermon_qp->recv.doorbell ) );
1219: DBGC ( hermon, "Hermon %p QPN %#lx send CQN %#lx receive CQN %#lx\n",
1220: hermon, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn );
1221: ib_qp_set_drvdata ( qp, hermon_qp );
1222: return 0;
1223:
1224: hermon_cmd_2rst_qp ( hermon, qp->qpn );
1225: err_rst2init_qp:
1226: hermon_free_mtt ( hermon, &hermon_qp->mtt );
1227: err_alloc_mtt:
1228: free_dma ( hermon_qp->wqe, hermon_qp->wqe_size );
1229: err_alloc_wqe:
1230: free_dma ( hermon_qp->recv.doorbell,
1231: sizeof ( hermon_qp->recv.doorbell[0] ) );
1232: err_recv_doorbell:
1233: free ( hermon_qp );
1234: err_hermon_qp:
1235: hermon_free_qpn ( ibdev, qp );
1236: err_alloc_qpn:
1237: return rc;
1238: }
1239:
1240: /**
1241: * Modify queue pair
1242: *
1243: * @v ibdev Infiniband device
1244: * @v qp Queue pair
1245: * @ret rc Return status code
1246: */
1247: static int hermon_modify_qp ( struct ib_device *ibdev,
1248: struct ib_queue_pair *qp ) {
1249: struct hermon *hermon = ib_get_drvdata ( ibdev );
1250: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
1251: struct hermonprm_qp_ee_state_transitions qpctx;
1252: int rc;
1253:
1254: /* Transition queue to RTR state, if applicable */
1255: if ( hermon_qp->state < HERMON_QP_ST_RTR ) {
1256: memset ( &qpctx, 0, sizeof ( qpctx ) );
1257: MLX_FILL_2 ( &qpctx, 4,
1258: qpc_eec_data.mtu,
1259: ( ( qp->type == IB_QPT_ETH ) ?
1260: HERMON_MTU_ETH : HERMON_MTU_2048 ),
1261: qpc_eec_data.msg_max, 31 );
1262: MLX_FILL_1 ( &qpctx, 7,
1263: qpc_eec_data.remote_qpn_een, qp->av.qpn );
1264: MLX_FILL_1 ( &qpctx, 9,
1265: qpc_eec_data.primary_address_path.rlid,
1266: qp->av.lid );
1267: MLX_FILL_1 ( &qpctx, 10,
1268: qpc_eec_data.primary_address_path.max_stat_rate,
1269: hermon_rate ( &qp->av ) );
1270: memcpy ( &qpctx.u.dwords[12], &qp->av.gid,
1271: sizeof ( qp->av.gid ) );
1272: MLX_FILL_1 ( &qpctx, 16,
1273: qpc_eec_data.primary_address_path.sched_queue,
1274: hermon_sched_queue ( ibdev, qp ) );
1275: MLX_FILL_1 ( &qpctx, 39,
1276: qpc_eec_data.next_rcv_psn, qp->recv.psn );
1277: if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn,
1278: &qpctx ) ) != 0 ) {
1279: DBGC ( hermon, "Hermon %p QPN %#lx INIT2RTR_QP failed:"
1280: " %s\n", hermon, qp->qpn, strerror ( rc ) );
1281: return rc;
1282: }
1283: hermon_qp->state = HERMON_QP_ST_RTR;
1284: }
1285:
1286: /* Transition queue to RTS state */
1287: if ( hermon_qp->state < HERMON_QP_ST_RTS ) {
1288: memset ( &qpctx, 0, sizeof ( qpctx ) );
1289: MLX_FILL_1 ( &qpctx, 10,
1290: qpc_eec_data.primary_address_path.ack_timeout,
1291: 14 /* 4.096us * 2^(14) = 67ms */ );
1292: MLX_FILL_2 ( &qpctx, 30,
1293: qpc_eec_data.retry_count, HERMON_RETRY_MAX,
1294: qpc_eec_data.rnr_retry, HERMON_RETRY_MAX );
1295: MLX_FILL_1 ( &qpctx, 32,
1296: qpc_eec_data.next_send_psn, qp->send.psn );
1297: if ( ( rc = hermon_cmd_rtr2rts_qp ( hermon, qp->qpn,
1298: &qpctx ) ) != 0 ) {
1299: DBGC ( hermon, "Hermon %p QPN %#lx RTR2RTS_QP failed: "
1300: "%s\n", hermon, qp->qpn, strerror ( rc ) );
1301: return rc;
1302: }
1303: hermon_qp->state = HERMON_QP_ST_RTS;
1304: }
1305:
1306: /* Update parameters in RTS state */
1307: memset ( &qpctx, 0, sizeof ( qpctx ) );
1308: MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY );
1309: MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
1310: if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
1311: DBGC ( hermon, "Hermon %p QPN %#lx RTS2RTS_QP failed: %s\n",
1312: hermon, qp->qpn, strerror ( rc ) );
1313: return rc;
1314: }
1315:
1316: return 0;
1317: }
1318:
1319: /**
1320: * Destroy queue pair
1321: *
1322: * @v ibdev Infiniband device
1323: * @v qp Queue pair
1324: */
1325: static void hermon_destroy_qp ( struct ib_device *ibdev,
1326: struct ib_queue_pair *qp ) {
1327: struct hermon *hermon = ib_get_drvdata ( ibdev );
1328: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
1329: int rc;
1330:
1331: /* Take ownership back from hardware */
1332: if ( ( rc = hermon_cmd_2rst_qp ( hermon, qp->qpn ) ) != 0 ) {
1333: DBGC ( hermon, "Hermon %p QPN %#lx FATAL 2RST_QP failed: %s\n",
1334: hermon, qp->qpn, strerror ( rc ) );
1335: /* Leak memory and return; at least we avoid corruption */
1336: return;
1337: }
1338:
1339: /* Free MTT entries */
1340: hermon_free_mtt ( hermon, &hermon_qp->mtt );
1341:
1342: /* Free memory */
1343: free_dma ( hermon_qp->wqe, hermon_qp->wqe_size );
1344: free_dma ( hermon_qp->recv.doorbell,
1345: sizeof ( hermon_qp->recv.doorbell[0] ) );
1346: free ( hermon_qp );
1347:
1348: /* Mark queue number as free */
1349: hermon_free_qpn ( ibdev, qp );
1350:
1351: ib_qp_set_drvdata ( qp, NULL );
1352: }
1353:
1354: /***************************************************************************
1355: *
1356: * Work request operations
1357: *
1358: ***************************************************************************
1359: */
1360:
1361: /**
1362: * Construct UD send work queue entry
1363: *
1364: * @v ibdev Infiniband device
1365: * @v qp Queue pair
1366: * @v av Address vector
1367: * @v iobuf I/O buffer
1368: * @v wqe Send work queue entry
1369: * @ret opcode Control opcode
1370: */
1371: static __attribute__ (( unused )) unsigned int
1372: hermon_fill_nop_send_wqe ( struct ib_device *ibdev __unused,
1373: struct ib_queue_pair *qp __unused,
1374: struct ib_address_vector *av __unused,
1375: struct io_buffer *iobuf __unused,
1376: union hermon_send_wqe *wqe ) {
1377:
1378: MLX_FILL_1 ( &wqe->ctrl, 1, ds, ( sizeof ( wqe->ctrl ) / 16 ) );
1379: MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ );
1380: return HERMON_OPCODE_NOP;
1381: }
1382:
1383: /**
1384: * Construct UD send work queue entry
1385: *
1386: * @v ibdev Infiniband device
1387: * @v qp Queue pair
1388: * @v av Address vector
1389: * @v iobuf I/O buffer
1390: * @v wqe Send work queue entry
1391: * @ret opcode Control opcode
1392: */
1393: static unsigned int
1394: hermon_fill_ud_send_wqe ( struct ib_device *ibdev,
1395: struct ib_queue_pair *qp __unused,
1396: struct ib_address_vector *av,
1397: struct io_buffer *iobuf,
1398: union hermon_send_wqe *wqe ) {
1399: struct hermon *hermon = ib_get_drvdata ( ibdev );
1400:
1401: MLX_FILL_1 ( &wqe->ud.ctrl, 1, ds,
1402: ( ( offsetof ( typeof ( wqe->ud ), data[1] ) / 16 ) ) );
1403: MLX_FILL_1 ( &wqe->ud.ctrl, 2, c, 0x03 /* generate completion */ );
1404: MLX_FILL_2 ( &wqe->ud.ud, 0,
1405: ud_address_vector.pd, HERMON_GLOBAL_PD,
1406: ud_address_vector.port_number, ibdev->port );
1407: MLX_FILL_2 ( &wqe->ud.ud, 1,
1408: ud_address_vector.rlid, av->lid,
1409: ud_address_vector.g, av->gid_present );
1410: MLX_FILL_1 ( &wqe->ud.ud, 2,
1411: ud_address_vector.max_stat_rate, hermon_rate ( av ) );
1412: MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, av->sl );
1413: memcpy ( &wqe->ud.ud.u.dwords[4], &av->gid, sizeof ( av->gid ) );
1414: MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, av->qpn );
1415: MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, av->qkey );
1416: MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) );
1417: MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, hermon->lkey );
1418: MLX_FILL_H ( &wqe->ud.data[0], 2,
1419: local_address_h, virt_to_bus ( iobuf->data ) );
1420: MLX_FILL_1 ( &wqe->ud.data[0], 3,
1421: local_address_l, virt_to_bus ( iobuf->data ) );
1422: return HERMON_OPCODE_SEND;
1423: }
1424:
1425: /**
1426: * Construct MLX send work queue entry
1427: *
1428: * @v ibdev Infiniband device
1429: * @v qp Queue pair
1430: * @v av Address vector
1431: * @v iobuf I/O buffer
1432: * @v wqe Send work queue entry
1433: * @ret opcode Control opcode
1434: */
1435: static unsigned int
1436: hermon_fill_mlx_send_wqe ( struct ib_device *ibdev,
1437: struct ib_queue_pair *qp,
1438: struct ib_address_vector *av,
1439: struct io_buffer *iobuf,
1440: union hermon_send_wqe *wqe ) {
1441: struct hermon *hermon = ib_get_drvdata ( ibdev );
1442: struct io_buffer headers;
1443:
1444: /* Construct IB headers */
1445: iob_populate ( &headers, &wqe->mlx.headers, 0,
1446: sizeof ( wqe->mlx.headers ) );
1447: iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) );
1448: ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), av );
1449:
1450: /* Fill work queue entry */
1451: MLX_FILL_1 ( &wqe->mlx.ctrl, 1, ds,
1452: ( ( offsetof ( typeof ( wqe->mlx ), data[2] ) / 16 ) ) );
1453: MLX_FILL_5 ( &wqe->mlx.ctrl, 2,
1454: c, 0x03 /* generate completion */,
1455: icrc, 0 /* generate ICRC */,
1456: max_statrate, hermon_rate ( av ),
1457: slr, 0,
1458: v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) );
1459: MLX_FILL_1 ( &wqe->mlx.ctrl, 3, rlid, av->lid );
1460: MLX_FILL_1 ( &wqe->mlx.data[0], 0,
1461: byte_count, iob_len ( &headers ) );
1462: MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, hermon->lkey );
1463: MLX_FILL_H ( &wqe->mlx.data[0], 2,
1464: local_address_h, virt_to_bus ( headers.data ) );
1465: MLX_FILL_1 ( &wqe->mlx.data[0], 3,
1466: local_address_l, virt_to_bus ( headers.data ) );
1467: MLX_FILL_1 ( &wqe->mlx.data[1], 0,
1468: byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) );
1469: MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, hermon->lkey );
1470: MLX_FILL_H ( &wqe->mlx.data[1], 2,
1471: local_address_h, virt_to_bus ( iobuf->data ) );
1472: MLX_FILL_1 ( &wqe->mlx.data[1], 3,
1473: local_address_l, virt_to_bus ( iobuf->data ) );
1474: return HERMON_OPCODE_SEND;
1475: }
1476:
1477: /**
1478: * Construct RC send work queue entry
1479: *
1480: * @v ibdev Infiniband device
1481: * @v qp Queue pair
1482: * @v av Address vector
1483: * @v iobuf I/O buffer
1484: * @v wqe Send work queue entry
1485: * @ret opcode Control opcode
1486: */
1487: static unsigned int
1488: hermon_fill_rc_send_wqe ( struct ib_device *ibdev,
1489: struct ib_queue_pair *qp __unused,
1490: struct ib_address_vector *av __unused,
1491: struct io_buffer *iobuf,
1492: union hermon_send_wqe *wqe ) {
1493: struct hermon *hermon = ib_get_drvdata ( ibdev );
1494:
1495: MLX_FILL_1 ( &wqe->rc.ctrl, 1, ds,
1496: ( ( offsetof ( typeof ( wqe->rc ), data[1] ) / 16 ) ) );
1497: MLX_FILL_1 ( &wqe->rc.ctrl, 2, c, 0x03 /* generate completion */ );
1498: MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) );
1499: MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, hermon->lkey );
1500: MLX_FILL_H ( &wqe->rc.data[0], 2,
1501: local_address_h, virt_to_bus ( iobuf->data ) );
1502: MLX_FILL_1 ( &wqe->rc.data[0], 3,
1503: local_address_l, virt_to_bus ( iobuf->data ) );
1504: return HERMON_OPCODE_SEND;
1505: }
1506:
1507: /**
1508: * Construct Ethernet send work queue entry
1509: *
1510: * @v ibdev Infiniband device
1511: * @v qp Queue pair
1512: * @v av Address vector
1513: * @v iobuf I/O buffer
1514: * @v wqe Send work queue entry
1515: * @ret opcode Control opcode
1516: */
1517: static unsigned int
1518: hermon_fill_eth_send_wqe ( struct ib_device *ibdev,
1519: struct ib_queue_pair *qp __unused,
1520: struct ib_address_vector *av __unused,
1521: struct io_buffer *iobuf,
1522: union hermon_send_wqe *wqe ) {
1523: struct hermon *hermon = ib_get_drvdata ( ibdev );
1524:
1525: /* Fill work queue entry */
1526: MLX_FILL_1 ( &wqe->eth.ctrl, 1, ds,
1527: ( ( offsetof ( typeof ( wqe->mlx ), data[1] ) / 16 ) ) );
1528: MLX_FILL_2 ( &wqe->eth.ctrl, 2,
1529: c, 0x03 /* generate completion */,
1530: s, 1 /* inhibit ICRC */ );
1531: MLX_FILL_1 ( &wqe->eth.data[0], 0,
1532: byte_count, iob_len ( iobuf ) );
1533: MLX_FILL_1 ( &wqe->eth.data[0], 1, l_key, hermon->lkey );
1534: MLX_FILL_H ( &wqe->eth.data[0], 2,
1535: local_address_h, virt_to_bus ( iobuf->data ) );
1536: MLX_FILL_1 ( &wqe->eth.data[0], 3,
1537: local_address_l, virt_to_bus ( iobuf->data ) );
1538: return HERMON_OPCODE_SEND;
1539: }
1540:
1541: /** Work queue entry constructors */
1542: static unsigned int
1543: ( * hermon_fill_send_wqe[] ) ( struct ib_device *ibdev,
1544: struct ib_queue_pair *qp,
1545: struct ib_address_vector *av,
1546: struct io_buffer *iobuf,
1547: union hermon_send_wqe *wqe ) = {
1548: [IB_QPT_SMI] = hermon_fill_mlx_send_wqe,
1549: [IB_QPT_GSI] = hermon_fill_mlx_send_wqe,
1550: [IB_QPT_UD] = hermon_fill_ud_send_wqe,
1551: [IB_QPT_RC] = hermon_fill_rc_send_wqe,
1552: [IB_QPT_ETH] = hermon_fill_eth_send_wqe,
1553: };
1554:
1555: /**
1556: * Post send work queue entry
1557: *
1558: * @v ibdev Infiniband device
1559: * @v qp Queue pair
1560: * @v av Address vector
1561: * @v iobuf I/O buffer
1562: * @ret rc Return status code
1563: */
1564: static int hermon_post_send ( struct ib_device *ibdev,
1565: struct ib_queue_pair *qp,
1566: struct ib_address_vector *av,
1567: struct io_buffer *iobuf ) {
1568: struct hermon *hermon = ib_get_drvdata ( ibdev );
1569: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
1570: struct ib_work_queue *wq = &qp->send;
1571: struct hermon_send_work_queue *hermon_send_wq = &hermon_qp->send;
1572: union hermon_send_wqe *wqe;
1573: union hermonprm_doorbell_register db_reg;
1574: unsigned long wqe_idx_mask;
1575: unsigned long wqe_idx;
1576: unsigned int owner;
1577: unsigned int opcode;
1578:
1579: /* Allocate work queue entry */
1580: wqe_idx = ( wq->next_idx & ( hermon_send_wq->num_wqes - 1 ) );
1581: owner = ( ( wq->next_idx & hermon_send_wq->num_wqes ) ? 1 : 0 );
1582: wqe_idx_mask = ( wq->num_wqes - 1 );
1583: if ( wq->iobufs[ wqe_idx & wqe_idx_mask ] ) {
1584: DBGC ( hermon, "Hermon %p QPN %#lx send queue full",
1585: hermon, qp->qpn );
1586: return -ENOBUFS;
1587: }
1588: wq->iobufs[ wqe_idx & wqe_idx_mask ] = iobuf;
1589: wqe = &hermon_send_wq->wqe[wqe_idx];
1590:
1591: /* Construct work queue entry */
1592: memset ( ( ( ( void * ) wqe ) + 4 /* avoid ctrl.owner */ ), 0,
1593: ( sizeof ( *wqe ) - 4 ) );
1594: assert ( qp->type < ( sizeof ( hermon_fill_send_wqe ) /
1595: sizeof ( hermon_fill_send_wqe[0] ) ) );
1596: assert ( hermon_fill_send_wqe[qp->type] != NULL );
1597: opcode = hermon_fill_send_wqe[qp->type] ( ibdev, qp, av, iobuf, wqe );
1598: barrier();
1599: MLX_FILL_2 ( &wqe->ctrl, 0,
1600: opcode, opcode,
1601: owner, owner );
1602: DBGCP ( hermon, "Hermon %p QPN %#lx posting send WQE %#lx:\n",
1603: hermon, qp->qpn, wqe_idx );
1604: DBGCP_HDA ( hermon, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) );
1605:
1606: /* Ring doorbell register */
1607: MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn );
1608: barrier();
1609: writel ( db_reg.dword[0], hermon_send_wq->doorbell );
1610:
1611: /* Update work queue's index */
1612: wq->next_idx++;
1613:
1614: return 0;
1615: }
1616:
1617: /**
1618: * Post receive work queue entry
1619: *
1620: * @v ibdev Infiniband device
1621: * @v qp Queue pair
1622: * @v iobuf I/O buffer
1623: * @ret rc Return status code
1624: */
1625: static int hermon_post_recv ( struct ib_device *ibdev,
1626: struct ib_queue_pair *qp,
1627: struct io_buffer *iobuf ) {
1628: struct hermon *hermon = ib_get_drvdata ( ibdev );
1629: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
1630: struct ib_work_queue *wq = &qp->recv;
1631: struct hermon_recv_work_queue *hermon_recv_wq = &hermon_qp->recv;
1632: struct hermonprm_recv_wqe *wqe;
1633: unsigned int wqe_idx_mask;
1634:
1635: /* Allocate work queue entry */
1636: wqe_idx_mask = ( wq->num_wqes - 1 );
1637: if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
1638: DBGC ( hermon, "Hermon %p QPN %#lx receive queue full",
1639: hermon, qp->qpn );
1640: return -ENOBUFS;
1641: }
1642: wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
1643: wqe = &hermon_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
1644:
1645: /* Construct work queue entry */
1646: MLX_FILL_1 ( &wqe->data[0], 0, byte_count, iob_tailroom ( iobuf ) );
1647: MLX_FILL_1 ( &wqe->data[0], 1, l_key, hermon->lkey );
1648: MLX_FILL_H ( &wqe->data[0], 2,
1649: local_address_h, virt_to_bus ( iobuf->data ) );
1650: MLX_FILL_1 ( &wqe->data[0], 3,
1651: local_address_l, virt_to_bus ( iobuf->data ) );
1652:
1653: /* Update work queue's index */
1654: wq->next_idx++;
1655:
1656: /* Update doorbell record */
1657: barrier();
1658: MLX_FILL_1 ( hermon_recv_wq->doorbell, 0, receive_wqe_counter,
1659: ( wq->next_idx & 0xffff ) );
1660:
1661: return 0;
1662: }
1663:
1664: /**
1665: * Handle completion
1666: *
1667: * @v ibdev Infiniband device
1668: * @v cq Completion queue
1669: * @v cqe Hardware completion queue entry
1670: * @ret rc Return status code
1671: */
1672: static int hermon_complete ( struct ib_device *ibdev,
1673: struct ib_completion_queue *cq,
1674: union hermonprm_completion_entry *cqe ) {
1675: struct hermon *hermon = ib_get_drvdata ( ibdev );
1676: struct ib_work_queue *wq;
1677: struct ib_queue_pair *qp;
1678: struct io_buffer *iobuf;
1679: struct ib_address_vector recv_av;
1680: struct ib_global_route_header *grh;
1681: struct ib_address_vector *av;
1682: unsigned int opcode;
1683: unsigned long qpn;
1684: int is_send;
1685: unsigned long wqe_idx;
1686: unsigned long wqe_idx_mask;
1687: size_t len;
1688: int rc = 0;
1689:
1690: /* Parse completion */
1691: qpn = MLX_GET ( &cqe->normal, qpn );
1692: is_send = MLX_GET ( &cqe->normal, s_r );
1693: opcode = MLX_GET ( &cqe->normal, opcode );
1694: if ( opcode >= HERMON_OPCODE_RECV_ERROR ) {
1695: /* "s" field is not valid for error opcodes */
1696: is_send = ( opcode == HERMON_OPCODE_SEND_ERROR );
1697: DBGC ( hermon, "Hermon %p CQN %#lx syndrome %x vendor %x\n",
1698: hermon, cq->cqn, MLX_GET ( &cqe->error, syndrome ),
1699: MLX_GET ( &cqe->error, vendor_error_syndrome ) );
1700: rc = -EIO;
1701: /* Don't return immediately; propagate error to completer */
1702: }
1703:
1704: /* Identify work queue */
1705: wq = ib_find_wq ( cq, qpn, is_send );
1706: if ( ! wq ) {
1707: DBGC ( hermon, "Hermon %p CQN %#lx unknown %s QPN %#lx\n",
1708: hermon, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
1709: return -EIO;
1710: }
1711: qp = wq->qp;
1712:
1713: /* Identify work queue entry */
1714: wqe_idx = MLX_GET ( &cqe->normal, wqe_counter );
1715: wqe_idx_mask = ( wq->num_wqes - 1 );
1716: DBGCP ( hermon, "Hermon %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n",
1717: hermon, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
1718: wqe_idx );
1719: DBGCP_HDA ( hermon, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
1720:
1721: /* Identify I/O buffer */
1722: iobuf = wq->iobufs[ wqe_idx & wqe_idx_mask ];
1723: if ( ! iobuf ) {
1724: DBGC ( hermon, "Hermon %p CQN %#lx QPN %#lx empty %s WQE "
1725: "%#lx\n", hermon, cq->cqn, qp->qpn,
1726: ( is_send ? "send" : "recv" ), wqe_idx );
1727: return -EIO;
1728: }
1729: wq->iobufs[ wqe_idx & wqe_idx_mask ] = NULL;
1730:
1731: if ( is_send ) {
1732: /* Hand off to completion handler */
1733: ib_complete_send ( ibdev, qp, iobuf, rc );
1734: } else {
1735: /* Set received length */
1736: len = MLX_GET ( &cqe->normal, byte_cnt );
1737: assert ( len <= iob_tailroom ( iobuf ) );
1738: iob_put ( iobuf, len );
1739: memset ( &recv_av, 0, sizeof ( recv_av ) );
1740: switch ( qp->type ) {
1741: case IB_QPT_SMI:
1742: case IB_QPT_GSI:
1743: case IB_QPT_UD:
1744: assert ( iob_len ( iobuf ) >= sizeof ( *grh ) );
1745: grh = iobuf->data;
1746: iob_pull ( iobuf, sizeof ( *grh ) );
1747: /* Construct address vector */
1748: av = &recv_av;
1749: av->qpn = MLX_GET ( &cqe->normal, srq_rqpn );
1750: av->lid = MLX_GET ( &cqe->normal, slid_smac47_32 );
1751: av->sl = MLX_GET ( &cqe->normal, sl );
1752: av->gid_present = MLX_GET ( &cqe->normal, g );
1753: memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) );
1754: break;
1755: case IB_QPT_RC:
1756: av = &qp->av;
1757: break;
1758: case IB_QPT_ETH:
1759: /* Construct address vector */
1760: av = &recv_av;
1761: av->vlan_present = MLX_GET ( &cqe->normal, vlan );
1762: av->vlan = MLX_GET ( &cqe->normal, vid );
1763: break;
1764: default:
1765: assert ( 0 );
1766: return -EINVAL;
1767: }
1768: /* Hand off to completion handler */
1769: ib_complete_recv ( ibdev, qp, av, iobuf, rc );
1770: }
1771:
1772: return rc;
1773: }
1774:
1775: /**
1776: * Poll completion queue
1777: *
1778: * @v ibdev Infiniband device
1779: * @v cq Completion queue
1780: */
1781: static void hermon_poll_cq ( struct ib_device *ibdev,
1782: struct ib_completion_queue *cq ) {
1783: struct hermon *hermon = ib_get_drvdata ( ibdev );
1784: struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
1785: union hermonprm_completion_entry *cqe;
1786: unsigned int cqe_idx_mask;
1787: int rc;
1788:
1789: while ( 1 ) {
1790: /* Look for completion entry */
1791: cqe_idx_mask = ( cq->num_cqes - 1 );
1792: cqe = &hermon_cq->cqe[cq->next_idx & cqe_idx_mask];
1793: if ( MLX_GET ( &cqe->normal, owner ) ^
1794: ( ( cq->next_idx & cq->num_cqes ) ? 1 : 0 ) ) {
1795: /* Entry still owned by hardware; end of poll */
1796: break;
1797: }
1798:
1799: /* Handle completion */
1800: if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) {
1801: DBGC ( hermon, "Hermon %p CQN %#lx failed to complete:"
1802: " %s\n", hermon, cq->cqn, strerror ( rc ) );
1803: DBGC_HDA ( hermon, virt_to_phys ( cqe ),
1804: cqe, sizeof ( *cqe ) );
1805: }
1806:
1807: /* Update completion queue's index */
1808: cq->next_idx++;
1809:
1810: /* Update doorbell record */
1811: MLX_FILL_1 ( hermon_cq->doorbell, 0, update_ci,
1812: ( cq->next_idx & 0x00ffffffUL ) );
1813: }
1814: }
1815:
1816: /***************************************************************************
1817: *
1818: * Event queues
1819: *
1820: ***************************************************************************
1821: */
1822:
1823: /**
1824: * Create event queue
1825: *
1826: * @v hermon Hermon device
1827: * @ret rc Return status code
1828: */
1829: static int hermon_create_eq ( struct hermon *hermon ) {
1830: struct hermon_event_queue *hermon_eq = &hermon->eq;
1831: struct hermonprm_eqc eqctx;
1832: struct hermonprm_event_mask mask;
1833: unsigned int i;
1834: int rc;
1835:
1836: /* Select event queue number */
1837: hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars );
1838: if ( hermon_eq->eqn < hermon->cap.reserved_eqs )
1839: hermon_eq->eqn = hermon->cap.reserved_eqs;
1840:
1841: /* Calculate doorbell address */
1842: hermon_eq->doorbell =
1843: ( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) );
1844:
1845: /* Allocate event queue itself */
1846: hermon_eq->eqe_size =
1847: ( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) );
1848: hermon_eq->eqe = malloc_dma ( hermon_eq->eqe_size,
1849: sizeof ( hermon_eq->eqe[0] ) );
1850: if ( ! hermon_eq->eqe ) {
1851: rc = -ENOMEM;
1852: goto err_eqe;
1853: }
1854: memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size );
1855: for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) {
1856: MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 );
1857: }
1858: barrier();
1859:
1860: /* Allocate MTT entries */
1861: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe,
1862: hermon_eq->eqe_size,
1863: &hermon_eq->mtt ) ) != 0 )
1864: goto err_alloc_mtt;
1865:
1866: /* Hand queue over to hardware */
1867: memset ( &eqctx, 0, sizeof ( eqctx ) );
1868: MLX_FILL_2 ( &eqctx, 0,
1869: st, 0xa /* "Fired" */,
1870: oi, 1 );
1871: MLX_FILL_1 ( &eqctx, 2,
1872: page_offset, ( hermon_eq->mtt.page_offset >> 5 ) );
1873: MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) );
1874: MLX_FILL_H ( &eqctx, 6, mtt_base_addr_h,
1875: hermon_eq->mtt.mtt_base_addr );
1876: MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l,
1877: ( hermon_eq->mtt.mtt_base_addr >> 3 ) );
1878: if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn,
1879: &eqctx ) ) != 0 ) {
1880: DBGC ( hermon, "Hermon %p EQN %#lx SW2HW_EQ failed: %s\n",
1881: hermon, hermon_eq->eqn, strerror ( rc ) );
1882: goto err_sw2hw_eq;
1883: }
1884:
1885: /* Map all events to this event queue */
1886: memset ( &mask, 0xff, sizeof ( mask ) );
1887: if ( ( rc = hermon_cmd_map_eq ( hermon,
1888: ( HERMON_MAP_EQ | hermon_eq->eqn ),
1889: &mask ) ) != 0 ) {
1890: DBGC ( hermon, "Hermon %p EQN %#lx MAP_EQ failed: %s\n",
1891: hermon, hermon_eq->eqn, strerror ( rc ) );
1892: goto err_map_eq;
1893: }
1894:
1895: DBGC ( hermon, "Hermon %p EQN %#lx ring [%08lx,%08lx), doorbell "
1896: "%08lx\n", hermon, hermon_eq->eqn,
1897: virt_to_phys ( hermon_eq->eqe ),
1898: ( virt_to_phys ( hermon_eq->eqe ) + hermon_eq->eqe_size ),
1899: virt_to_phys ( hermon_eq->doorbell ) );
1900: return 0;
1901:
1902: err_map_eq:
1903: hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx );
1904: err_sw2hw_eq:
1905: hermon_free_mtt ( hermon, &hermon_eq->mtt );
1906: err_alloc_mtt:
1907: free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
1908: err_eqe:
1909: memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
1910: return rc;
1911: }
1912:
1913: /**
1914: * Destroy event queue
1915: *
1916: * @v hermon Hermon device
1917: */
1918: static void hermon_destroy_eq ( struct hermon *hermon ) {
1919: struct hermon_event_queue *hermon_eq = &hermon->eq;
1920: struct hermonprm_eqc eqctx;
1921: struct hermonprm_event_mask mask;
1922: int rc;
1923:
1924: /* Unmap events from event queue */
1925: memset ( &mask, 0xff, sizeof ( mask ) );
1926: if ( ( rc = hermon_cmd_map_eq ( hermon,
1927: ( HERMON_UNMAP_EQ | hermon_eq->eqn ),
1928: &mask ) ) != 0 ) {
1929: DBGC ( hermon, "Hermon %p EQN %#lx FATAL MAP_EQ failed to "
1930: "unmap: %s\n", hermon, hermon_eq->eqn, strerror ( rc ) );
1931: /* Continue; HCA may die but system should survive */
1932: }
1933:
1934: /* Take ownership back from hardware */
1935: if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn,
1936: &eqctx ) ) != 0 ) {
1937: DBGC ( hermon, "Hermon %p EQN %#lx FATAL HW2SW_EQ failed: %s\n",
1938: hermon, hermon_eq->eqn, strerror ( rc ) );
1939: /* Leak memory and return; at least we avoid corruption */
1940: return;
1941: }
1942:
1943: /* Free MTT entries */
1944: hermon_free_mtt ( hermon, &hermon_eq->mtt );
1945:
1946: /* Free memory */
1947: free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
1948: memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
1949: }
1950:
1951: /**
1952: * Handle port state event
1953: *
1954: * @v hermon Hermon device
1955: * @v eqe Port state change event queue entry
1956: */
1957: static void hermon_event_port_state_change ( struct hermon *hermon,
1958: union hermonprm_event_entry *eqe){
1959: unsigned int port;
1960: int link_up;
1961:
1962: /* Get port and link status */
1963: port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
1964: link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
1965: DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ),
1966: ( link_up ? "up" : "down" ) );
1967:
1968: /* Sanity check */
1969: if ( port >= hermon->cap.num_ports ) {
1970: DBGC ( hermon, "Hermon %p port %d does not exist!\n",
1971: hermon, ( port + 1 ) );
1972: return;
1973: }
1974:
1975: /* Notify device of port state change */
1976: hermon->port[port].type->state_change ( hermon, &hermon->port[port],
1977: link_up );
1978: }
1979:
1980: /**
1981: * Poll event queue
1982: *
1983: * @v ibdev Infiniband device
1984: */
1985: static void hermon_poll_eq ( struct ib_device *ibdev ) {
1986: struct hermon *hermon = ib_get_drvdata ( ibdev );
1987: struct hermon_event_queue *hermon_eq = &hermon->eq;
1988: union hermonprm_event_entry *eqe;
1989: union hermonprm_doorbell_register db_reg;
1990: unsigned int eqe_idx_mask;
1991: unsigned int event_type;
1992:
1993: /* No event is generated upon reaching INIT, so we must poll
1994: * separately for link state changes while we remain DOWN.
1995: */
1996: if ( ib_is_open ( ibdev ) &&
1997: ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) {
1998: ib_smc_update ( ibdev, hermon_mad );
1999: }
2000:
2001: /* Poll event queue */
2002: while ( 1 ) {
2003: /* Look for event entry */
2004: eqe_idx_mask = ( HERMON_NUM_EQES - 1 );
2005: eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask];
2006: if ( MLX_GET ( &eqe->generic, owner ) ^
2007: ( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) {
2008: /* Entry still owned by hardware; end of poll */
2009: break;
2010: }
2011: DBGCP ( hermon, "Hermon %p EQN %#lx event:\n",
2012: hermon, hermon_eq->eqn );
2013: DBGCP_HDA ( hermon, virt_to_phys ( eqe ),
2014: eqe, sizeof ( *eqe ) );
2015:
2016: /* Handle event */
2017: event_type = MLX_GET ( &eqe->generic, event_type );
2018: switch ( event_type ) {
2019: case HERMON_EV_PORT_STATE_CHANGE:
2020: hermon_event_port_state_change ( hermon, eqe );
2021: break;
2022: default:
2023: DBGC ( hermon, "Hermon %p EQN %#lx unrecognised event "
2024: "type %#x:\n",
2025: hermon, hermon_eq->eqn, event_type );
2026: DBGC_HDA ( hermon, virt_to_phys ( eqe ),
2027: eqe, sizeof ( *eqe ) );
2028: break;
2029: }
2030:
2031: /* Update event queue's index */
2032: hermon_eq->next_idx++;
2033:
2034: /* Ring doorbell */
2035: MLX_FILL_1 ( &db_reg.event, 0,
2036: ci, ( hermon_eq->next_idx & 0x00ffffffUL ) );
2037: writel ( db_reg.dword[0], hermon_eq->doorbell );
2038: }
2039: }
2040:
2041: /***************************************************************************
2042: *
2043: * Infiniband link-layer operations
2044: *
2045: ***************************************************************************
2046: */
2047:
2048: /**
2049: * Initialise Infiniband link
2050: *
2051: * @v ibdev Infiniband device
2052: * @ret rc Return status code
2053: */
2054: static int hermon_open ( struct ib_device *ibdev ) {
2055: struct hermon *hermon = ib_get_drvdata ( ibdev );
2056: union hermonprm_set_port set_port;
2057: int rc;
2058:
2059: /* Set port parameters */
2060: memset ( &set_port, 0, sizeof ( set_port ) );
2061: MLX_FILL_8 ( &set_port.ib, 0,
2062: mmc, 1,
2063: mvc, 1,
2064: mp, 1,
2065: mg, 1,
2066: mtu_cap, IB_MTU_2048,
2067: vl_cap, IB_VL_0,
2068: rcm, 1,
2069: lss, 1 );
2070: MLX_FILL_2 ( &set_port.ib, 10,
2071: max_pkey, 1,
2072: max_gid, 1 );
2073: MLX_FILL_1 ( &set_port.ib, 28,
2074: link_speed_supported, 1 );
2075: if ( ( rc = hermon_cmd_set_port ( hermon, 0, ibdev->port,
2076: &set_port ) ) != 0 ) {
2077: DBGC ( hermon, "Hermon %p port %d could not set port: %s\n",
2078: hermon, ibdev->port, strerror ( rc ) );
2079: return rc;
2080: }
2081:
2082: /* Initialise port */
2083: if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
2084: DBGC ( hermon, "Hermon %p port %d could not initialise port: "
2085: "%s\n", hermon, ibdev->port, strerror ( rc ) );
2086: return rc;
2087: }
2088:
2089: /* Update MAD parameters */
2090: ib_smc_update ( ibdev, hermon_mad );
2091:
2092: return 0;
2093: }
2094:
2095: /**
2096: * Close Infiniband link
2097: *
2098: * @v ibdev Infiniband device
2099: */
2100: static void hermon_close ( struct ib_device *ibdev ) {
2101: struct hermon *hermon = ib_get_drvdata ( ibdev );
2102: int rc;
2103:
2104: if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
2105: DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
2106: hermon, ibdev->port, strerror ( rc ) );
2107: /* Nothing we can do about this */
2108: }
2109: }
2110:
2111: /**
2112: * Inform embedded subnet management agent of a received MAD
2113: *
2114: * @v ibdev Infiniband device
2115: * @v mad MAD
2116: * @ret rc Return status code
2117: */
2118: static int hermon_inform_sma ( struct ib_device *ibdev,
2119: union ib_mad *mad ) {
2120: int rc;
2121:
2122: /* Send the MAD to the embedded SMA */
2123: if ( ( rc = hermon_mad ( ibdev, mad ) ) != 0 )
2124: return rc;
2125:
2126: /* Update parameters held in software */
2127: ib_smc_update ( ibdev, hermon_mad );
2128:
2129: return 0;
2130: }
2131:
2132: /***************************************************************************
2133: *
2134: * Multicast group operations
2135: *
2136: ***************************************************************************
2137: */
2138:
2139: /**
2140: * Attach to multicast group
2141: *
2142: * @v ibdev Infiniband device
2143: * @v qp Queue pair
2144: * @v gid Multicast GID
2145: * @ret rc Return status code
2146: */
2147: static int hermon_mcast_attach ( struct ib_device *ibdev,
2148: struct ib_queue_pair *qp,
2149: union ib_gid *gid ) {
2150: struct hermon *hermon = ib_get_drvdata ( ibdev );
2151: struct hermonprm_mgm_hash hash;
2152: struct hermonprm_mcg_entry mcg;
2153: unsigned int index;
2154: int rc;
2155:
2156: /* Generate hash table index */
2157: if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
2158: DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
2159: hermon, strerror ( rc ) );
2160: return rc;
2161: }
2162: index = MLX_GET ( &hash, hash );
2163:
2164: /* Check for existing hash table entry */
2165: if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) {
2166: DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n",
2167: hermon, index, strerror ( rc ) );
2168: return rc;
2169: }
2170: if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) {
2171: /* FIXME: this implementation allows only a single QP
2172: * per multicast group, and doesn't handle hash
2173: * collisions. Sufficient for IPoIB but may need to
2174: * be extended in future.
2175: */
2176: DBGC ( hermon, "Hermon %p MGID index %#x already in use\n",
2177: hermon, index );
2178: return -EBUSY;
2179: }
2180:
2181: /* Update hash table entry */
2182: MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 );
2183: MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn );
2184: memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) );
2185: if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
2186: DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
2187: hermon, index, strerror ( rc ) );
2188: return rc;
2189: }
2190:
2191: return 0;
2192: }
2193:
2194: /**
2195: * Detach from multicast group
2196: *
2197: * @v ibdev Infiniband device
2198: * @v qp Queue pair
2199: * @v gid Multicast GID
2200: */
2201: static void hermon_mcast_detach ( struct ib_device *ibdev,
2202: struct ib_queue_pair *qp __unused,
2203: union ib_gid *gid ) {
2204: struct hermon *hermon = ib_get_drvdata ( ibdev );
2205: struct hermonprm_mgm_hash hash;
2206: struct hermonprm_mcg_entry mcg;
2207: unsigned int index;
2208: int rc;
2209:
2210: /* Generate hash table index */
2211: if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
2212: DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
2213: hermon, strerror ( rc ) );
2214: return;
2215: }
2216: index = MLX_GET ( &hash, hash );
2217:
2218: /* Clear hash table entry */
2219: memset ( &mcg, 0, sizeof ( mcg ) );
2220: if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
2221: DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
2222: hermon, index, strerror ( rc ) );
2223: return;
2224: }
2225: }
2226:
2227: /** Hermon Infiniband operations */
2228: static struct ib_device_operations hermon_ib_operations = {
2229: .create_cq = hermon_create_cq,
2230: .destroy_cq = hermon_destroy_cq,
2231: .create_qp = hermon_create_qp,
2232: .modify_qp = hermon_modify_qp,
2233: .destroy_qp = hermon_destroy_qp,
2234: .post_send = hermon_post_send,
2235: .post_recv = hermon_post_recv,
2236: .poll_cq = hermon_poll_cq,
2237: .poll_eq = hermon_poll_eq,
2238: .open = hermon_open,
2239: .close = hermon_close,
2240: .mcast_attach = hermon_mcast_attach,
2241: .mcast_detach = hermon_mcast_detach,
2242: .set_port_info = hermon_inform_sma,
2243: .set_pkey_table = hermon_inform_sma,
2244: };
2245:
2246: /**
2247: * Register Hermon Infiniband device
2248: *
2249: * @v hermon Hermon device
2250: * @v port Hermon port
2251: * @ret rc Return status code
2252: */
2253: static int hermon_register_ibdev ( struct hermon *hermon,
2254: struct hermon_port *port ) {
2255: struct ib_device *ibdev = port->ibdev;
2256: int rc;
2257:
2258: /* Initialise parameters using SMC */
2259: ib_smc_init ( ibdev, hermon_mad );
2260:
2261: /* Register Infiniband device */
2262: if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
2263: DBGC ( hermon, "Hermon %p port %d could not register IB "
2264: "device: %s\n", hermon, ibdev->port, strerror ( rc ) );
2265: return rc;
2266: }
2267:
2268: return 0;
2269: }
2270:
2271: /**
2272: * Handle Hermon Infiniband device port state change
2273: *
2274: * @v hermon Hermon device
2275: * @v port Hermon port
2276: * @v link_up Link is up
2277: */
2278: static void hermon_state_change_ibdev ( struct hermon *hermon __unused,
2279: struct hermon_port *port,
2280: int link_up __unused ) {
2281: struct ib_device *ibdev = port->ibdev;
2282:
2283: /* Update MAD parameters */
2284: ib_smc_update ( ibdev, hermon_mad );
2285: }
2286:
2287: /**
2288: * Unregister Hermon Infiniband device
2289: *
2290: * @v hermon Hermon device
2291: * @v port Hermon port
2292: */
2293: static void hermon_unregister_ibdev ( struct hermon *hermon __unused,
2294: struct hermon_port *port ) {
2295: struct ib_device *ibdev = port->ibdev;
2296:
2297: unregister_ibdev ( ibdev );
2298: }
2299:
2300: /** Hermon Infiniband port type */
2301: static struct hermon_port_type hermon_port_type_ib = {
2302: .register_dev = hermon_register_ibdev,
2303: .state_change = hermon_state_change_ibdev,
2304: .unregister_dev = hermon_unregister_ibdev,
2305: };
2306:
2307: /***************************************************************************
2308: *
2309: * Ethernet operation
2310: *
2311: ***************************************************************************
2312: */
2313:
2314: /** Number of Hermon Ethernet send work queue entries */
2315: #define HERMON_ETH_NUM_SEND_WQES 2
2316:
2317: /** Number of Hermon Ethernet receive work queue entries */
2318: #define HERMON_ETH_NUM_RECV_WQES 4
2319:
2320: /** Number of Hermon Ethernet completion entries */
2321: #define HERMON_ETH_NUM_CQES 8
2322:
2323: /**
2324: * Transmit packet via Hermon Ethernet device
2325: *
2326: * @v netdev Network device
2327: * @v iobuf I/O buffer
2328: * @ret rc Return status code
2329: */
2330: static int hermon_eth_transmit ( struct net_device *netdev,
2331: struct io_buffer *iobuf ) {
2332: struct hermon_port *port = netdev->priv;
2333: struct ib_device *ibdev = port->ibdev;
2334: struct hermon *hermon = ib_get_drvdata ( ibdev );
2335: int rc;
2336:
2337: /* Transmit packet */
2338: if ( ( rc = ib_post_send ( ibdev, port->eth_qp, NULL,
2339: iobuf ) ) != 0 ) {
2340: DBGC ( hermon, "Hermon %p port %d could not transmit: %s\n",
2341: hermon, ibdev->port, strerror ( rc ) );
2342: return rc;
2343: }
2344:
2345: return 0;
2346: }
2347:
2348: /**
2349: * Handle Hermon Ethernet device send completion
2350: *
2351: * @v ibdev Infiniband device
2352: * @v qp Queue pair
2353: * @v iobuf I/O buffer
2354: * @v rc Completion status code
2355: */
2356: static void hermon_eth_complete_send ( struct ib_device *ibdev __unused,
2357: struct ib_queue_pair *qp,
2358: struct io_buffer *iobuf, int rc ) {
2359: struct net_device *netdev = ib_qp_get_ownerdata ( qp );
2360:
2361: netdev_tx_complete_err ( netdev, iobuf, rc );
2362: }
2363:
2364: /**
2365: * Handle Hermon Ethernet device receive completion
2366: *
2367: * @v ibdev Infiniband device
2368: * @v qp Queue pair
2369: * @v av Address vector, or NULL
2370: * @v iobuf I/O buffer
2371: * @v rc Completion status code
2372: */
2373: static void hermon_eth_complete_recv ( struct ib_device *ibdev __unused,
2374: struct ib_queue_pair *qp,
2375: struct ib_address_vector *av,
2376: struct io_buffer *iobuf, int rc ) {
2377: struct net_device *netdev = ib_qp_get_ownerdata ( qp );
2378: struct net_device *vlan;
2379:
2380: /* Find VLAN device, if applicable */
2381: if ( av->vlan_present ) {
2382: if ( ( vlan = vlan_find ( netdev, av->vlan ) ) != NULL ) {
2383: netdev = vlan;
2384: } else if ( rc == 0 ) {
2385: rc = -ENODEV;
2386: }
2387: }
2388:
2389: /* Hand off to network layer */
2390: if ( rc == 0 ) {
2391: netdev_rx ( netdev, iobuf );
2392: } else {
2393: netdev_rx_err ( netdev, iobuf, rc );
2394: }
2395: }
2396:
2397: /** Hermon Ethernet device completion operations */
2398: static struct ib_completion_queue_operations hermon_eth_cq_op = {
2399: .complete_send = hermon_eth_complete_send,
2400: .complete_recv = hermon_eth_complete_recv,
2401: };
2402:
2403: /**
2404: * Poll Hermon Ethernet device
2405: *
2406: * @v netdev Network device
2407: */
2408: static void hermon_eth_poll ( struct net_device *netdev ) {
2409: struct hermon_port *port = netdev->priv;
2410: struct ib_device *ibdev = port->ibdev;
2411:
2412: ib_poll_eq ( ibdev );
2413: }
2414:
2415: /**
2416: * Open Hermon Ethernet device
2417: *
2418: * @v netdev Network device
2419: * @ret rc Return status code
2420: */
2421: static int hermon_eth_open ( struct net_device *netdev ) {
2422: struct hermon_port *port = netdev->priv;
2423: struct ib_device *ibdev = port->ibdev;
2424: struct hermon *hermon = ib_get_drvdata ( ibdev );
2425: union hermonprm_set_port set_port;
2426: int rc;
2427:
2428: /* Allocate completion queue */
2429: port->eth_cq = ib_create_cq ( ibdev, HERMON_ETH_NUM_CQES,
2430: &hermon_eth_cq_op );
2431: if ( ! port->eth_cq ) {
2432: DBGC ( hermon, "Hermon %p port %d could not create completion "
2433: "queue\n", hermon, ibdev->port );
2434: rc = -ENOMEM;
2435: goto err_create_cq;
2436: }
2437:
2438: /* Allocate queue pair */
2439: port->eth_qp = ib_create_qp ( ibdev, IB_QPT_ETH,
2440: HERMON_ETH_NUM_SEND_WQES, port->eth_cq,
2441: HERMON_ETH_NUM_RECV_WQES, port->eth_cq );
2442: if ( ! port->eth_qp ) {
2443: DBGC ( hermon, "Hermon %p port %d could not create queue "
2444: "pair\n", hermon, ibdev->port );
2445: rc = -ENOMEM;
2446: goto err_create_qp;
2447: }
2448: ib_qp_set_ownerdata ( port->eth_qp, netdev );
2449:
2450: /* Activate queue pair */
2451: if ( ( rc = ib_modify_qp ( ibdev, port->eth_qp ) ) != 0 ) {
2452: DBGC ( hermon, "Hermon %p port %d could not modify queue "
2453: "pair: %s\n", hermon, ibdev->port, strerror ( rc ) );
2454: goto err_modify_qp;
2455: }
2456:
2457: /* Fill receive rings */
2458: ib_refill_recv ( ibdev, port->eth_qp );
2459:
2460: /* Set port general parameters */
2461: memset ( &set_port, 0, sizeof ( set_port ) );
2462: MLX_FILL_3 ( &set_port.general, 0,
2463: v_mtu, 1,
2464: v_pprx, 1,
2465: v_pptx, 1 );
2466: MLX_FILL_1 ( &set_port.general, 1,
2467: mtu, ( ETH_FRAME_LEN + 40 /* Used by card */ ) );
2468: MLX_FILL_1 ( &set_port.general, 2,
2469: pfctx, ( 1 << FCOE_VLAN_PRIORITY ) );
2470: MLX_FILL_1 ( &set_port.general, 3,
2471: pfcrx, ( 1 << FCOE_VLAN_PRIORITY ) );
2472: if ( ( rc = hermon_cmd_set_port ( hermon, 1,
2473: ( HERMON_SET_PORT_GENERAL_PARAM |
2474: ibdev->port ),
2475: &set_port ) ) != 0 ) {
2476: DBGC ( hermon, "Hermon %p port %d could not set port general "
2477: "parameters: %s\n",
2478: hermon, ibdev->port, strerror ( rc ) );
2479: goto err_set_port_general_params;
2480: }
2481:
2482: /* Set port receive QP */
2483: memset ( &set_port, 0, sizeof ( set_port ) );
2484: MLX_FILL_1 ( &set_port.rqp_calc, 0, base_qpn, port->eth_qp->qpn );
2485: MLX_FILL_1 ( &set_port.rqp_calc, 2,
2486: mac_miss_index, 128 /* MAC misses go to promisc QP */ );
2487: MLX_FILL_2 ( &set_port.rqp_calc, 3,
2488: vlan_miss_index, 127 /* VLAN misses go to promisc QP */,
2489: no_vlan_index, 126 /* VLAN-free go to promisc QP */ );
2490: MLX_FILL_2 ( &set_port.rqp_calc, 5,
2491: promisc_qpn, port->eth_qp->qpn,
2492: en_uc_promisc, 1 );
2493: MLX_FILL_2 ( &set_port.rqp_calc, 6,
2494: def_mcast_qpn, port->eth_qp->qpn,
2495: mc_promisc_mode, 2 /* Receive all multicasts */ );
2496: if ( ( rc = hermon_cmd_set_port ( hermon, 1,
2497: ( HERMON_SET_PORT_RECEIVE_QP |
2498: ibdev->port ),
2499: &set_port ) ) != 0 ) {
2500: DBGC ( hermon, "Hermon %p port %d could not set port receive "
2501: "QP: %s\n", hermon, ibdev->port, strerror ( rc ) );
2502: goto err_set_port_receive_qp;
2503: }
2504:
2505: /* Initialise port */
2506: if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
2507: DBGC ( hermon, "Hermon %p port %d could not initialise port: "
2508: "%s\n", hermon, ibdev->port, strerror ( rc ) );
2509: goto err_init_port;
2510: }
2511:
2512: return 0;
2513:
2514: err_init_port:
2515: err_set_port_receive_qp:
2516: err_set_port_general_params:
2517: err_modify_qp:
2518: ib_destroy_qp ( ibdev, port->eth_qp );
2519: err_create_qp:
2520: ib_destroy_cq ( ibdev, port->eth_cq );
2521: err_create_cq:
2522: return rc;
2523: }
2524:
2525: /**
2526: * Close Hermon Ethernet device
2527: *
2528: * @v netdev Network device
2529: */
2530: static void hermon_eth_close ( struct net_device *netdev ) {
2531: struct hermon_port *port = netdev->priv;
2532: struct ib_device *ibdev = port->ibdev;
2533: struct hermon *hermon = ib_get_drvdata ( ibdev );
2534: int rc;
2535:
2536: /* Close port */
2537: if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
2538: DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
2539: hermon, ibdev->port, strerror ( rc ) );
2540: /* Nothing we can do about this */
2541: }
2542:
2543: /* Tear down the queues */
2544: ib_destroy_qp ( ibdev, port->eth_qp );
2545: ib_destroy_cq ( ibdev, port->eth_cq );
2546: }
2547:
2548: /** Hermon Ethernet network device operations */
2549: static struct net_device_operations hermon_eth_operations = {
2550: .open = hermon_eth_open,
2551: .close = hermon_eth_close,
2552: .transmit = hermon_eth_transmit,
2553: .poll = hermon_eth_poll,
2554: };
2555:
2556: /**
2557: * Register Hermon Ethernet device
2558: *
2559: * @v hermon Hermon device
2560: * @v port Hermon port
2561: * @ret rc Return status code
2562: */
2563: static int hermon_register_netdev ( struct hermon *hermon,
2564: struct hermon_port *port ) {
2565: struct net_device *netdev = port->netdev;
2566: struct ib_device *ibdev = port->ibdev;
2567: struct hermonprm_query_port_cap query_port;
2568: union {
2569: uint8_t bytes[8];
2570: uint32_t dwords[2];
2571: } mac;
2572: int rc;
2573:
2574: /* Retrieve MAC address */
2575: if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port,
2576: &query_port ) ) != 0 ) {
2577: DBGC ( hermon, "Hermon %p port %d could not query port: %s\n",
2578: hermon, ibdev->port, strerror ( rc ) );
2579: return rc;
2580: }
2581: mac.dwords[0] = htonl ( MLX_GET ( &query_port, mac_47_32 ) );
2582: mac.dwords[1] = htonl ( MLX_GET ( &query_port, mac_31_0 ) );
2583: memcpy ( netdev->hw_addr,
2584: &mac.bytes[ sizeof ( mac.bytes ) - ETH_ALEN ], ETH_ALEN );
2585:
2586: /* Register network device */
2587: if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
2588: DBGC ( hermon, "Hermon %p port %d could not register network "
2589: "device: %s\n", hermon, ibdev->port, strerror ( rc ) );
2590: return rc;
2591: }
2592:
2593: return 0;
2594: }
2595:
2596: /**
2597: * Handle Hermon Ethernet device port state change
2598: *
2599: * @v hermon Hermon device
2600: * @v port Hermon port
2601: * @v link_up Link is up
2602: */
2603: static void hermon_state_change_netdev ( struct hermon *hermon __unused,
2604: struct hermon_port *port,
2605: int link_up ) {
2606: struct net_device *netdev = port->netdev;
2607:
2608: if ( link_up ) {
2609: netdev_link_up ( netdev );
2610: } else {
2611: netdev_link_down ( netdev );
2612: }
2613: }
2614:
2615: /**
2616: * Unregister Hermon Ethernet device
2617: *
2618: * @v hermon Hermon device
2619: * @v port Hermon port
2620: */
2621: static void hermon_unregister_netdev ( struct hermon *hermon __unused,
2622: struct hermon_port *port ) {
2623: struct net_device *netdev = port->netdev;
2624:
2625: unregister_netdev ( netdev );
2626: }
2627:
2628: /** Hermon Ethernet port type */
2629: static struct hermon_port_type hermon_port_type_eth = {
2630: .register_dev = hermon_register_netdev,
2631: .state_change = hermon_state_change_netdev,
2632: .unregister_dev = hermon_unregister_netdev,
2633: };
2634:
2635: /***************************************************************************
2636: *
2637: * Port type detection
2638: *
2639: ***************************************************************************
2640: */
2641:
2642: /** Timeout for port sensing */
2643: #define HERMON_SENSE_PORT_TIMEOUT ( TICKS_PER_SEC / 2 )
2644:
2645: /**
2646: * Name port type
2647: *
2648: * @v port_type Port type
2649: * @v port_type_name Port type name
2650: */
2651: static inline const char * hermon_name_port_type ( unsigned int port_type ) {
2652: switch ( port_type ) {
2653: case HERMON_PORT_TYPE_UNKNOWN: return "unknown";
2654: case HERMON_PORT_TYPE_IB: return "Infiniband";
2655: case HERMON_PORT_TYPE_ETH: return "Ethernet";
2656: default: return "INVALID";
2657: }
2658: }
2659:
2660: /**
2661: * Sense port type
2662: *
2663: * @v hermon Hermon device
2664: * @v port Hermon port
2665: * @ret port_type Port type, or negative error
2666: */
2667: static int hermon_sense_port_type ( struct hermon *hermon,
2668: struct hermon_port *port ) {
2669: struct ib_device *ibdev = port->ibdev;
2670: struct hermonprm_sense_port sense_port;
2671: int port_type;
2672: int rc;
2673:
2674: /* If DPDP is not supported, always assume Infiniband */
2675: if ( ! hermon->cap.dpdp ) {
2676: port_type = HERMON_PORT_TYPE_IB;
2677: DBGC ( hermon, "Hermon %p port %d does not support DPDP; "
2678: "assuming an %s network\n", hermon, ibdev->port,
2679: hermon_name_port_type ( port_type ) );
2680: return port_type;
2681: }
2682:
2683: /* Sense the port type */
2684: if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
2685: &sense_port ) ) != 0 ) {
2686: DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
2687: hermon, ibdev->port, strerror ( rc ) );
2688: return rc;
2689: }
2690: port_type = MLX_GET ( &sense_port, port_type );
2691:
2692: DBGC ( hermon, "Hermon %p port %d sensed an %s network\n",
2693: hermon, ibdev->port, hermon_name_port_type ( port_type ) );
2694: return port_type;
2695: }
2696:
2697: /**
2698: * Set port type
2699: *
2700: * @v hermon Hermon device
2701: * @v port Hermon port
2702: * @ret rc Return status code
2703: */
2704: static int hermon_set_port_type ( struct hermon *hermon,
2705: struct hermon_port *port ) {
2706: struct ib_device *ibdev = port->ibdev;
2707: struct hermonprm_query_port_cap query_port;
2708: int ib_supported;
2709: int eth_supported;
2710: int port_type;
2711: unsigned long start;
2712: unsigned long elapsed;
2713: int rc;
2714:
2715: /* Check to see which types are supported */
2716: if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port,
2717: &query_port ) ) != 0 ) {
2718: DBGC ( hermon, "Hermon %p port %d could not query port: %s\n",
2719: hermon, ibdev->port, strerror ( rc ) );
2720: return rc;
2721: }
2722: ib_supported = MLX_GET ( &query_port, ib );
2723: eth_supported = MLX_GET ( &query_port, eth );
2724: DBGC ( hermon, "Hermon %p port %d supports%s%s%s\n",
2725: hermon, ibdev->port, ( ib_supported ? " Infiniband" : "" ),
2726: ( ( ib_supported && eth_supported ) ? " and" : "" ),
2727: ( eth_supported ? " Ethernet" : "" ) );
2728:
2729: /* Sense network, if applicable */
2730: if ( ib_supported && eth_supported ) {
2731:
2732: /* Both types are supported; try sensing network */
2733: start = currticks();
2734: do {
2735: /* Try sensing port */
2736: port_type = hermon_sense_port_type ( hermon, port );
2737: if ( port_type < 0 ) {
2738: rc = port_type;
2739: return rc;
2740: }
2741: } while ( ( port_type == HERMON_PORT_TYPE_UNKNOWN ) &&
2742: ( ( elapsed = ( currticks() - start ) ) <
2743: HERMON_SENSE_PORT_TIMEOUT ) );
2744:
2745: /* Set port type based on sensed network, defaulting
2746: * to Infiniband if nothing was sensed.
2747: */
2748: switch ( port_type ) {
2749: case HERMON_PORT_TYPE_ETH:
2750: port->type = &hermon_port_type_eth;
2751: break;
2752: case HERMON_PORT_TYPE_IB:
2753: case HERMON_PORT_TYPE_UNKNOWN:
2754: port->type = &hermon_port_type_ib;
2755: break;
2756: default:
2757: return -EINVAL;
2758: }
2759:
2760: } else if ( eth_supported ) {
2761: port->type = &hermon_port_type_eth;
2762: } else {
2763: port->type = &hermon_port_type_ib;
2764: }
2765:
2766: assert ( port->type != NULL );
2767: return 0;
2768: }
2769:
2770: /***************************************************************************
2771: *
2772: * Firmware control
2773: *
2774: ***************************************************************************
2775: */
2776:
2777: /**
2778: * Map virtual to physical address for firmware usage
2779: *
2780: * @v hermon Hermon device
2781: * @v map Mapping function
2782: * @v va Virtual address
2783: * @v pa Physical address
2784: * @v len Length of region
2785: * @ret rc Return status code
2786: */
2787: static int hermon_map_vpm ( struct hermon *hermon,
2788: int ( *map ) ( struct hermon *hermon,
2789: const struct hermonprm_virtual_physical_mapping* ),
2790: uint64_t va, physaddr_t pa, size_t len ) {
2791: struct hermonprm_virtual_physical_mapping mapping;
2792: physaddr_t start;
2793: physaddr_t low;
2794: physaddr_t high;
2795: physaddr_t end;
2796: size_t size;
2797: int rc;
2798:
2799: /* Sanity checks */
2800: assert ( ( va & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
2801: assert ( ( pa & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
2802: assert ( ( len & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
2803:
2804: /* Calculate starting points */
2805: start = pa;
2806: end = ( start + len );
2807: size = ( 1UL << ( fls ( start ^ end ) - 1 ) );
2808: low = high = ( end & ~( size - 1 ) );
2809: assert ( start < low );
2810: assert ( high <= end );
2811:
2812: /* These mappings tend to generate huge volumes of
2813: * uninteresting debug data, which basically makes it
2814: * impossible to use debugging otherwise.
2815: */
2816: DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
2817:
2818: /* Map blocks in descending order of size */
2819: while ( size >= HERMON_PAGE_SIZE ) {
2820:
2821: /* Find the next candidate block */
2822: if ( ( low - size ) >= start ) {
2823: low -= size;
2824: pa = low;
2825: } else if ( ( high + size ) <= end ) {
2826: pa = high;
2827: high += size;
2828: } else {
2829: size >>= 1;
2830: continue;
2831: }
2832: assert ( ( va & ( size - 1 ) ) == 0 );
2833: assert ( ( pa & ( size - 1 ) ) == 0 );
2834:
2835: /* Map this block */
2836: memset ( &mapping, 0, sizeof ( mapping ) );
2837: MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) );
2838: MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) );
2839: MLX_FILL_H ( &mapping, 2, pa_h, pa );
2840: MLX_FILL_2 ( &mapping, 3,
2841: log2size, ( ( fls ( size ) - 1 ) - 12 ),
2842: pa_l, ( pa >> 12 ) );
2843: if ( ( rc = map ( hermon, &mapping ) ) != 0 ) {
2844: DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
2845: DBGC ( hermon, "Hermon %p could not map %08llx+%zx to "
2846: "%08lx: %s\n",
2847: hermon, va, size, pa, strerror ( rc ) );
2848: return rc;
2849: }
2850: va += size;
2851: }
2852: assert ( low == start );
2853: assert ( high == end );
2854:
2855: DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
2856: return 0;
2857: }
2858:
2859: /**
2860: * Start firmware running
2861: *
2862: * @v hermon Hermon device
2863: * @ret rc Return status code
2864: */
2865: static int hermon_start_firmware ( struct hermon *hermon ) {
2866: struct hermonprm_query_fw fw;
2867: unsigned int fw_pages;
2868: size_t fw_size;
2869: physaddr_t fw_base;
2870: int rc;
2871:
2872: /* Get firmware parameters */
2873: if ( ( rc = hermon_cmd_query_fw ( hermon, &fw ) ) != 0 ) {
2874: DBGC ( hermon, "Hermon %p could not query firmware: %s\n",
2875: hermon, strerror ( rc ) );
2876: goto err_query_fw;
2877: }
2878: DBGC ( hermon, "Hermon %p firmware version %d.%d.%d\n", hermon,
2879: MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
2880: MLX_GET ( &fw, fw_rev_subminor ) );
2881: fw_pages = MLX_GET ( &fw, fw_pages );
2882: DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n",
2883: hermon, fw_pages, ( fw_pages * 4 ) );
2884:
2885: /* Allocate firmware pages and map firmware area */
2886: fw_size = ( fw_pages * HERMON_PAGE_SIZE );
2887: hermon->firmware_area = umalloc ( fw_size );
2888: if ( ! hermon->firmware_area ) {
2889: rc = -ENOMEM;
2890: goto err_alloc_fa;
2891: }
2892: fw_base = user_to_phys ( hermon->firmware_area, 0 );
2893: DBGC ( hermon, "Hermon %p firmware area at physical [%08lx,%08lx)\n",
2894: hermon, fw_base, ( fw_base + fw_size ) );
2895: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_fa,
2896: 0, fw_base, fw_size ) ) != 0 ) {
2897: DBGC ( hermon, "Hermon %p could not map firmware: %s\n",
2898: hermon, strerror ( rc ) );
2899: goto err_map_fa;
2900: }
2901:
2902: /* Start firmware */
2903: if ( ( rc = hermon_cmd_run_fw ( hermon ) ) != 0 ) {
2904: DBGC ( hermon, "Hermon %p could not run firmware: %s\n",
2905: hermon, strerror ( rc ) );
2906: goto err_run_fw;
2907: }
2908:
2909: DBGC ( hermon, "Hermon %p firmware started\n", hermon );
2910: return 0;
2911:
2912: err_run_fw:
2913: err_map_fa:
2914: hermon_cmd_unmap_fa ( hermon );
2915: ufree ( hermon->firmware_area );
2916: hermon->firmware_area = UNULL;
2917: err_alloc_fa:
2918: err_query_fw:
2919: return rc;
2920: }
2921:
2922: /**
2923: * Stop firmware running
2924: *
2925: * @v hermon Hermon device
2926: */
2927: static void hermon_stop_firmware ( struct hermon *hermon ) {
2928: int rc;
2929:
2930: if ( ( rc = hermon_cmd_unmap_fa ( hermon ) ) != 0 ) {
2931: DBGC ( hermon, "Hermon %p FATAL could not stop firmware: %s\n",
2932: hermon, strerror ( rc ) );
2933: /* Leak memory and return; at least we avoid corruption */
2934: return;
2935: }
2936: ufree ( hermon->firmware_area );
2937: hermon->firmware_area = UNULL;
2938: }
2939:
2940: /***************************************************************************
2941: *
2942: * Infinihost Context Memory management
2943: *
2944: ***************************************************************************
2945: */
2946:
2947: /**
2948: * Get device limits
2949: *
2950: * @v hermon Hermon device
2951: * @ret rc Return status code
2952: */
2953: static int hermon_get_cap ( struct hermon *hermon ) {
2954: struct hermonprm_query_dev_cap dev_cap;
2955: int rc;
2956:
2957: if ( ( rc = hermon_cmd_query_dev_cap ( hermon, &dev_cap ) ) != 0 ) {
2958: DBGC ( hermon, "Hermon %p could not get device limits: %s\n",
2959: hermon, strerror ( rc ) );
2960: return rc;
2961: }
2962:
2963: hermon->cap.cmpt_entry_size = MLX_GET ( &dev_cap, c_mpt_entry_sz );
2964: hermon->cap.reserved_qps =
2965: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_qps ) );
2966: hermon->cap.qpc_entry_size = MLX_GET ( &dev_cap, qpc_entry_sz );
2967: hermon->cap.altc_entry_size = MLX_GET ( &dev_cap, altc_entry_sz );
2968: hermon->cap.auxc_entry_size = MLX_GET ( &dev_cap, aux_entry_sz );
2969: hermon->cap.reserved_srqs =
2970: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_srqs ) );
2971: hermon->cap.srqc_entry_size = MLX_GET ( &dev_cap, srq_entry_sz );
2972: hermon->cap.reserved_cqs =
2973: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_cqs ) );
2974: hermon->cap.cqc_entry_size = MLX_GET ( &dev_cap, cqc_entry_sz );
2975: hermon->cap.reserved_eqs = MLX_GET ( &dev_cap, num_rsvd_eqs );
2976: if ( hermon->cap.reserved_eqs == 0 ) {
2977: /* Backward compatibility */
2978: hermon->cap.reserved_eqs =
2979: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_eqs ) );
2980: }
2981: hermon->cap.eqc_entry_size = MLX_GET ( &dev_cap, eqc_entry_sz );
2982: hermon->cap.reserved_mtts =
2983: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mtts ) );
2984: hermon->cap.mtt_entry_size = MLX_GET ( &dev_cap, mtt_entry_sz );
2985: hermon->cap.reserved_mrws =
2986: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
2987: hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
2988: hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
2989: hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
2990: hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );
2991:
2992: /* Sanity check */
2993: if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
2994: DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
2995: hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
2996: hermon->cap.num_ports = HERMON_MAX_PORTS;
2997: }
2998:
2999: return 0;
3000: }
3001:
3002: /**
3003: * Align ICM table
3004: *
3005: * @v icm_offset Current ICM offset
3006: * @v len ICM table length
3007: * @ret icm_offset ICM offset
3008: */
3009: static uint64_t icm_align ( uint64_t icm_offset, size_t len ) {
3010:
3011: /* Round up to a multiple of the table size */
3012: assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
3013: return ( ( icm_offset + len - 1 ) & ~( ( ( uint64_t ) len ) - 1 ) );
3014: }
3015:
3016: /**
3017: * Allocate ICM
3018: *
3019: * @v hermon Hermon device
3020: * @v init_hca INIT_HCA structure to fill in
3021: * @ret rc Return status code
3022: */
3023: static int hermon_alloc_icm ( struct hermon *hermon,
3024: struct hermonprm_init_hca *init_hca ) {
3025: struct hermonprm_scalar_parameter icm_size;
3026: struct hermonprm_scalar_parameter icm_aux_size;
3027: uint64_t icm_offset = 0;
3028: unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs;
3029: unsigned int log_num_mtts, log_num_mpts, log_num_mcs;
3030: size_t cmpt_max_len;
3031: size_t icm_len, icm_aux_len;
3032: size_t len;
3033: physaddr_t icm_phys;
3034: int i;
3035: int rc;
3036:
3037: /*
3038: * Start by carving up the ICM virtual address space
3039: *
3040: */
3041:
3042: /* Calculate number of each object type within ICM */
3043: log_num_qps = fls ( hermon->cap.reserved_qps +
3044: HERMON_RSVD_SPECIAL_QPS + HERMON_MAX_QPS - 1 );
3045: log_num_srqs = fls ( hermon->cap.reserved_srqs - 1 );
3046: log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 );
3047: log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 );
3048: log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 );
3049: log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 );
3050: log_num_mcs = HERMON_LOG_MULTICAST_HASH_SIZE;
3051:
3052: /* ICM starts with the cMPT tables, which are sparse */
3053: cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES *
3054: ( ( uint64_t ) hermon->cap.cmpt_entry_size ) );
3055: len = ( ( ( ( 1 << log_num_qps ) * hermon->cap.cmpt_entry_size ) +
3056: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
3057: hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset;
3058: hermon->icm_map[HERMON_ICM_QP_CMPT].len = len;
3059: icm_offset += cmpt_max_len;
3060: len = ( ( ( ( 1 << log_num_srqs ) * hermon->cap.cmpt_entry_size ) +
3061: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
3062: hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset;
3063: hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = len;
3064: icm_offset += cmpt_max_len;
3065: len = ( ( ( ( 1 << log_num_cqs ) * hermon->cap.cmpt_entry_size ) +
3066: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
3067: hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset;
3068: hermon->icm_map[HERMON_ICM_CQ_CMPT].len = len;
3069: icm_offset += cmpt_max_len;
3070: len = ( ( ( ( 1 << log_num_eqs ) * hermon->cap.cmpt_entry_size ) +
3071: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
3072: hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset;
3073: hermon->icm_map[HERMON_ICM_EQ_CMPT].len = len;
3074: icm_offset += cmpt_max_len;
3075:
3076: hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset;
3077:
3078: /* Queue pair contexts */
3079: len = ( ( 1 << log_num_qps ) * hermon->cap.qpc_entry_size );
3080: icm_offset = icm_align ( icm_offset, len );
3081: MLX_FILL_1 ( init_hca, 12,
3082: qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h,
3083: ( icm_offset >> 32 ) );
3084: MLX_FILL_2 ( init_hca, 13,
3085: qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l,
3086: ( icm_offset >> 5 ),
3087: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
3088: log_num_qps );
3089: DBGC ( hermon, "Hermon %p ICM QPC is %d x %#zx at [%08llx,%08llx)\n",
3090: hermon, ( 1 << log_num_qps ), hermon->cap.qpc_entry_size,
3091: icm_offset, ( icm_offset + len ) );
3092: icm_offset += len;
3093:
3094: /* Extended alternate path contexts */
3095: len = ( ( 1 << log_num_qps ) * hermon->cap.altc_entry_size );
3096: icm_offset = icm_align ( icm_offset, len );
3097: MLX_FILL_1 ( init_hca, 24,
3098: qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h,
3099: ( icm_offset >> 32 ) );
3100: MLX_FILL_1 ( init_hca, 25,
3101: qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l,
3102: icm_offset );
3103: DBGC ( hermon, "Hermon %p ICM ALTC is %d x %#zx at [%08llx,%08llx)\n",
3104: hermon, ( 1 << log_num_qps ), hermon->cap.altc_entry_size,
3105: icm_offset, ( icm_offset + len ) );
3106: icm_offset += len;
3107:
3108: /* Extended auxiliary contexts */
3109: len = ( ( 1 << log_num_qps ) * hermon->cap.auxc_entry_size );
3110: icm_offset = icm_align ( icm_offset, len );
3111: MLX_FILL_1 ( init_hca, 28,
3112: qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h,
3113: ( icm_offset >> 32 ) );
3114: MLX_FILL_1 ( init_hca, 29,
3115: qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l,
3116: icm_offset );
3117: DBGC ( hermon, "Hermon %p ICM AUXC is %d x %#zx at [%08llx,%08llx)\n",
3118: hermon, ( 1 << log_num_qps ), hermon->cap.auxc_entry_size,
3119: icm_offset, ( icm_offset + len ) );
3120: icm_offset += len;
3121:
3122: /* Shared receive queue contexts */
3123: len = ( ( 1 << log_num_srqs ) * hermon->cap.srqc_entry_size );
3124: icm_offset = icm_align ( icm_offset, len );
3125: MLX_FILL_1 ( init_hca, 18,
3126: qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h,
3127: ( icm_offset >> 32 ) );
3128: MLX_FILL_2 ( init_hca, 19,
3129: qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l,
3130: ( icm_offset >> 5 ),
3131: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
3132: log_num_srqs );
3133: DBGC ( hermon, "Hermon %p ICM SRQC is %d x %#zx at [%08llx,%08llx)\n",
3134: hermon, ( 1 << log_num_srqs ), hermon->cap.srqc_entry_size,
3135: icm_offset, ( icm_offset + len ) );
3136: icm_offset += len;
3137:
3138: /* Completion queue contexts */
3139: len = ( ( 1 << log_num_cqs ) * hermon->cap.cqc_entry_size );
3140: icm_offset = icm_align ( icm_offset, len );
3141: MLX_FILL_1 ( init_hca, 20,
3142: qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h,
3143: ( icm_offset >> 32 ) );
3144: MLX_FILL_2 ( init_hca, 21,
3145: qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l,
3146: ( icm_offset >> 5 ),
3147: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
3148: log_num_cqs );
3149: DBGC ( hermon, "Hermon %p ICM CQC is %d x %#zx at [%08llx,%08llx)\n",
3150: hermon, ( 1 << log_num_cqs ), hermon->cap.cqc_entry_size,
3151: icm_offset, ( icm_offset + len ) );
3152: icm_offset += len;
3153:
3154: /* Event queue contexts */
3155: len = ( ( 1 << log_num_eqs ) * hermon->cap.eqc_entry_size );
3156: icm_offset = icm_align ( icm_offset, len );
3157: MLX_FILL_1 ( init_hca, 32,
3158: qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h,
3159: ( icm_offset >> 32 ) );
3160: MLX_FILL_2 ( init_hca, 33,
3161: qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l,
3162: ( icm_offset >> 5 ),
3163: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq,
3164: log_num_eqs );
3165: DBGC ( hermon, "Hermon %p ICM EQC is %d x %#zx at [%08llx,%08llx)\n",
3166: hermon, ( 1 << log_num_eqs ), hermon->cap.eqc_entry_size,
3167: icm_offset, ( icm_offset + len ) );
3168: icm_offset += len;
3169:
3170: /* Memory translation table */
3171: len = ( ( 1 << log_num_mtts ) * hermon->cap.mtt_entry_size );
3172: icm_offset = icm_align ( icm_offset, len );
3173: MLX_FILL_1 ( init_hca, 64,
3174: tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) );
3175: MLX_FILL_1 ( init_hca, 65,
3176: tpt_parameters.mtt_base_addr_l, icm_offset );
3177: DBGC ( hermon, "Hermon %p ICM MTT is %d x %#zx at [%08llx,%08llx)\n",
3178: hermon, ( 1 << log_num_mtts ), hermon->cap.mtt_entry_size,
3179: icm_offset, ( icm_offset + len ) );
3180: icm_offset += len;
3181:
3182: /* Memory protection table */
3183: len = ( ( 1 << log_num_mpts ) * hermon->cap.dmpt_entry_size );
3184: icm_offset = icm_align ( icm_offset, len );
3185: MLX_FILL_1 ( init_hca, 60,
3186: tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) );
3187: MLX_FILL_1 ( init_hca, 61,
3188: tpt_parameters.dmpt_base_adr_l, icm_offset );
3189: MLX_FILL_1 ( init_hca, 62,
3190: tpt_parameters.log_dmpt_sz, log_num_mpts );
3191: DBGC ( hermon, "Hermon %p ICM DMPT is %d x %#zx at [%08llx,%08llx)\n",
3192: hermon, ( 1 << log_num_mpts ), hermon->cap.dmpt_entry_size,
3193: icm_offset, ( icm_offset + len ) );
3194: icm_offset += len;
3195:
3196: /* Multicast table */
3197: len = ( ( 1 << log_num_mcs ) * sizeof ( struct hermonprm_mcg_entry ) );
3198: icm_offset = icm_align ( icm_offset, len );
3199: MLX_FILL_1 ( init_hca, 48,
3200: multicast_parameters.mc_base_addr_h,
3201: ( icm_offset >> 32 ) );
3202: MLX_FILL_1 ( init_hca, 49,
3203: multicast_parameters.mc_base_addr_l, icm_offset );
3204: MLX_FILL_1 ( init_hca, 52,
3205: multicast_parameters.log_mc_table_entry_sz,
3206: fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) );
3207: MLX_FILL_1 ( init_hca, 53,
3208: multicast_parameters.log_mc_table_hash_sz, log_num_mcs );
3209: MLX_FILL_1 ( init_hca, 54,
3210: multicast_parameters.log_mc_table_sz, log_num_mcs );
3211: DBGC ( hermon, "Hermon %p ICM MC is %d x %#zx at [%08llx,%08llx)\n",
3212: hermon, ( 1 << log_num_mcs ),
3213: sizeof ( struct hermonprm_mcg_entry ),
3214: icm_offset, ( icm_offset + len ) );
3215: icm_offset += len;
3216:
3217:
3218: hermon->icm_map[HERMON_ICM_OTHER].len =
3219: ( icm_offset - hermon->icm_map[HERMON_ICM_OTHER].offset );
3220:
3221: /*
3222: * Allocate and map physical memory for (portions of) ICM
3223: *
3224: * Map is:
3225: * ICM AUX area (aligned to its own size)
3226: * cMPT areas
3227: * Other areas
3228: */
3229:
3230: /* Calculate physical memory required for ICM */
3231: icm_len = 0;
3232: for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
3233: icm_len += hermon->icm_map[i].len;
3234: }
3235:
3236: /* Get ICM auxiliary area size */
3237: memset ( &icm_size, 0, sizeof ( icm_size ) );
3238: MLX_FILL_1 ( &icm_size, 0, value_hi, ( icm_offset >> 32 ) );
3239: MLX_FILL_1 ( &icm_size, 1, value, icm_offset );
3240: if ( ( rc = hermon_cmd_set_icm_size ( hermon, &icm_size,
3241: &icm_aux_size ) ) != 0 ) {
3242: DBGC ( hermon, "Hermon %p could not set ICM size: %s\n",
3243: hermon, strerror ( rc ) );
3244: goto err_set_icm_size;
3245: }
3246: icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * HERMON_PAGE_SIZE );
3247:
3248: /* Allocate ICM data and auxiliary area */
3249: DBGC ( hermon, "Hermon %p requires %zd kB ICM and %zd kB AUX ICM\n",
3250: hermon, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
3251: hermon->icm = umalloc ( icm_aux_len + icm_len );
3252: if ( ! hermon->icm ) {
3253: rc = -ENOMEM;
3254: goto err_alloc;
3255: }
3256: icm_phys = user_to_phys ( hermon->icm, 0 );
3257:
3258: /* Map ICM auxiliary area */
3259: DBGC ( hermon, "Hermon %p mapping ICM AUX => %08lx\n",
3260: hermon, icm_phys );
3261: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm_aux,
3262: 0, icm_phys, icm_aux_len ) ) != 0 ) {
3263: DBGC ( hermon, "Hermon %p could not map AUX ICM: %s\n",
3264: hermon, strerror ( rc ) );
3265: goto err_map_icm_aux;
3266: }
3267: icm_phys += icm_aux_len;
3268:
3269: /* MAP ICM area */
3270: for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
3271: DBGC ( hermon, "Hermon %p mapping ICM %llx+%zx => %08lx\n",
3272: hermon, hermon->icm_map[i].offset,
3273: hermon->icm_map[i].len, icm_phys );
3274: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm,
3275: hermon->icm_map[i].offset,
3276: icm_phys,
3277: hermon->icm_map[i].len ) ) != 0 ){
3278: DBGC ( hermon, "Hermon %p could not map ICM: %s\n",
3279: hermon, strerror ( rc ) );
3280: goto err_map_icm;
3281: }
3282: icm_phys += hermon->icm_map[i].len;
3283: }
3284:
3285: return 0;
3286:
3287: err_map_icm:
3288: assert ( i == 0 ); /* We don't handle partial failure at present */
3289: err_map_icm_aux:
3290: hermon_cmd_unmap_icm_aux ( hermon );
3291: ufree ( hermon->icm );
3292: hermon->icm = UNULL;
3293: err_alloc:
3294: err_set_icm_size:
3295: return rc;
3296: }
3297:
3298: /**
3299: * Free ICM
3300: *
3301: * @v hermon Hermon device
3302: */
3303: static void hermon_free_icm ( struct hermon *hermon ) {
3304: struct hermonprm_scalar_parameter unmap_icm;
3305: int i;
3306:
3307: for ( i = ( HERMON_ICM_NUM_REGIONS - 1 ) ; i >= 0 ; i-- ) {
3308: memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
3309: MLX_FILL_1 ( &unmap_icm, 0, value_hi,
3310: ( hermon->icm_map[i].offset >> 32 ) );
3311: MLX_FILL_1 ( &unmap_icm, 1, value,
3312: hermon->icm_map[i].offset );
3313: hermon_cmd_unmap_icm ( hermon,
3314: ( 1 << fls ( ( hermon->icm_map[i].len /
3315: HERMON_PAGE_SIZE ) - 1)),
3316: &unmap_icm );
3317: }
3318: hermon_cmd_unmap_icm_aux ( hermon );
3319: ufree ( hermon->icm );
3320: hermon->icm = UNULL;
3321: }
3322:
3323: /***************************************************************************
3324: *
3325: * BOFM interface
3326: *
3327: ***************************************************************************
3328: */
3329:
3330: /**
3331: * Harvest Ethernet MAC for BOFM
3332: *
3333: * @v bofm BOFM device
3334: * @v mport Multi-port index
3335: * @v mac MAC to fill in
3336: * @ret rc Return status code
3337: */
3338: static int hermon_bofm_harvest ( struct bofm_device *bofm, unsigned int mport,
3339: uint8_t *mac ) {
3340: struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
3341: struct hermonprm_mod_stat_cfg stat_cfg;
3342: union {
3343: uint8_t bytes[8];
3344: uint32_t dwords[2];
3345: } buf;
3346: int rc;
3347:
3348: /* Query static configuration */
3349: if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
3350: HERMON_MOD_STAT_CFG_QUERY,
3351: HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
3352: &stat_cfg ) ) != 0 ) {
3353: DBGC ( hermon, "Hermon %p port %d could not query "
3354: "configuration: %s\n", hermon, mport, strerror ( rc ) );
3355: return rc;
3356: }
3357:
3358: /* Retrieve MAC address */
3359: buf.dwords[0] = htonl ( MLX_GET ( &stat_cfg, mac_high ) );
3360: buf.dwords[1] = htonl ( MLX_GET ( &stat_cfg, mac_low ) );
3361: memcpy ( mac, &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ],
3362: ETH_ALEN );
3363:
3364: DBGC ( hermon, "Hermon %p port %d harvested MAC address %s\n",
3365: hermon, mport, eth_ntoa ( mac ) );
3366:
3367: return 0;
3368: }
3369:
3370: /**
3371: * Update Ethernet MAC for BOFM
3372: *
3373: * @v bofm BOFM device
3374: * @v mport Multi-port index
3375: * @v mac MAC to fill in
3376: * @ret rc Return status code
3377: */
3378: static int hermon_bofm_update ( struct bofm_device *bofm, unsigned int mport,
3379: const uint8_t *mac ) {
3380: struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
3381: struct hermonprm_mod_stat_cfg stat_cfg;
3382: union {
3383: uint8_t bytes[8];
3384: uint32_t dwords[2];
3385: uint64_t qword;
3386: } buf;
3387: uint8_t *mac_copy = &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ];
3388: int rc;
3389:
3390: /* Prepare MAC address */
3391: memset ( &buf, 0, sizeof ( buf ) );
3392: memcpy ( mac_copy, mac, ETH_ALEN );
3393:
3394: /* Current BOFM versions are unable to create entries with
3395: * mport>1, which means that only the port 1 MAC address can
3396: * be explicitly specified. Work around this by using the
3397: * provided MAC address as a base address for all subsequent
3398: * ports. For example, if BOFM assigns the address
3399: *
3400: * 00:1A:64:76:00:09 for port 1
3401: *
3402: * then we will assign the addresses
3403: *
3404: * 00:1A:64:76:00:09 for port 1
3405: * 00:1A:64:76:00:0a for port 2
3406: *
3407: * Note that hermon->cap.num_ports is not yet defined at this
3408: * point.
3409: */
3410: for ( ; mport <= HERMON_MAX_PORTS ; mport++ ) {
3411:
3412: /* Modify static configuration */
3413: memset ( &stat_cfg, 0, sizeof ( stat_cfg ) );
3414: MLX_FILL_2 ( &stat_cfg, 36,
3415: mac_m, 1,
3416: mac_high, ntohl ( buf.dwords[0] ) );
3417: MLX_FILL_1 ( &stat_cfg, 37, mac_low, ntohl ( buf.dwords[1] ) );
3418: if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
3419: HERMON_MOD_STAT_CFG_SET,
3420: HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
3421: &stat_cfg ) ) != 0 ) {
3422: DBGC ( hermon, "Hermon %p port %d could not modify "
3423: "configuration: %s\n",
3424: hermon, mport, strerror ( rc ) );
3425: return rc;
3426: }
3427:
3428: DBGC ( hermon, "Hermon %p port %d updated MAC address to %s\n",
3429: hermon, mport, eth_ntoa ( mac_copy ) );
3430:
3431: /* Increment MAC address */
3432: buf.qword = cpu_to_be64 ( be64_to_cpu ( buf.qword ) + 1 );
3433: }
3434:
3435: return 0;
3436: }
3437:
3438: /** Hermon BOFM operations */
3439: static struct bofm_operations hermon_bofm_operations = {
3440: .harvest = hermon_bofm_harvest,
3441: .update = hermon_bofm_update,
3442: };
3443:
3444: /***************************************************************************
3445: *
3446: * PCI interface
3447: *
3448: ***************************************************************************
3449: */
3450:
3451: /**
3452: * Set up memory protection table
3453: *
3454: * @v hermon Hermon device
3455: * @ret rc Return status code
3456: */
3457: static int hermon_setup_mpt ( struct hermon *hermon ) {
3458: struct hermonprm_mpt mpt;
3459: uint32_t key;
3460: int rc;
3461:
3462: /* Derive key */
3463: key = ( hermon->cap.reserved_mrws | HERMON_MKEY_PREFIX );
3464: hermon->lkey = ( ( key << 8 ) | ( key >> 24 ) );
3465:
3466: /* Initialise memory protection table */
3467: memset ( &mpt, 0, sizeof ( mpt ) );
3468: MLX_FILL_7 ( &mpt, 0,
3469: atomic, 1,
3470: rw, 1,
3471: rr, 1,
3472: lw, 1,
3473: lr, 1,
3474: pa, 1,
3475: r_w, 1 );
3476: MLX_FILL_1 ( &mpt, 2, mem_key, key );
3477: MLX_FILL_1 ( &mpt, 3,
3478: pd, HERMON_GLOBAL_PD );
3479: MLX_FILL_1 ( &mpt, 10, len64, 1 );
3480: if ( ( rc = hermon_cmd_sw2hw_mpt ( hermon,
3481: hermon->cap.reserved_mrws,
3482: &mpt ) ) != 0 ) {
3483: DBGC ( hermon, "Hermon %p could not set up MPT: %s\n",
3484: hermon, strerror ( rc ) );
3485: return rc;
3486: }
3487:
3488: return 0;
3489: }
3490:
3491: /**
3492: * Configure special queue pairs
3493: *
3494: * @v hermon Hermon device
3495: * @ret rc Return status code
3496: */
3497: static int hermon_configure_special_qps ( struct hermon *hermon ) {
3498: int rc;
3499:
3500: /* Special QP block must be aligned on its own size */
3501: hermon->special_qpn_base = ( ( hermon->cap.reserved_qps +
3502: HERMON_NUM_SPECIAL_QPS - 1 )
3503: & ~( HERMON_NUM_SPECIAL_QPS - 1 ) );
3504: hermon->qpn_base = ( hermon->special_qpn_base +
3505: HERMON_NUM_SPECIAL_QPS );
3506: DBGC ( hermon, "Hermon %p special QPs at [%lx,%lx]\n", hermon,
3507: hermon->special_qpn_base, ( hermon->qpn_base - 1 ) );
3508:
3509: /* Issue command to configure special QPs */
3510: if ( ( rc = hermon_cmd_conf_special_qp ( hermon, 0x00,
3511: hermon->special_qpn_base ) ) != 0 ) {
3512: DBGC ( hermon, "Hermon %p could not configure special QPs: "
3513: "%s\n", hermon, strerror ( rc ) );
3514: return rc;
3515: }
3516:
3517: return 0;
3518: }
3519:
3520: /**
3521: * Reset device
3522: *
3523: * @v hermon Hermon device
3524: * @v pci PCI device
3525: */
3526: static void hermon_reset ( struct hermon *hermon,
3527: struct pci_device *pci ) {
3528: struct pci_config_backup backup;
3529: static const uint8_t backup_exclude[] =
3530: PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
3531:
3532: pci_backup ( pci, &backup, backup_exclude );
3533: writel ( HERMON_RESET_MAGIC,
3534: ( hermon->config + HERMON_RESET_OFFSET ) );
3535: mdelay ( HERMON_RESET_WAIT_TIME_MS );
3536: pci_restore ( pci, &backup, backup_exclude );
3537: }
3538:
3539: /**
3540: * Allocate Hermon device
3541: *
3542: * @v pci PCI device
3543: * @v id PCI ID
3544: * @ret rc Return status code
3545: */
3546: static struct hermon * hermon_alloc ( void ) {
3547: struct hermon *hermon;
3548:
3549: /* Allocate Hermon device */
3550: hermon = zalloc ( sizeof ( *hermon ) );
3551: if ( ! hermon )
3552: goto err_hermon;
3553:
3554: /* Allocate space for mailboxes */
3555: hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE,
3556: HERMON_MBOX_ALIGN );
3557: if ( ! hermon->mailbox_in )
3558: goto err_mailbox_in;
3559: hermon->mailbox_out = malloc_dma ( HERMON_MBOX_SIZE,
3560: HERMON_MBOX_ALIGN );
3561: if ( ! hermon->mailbox_out )
3562: goto err_mailbox_out;
3563:
3564: return hermon;
3565:
3566: free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
3567: err_mailbox_out:
3568: free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
3569: err_mailbox_in:
3570: free ( hermon );
3571: err_hermon:
3572: return NULL;
3573: }
3574:
3575: /**
3576: * Free Hermon device
3577: *
3578: * @v hermon Hermon device
3579: */
3580: static void hermon_free ( struct hermon *hermon ) {
3581:
3582: free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
3583: free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
3584: free ( hermon );
3585: }
3586:
3587: /**
3588: * Initialise Hermon PCI parameters
3589: *
3590: * @v hermon Hermon device
3591: * @v pci PCI device
3592: */
3593: static void hermon_pci_init ( struct hermon *hermon, struct pci_device *pci ) {
3594:
3595: /* Fix up PCI device */
3596: adjust_pci_device ( pci );
3597:
3598: /* Get PCI BARs */
3599: hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR),
3600: HERMON_PCI_CONFIG_BAR_SIZE );
3601: hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ),
3602: HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE );
3603: }
3604:
3605: /**
3606: * Probe PCI device
3607: *
3608: * @v pci PCI device
3609: * @v id PCI ID
3610: * @ret rc Return status code
3611: */
3612: static int hermon_probe ( struct pci_device *pci ) {
3613: struct hermon *hermon;
3614: struct ib_device *ibdev;
3615: struct net_device *netdev;
3616: struct hermon_port *port;
3617: struct hermonprm_init_hca init_hca;
3618: unsigned int i;
3619: int rc;
3620:
3621: /* Allocate Hermon device */
3622: hermon = hermon_alloc();
3623: if ( ! hermon ) {
3624: rc = -ENOMEM;
3625: goto err_alloc;
3626: }
3627: pci_set_drvdata ( pci, hermon );
3628:
3629: /* Initialise PCI parameters */
3630: hermon_pci_init ( hermon, pci );
3631:
3632: /* Reset device */
3633: hermon_reset ( hermon, pci );
3634:
3635: /* Start firmware */
3636: if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 )
3637: goto err_start_firmware;
3638:
3639: /* Get device limits */
3640: if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
3641: goto err_get_cap;
3642:
3643: /* Allocate Infiniband devices */
3644: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
3645: ibdev = alloc_ibdev ( 0 );
3646: if ( ! ibdev ) {
3647: rc = -ENOMEM;
3648: goto err_alloc_ibdev;
3649: }
3650: hermon->port[i].ibdev = ibdev;
3651: ibdev->op = &hermon_ib_operations;
3652: ibdev->dev = &pci->dev;
3653: ibdev->port = ( HERMON_PORT_BASE + i );
3654: ib_set_drvdata ( ibdev, hermon );
3655: }
3656:
3657: /* Allocate network devices */
3658: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
3659: netdev = alloc_etherdev ( 0 );
3660: if ( ! netdev ) {
3661: rc = -ENOMEM;
3662: goto err_alloc_netdev;
3663: }
3664: hermon->port[i].netdev = netdev;
3665: netdev_init ( netdev, &hermon_eth_operations );
3666: netdev->dev = &pci->dev;
3667: netdev->priv = &hermon->port[i];
3668: }
3669:
3670: /* Allocate ICM */
3671: memset ( &init_hca, 0, sizeof ( init_hca ) );
3672: if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 )
3673: goto err_alloc_icm;
3674:
3675: /* Initialise HCA */
3676: MLX_FILL_1 ( &init_hca, 0, version, 0x02 /* "Must be 0x02" */ );
3677: MLX_FILL_1 ( &init_hca, 5, udp, 1 );
3678: MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 8 );
3679: if ( ( rc = hermon_cmd_init_hca ( hermon, &init_hca ) ) != 0 ) {
3680: DBGC ( hermon, "Hermon %p could not initialise HCA: %s\n",
3681: hermon, strerror ( rc ) );
3682: goto err_init_hca;
3683: }
3684:
3685: /* Set up memory protection */
3686: if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
3687: goto err_setup_mpt;
3688: for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
3689: hermon->port[i].ibdev->rdma_key = hermon->lkey;
3690:
3691: /* Set up event queue */
3692: if ( ( rc = hermon_create_eq ( hermon ) ) != 0 )
3693: goto err_create_eq;
3694:
3695: /* Configure special QPs */
3696: if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
3697: goto err_conf_special_qps;
3698:
3699: /* Determine port types */
3700: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
3701: port = &hermon->port[i];
3702: if ( ( rc = hermon_set_port_type ( hermon, port ) ) != 0 )
3703: goto err_set_port_type;
3704: }
3705:
3706: /* Register devices */
3707: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
3708: port = &hermon->port[i];
3709: if ( ( rc = port->type->register_dev ( hermon, port ) ) != 0 )
3710: goto err_register;
3711: }
3712:
3713: return 0;
3714:
3715: i = hermon->cap.num_ports;
3716: err_register:
3717: for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
3718: port = &hermon->port[i];
3719: port->type->unregister_dev ( hermon, port );
3720: }
3721: err_set_port_type:
3722: err_conf_special_qps:
3723: hermon_destroy_eq ( hermon );
3724: err_create_eq:
3725: err_setup_mpt:
3726: hermon_cmd_close_hca ( hermon );
3727: err_init_hca:
3728: hermon_free_icm ( hermon );
3729: err_alloc_icm:
3730: i = hermon->cap.num_ports;
3731: err_alloc_netdev:
3732: for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
3733: netdev_nullify ( hermon->port[i].netdev );
3734: netdev_put ( hermon->port[i].netdev );
3735: }
3736: i = hermon->cap.num_ports;
3737: err_alloc_ibdev:
3738: for ( i-- ; ( signed int ) i >= 0 ; i-- )
3739: ibdev_put ( hermon->port[i].ibdev );
3740: err_get_cap:
3741: hermon_stop_firmware ( hermon );
3742: err_start_firmware:
3743: hermon_free ( hermon );
3744: err_alloc:
3745: return rc;
3746: }
3747:
3748: /**
3749: * Remove PCI device
3750: *
3751: * @v pci PCI device
3752: */
3753: static void hermon_remove ( struct pci_device *pci ) {
3754: struct hermon *hermon = pci_get_drvdata ( pci );
3755: struct hermon_port *port;
3756: int i;
3757:
3758: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
3759: port = &hermon->port[i];
3760: port->type->unregister_dev ( hermon, port );
3761: }
3762: hermon_destroy_eq ( hermon );
3763: hermon_cmd_close_hca ( hermon );
3764: hermon_free_icm ( hermon );
3765: hermon_stop_firmware ( hermon );
3766: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
3767: netdev_nullify ( hermon->port[i].netdev );
3768: netdev_put ( hermon->port[i].netdev );
3769: }
3770: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
3771: ibdev_put ( hermon->port[i].ibdev );
3772: hermon_free ( hermon );
3773: }
3774:
3775: /**
3776: * Probe PCI device for BOFM
3777: *
3778: * @v pci PCI device
3779: * @v id PCI ID
3780: * @ret rc Return status code
3781: */
3782: static int hermon_bofm_probe ( struct pci_device *pci ) {
3783: struct hermon *hermon;
3784: int rc;
3785:
3786: /* Allocate Hermon device */
3787: hermon = hermon_alloc();
3788: if ( ! hermon ) {
3789: rc = -ENOMEM;
3790: goto err_alloc;
3791: }
3792: pci_set_drvdata ( pci, hermon );
3793:
3794: /* Initialise PCI parameters */
3795: hermon_pci_init ( hermon, pci );
3796:
3797: /* Initialise BOFM device */
3798: bofm_init ( &hermon->bofm, pci, &hermon_bofm_operations );
3799:
3800: /* Register BOFM device */
3801: if ( ( rc = bofm_register ( &hermon->bofm ) ) != 0 ) {
3802: DBGC ( hermon, "Hermon %p could not register BOFM device: "
3803: "%s\n", hermon, strerror ( rc ) );
3804: goto err_bofm_register;
3805: }
3806:
3807: return 0;
3808:
3809: err_bofm_register:
3810: hermon_free ( hermon );
3811: err_alloc:
3812: return rc;
3813: }
3814:
3815: /**
3816: * Remove PCI device for BOFM
3817: *
3818: * @v pci PCI device
3819: */
3820: static void hermon_bofm_remove ( struct pci_device *pci ) {
3821: struct hermon *hermon = pci_get_drvdata ( pci );
3822:
3823: bofm_unregister ( &hermon->bofm );
3824: hermon_free ( hermon );
3825: }
3826:
3827: static struct pci_device_id hermon_nics[] = {
3828: PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ),
3829: PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ),
3830: PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ),
3831: PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ),
3832: PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ),
3833: PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
3834: PCI_ROM ( 0x15b3, 0x6368, "mt25448", "MT25448 HCA driver", 0 ),
3835: PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ),
3836: PCI_ROM ( 0x15b3, 0x6372, "mt25458", "MT25458 HCA driver", 0 ),
3837: PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ),
3838: PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ),
3839: PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ),
3840: };
3841:
3842: struct pci_driver hermon_driver __pci_driver = {
3843: .ids = hermon_nics,
3844: .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
3845: .probe = hermon_probe,
3846: .remove = hermon_remove,
3847: };
3848:
3849: struct pci_driver hermon_bofm_driver __bofm_driver = {
3850: .ids = hermon_nics,
3851: .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
3852: .probe = hermon_bofm_probe,
3853: .remove = hermon_bofm_remove,
3854: };
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.