![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to hermon.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / drivers / infiniband|
Return to hermon.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / drivers / infiniband |
1.1 ! root 1: /* ! 2: * Copyright (C) 2008 Michael Brown <[email protected]>. ! 3: * Copyright (C) 2008 Mellanox Technologies Ltd. ! 4: * ! 5: * This program is free software; you can redistribute it and/or ! 6: * modify it under the terms of the GNU General Public License as ! 7: * published by the Free Software Foundation; either version 2 of the ! 8: * License, or any later version. ! 9: * ! 10: * This program is distributed in the hope that it will be useful, but ! 11: * WITHOUT ANY WARRANTY; without even the implied warranty of ! 12: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ! 13: * General Public License for more details. ! 14: * ! 15: * You should have received a copy of the GNU General Public License ! 16: * along with this program; if not, write to the Free Software ! 17: * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ! 18: */ ! 19: ! 20: FILE_LICENCE ( GPL2_OR_LATER ); ! 21: ! 22: #include <stdint.h> ! 23: #include <stdlib.h> ! 24: #include <stdio.h> ! 25: #include <string.h> ! 26: #include <strings.h> ! 27: #include <unistd.h> ! 28: #include <errno.h> ! 29: #include <byteswap.h> ! 30: #include <ipxe/io.h> ! 31: #include <ipxe/pci.h> ! 32: #include <ipxe/pcibackup.h> ! 33: #include <ipxe/malloc.h> ! 34: #include <ipxe/umalloc.h> ! 35: #include <ipxe/iobuf.h> ! 36: #include <ipxe/netdevice.h> ! 37: #include <ipxe/infiniband.h> ! 38: #include <ipxe/ib_smc.h> ! 39: #include <ipxe/if_ether.h> ! 40: #include <ipxe/ethernet.h> ! 41: #include <ipxe/fcoe.h> ! 42: #include <ipxe/vlan.h> ! 43: #include <ipxe/bofm.h> ! 44: #include "hermon.h" ! 45: ! 46: /** ! 47: * @file ! 48: * ! 49: * Mellanox Hermon Infiniband HCA ! 50: * ! 51: */ ! 52: ! 53: /*************************************************************************** ! 54: * ! 55: * Queue number allocation ! 56: * ! 57: *************************************************************************** ! 58: */ ! 59: ! 60: /** ! 61: * Allocate offsets within usage bitmask ! 62: * ! 63: * @v bits Usage bitmask ! 64: * @v bits_len Length of usage bitmask ! 65: * @v num_bits Number of contiguous bits to allocate within bitmask ! 66: * @ret bit First free bit within bitmask, or negative error ! 67: */ ! 68: static int hermon_bitmask_alloc ( hermon_bitmask_t *bits, ! 69: unsigned int bits_len, ! 70: unsigned int num_bits ) { ! 71: unsigned int bit = 0; ! 72: hermon_bitmask_t mask = 1; ! 73: unsigned int found = 0; ! 74: ! 75: /* Search bits for num_bits contiguous free bits */ ! 76: while ( bit < bits_len ) { ! 77: if ( ( mask & *bits ) == 0 ) { ! 78: if ( ++found == num_bits ) ! 79: goto found; ! 80: } else { ! 81: found = 0; ! 82: } ! 83: bit++; ! 84: mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) ); ! 85: if ( mask == 1 ) ! 86: bits++; ! 87: } ! 88: return -ENFILE; ! 89: ! 90: found: ! 91: /* Mark bits as in-use */ ! 92: do { ! 93: *bits |= mask; ! 94: if ( mask == 1 ) ! 95: bits--; ! 96: mask = ( mask >> 1 ) | ( mask << ( 8 * sizeof ( mask ) - 1 ) ); ! 97: } while ( --found ); ! 98: ! 99: return ( bit - num_bits + 1 ); ! 100: } ! 101: ! 102: /** ! 103: * Free offsets within usage bitmask ! 104: * ! 105: * @v bits Usage bitmask ! 106: * @v bit Starting bit within bitmask ! 107: * @v num_bits Number of contiguous bits to free within bitmask ! 108: */ ! 109: static void hermon_bitmask_free ( hermon_bitmask_t *bits, ! 110: int bit, unsigned int num_bits ) { ! 111: hermon_bitmask_t mask; ! 112: ! 113: for ( ; num_bits ; bit++, num_bits-- ) { ! 114: mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) ); ! 115: bits[ ( bit / ( 8 * sizeof ( mask ) ) ) ] &= ~mask; ! 116: } ! 117: } ! 118: ! 119: /*************************************************************************** ! 120: * ! 121: * HCA commands ! 122: * ! 123: *************************************************************************** ! 124: */ ! 125: ! 126: /** ! 127: * Wait for Hermon command completion ! 128: * ! 129: * @v hermon Hermon device ! 130: * @v hcr HCA command registers ! 131: * @ret rc Return status code ! 132: */ ! 133: static int hermon_cmd_wait ( struct hermon *hermon, ! 134: struct hermonprm_hca_command_register *hcr ) { ! 135: unsigned int wait; ! 136: ! 137: for ( wait = HERMON_HCR_MAX_WAIT_MS ; wait ; wait-- ) { ! 138: hcr->u.dwords[6] = ! 139: readl ( hermon->config + HERMON_HCR_REG ( 6 ) ); ! 140: if ( ( MLX_GET ( hcr, go ) == 0 ) && ! 141: ( MLX_GET ( hcr, t ) == hermon->toggle ) ) ! 142: return 0; ! 143: mdelay ( 1 ); ! 144: } ! 145: return -EBUSY; ! 146: } ! 147: ! 148: /** ! 149: * Issue HCA command ! 150: * ! 151: * @v hermon Hermon device ! 152: * @v command Command opcode, flags and input/output lengths ! 153: * @v op_mod Opcode modifier (0 if no modifier applicable) ! 154: * @v in Input parameters ! 155: * @v in_mod Input modifier (0 if no modifier applicable) ! 156: * @v out Output parameters ! 157: * @ret rc Return status code ! 158: */ ! 159: static int hermon_cmd ( struct hermon *hermon, unsigned long command, ! 160: unsigned int op_mod, const void *in, ! 161: unsigned int in_mod, void *out ) { ! 162: struct hermonprm_hca_command_register hcr; ! 163: unsigned int opcode = HERMON_HCR_OPCODE ( command ); ! 164: size_t in_len = HERMON_HCR_IN_LEN ( command ); ! 165: size_t out_len = HERMON_HCR_OUT_LEN ( command ); ! 166: void *in_buffer; ! 167: void *out_buffer; ! 168: unsigned int status; ! 169: unsigned int i; ! 170: int rc; ! 171: ! 172: assert ( in_len <= HERMON_MBOX_SIZE ); ! 173: assert ( out_len <= HERMON_MBOX_SIZE ); ! 174: ! 175: DBGC2 ( hermon, "Hermon %p command %02x in %zx%s out %zx%s\n", ! 176: hermon, opcode, in_len, ! 177: ( ( command & HERMON_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len, ! 178: ( ( command & HERMON_HCR_OUT_MBOX ) ? "(mbox)" : "" ) ); ! 179: ! 180: /* Check that HCR is free */ ! 181: if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) { ! 182: DBGC ( hermon, "Hermon %p command interface locked\n", ! 183: hermon ); ! 184: return rc; ! 185: } ! 186: ! 187: /* Flip HCR toggle */ ! 188: hermon->toggle = ( 1 - hermon->toggle ); ! 189: ! 190: /* Prepare HCR */ ! 191: memset ( &hcr, 0, sizeof ( hcr ) ); ! 192: in_buffer = &hcr.u.dwords[0]; ! 193: if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) { ! 194: memset ( hermon->mailbox_in, 0, HERMON_MBOX_SIZE ); ! 195: in_buffer = hermon->mailbox_in; ! 196: MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) ); ! 197: MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) ); ! 198: } ! 199: memcpy ( in_buffer, in, in_len ); ! 200: MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod ); ! 201: out_buffer = &hcr.u.dwords[3]; ! 202: if ( out_len && ( command & HERMON_HCR_OUT_MBOX ) ) { ! 203: out_buffer = hermon->mailbox_out; ! 204: MLX_FILL_H ( &hcr, 3, out_param_h, ! 205: virt_to_bus ( out_buffer ) ); ! 206: MLX_FILL_1 ( &hcr, 4, out_param_l, ! 207: virt_to_bus ( out_buffer ) ); ! 208: } ! 209: MLX_FILL_4 ( &hcr, 6, ! 210: opcode, opcode, ! 211: opcode_modifier, op_mod, ! 212: go, 1, ! 213: t, hermon->toggle ); ! 214: DBGC ( hermon, "Hermon %p issuing command %04x\n", ! 215: hermon, opcode ); ! 216: DBGC2_HDA ( hermon, virt_to_phys ( hermon->config + HERMON_HCR_BASE ), ! 217: &hcr, sizeof ( hcr ) ); ! 218: if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) { ! 219: DBGC2 ( hermon, "Input mailbox:\n" ); ! 220: DBGC2_HDA ( hermon, virt_to_phys ( in_buffer ), in_buffer, ! 221: ( ( in_len < 512 ) ? in_len : 512 ) ); ! 222: } ! 223: ! 224: /* Issue command */ ! 225: for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ; ! 226: i++ ) { ! 227: writel ( hcr.u.dwords[i], ! 228: hermon->config + HERMON_HCR_REG ( i ) ); ! 229: barrier(); ! 230: } ! 231: ! 232: /* Wait for command completion */ ! 233: if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) { ! 234: DBGC ( hermon, "Hermon %p timed out waiting for command:\n", ! 235: hermon ); ! 236: DBGC_HDA ( hermon, ! 237: virt_to_phys ( hermon->config + HERMON_HCR_BASE ), ! 238: &hcr, sizeof ( hcr ) ); ! 239: return rc; ! 240: } ! 241: ! 242: /* Check command status */ ! 243: status = MLX_GET ( &hcr, status ); ! 244: if ( status != 0 ) { ! 245: DBGC ( hermon, "Hermon %p command failed with status %02x:\n", ! 246: hermon, status ); ! 247: DBGC_HDA ( hermon, ! 248: virt_to_phys ( hermon->config + HERMON_HCR_BASE ), ! 249: &hcr, sizeof ( hcr ) ); ! 250: return -EIO; ! 251: } ! 252: ! 253: /* Read output parameters, if any */ ! 254: hcr.u.dwords[3] = readl ( hermon->config + HERMON_HCR_REG ( 3 ) ); ! 255: hcr.u.dwords[4] = readl ( hermon->config + HERMON_HCR_REG ( 4 ) ); ! 256: memcpy ( out, out_buffer, out_len ); ! 257: if ( out_len ) { ! 258: DBGC2 ( hermon, "Output%s:\n", ! 259: ( command & HERMON_HCR_OUT_MBOX ) ? " mailbox" : "" ); ! 260: DBGC2_HDA ( hermon, virt_to_phys ( out_buffer ), out_buffer, ! 261: ( ( out_len < 512 ) ? out_len : 512 ) ); ! 262: } ! 263: ! 264: return 0; ! 265: } ! 266: ! 267: static inline int ! 268: hermon_cmd_query_dev_cap ( struct hermon *hermon, ! 269: struct hermonprm_query_dev_cap *dev_cap ) { ! 270: return hermon_cmd ( hermon, ! 271: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_DEV_CAP, ! 272: 1, sizeof ( *dev_cap ) ), ! 273: 0, NULL, 0, dev_cap ); ! 274: } ! 275: ! 276: static inline int ! 277: hermon_cmd_query_fw ( struct hermon *hermon, struct hermonprm_query_fw *fw ) { ! 278: return hermon_cmd ( hermon, ! 279: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_FW, ! 280: 1, sizeof ( *fw ) ), ! 281: 0, NULL, 0, fw ); ! 282: } ! 283: ! 284: static inline int ! 285: hermon_cmd_init_hca ( struct hermon *hermon, ! 286: const struct hermonprm_init_hca *init_hca ) { ! 287: return hermon_cmd ( hermon, ! 288: HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_HCA, ! 289: 1, sizeof ( *init_hca ) ), ! 290: 0, init_hca, 0, NULL ); ! 291: } ! 292: ! 293: static inline int ! 294: hermon_cmd_close_hca ( struct hermon *hermon ) { ! 295: return hermon_cmd ( hermon, ! 296: HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_HCA ), ! 297: 0, NULL, 0, NULL ); ! 298: } ! 299: ! 300: static inline int ! 301: hermon_cmd_init_port ( struct hermon *hermon, unsigned int port ) { ! 302: return hermon_cmd ( hermon, ! 303: HERMON_HCR_VOID_CMD ( HERMON_HCR_INIT_PORT ), ! 304: 0, NULL, port, NULL ); ! 305: } ! 306: ! 307: static inline int ! 308: hermon_cmd_close_port ( struct hermon *hermon, unsigned int port ) { ! 309: return hermon_cmd ( hermon, ! 310: HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_PORT ), ! 311: 0, NULL, port, NULL ); ! 312: } ! 313: ! 314: static inline int ! 315: hermon_cmd_set_port ( struct hermon *hermon, int is_ethernet, ! 316: unsigned int port_selector, ! 317: const union hermonprm_set_port *set_port ) { ! 318: return hermon_cmd ( hermon, ! 319: HERMON_HCR_IN_CMD ( HERMON_HCR_SET_PORT, ! 320: 1, sizeof ( *set_port ) ), ! 321: is_ethernet, set_port, port_selector, NULL ); ! 322: } ! 323: ! 324: static inline int ! 325: hermon_cmd_sw2hw_mpt ( struct hermon *hermon, unsigned int index, ! 326: const struct hermonprm_mpt *mpt ) { ! 327: return hermon_cmd ( hermon, ! 328: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_MPT, ! 329: 1, sizeof ( *mpt ) ), ! 330: 0, mpt, index, NULL ); ! 331: } ! 332: ! 333: static inline int ! 334: hermon_cmd_write_mtt ( struct hermon *hermon, ! 335: const struct hermonprm_write_mtt *write_mtt ) { ! 336: return hermon_cmd ( hermon, ! 337: HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MTT, ! 338: 1, sizeof ( *write_mtt ) ), ! 339: 0, write_mtt, 1, NULL ); ! 340: } ! 341: ! 342: static inline int ! 343: hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map, ! 344: const struct hermonprm_event_mask *mask ) { ! 345: return hermon_cmd ( hermon, ! 346: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ, ! 347: 0, sizeof ( *mask ) ), ! 348: 0, mask, index_map, NULL ); ! 349: } ! 350: ! 351: static inline int ! 352: hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index, ! 353: const struct hermonprm_eqc *eqctx ) { ! 354: return hermon_cmd ( hermon, ! 355: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ, ! 356: 1, sizeof ( *eqctx ) ), ! 357: 0, eqctx, index, NULL ); ! 358: } ! 359: ! 360: static inline int ! 361: hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index, ! 362: struct hermonprm_eqc *eqctx ) { ! 363: return hermon_cmd ( hermon, ! 364: HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ, ! 365: 1, sizeof ( *eqctx ) ), ! 366: 1, NULL, index, eqctx ); ! 367: } ! 368: ! 369: static inline int ! 370: hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index, ! 371: struct hermonprm_eqc *eqctx ) { ! 372: return hermon_cmd ( hermon, ! 373: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ, ! 374: 1, sizeof ( *eqctx ) ), ! 375: 0, NULL, index, eqctx ); ! 376: } ! 377: ! 378: static inline int ! 379: hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn, ! 380: const struct hermonprm_completion_queue_context *cqctx ){ ! 381: return hermon_cmd ( hermon, ! 382: HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_CQ, ! 383: 1, sizeof ( *cqctx ) ), ! 384: 0, cqctx, cqn, NULL ); ! 385: } ! 386: ! 387: static inline int ! 388: hermon_cmd_hw2sw_cq ( struct hermon *hermon, unsigned long cqn, ! 389: struct hermonprm_completion_queue_context *cqctx ) { ! 390: return hermon_cmd ( hermon, ! 391: HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_CQ, ! 392: 1, sizeof ( *cqctx ) ), ! 393: 0, NULL, cqn, cqctx ); ! 394: } ! 395: ! 396: static inline int ! 397: hermon_cmd_query_cq ( struct hermon *hermon, unsigned long cqn, ! 398: struct hermonprm_completion_queue_context *cqctx ) { ! 399: return hermon_cmd ( hermon, ! 400: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_CQ, ! 401: 1, sizeof ( *cqctx ) ), ! 402: 0, NULL, cqn, cqctx ); ! 403: } ! 404: ! 405: static inline int ! 406: hermon_cmd_rst2init_qp ( struct hermon *hermon, unsigned long qpn, ! 407: const struct hermonprm_qp_ee_state_transitions *ctx ){ ! 408: return hermon_cmd ( hermon, ! 409: HERMON_HCR_IN_CMD ( HERMON_HCR_RST2INIT_QP, ! 410: 1, sizeof ( *ctx ) ), ! 411: 0, ctx, qpn, NULL ); ! 412: } ! 413: ! 414: static inline int ! 415: hermon_cmd_init2rtr_qp ( struct hermon *hermon, unsigned long qpn, ! 416: const struct hermonprm_qp_ee_state_transitions *ctx ){ ! 417: return hermon_cmd ( hermon, ! 418: HERMON_HCR_IN_CMD ( HERMON_HCR_INIT2RTR_QP, ! 419: 1, sizeof ( *ctx ) ), ! 420: 0, ctx, qpn, NULL ); ! 421: } ! 422: ! 423: static inline int ! 424: hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn, ! 425: const struct hermonprm_qp_ee_state_transitions *ctx ) { ! 426: return hermon_cmd ( hermon, ! 427: HERMON_HCR_IN_CMD ( HERMON_HCR_RTR2RTS_QP, ! 428: 1, sizeof ( *ctx ) ), ! 429: 0, ctx, qpn, NULL ); ! 430: } ! 431: ! 432: static inline int ! 433: hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn, ! 434: const struct hermonprm_qp_ee_state_transitions *ctx ) { ! 435: return hermon_cmd ( hermon, ! 436: HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP, ! 437: 1, sizeof ( *ctx ) ), ! 438: 0, ctx, qpn, NULL ); ! 439: } ! 440: ! 441: static inline int ! 442: hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) { ! 443: return hermon_cmd ( hermon, ! 444: HERMON_HCR_VOID_CMD ( HERMON_HCR_2RST_QP ), ! 445: 0x03, NULL, qpn, NULL ); ! 446: } ! 447: ! 448: static inline int ! 449: hermon_cmd_query_qp ( struct hermon *hermon, unsigned long qpn, ! 450: struct hermonprm_qp_ee_state_transitions *ctx ) { ! 451: return hermon_cmd ( hermon, ! 452: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_QP, ! 453: 1, sizeof ( *ctx ) ), ! 454: 0, NULL, qpn, ctx ); ! 455: } ! 456: ! 457: static inline int ! 458: hermon_cmd_conf_special_qp ( struct hermon *hermon, unsigned int internal_qps, ! 459: unsigned long base_qpn ) { ! 460: return hermon_cmd ( hermon, ! 461: HERMON_HCR_VOID_CMD ( HERMON_HCR_CONF_SPECIAL_QP ), ! 462: internal_qps, NULL, base_qpn, NULL ); ! 463: } ! 464: ! 465: static inline int ! 466: hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port, ! 467: union hermonprm_mad *mad ) { ! 468: return hermon_cmd ( hermon, ! 469: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC, ! 470: 1, sizeof ( *mad ), ! 471: 1, sizeof ( *mad ) ), ! 472: 0x03, mad, port, mad ); ! 473: } ! 474: ! 475: static inline int ! 476: hermon_cmd_read_mcg ( struct hermon *hermon, unsigned int index, ! 477: struct hermonprm_mcg_entry *mcg ) { ! 478: return hermon_cmd ( hermon, ! 479: HERMON_HCR_OUT_CMD ( HERMON_HCR_READ_MCG, ! 480: 1, sizeof ( *mcg ) ), ! 481: 0, NULL, index, mcg ); ! 482: } ! 483: ! 484: static inline int ! 485: hermon_cmd_write_mcg ( struct hermon *hermon, unsigned int index, ! 486: const struct hermonprm_mcg_entry *mcg ) { ! 487: return hermon_cmd ( hermon, ! 488: HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MCG, ! 489: 1, sizeof ( *mcg ) ), ! 490: 0, mcg, index, NULL ); ! 491: } ! 492: ! 493: static inline int ! 494: hermon_cmd_mgid_hash ( struct hermon *hermon, const union ib_gid *gid, ! 495: struct hermonprm_mgm_hash *hash ) { ! 496: return hermon_cmd ( hermon, ! 497: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MGID_HASH, ! 498: 1, sizeof ( *gid ), ! 499: 0, sizeof ( *hash ) ), ! 500: 0, gid, 0, hash ); ! 501: } ! 502: ! 503: static inline int ! 504: hermon_cmd_mod_stat_cfg ( struct hermon *hermon, unsigned int mode, ! 505: unsigned int input_mod, ! 506: struct hermonprm_scalar_parameter *portion ) { ! 507: return hermon_cmd ( hermon, ! 508: HERMON_HCR_INOUT_CMD ( HERMON_HCR_MOD_STAT_CFG, ! 509: 0, sizeof ( *portion ), ! 510: 0, sizeof ( *portion ) ), ! 511: mode, portion, input_mod, portion ); ! 512: } ! 513: ! 514: static inline int ! 515: hermon_cmd_query_port ( struct hermon *hermon, unsigned int port, ! 516: struct hermonprm_query_port_cap *query_port ) { ! 517: return hermon_cmd ( hermon, ! 518: HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_PORT, ! 519: 1, sizeof ( *query_port ) ), ! 520: 0, NULL, port, query_port ); ! 521: } ! 522: ! 523: static inline int ! 524: hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port, ! 525: struct hermonprm_sense_port *port_type ) { ! 526: return hermon_cmd ( hermon, ! 527: HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT, ! 528: 0, sizeof ( *port_type ) ), ! 529: 0, NULL, port, port_type ); ! 530: } ! 531: ! 532: static inline int ! 533: hermon_cmd_run_fw ( struct hermon *hermon ) { ! 534: return hermon_cmd ( hermon, ! 535: HERMON_HCR_VOID_CMD ( HERMON_HCR_RUN_FW ), ! 536: 0, NULL, 0, NULL ); ! 537: } ! 538: ! 539: static inline int ! 540: hermon_cmd_unmap_icm ( struct hermon *hermon, unsigned int page_count, ! 541: const struct hermonprm_scalar_parameter *offset ) { ! 542: return hermon_cmd ( hermon, ! 543: HERMON_HCR_IN_CMD ( HERMON_HCR_UNMAP_ICM, ! 544: 0, sizeof ( *offset ) ), ! 545: 0, offset, page_count, NULL ); ! 546: } ! 547: ! 548: static inline int ! 549: hermon_cmd_map_icm ( struct hermon *hermon, ! 550: const struct hermonprm_virtual_physical_mapping *map ) { ! 551: return hermon_cmd ( hermon, ! 552: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM, ! 553: 1, sizeof ( *map ) ), ! 554: 0, map, 1, NULL ); ! 555: } ! 556: ! 557: static inline int ! 558: hermon_cmd_unmap_icm_aux ( struct hermon *hermon ) { ! 559: return hermon_cmd ( hermon, ! 560: HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_ICM_AUX ), ! 561: 0, NULL, 0, NULL ); ! 562: } ! 563: ! 564: static inline int ! 565: hermon_cmd_map_icm_aux ( struct hermon *hermon, ! 566: const struct hermonprm_virtual_physical_mapping *map ) { ! 567: return hermon_cmd ( hermon, ! 568: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM_AUX, ! 569: 1, sizeof ( *map ) ), ! 570: 0, map, 1, NULL ); ! 571: } ! 572: ! 573: static inline int ! 574: hermon_cmd_set_icm_size ( struct hermon *hermon, ! 575: const struct hermonprm_scalar_parameter *icm_size, ! 576: struct hermonprm_scalar_parameter *icm_aux_size ) { ! 577: return hermon_cmd ( hermon, ! 578: HERMON_HCR_INOUT_CMD ( HERMON_HCR_SET_ICM_SIZE, ! 579: 0, sizeof ( *icm_size ), ! 580: 0, sizeof (*icm_aux_size) ), ! 581: 0, icm_size, 0, icm_aux_size ); ! 582: } ! 583: ! 584: static inline int ! 585: hermon_cmd_unmap_fa ( struct hermon *hermon ) { ! 586: return hermon_cmd ( hermon, ! 587: HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_FA ), ! 588: 0, NULL, 0, NULL ); ! 589: } ! 590: ! 591: static inline int ! 592: hermon_cmd_map_fa ( struct hermon *hermon, ! 593: const struct hermonprm_virtual_physical_mapping *map ) { ! 594: return hermon_cmd ( hermon, ! 595: HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_FA, ! 596: 1, sizeof ( *map ) ), ! 597: 0, map, 1, NULL ); ! 598: } ! 599: ! 600: /*************************************************************************** ! 601: * ! 602: * Memory translation table operations ! 603: * ! 604: *************************************************************************** ! 605: */ ! 606: ! 607: /** ! 608: * Allocate MTT entries ! 609: * ! 610: * @v hermon Hermon device ! 611: * @v memory Memory to map into MTT ! 612: * @v len Length of memory to map ! 613: * @v mtt MTT descriptor to fill in ! 614: * @ret rc Return status code ! 615: */ ! 616: static int hermon_alloc_mtt ( struct hermon *hermon, ! 617: const void *memory, size_t len, ! 618: struct hermon_mtt *mtt ) { ! 619: struct hermonprm_write_mtt write_mtt; ! 620: physaddr_t start; ! 621: physaddr_t addr; ! 622: unsigned int page_offset; ! 623: unsigned int num_pages; ! 624: int mtt_offset; ! 625: unsigned int mtt_base_addr; ! 626: unsigned int i; ! 627: int rc; ! 628: ! 629: /* Find available MTT entries */ ! 630: start = virt_to_phys ( memory ); ! 631: page_offset = ( start & ( HERMON_PAGE_SIZE - 1 ) ); ! 632: start -= page_offset; ! 633: len += page_offset; ! 634: num_pages = ( ( len + HERMON_PAGE_SIZE - 1 ) / HERMON_PAGE_SIZE ); ! 635: mtt_offset = hermon_bitmask_alloc ( hermon->mtt_inuse, HERMON_MAX_MTTS, ! 636: num_pages ); ! 637: if ( mtt_offset < 0 ) { ! 638: DBGC ( hermon, "Hermon %p could not allocate %d MTT entries\n", ! 639: hermon, num_pages ); ! 640: rc = mtt_offset; ! 641: goto err_mtt_offset; ! 642: } ! 643: mtt_base_addr = ( ( hermon->cap.reserved_mtts + mtt_offset ) * ! 644: hermon->cap.mtt_entry_size ); ! 645: addr = start; ! 646: ! 647: /* Fill in MTT structure */ ! 648: mtt->mtt_offset = mtt_offset; ! 649: mtt->num_pages = num_pages; ! 650: mtt->mtt_base_addr = mtt_base_addr; ! 651: mtt->page_offset = page_offset; ! 652: ! 653: /* Construct and issue WRITE_MTT commands */ ! 654: for ( i = 0 ; i < num_pages ; i++ ) { ! 655: memset ( &write_mtt, 0, sizeof ( write_mtt ) ); ! 656: MLX_FILL_1 ( &write_mtt.mtt_base_addr, 1, ! 657: value, mtt_base_addr ); ! 658: MLX_FILL_H ( &write_mtt.mtt, 0, ptag_h, addr ); ! 659: MLX_FILL_2 ( &write_mtt.mtt, 1, ! 660: p, 1, ! 661: ptag_l, ( addr >> 3 ) ); ! 662: if ( ( rc = hermon_cmd_write_mtt ( hermon, ! 663: &write_mtt ) ) != 0 ) { ! 664: DBGC ( hermon, "Hermon %p could not write MTT at %x\n", ! 665: hermon, mtt_base_addr ); ! 666: goto err_write_mtt; ! 667: } ! 668: addr += HERMON_PAGE_SIZE; ! 669: mtt_base_addr += hermon->cap.mtt_entry_size; ! 670: } ! 671: ! 672: DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] for " ! 673: "[%08lx,%08lx,%08lx,%08lx)\n", hermon, mtt->mtt_offset, ! 674: ( mtt->mtt_offset + mtt->num_pages - 1 ), start, ! 675: ( start + page_offset ), ( start + len ), addr ); ! 676: ! 677: return 0; ! 678: ! 679: err_write_mtt: ! 680: hermon_bitmask_free ( hermon->mtt_inuse, mtt_offset, num_pages ); ! 681: err_mtt_offset: ! 682: return rc; ! 683: } ! 684: ! 685: /** ! 686: * Free MTT entries ! 687: * ! 688: * @v hermon Hermon device ! 689: * @v mtt MTT descriptor ! 690: */ ! 691: static void hermon_free_mtt ( struct hermon *hermon, ! 692: struct hermon_mtt *mtt ) { ! 693: ! 694: DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] freed\n", ! 695: hermon, mtt->mtt_offset, ! 696: ( mtt->mtt_offset + mtt->num_pages - 1 ) ); ! 697: hermon_bitmask_free ( hermon->mtt_inuse, mtt->mtt_offset, ! 698: mtt->num_pages ); ! 699: } ! 700: ! 701: /*************************************************************************** ! 702: * ! 703: * Static configuration operations ! 704: * ! 705: *************************************************************************** ! 706: */ ! 707: ! 708: /** ! 709: * Calculate offset within static configuration ! 710: * ! 711: * @v field Field ! 712: * @ret offset Offset ! 713: */ ! 714: #define HERMON_MOD_STAT_CFG_OFFSET( field ) \ ! 715: ( ( MLX_BIT_OFFSET ( struct hermonprm_mod_stat_cfg_st, field ) / 8 ) \ ! 716: & ~( sizeof ( struct hermonprm_scalar_parameter ) - 1 ) ) ! 717: ! 718: /** ! 719: * Query or modify static configuration ! 720: * ! 721: * @v hermon Hermon device ! 722: * @v port Port ! 723: * @v mode Command mode ! 724: * @v offset Offset within static configuration ! 725: * @v stat_cfg Static configuration ! 726: * @ret rc Return status code ! 727: */ ! 728: static int hermon_mod_stat_cfg ( struct hermon *hermon, unsigned int port, ! 729: unsigned int mode, unsigned int offset, ! 730: struct hermonprm_mod_stat_cfg *stat_cfg ) { ! 731: struct hermonprm_scalar_parameter *portion = ! 732: ( ( void * ) &stat_cfg->u.bytes[offset] ); ! 733: struct hermonprm_mod_stat_cfg_input_mod mod; ! 734: int rc; ! 735: ! 736: /* Sanity check */ ! 737: assert ( ( offset % sizeof ( *portion ) ) == 0 ); ! 738: ! 739: /* Construct input modifier */ ! 740: memset ( &mod, 0, sizeof ( mod ) ); ! 741: MLX_FILL_2 ( &mod, 0, ! 742: portnum, port, ! 743: offset, offset ); ! 744: ! 745: /* Issue command */ ! 746: if ( ( rc = hermon_cmd_mod_stat_cfg ( hermon, mode, ! 747: be32_to_cpu ( mod.u.dwords[0] ), ! 748: portion ) ) != 0 ) ! 749: return rc; ! 750: ! 751: return 0; ! 752: } ! 753: ! 754: /*************************************************************************** ! 755: * ! 756: * MAD operations ! 757: * ! 758: *************************************************************************** ! 759: */ ! 760: ! 761: /** ! 762: * Issue management datagram ! 763: * ! 764: * @v ibdev Infiniband device ! 765: * @v mad Management datagram ! 766: * @ret rc Return status code ! 767: */ ! 768: static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) { ! 769: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 770: union hermonprm_mad mad_ifc; ! 771: int rc; ! 772: ! 773: linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ), ! 774: mad_size_mismatch ); ! 775: ! 776: /* Copy in request packet */ ! 777: memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) ); ! 778: ! 779: /* Issue MAD */ ! 780: if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port, ! 781: &mad_ifc ) ) != 0 ) { ! 782: DBGC ( hermon, "Hermon %p port %d could not issue MAD IFC: " ! 783: "%s\n", hermon, ibdev->port, strerror ( rc ) ); ! 784: return rc; ! 785: } ! 786: ! 787: /* Copy out reply packet */ ! 788: memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) ); ! 789: ! 790: if ( mad->hdr.status != 0 ) { ! 791: DBGC ( hermon, "Hermon %p port %d MAD IFC status %04x\n", ! 792: hermon, ibdev->port, ntohs ( mad->hdr.status ) ); ! 793: return -EIO; ! 794: } ! 795: return 0; ! 796: } ! 797: ! 798: /*************************************************************************** ! 799: * ! 800: * Completion queue operations ! 801: * ! 802: *************************************************************************** ! 803: */ ! 804: ! 805: /** ! 806: * Dump completion queue context (for debugging only) ! 807: * ! 808: * @v hermon Hermon device ! 809: * @v cq Completion queue ! 810: * @ret rc Return status code ! 811: */ ! 812: static __attribute__ (( unused )) int ! 813: hermon_dump_cqctx ( struct hermon *hermon, struct ib_completion_queue *cq ) { ! 814: struct hermonprm_completion_queue_context cqctx; ! 815: int rc; ! 816: ! 817: memset ( &cqctx, 0, sizeof ( cqctx ) ); ! 818: if ( ( rc = hermon_cmd_query_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) { ! 819: DBGC ( hermon, "Hermon %p CQN %#lx QUERY_CQ failed: %s\n", ! 820: hermon, cq->cqn, strerror ( rc ) ); ! 821: return rc; ! 822: } ! 823: DBGC ( hermon, "Hermon %p CQN %#lx context:\n", hermon, cq->cqn ); ! 824: DBGC_HDA ( hermon, 0, &cqctx, sizeof ( cqctx ) ); ! 825: ! 826: return 0; ! 827: } ! 828: ! 829: /** ! 830: * Create completion queue ! 831: * ! 832: * @v ibdev Infiniband device ! 833: * @v cq Completion queue ! 834: * @ret rc Return status code ! 835: */ ! 836: static int hermon_create_cq ( struct ib_device *ibdev, ! 837: struct ib_completion_queue *cq ) { ! 838: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 839: struct hermon_completion_queue *hermon_cq; ! 840: struct hermonprm_completion_queue_context cqctx; ! 841: int cqn_offset; ! 842: unsigned int i; ! 843: int rc; ! 844: ! 845: /* Find a free completion queue number */ ! 846: cqn_offset = hermon_bitmask_alloc ( hermon->cq_inuse, ! 847: HERMON_MAX_CQS, 1 ); ! 848: if ( cqn_offset < 0 ) { ! 849: DBGC ( hermon, "Hermon %p out of completion queues\n", ! 850: hermon ); ! 851: rc = cqn_offset; ! 852: goto err_cqn_offset; ! 853: } ! 854: cq->cqn = ( hermon->cap.reserved_cqs + cqn_offset ); ! 855: ! 856: /* Allocate control structures */ ! 857: hermon_cq = zalloc ( sizeof ( *hermon_cq ) ); ! 858: if ( ! hermon_cq ) { ! 859: rc = -ENOMEM; ! 860: goto err_hermon_cq; ! 861: } ! 862: ! 863: /* Allocate doorbell */ ! 864: hermon_cq->doorbell = malloc_dma ( sizeof ( hermon_cq->doorbell[0] ), ! 865: sizeof ( hermon_cq->doorbell[0] ) ); ! 866: if ( ! hermon_cq->doorbell ) { ! 867: rc = -ENOMEM; ! 868: goto err_doorbell; ! 869: } ! 870: memset ( hermon_cq->doorbell, 0, sizeof ( hermon_cq->doorbell[0] ) ); ! 871: ! 872: /* Allocate completion queue itself */ ! 873: hermon_cq->cqe_size = ( cq->num_cqes * sizeof ( hermon_cq->cqe[0] ) ); ! 874: hermon_cq->cqe = malloc_dma ( hermon_cq->cqe_size, ! 875: sizeof ( hermon_cq->cqe[0] ) ); ! 876: if ( ! hermon_cq->cqe ) { ! 877: rc = -ENOMEM; ! 878: goto err_cqe; ! 879: } ! 880: memset ( hermon_cq->cqe, 0, hermon_cq->cqe_size ); ! 881: for ( i = 0 ; i < cq->num_cqes ; i++ ) { ! 882: MLX_FILL_1 ( &hermon_cq->cqe[i].normal, 7, owner, 1 ); ! 883: } ! 884: barrier(); ! 885: ! 886: /* Allocate MTT entries */ ! 887: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_cq->cqe, ! 888: hermon_cq->cqe_size, ! 889: &hermon_cq->mtt ) ) != 0 ) ! 890: goto err_alloc_mtt; ! 891: ! 892: /* Hand queue over to hardware */ ! 893: memset ( &cqctx, 0, sizeof ( cqctx ) ); ! 894: MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ ); ! 895: MLX_FILL_1 ( &cqctx, 2, ! 896: page_offset, ( hermon_cq->mtt.page_offset >> 5 ) ); ! 897: MLX_FILL_2 ( &cqctx, 3, ! 898: usr_page, HERMON_UAR_NON_EQ_PAGE, ! 899: log_cq_size, fls ( cq->num_cqes - 1 ) ); ! 900: MLX_FILL_1 ( &cqctx, 5, c_eqn, hermon->eq.eqn ); ! 901: MLX_FILL_H ( &cqctx, 6, mtt_base_addr_h, ! 902: hermon_cq->mtt.mtt_base_addr ); ! 903: MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l, ! 904: ( hermon_cq->mtt.mtt_base_addr >> 3 ) ); ! 905: MLX_FILL_H ( &cqctx, 14, db_record_addr_h, ! 906: virt_to_phys ( hermon_cq->doorbell ) ); ! 907: MLX_FILL_1 ( &cqctx, 15, db_record_addr_l, ! 908: ( virt_to_phys ( hermon_cq->doorbell ) >> 3 ) ); ! 909: if ( ( rc = hermon_cmd_sw2hw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) { ! 910: DBGC ( hermon, "Hermon %p CQN %#lx SW2HW_CQ failed: %s\n", ! 911: hermon, cq->cqn, strerror ( rc ) ); ! 912: goto err_sw2hw_cq; ! 913: } ! 914: ! 915: DBGC ( hermon, "Hermon %p CQN %#lx ring [%08lx,%08lx), doorbell " ! 916: "%08lx\n", hermon, cq->cqn, virt_to_phys ( hermon_cq->cqe ), ! 917: ( virt_to_phys ( hermon_cq->cqe ) + hermon_cq->cqe_size ), ! 918: virt_to_phys ( hermon_cq->doorbell ) ); ! 919: ib_cq_set_drvdata ( cq, hermon_cq ); ! 920: return 0; ! 921: ! 922: err_sw2hw_cq: ! 923: hermon_free_mtt ( hermon, &hermon_cq->mtt ); ! 924: err_alloc_mtt: ! 925: free_dma ( hermon_cq->cqe, hermon_cq->cqe_size ); ! 926: err_cqe: ! 927: free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) ); ! 928: err_doorbell: ! 929: free ( hermon_cq ); ! 930: err_hermon_cq: ! 931: hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 ); ! 932: err_cqn_offset: ! 933: return rc; ! 934: } ! 935: ! 936: /** ! 937: * Destroy completion queue ! 938: * ! 939: * @v ibdev Infiniband device ! 940: * @v cq Completion queue ! 941: */ ! 942: static void hermon_destroy_cq ( struct ib_device *ibdev, ! 943: struct ib_completion_queue *cq ) { ! 944: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 945: struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq ); ! 946: struct hermonprm_completion_queue_context cqctx; ! 947: int cqn_offset; ! 948: int rc; ! 949: ! 950: /* Take ownership back from hardware */ ! 951: if ( ( rc = hermon_cmd_hw2sw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) { ! 952: DBGC ( hermon, "Hermon %p CQN %#lx FATAL HW2SW_CQ failed: " ! 953: "%s\n", hermon, cq->cqn, strerror ( rc ) ); ! 954: /* Leak memory and return; at least we avoid corruption */ ! 955: return; ! 956: } ! 957: ! 958: /* Free MTT entries */ ! 959: hermon_free_mtt ( hermon, &hermon_cq->mtt ); ! 960: ! 961: /* Free memory */ ! 962: free_dma ( hermon_cq->cqe, hermon_cq->cqe_size ); ! 963: free_dma ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) ); ! 964: free ( hermon_cq ); ! 965: ! 966: /* Mark queue number as free */ ! 967: cqn_offset = ( cq->cqn - hermon->cap.reserved_cqs ); ! 968: hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 ); ! 969: ! 970: ib_cq_set_drvdata ( cq, NULL ); ! 971: } ! 972: ! 973: /*************************************************************************** ! 974: * ! 975: * Queue pair operations ! 976: * ! 977: *************************************************************************** ! 978: */ ! 979: ! 980: /** ! 981: * Assign queue pair number ! 982: * ! 983: * @v ibdev Infiniband device ! 984: * @v qp Queue pair ! 985: * @ret rc Return status code ! 986: */ ! 987: static int hermon_alloc_qpn ( struct ib_device *ibdev, ! 988: struct ib_queue_pair *qp ) { ! 989: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 990: unsigned int port_offset; ! 991: int qpn_offset; ! 992: ! 993: /* Calculate queue pair number */ ! 994: port_offset = ( ibdev->port - HERMON_PORT_BASE ); ! 995: ! 996: switch ( qp->type ) { ! 997: case IB_QPT_SMI: ! 998: qp->qpn = ( hermon->special_qpn_base + port_offset ); ! 999: return 0; ! 1000: case IB_QPT_GSI: ! 1001: qp->qpn = ( hermon->special_qpn_base + 2 + port_offset ); ! 1002: return 0; ! 1003: case IB_QPT_UD: ! 1004: case IB_QPT_RC: ! 1005: case IB_QPT_ETH: ! 1006: /* Find a free queue pair number */ ! 1007: qpn_offset = hermon_bitmask_alloc ( hermon->qp_inuse, ! 1008: HERMON_MAX_QPS, 1 ); ! 1009: if ( qpn_offset < 0 ) { ! 1010: DBGC ( hermon, "Hermon %p out of queue pairs\n", ! 1011: hermon ); ! 1012: return qpn_offset; ! 1013: } ! 1014: qp->qpn = ( ( random() & HERMON_QPN_RANDOM_MASK ) | ! 1015: ( hermon->qpn_base + qpn_offset ) ); ! 1016: return 0; ! 1017: default: ! 1018: DBGC ( hermon, "Hermon %p unsupported QP type %d\n", ! 1019: hermon, qp->type ); ! 1020: return -ENOTSUP; ! 1021: } ! 1022: } ! 1023: ! 1024: /** ! 1025: * Free queue pair number ! 1026: * ! 1027: * @v ibdev Infiniband device ! 1028: * @v qp Queue pair ! 1029: */ ! 1030: static void hermon_free_qpn ( struct ib_device *ibdev, ! 1031: struct ib_queue_pair *qp ) { ! 1032: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1033: int qpn_offset; ! 1034: ! 1035: qpn_offset = ( ( qp->qpn & ~HERMON_QPN_RANDOM_MASK ) ! 1036: - hermon->qpn_base ); ! 1037: if ( qpn_offset >= 0 ) ! 1038: hermon_bitmask_free ( hermon->qp_inuse, qpn_offset, 1 ); ! 1039: } ! 1040: ! 1041: /** ! 1042: * Calculate transmission rate ! 1043: * ! 1044: * @v av Address vector ! 1045: * @ret hermon_rate Hermon rate ! 1046: */ ! 1047: static unsigned int hermon_rate ( struct ib_address_vector *av ) { ! 1048: return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) ) ! 1049: ? ( av->rate + 5 ) : 0 ); ! 1050: } ! 1051: ! 1052: /** ! 1053: * Calculate schedule queue ! 1054: * ! 1055: * @v ibdev Infiniband device ! 1056: * @v qp Queue pair ! 1057: * @ret sched_queue Schedule queue ! 1058: */ ! 1059: static unsigned int hermon_sched_queue ( struct ib_device *ibdev, ! 1060: struct ib_queue_pair *qp ) { ! 1061: return ( ( ( qp->type == IB_QPT_SMI ) ? ! 1062: HERMON_SCHED_QP0 : HERMON_SCHED_DEFAULT ) | ! 1063: ( ( ibdev->port - 1 ) << 6 ) ); ! 1064: } ! 1065: ! 1066: /** Queue pair transport service type map */ ! 1067: static uint8_t hermon_qp_st[] = { ! 1068: [IB_QPT_SMI] = HERMON_ST_MLX, ! 1069: [IB_QPT_GSI] = HERMON_ST_MLX, ! 1070: [IB_QPT_UD] = HERMON_ST_UD, ! 1071: [IB_QPT_RC] = HERMON_ST_RC, ! 1072: [IB_QPT_ETH] = HERMON_ST_MLX, ! 1073: }; ! 1074: ! 1075: /** ! 1076: * Dump queue pair context (for debugging only) ! 1077: * ! 1078: * @v hermon Hermon device ! 1079: * @v qp Queue pair ! 1080: * @ret rc Return status code ! 1081: */ ! 1082: static __attribute__ (( unused )) int ! 1083: hermon_dump_qpctx ( struct hermon *hermon, struct ib_queue_pair *qp ) { ! 1084: struct hermonprm_qp_ee_state_transitions qpctx; ! 1085: int rc; ! 1086: ! 1087: memset ( &qpctx, 0, sizeof ( qpctx ) ); ! 1088: if ( ( rc = hermon_cmd_query_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ) { ! 1089: DBGC ( hermon, "Hermon %p QPN %#lx QUERY_QP failed: %s\n", ! 1090: hermon, qp->qpn, strerror ( rc ) ); ! 1091: return rc; ! 1092: } ! 1093: DBGC ( hermon, "Hermon %p QPN %#lx context:\n", hermon, qp->qpn ); ! 1094: DBGC_HDA ( hermon, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) ); ! 1095: ! 1096: return 0; ! 1097: } ! 1098: ! 1099: /** ! 1100: * Create queue pair ! 1101: * ! 1102: * @v ibdev Infiniband device ! 1103: * @v qp Queue pair ! 1104: * @ret rc Return status code ! 1105: */ ! 1106: static int hermon_create_qp ( struct ib_device *ibdev, ! 1107: struct ib_queue_pair *qp ) { ! 1108: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1109: struct hermon_queue_pair *hermon_qp; ! 1110: struct hermonprm_qp_ee_state_transitions qpctx; ! 1111: int rc; ! 1112: ! 1113: /* Calculate queue pair number */ ! 1114: if ( ( rc = hermon_alloc_qpn ( ibdev, qp ) ) != 0 ) ! 1115: goto err_alloc_qpn; ! 1116: ! 1117: /* Allocate control structures */ ! 1118: hermon_qp = zalloc ( sizeof ( *hermon_qp ) ); ! 1119: if ( ! hermon_qp ) { ! 1120: rc = -ENOMEM; ! 1121: goto err_hermon_qp; ! 1122: } ! 1123: ! 1124: /* Allocate doorbells */ ! 1125: hermon_qp->recv.doorbell = ! 1126: malloc_dma ( sizeof ( hermon_qp->recv.doorbell[0] ), ! 1127: sizeof ( hermon_qp->recv.doorbell[0] ) ); ! 1128: if ( ! hermon_qp->recv.doorbell ) { ! 1129: rc = -ENOMEM; ! 1130: goto err_recv_doorbell; ! 1131: } ! 1132: memset ( hermon_qp->recv.doorbell, 0, ! 1133: sizeof ( hermon_qp->recv.doorbell[0] ) ); ! 1134: hermon_qp->send.doorbell = ! 1135: ( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE + ! 1136: HERMON_DB_POST_SND_OFFSET ); ! 1137: ! 1138: /* Allocate work queue buffer */ ! 1139: hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 + ! 1140: ( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) ); ! 1141: hermon_qp->send.num_wqes = ! 1142: ( 1 << fls ( hermon_qp->send.num_wqes - 1 ) ); /* round up */ ! 1143: hermon_qp->send.wqe_size = ( hermon_qp->send.num_wqes * ! 1144: sizeof ( hermon_qp->send.wqe[0] ) ); ! 1145: hermon_qp->recv.wqe_size = ( qp->recv.num_wqes * ! 1146: sizeof ( hermon_qp->recv.wqe[0] ) ); ! 1147: hermon_qp->wqe_size = ( hermon_qp->send.wqe_size + ! 1148: hermon_qp->recv.wqe_size ); ! 1149: hermon_qp->wqe = malloc_dma ( hermon_qp->wqe_size, ! 1150: sizeof ( hermon_qp->send.wqe[0] ) ); ! 1151: if ( ! hermon_qp->wqe ) { ! 1152: rc = -ENOMEM; ! 1153: goto err_alloc_wqe; ! 1154: } ! 1155: hermon_qp->send.wqe = hermon_qp->wqe; ! 1156: memset ( hermon_qp->send.wqe, 0xff, hermon_qp->send.wqe_size ); ! 1157: hermon_qp->recv.wqe = ( hermon_qp->wqe + hermon_qp->send.wqe_size ); ! 1158: memset ( hermon_qp->recv.wqe, 0, hermon_qp->recv.wqe_size ); ! 1159: ! 1160: /* Allocate MTT entries */ ! 1161: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_qp->wqe, ! 1162: hermon_qp->wqe_size, ! 1163: &hermon_qp->mtt ) ) != 0 ) { ! 1164: goto err_alloc_mtt; ! 1165: } ! 1166: ! 1167: /* Transition queue to INIT state */ ! 1168: memset ( &qpctx, 0, sizeof ( qpctx ) ); ! 1169: MLX_FILL_2 ( &qpctx, 2, ! 1170: qpc_eec_data.pm_state, HERMON_PM_STATE_MIGRATED, ! 1171: qpc_eec_data.st, hermon_qp_st[qp->type] ); ! 1172: MLX_FILL_1 ( &qpctx, 3, qpc_eec_data.pd, HERMON_GLOBAL_PD ); ! 1173: MLX_FILL_4 ( &qpctx, 4, ! 1174: qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ), ! 1175: qpc_eec_data.log_rq_stride, ! 1176: ( fls ( sizeof ( hermon_qp->recv.wqe[0] ) - 1 ) - 4 ), ! 1177: qpc_eec_data.log_sq_size, ! 1178: fls ( hermon_qp->send.num_wqes - 1 ), ! 1179: qpc_eec_data.log_sq_stride, ! 1180: ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) ); ! 1181: MLX_FILL_1 ( &qpctx, 5, ! 1182: qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE ); ! 1183: MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn ); ! 1184: MLX_FILL_4 ( &qpctx, 38, ! 1185: qpc_eec_data.rre, 1, ! 1186: qpc_eec_data.rwe, 1, ! 1187: qpc_eec_data.rae, 1, ! 1188: qpc_eec_data.page_offset, ! 1189: ( hermon_qp->mtt.page_offset >> 6 ) ); ! 1190: MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn ); ! 1191: MLX_FILL_H ( &qpctx, 42, qpc_eec_data.db_record_addr_h, ! 1192: virt_to_phys ( hermon_qp->recv.doorbell ) ); ! 1193: MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l, ! 1194: ( virt_to_phys ( hermon_qp->recv.doorbell ) >> 2 ) ); ! 1195: MLX_FILL_H ( &qpctx, 52, qpc_eec_data.mtt_base_addr_h, ! 1196: hermon_qp->mtt.mtt_base_addr ); ! 1197: MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l, ! 1198: ( hermon_qp->mtt.mtt_base_addr >> 3 ) ); ! 1199: if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn, ! 1200: &qpctx ) ) != 0 ) { ! 1201: DBGC ( hermon, "Hermon %p QPN %#lx RST2INIT_QP failed: %s\n", ! 1202: hermon, qp->qpn, strerror ( rc ) ); ! 1203: goto err_rst2init_qp; ! 1204: } ! 1205: hermon_qp->state = HERMON_QP_ST_INIT; ! 1206: ! 1207: DBGC ( hermon, "Hermon %p QPN %#lx send ring [%08lx,%08lx), doorbell " ! 1208: "%08lx\n", hermon, qp->qpn, ! 1209: virt_to_phys ( hermon_qp->send.wqe ), ! 1210: ( virt_to_phys ( hermon_qp->send.wqe ) + ! 1211: hermon_qp->send.wqe_size ), ! 1212: virt_to_phys ( hermon_qp->send.doorbell ) ); ! 1213: DBGC ( hermon, "Hermon %p QPN %#lx receive ring [%08lx,%08lx), " ! 1214: "doorbell %08lx\n", hermon, qp->qpn, ! 1215: virt_to_phys ( hermon_qp->recv.wqe ), ! 1216: ( virt_to_phys ( hermon_qp->recv.wqe ) + ! 1217: hermon_qp->recv.wqe_size ), ! 1218: virt_to_phys ( hermon_qp->recv.doorbell ) ); ! 1219: DBGC ( hermon, "Hermon %p QPN %#lx send CQN %#lx receive CQN %#lx\n", ! 1220: hermon, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn ); ! 1221: ib_qp_set_drvdata ( qp, hermon_qp ); ! 1222: return 0; ! 1223: ! 1224: hermon_cmd_2rst_qp ( hermon, qp->qpn ); ! 1225: err_rst2init_qp: ! 1226: hermon_free_mtt ( hermon, &hermon_qp->mtt ); ! 1227: err_alloc_mtt: ! 1228: free_dma ( hermon_qp->wqe, hermon_qp->wqe_size ); ! 1229: err_alloc_wqe: ! 1230: free_dma ( hermon_qp->recv.doorbell, ! 1231: sizeof ( hermon_qp->recv.doorbell[0] ) ); ! 1232: err_recv_doorbell: ! 1233: free ( hermon_qp ); ! 1234: err_hermon_qp: ! 1235: hermon_free_qpn ( ibdev, qp ); ! 1236: err_alloc_qpn: ! 1237: return rc; ! 1238: } ! 1239: ! 1240: /** ! 1241: * Modify queue pair ! 1242: * ! 1243: * @v ibdev Infiniband device ! 1244: * @v qp Queue pair ! 1245: * @ret rc Return status code ! 1246: */ ! 1247: static int hermon_modify_qp ( struct ib_device *ibdev, ! 1248: struct ib_queue_pair *qp ) { ! 1249: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1250: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); ! 1251: struct hermonprm_qp_ee_state_transitions qpctx; ! 1252: int rc; ! 1253: ! 1254: /* Transition queue to RTR state, if applicable */ ! 1255: if ( hermon_qp->state < HERMON_QP_ST_RTR ) { ! 1256: memset ( &qpctx, 0, sizeof ( qpctx ) ); ! 1257: MLX_FILL_2 ( &qpctx, 4, ! 1258: qpc_eec_data.mtu, ! 1259: ( ( qp->type == IB_QPT_ETH ) ? ! 1260: HERMON_MTU_ETH : HERMON_MTU_2048 ), ! 1261: qpc_eec_data.msg_max, 31 ); ! 1262: MLX_FILL_1 ( &qpctx, 7, ! 1263: qpc_eec_data.remote_qpn_een, qp->av.qpn ); ! 1264: MLX_FILL_1 ( &qpctx, 9, ! 1265: qpc_eec_data.primary_address_path.rlid, ! 1266: qp->av.lid ); ! 1267: MLX_FILL_1 ( &qpctx, 10, ! 1268: qpc_eec_data.primary_address_path.max_stat_rate, ! 1269: hermon_rate ( &qp->av ) ); ! 1270: memcpy ( &qpctx.u.dwords[12], &qp->av.gid, ! 1271: sizeof ( qp->av.gid ) ); ! 1272: MLX_FILL_1 ( &qpctx, 16, ! 1273: qpc_eec_data.primary_address_path.sched_queue, ! 1274: hermon_sched_queue ( ibdev, qp ) ); ! 1275: MLX_FILL_1 ( &qpctx, 39, ! 1276: qpc_eec_data.next_rcv_psn, qp->recv.psn ); ! 1277: if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn, ! 1278: &qpctx ) ) != 0 ) { ! 1279: DBGC ( hermon, "Hermon %p QPN %#lx INIT2RTR_QP failed:" ! 1280: " %s\n", hermon, qp->qpn, strerror ( rc ) ); ! 1281: return rc; ! 1282: } ! 1283: hermon_qp->state = HERMON_QP_ST_RTR; ! 1284: } ! 1285: ! 1286: /* Transition queue to RTS state */ ! 1287: if ( hermon_qp->state < HERMON_QP_ST_RTS ) { ! 1288: memset ( &qpctx, 0, sizeof ( qpctx ) ); ! 1289: MLX_FILL_1 ( &qpctx, 10, ! 1290: qpc_eec_data.primary_address_path.ack_timeout, ! 1291: 14 /* 4.096us * 2^(14) = 67ms */ ); ! 1292: MLX_FILL_2 ( &qpctx, 30, ! 1293: qpc_eec_data.retry_count, HERMON_RETRY_MAX, ! 1294: qpc_eec_data.rnr_retry, HERMON_RETRY_MAX ); ! 1295: MLX_FILL_1 ( &qpctx, 32, ! 1296: qpc_eec_data.next_send_psn, qp->send.psn ); ! 1297: if ( ( rc = hermon_cmd_rtr2rts_qp ( hermon, qp->qpn, ! 1298: &qpctx ) ) != 0 ) { ! 1299: DBGC ( hermon, "Hermon %p QPN %#lx RTR2RTS_QP failed: " ! 1300: "%s\n", hermon, qp->qpn, strerror ( rc ) ); ! 1301: return rc; ! 1302: } ! 1303: hermon_qp->state = HERMON_QP_ST_RTS; ! 1304: } ! 1305: ! 1306: /* Update parameters in RTS state */ ! 1307: memset ( &qpctx, 0, sizeof ( qpctx ) ); ! 1308: MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY ); ! 1309: MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey ); ! 1310: if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){ ! 1311: DBGC ( hermon, "Hermon %p QPN %#lx RTS2RTS_QP failed: %s\n", ! 1312: hermon, qp->qpn, strerror ( rc ) ); ! 1313: return rc; ! 1314: } ! 1315: ! 1316: return 0; ! 1317: } ! 1318: ! 1319: /** ! 1320: * Destroy queue pair ! 1321: * ! 1322: * @v ibdev Infiniband device ! 1323: * @v qp Queue pair ! 1324: */ ! 1325: static void hermon_destroy_qp ( struct ib_device *ibdev, ! 1326: struct ib_queue_pair *qp ) { ! 1327: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1328: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); ! 1329: int rc; ! 1330: ! 1331: /* Take ownership back from hardware */ ! 1332: if ( ( rc = hermon_cmd_2rst_qp ( hermon, qp->qpn ) ) != 0 ) { ! 1333: DBGC ( hermon, "Hermon %p QPN %#lx FATAL 2RST_QP failed: %s\n", ! 1334: hermon, qp->qpn, strerror ( rc ) ); ! 1335: /* Leak memory and return; at least we avoid corruption */ ! 1336: return; ! 1337: } ! 1338: ! 1339: /* Free MTT entries */ ! 1340: hermon_free_mtt ( hermon, &hermon_qp->mtt ); ! 1341: ! 1342: /* Free memory */ ! 1343: free_dma ( hermon_qp->wqe, hermon_qp->wqe_size ); ! 1344: free_dma ( hermon_qp->recv.doorbell, ! 1345: sizeof ( hermon_qp->recv.doorbell[0] ) ); ! 1346: free ( hermon_qp ); ! 1347: ! 1348: /* Mark queue number as free */ ! 1349: hermon_free_qpn ( ibdev, qp ); ! 1350: ! 1351: ib_qp_set_drvdata ( qp, NULL ); ! 1352: } ! 1353: ! 1354: /*************************************************************************** ! 1355: * ! 1356: * Work request operations ! 1357: * ! 1358: *************************************************************************** ! 1359: */ ! 1360: ! 1361: /** ! 1362: * Construct UD send work queue entry ! 1363: * ! 1364: * @v ibdev Infiniband device ! 1365: * @v qp Queue pair ! 1366: * @v av Address vector ! 1367: * @v iobuf I/O buffer ! 1368: * @v wqe Send work queue entry ! 1369: * @ret opcode Control opcode ! 1370: */ ! 1371: static __attribute__ (( unused )) unsigned int ! 1372: hermon_fill_nop_send_wqe ( struct ib_device *ibdev __unused, ! 1373: struct ib_queue_pair *qp __unused, ! 1374: struct ib_address_vector *av __unused, ! 1375: struct io_buffer *iobuf __unused, ! 1376: union hermon_send_wqe *wqe ) { ! 1377: ! 1378: MLX_FILL_1 ( &wqe->ctrl, 1, ds, ( sizeof ( wqe->ctrl ) / 16 ) ); ! 1379: MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ ); ! 1380: return HERMON_OPCODE_NOP; ! 1381: } ! 1382: ! 1383: /** ! 1384: * Construct UD send work queue entry ! 1385: * ! 1386: * @v ibdev Infiniband device ! 1387: * @v qp Queue pair ! 1388: * @v av Address vector ! 1389: * @v iobuf I/O buffer ! 1390: * @v wqe Send work queue entry ! 1391: * @ret opcode Control opcode ! 1392: */ ! 1393: static unsigned int ! 1394: hermon_fill_ud_send_wqe ( struct ib_device *ibdev, ! 1395: struct ib_queue_pair *qp __unused, ! 1396: struct ib_address_vector *av, ! 1397: struct io_buffer *iobuf, ! 1398: union hermon_send_wqe *wqe ) { ! 1399: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1400: ! 1401: MLX_FILL_1 ( &wqe->ud.ctrl, 1, ds, ! 1402: ( ( offsetof ( typeof ( wqe->ud ), data[1] ) / 16 ) ) ); ! 1403: MLX_FILL_1 ( &wqe->ud.ctrl, 2, c, 0x03 /* generate completion */ ); ! 1404: MLX_FILL_2 ( &wqe->ud.ud, 0, ! 1405: ud_address_vector.pd, HERMON_GLOBAL_PD, ! 1406: ud_address_vector.port_number, ibdev->port ); ! 1407: MLX_FILL_2 ( &wqe->ud.ud, 1, ! 1408: ud_address_vector.rlid, av->lid, ! 1409: ud_address_vector.g, av->gid_present ); ! 1410: MLX_FILL_1 ( &wqe->ud.ud, 2, ! 1411: ud_address_vector.max_stat_rate, hermon_rate ( av ) ); ! 1412: MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, av->sl ); ! 1413: memcpy ( &wqe->ud.ud.u.dwords[4], &av->gid, sizeof ( av->gid ) ); ! 1414: MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, av->qpn ); ! 1415: MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, av->qkey ); ! 1416: MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) ); ! 1417: MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, hermon->lkey ); ! 1418: MLX_FILL_H ( &wqe->ud.data[0], 2, ! 1419: local_address_h, virt_to_bus ( iobuf->data ) ); ! 1420: MLX_FILL_1 ( &wqe->ud.data[0], 3, ! 1421: local_address_l, virt_to_bus ( iobuf->data ) ); ! 1422: return HERMON_OPCODE_SEND; ! 1423: } ! 1424: ! 1425: /** ! 1426: * Construct MLX send work queue entry ! 1427: * ! 1428: * @v ibdev Infiniband device ! 1429: * @v qp Queue pair ! 1430: * @v av Address vector ! 1431: * @v iobuf I/O buffer ! 1432: * @v wqe Send work queue entry ! 1433: * @ret opcode Control opcode ! 1434: */ ! 1435: static unsigned int ! 1436: hermon_fill_mlx_send_wqe ( struct ib_device *ibdev, ! 1437: struct ib_queue_pair *qp, ! 1438: struct ib_address_vector *av, ! 1439: struct io_buffer *iobuf, ! 1440: union hermon_send_wqe *wqe ) { ! 1441: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1442: struct io_buffer headers; ! 1443: ! 1444: /* Construct IB headers */ ! 1445: iob_populate ( &headers, &wqe->mlx.headers, 0, ! 1446: sizeof ( wqe->mlx.headers ) ); ! 1447: iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) ); ! 1448: ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), av ); ! 1449: ! 1450: /* Fill work queue entry */ ! 1451: MLX_FILL_1 ( &wqe->mlx.ctrl, 1, ds, ! 1452: ( ( offsetof ( typeof ( wqe->mlx ), data[2] ) / 16 ) ) ); ! 1453: MLX_FILL_5 ( &wqe->mlx.ctrl, 2, ! 1454: c, 0x03 /* generate completion */, ! 1455: icrc, 0 /* generate ICRC */, ! 1456: max_statrate, hermon_rate ( av ), ! 1457: slr, 0, ! 1458: v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) ); ! 1459: MLX_FILL_1 ( &wqe->mlx.ctrl, 3, rlid, av->lid ); ! 1460: MLX_FILL_1 ( &wqe->mlx.data[0], 0, ! 1461: byte_count, iob_len ( &headers ) ); ! 1462: MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, hermon->lkey ); ! 1463: MLX_FILL_H ( &wqe->mlx.data[0], 2, ! 1464: local_address_h, virt_to_bus ( headers.data ) ); ! 1465: MLX_FILL_1 ( &wqe->mlx.data[0], 3, ! 1466: local_address_l, virt_to_bus ( headers.data ) ); ! 1467: MLX_FILL_1 ( &wqe->mlx.data[1], 0, ! 1468: byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) ); ! 1469: MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, hermon->lkey ); ! 1470: MLX_FILL_H ( &wqe->mlx.data[1], 2, ! 1471: local_address_h, virt_to_bus ( iobuf->data ) ); ! 1472: MLX_FILL_1 ( &wqe->mlx.data[1], 3, ! 1473: local_address_l, virt_to_bus ( iobuf->data ) ); ! 1474: return HERMON_OPCODE_SEND; ! 1475: } ! 1476: ! 1477: /** ! 1478: * Construct RC send work queue entry ! 1479: * ! 1480: * @v ibdev Infiniband device ! 1481: * @v qp Queue pair ! 1482: * @v av Address vector ! 1483: * @v iobuf I/O buffer ! 1484: * @v wqe Send work queue entry ! 1485: * @ret opcode Control opcode ! 1486: */ ! 1487: static unsigned int ! 1488: hermon_fill_rc_send_wqe ( struct ib_device *ibdev, ! 1489: struct ib_queue_pair *qp __unused, ! 1490: struct ib_address_vector *av __unused, ! 1491: struct io_buffer *iobuf, ! 1492: union hermon_send_wqe *wqe ) { ! 1493: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1494: ! 1495: MLX_FILL_1 ( &wqe->rc.ctrl, 1, ds, ! 1496: ( ( offsetof ( typeof ( wqe->rc ), data[1] ) / 16 ) ) ); ! 1497: MLX_FILL_1 ( &wqe->rc.ctrl, 2, c, 0x03 /* generate completion */ ); ! 1498: MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) ); ! 1499: MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, hermon->lkey ); ! 1500: MLX_FILL_H ( &wqe->rc.data[0], 2, ! 1501: local_address_h, virt_to_bus ( iobuf->data ) ); ! 1502: MLX_FILL_1 ( &wqe->rc.data[0], 3, ! 1503: local_address_l, virt_to_bus ( iobuf->data ) ); ! 1504: return HERMON_OPCODE_SEND; ! 1505: } ! 1506: ! 1507: /** ! 1508: * Construct Ethernet send work queue entry ! 1509: * ! 1510: * @v ibdev Infiniband device ! 1511: * @v qp Queue pair ! 1512: * @v av Address vector ! 1513: * @v iobuf I/O buffer ! 1514: * @v wqe Send work queue entry ! 1515: * @ret opcode Control opcode ! 1516: */ ! 1517: static unsigned int ! 1518: hermon_fill_eth_send_wqe ( struct ib_device *ibdev, ! 1519: struct ib_queue_pair *qp __unused, ! 1520: struct ib_address_vector *av __unused, ! 1521: struct io_buffer *iobuf, ! 1522: union hermon_send_wqe *wqe ) { ! 1523: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1524: ! 1525: /* Fill work queue entry */ ! 1526: MLX_FILL_1 ( &wqe->eth.ctrl, 1, ds, ! 1527: ( ( offsetof ( typeof ( wqe->mlx ), data[1] ) / 16 ) ) ); ! 1528: MLX_FILL_2 ( &wqe->eth.ctrl, 2, ! 1529: c, 0x03 /* generate completion */, ! 1530: s, 1 /* inhibit ICRC */ ); ! 1531: MLX_FILL_1 ( &wqe->eth.data[0], 0, ! 1532: byte_count, iob_len ( iobuf ) ); ! 1533: MLX_FILL_1 ( &wqe->eth.data[0], 1, l_key, hermon->lkey ); ! 1534: MLX_FILL_H ( &wqe->eth.data[0], 2, ! 1535: local_address_h, virt_to_bus ( iobuf->data ) ); ! 1536: MLX_FILL_1 ( &wqe->eth.data[0], 3, ! 1537: local_address_l, virt_to_bus ( iobuf->data ) ); ! 1538: return HERMON_OPCODE_SEND; ! 1539: } ! 1540: ! 1541: /** Work queue entry constructors */ ! 1542: static unsigned int ! 1543: ( * hermon_fill_send_wqe[] ) ( struct ib_device *ibdev, ! 1544: struct ib_queue_pair *qp, ! 1545: struct ib_address_vector *av, ! 1546: struct io_buffer *iobuf, ! 1547: union hermon_send_wqe *wqe ) = { ! 1548: [IB_QPT_SMI] = hermon_fill_mlx_send_wqe, ! 1549: [IB_QPT_GSI] = hermon_fill_mlx_send_wqe, ! 1550: [IB_QPT_UD] = hermon_fill_ud_send_wqe, ! 1551: [IB_QPT_RC] = hermon_fill_rc_send_wqe, ! 1552: [IB_QPT_ETH] = hermon_fill_eth_send_wqe, ! 1553: }; ! 1554: ! 1555: /** ! 1556: * Post send work queue entry ! 1557: * ! 1558: * @v ibdev Infiniband device ! 1559: * @v qp Queue pair ! 1560: * @v av Address vector ! 1561: * @v iobuf I/O buffer ! 1562: * @ret rc Return status code ! 1563: */ ! 1564: static int hermon_post_send ( struct ib_device *ibdev, ! 1565: struct ib_queue_pair *qp, ! 1566: struct ib_address_vector *av, ! 1567: struct io_buffer *iobuf ) { ! 1568: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1569: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); ! 1570: struct ib_work_queue *wq = &qp->send; ! 1571: struct hermon_send_work_queue *hermon_send_wq = &hermon_qp->send; ! 1572: union hermon_send_wqe *wqe; ! 1573: union hermonprm_doorbell_register db_reg; ! 1574: unsigned long wqe_idx_mask; ! 1575: unsigned long wqe_idx; ! 1576: unsigned int owner; ! 1577: unsigned int opcode; ! 1578: ! 1579: /* Allocate work queue entry */ ! 1580: wqe_idx = ( wq->next_idx & ( hermon_send_wq->num_wqes - 1 ) ); ! 1581: owner = ( ( wq->next_idx & hermon_send_wq->num_wqes ) ? 1 : 0 ); ! 1582: wqe_idx_mask = ( wq->num_wqes - 1 ); ! 1583: if ( wq->iobufs[ wqe_idx & wqe_idx_mask ] ) { ! 1584: DBGC ( hermon, "Hermon %p QPN %#lx send queue full", ! 1585: hermon, qp->qpn ); ! 1586: return -ENOBUFS; ! 1587: } ! 1588: wq->iobufs[ wqe_idx & wqe_idx_mask ] = iobuf; ! 1589: wqe = &hermon_send_wq->wqe[wqe_idx]; ! 1590: ! 1591: /* Construct work queue entry */ ! 1592: memset ( ( ( ( void * ) wqe ) + 4 /* avoid ctrl.owner */ ), 0, ! 1593: ( sizeof ( *wqe ) - 4 ) ); ! 1594: assert ( qp->type < ( sizeof ( hermon_fill_send_wqe ) / ! 1595: sizeof ( hermon_fill_send_wqe[0] ) ) ); ! 1596: assert ( hermon_fill_send_wqe[qp->type] != NULL ); ! 1597: opcode = hermon_fill_send_wqe[qp->type] ( ibdev, qp, av, iobuf, wqe ); ! 1598: barrier(); ! 1599: MLX_FILL_2 ( &wqe->ctrl, 0, ! 1600: opcode, opcode, ! 1601: owner, owner ); ! 1602: DBGCP ( hermon, "Hermon %p QPN %#lx posting send WQE %#lx:\n", ! 1603: hermon, qp->qpn, wqe_idx ); ! 1604: DBGCP_HDA ( hermon, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) ); ! 1605: ! 1606: /* Ring doorbell register */ ! 1607: MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn ); ! 1608: barrier(); ! 1609: writel ( db_reg.dword[0], hermon_send_wq->doorbell ); ! 1610: ! 1611: /* Update work queue's index */ ! 1612: wq->next_idx++; ! 1613: ! 1614: return 0; ! 1615: } ! 1616: ! 1617: /** ! 1618: * Post receive work queue entry ! 1619: * ! 1620: * @v ibdev Infiniband device ! 1621: * @v qp Queue pair ! 1622: * @v iobuf I/O buffer ! 1623: * @ret rc Return status code ! 1624: */ ! 1625: static int hermon_post_recv ( struct ib_device *ibdev, ! 1626: struct ib_queue_pair *qp, ! 1627: struct io_buffer *iobuf ) { ! 1628: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1629: struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); ! 1630: struct ib_work_queue *wq = &qp->recv; ! 1631: struct hermon_recv_work_queue *hermon_recv_wq = &hermon_qp->recv; ! 1632: struct hermonprm_recv_wqe *wqe; ! 1633: unsigned int wqe_idx_mask; ! 1634: ! 1635: /* Allocate work queue entry */ ! 1636: wqe_idx_mask = ( wq->num_wqes - 1 ); ! 1637: if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) { ! 1638: DBGC ( hermon, "Hermon %p QPN %#lx receive queue full", ! 1639: hermon, qp->qpn ); ! 1640: return -ENOBUFS; ! 1641: } ! 1642: wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf; ! 1643: wqe = &hermon_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv; ! 1644: ! 1645: /* Construct work queue entry */ ! 1646: MLX_FILL_1 ( &wqe->data[0], 0, byte_count, iob_tailroom ( iobuf ) ); ! 1647: MLX_FILL_1 ( &wqe->data[0], 1, l_key, hermon->lkey ); ! 1648: MLX_FILL_H ( &wqe->data[0], 2, ! 1649: local_address_h, virt_to_bus ( iobuf->data ) ); ! 1650: MLX_FILL_1 ( &wqe->data[0], 3, ! 1651: local_address_l, virt_to_bus ( iobuf->data ) ); ! 1652: ! 1653: /* Update work queue's index */ ! 1654: wq->next_idx++; ! 1655: ! 1656: /* Update doorbell record */ ! 1657: barrier(); ! 1658: MLX_FILL_1 ( hermon_recv_wq->doorbell, 0, receive_wqe_counter, ! 1659: ( wq->next_idx & 0xffff ) ); ! 1660: ! 1661: return 0; ! 1662: } ! 1663: ! 1664: /** ! 1665: * Handle completion ! 1666: * ! 1667: * @v ibdev Infiniband device ! 1668: * @v cq Completion queue ! 1669: * @v cqe Hardware completion queue entry ! 1670: * @ret rc Return status code ! 1671: */ ! 1672: static int hermon_complete ( struct ib_device *ibdev, ! 1673: struct ib_completion_queue *cq, ! 1674: union hermonprm_completion_entry *cqe ) { ! 1675: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1676: struct ib_work_queue *wq; ! 1677: struct ib_queue_pair *qp; ! 1678: struct io_buffer *iobuf; ! 1679: struct ib_address_vector recv_av; ! 1680: struct ib_global_route_header *grh; ! 1681: struct ib_address_vector *av; ! 1682: unsigned int opcode; ! 1683: unsigned long qpn; ! 1684: int is_send; ! 1685: unsigned long wqe_idx; ! 1686: unsigned long wqe_idx_mask; ! 1687: size_t len; ! 1688: int rc = 0; ! 1689: ! 1690: /* Parse completion */ ! 1691: qpn = MLX_GET ( &cqe->normal, qpn ); ! 1692: is_send = MLX_GET ( &cqe->normal, s_r ); ! 1693: opcode = MLX_GET ( &cqe->normal, opcode ); ! 1694: if ( opcode >= HERMON_OPCODE_RECV_ERROR ) { ! 1695: /* "s" field is not valid for error opcodes */ ! 1696: is_send = ( opcode == HERMON_OPCODE_SEND_ERROR ); ! 1697: DBGC ( hermon, "Hermon %p CQN %#lx syndrome %x vendor %x\n", ! 1698: hermon, cq->cqn, MLX_GET ( &cqe->error, syndrome ), ! 1699: MLX_GET ( &cqe->error, vendor_error_syndrome ) ); ! 1700: rc = -EIO; ! 1701: /* Don't return immediately; propagate error to completer */ ! 1702: } ! 1703: ! 1704: /* Identify work queue */ ! 1705: wq = ib_find_wq ( cq, qpn, is_send ); ! 1706: if ( ! wq ) { ! 1707: DBGC ( hermon, "Hermon %p CQN %#lx unknown %s QPN %#lx\n", ! 1708: hermon, cq->cqn, ( is_send ? "send" : "recv" ), qpn ); ! 1709: return -EIO; ! 1710: } ! 1711: qp = wq->qp; ! 1712: ! 1713: /* Identify work queue entry */ ! 1714: wqe_idx = MLX_GET ( &cqe->normal, wqe_counter ); ! 1715: wqe_idx_mask = ( wq->num_wqes - 1 ); ! 1716: DBGCP ( hermon, "Hermon %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n", ! 1717: hermon, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ), ! 1718: wqe_idx ); ! 1719: DBGCP_HDA ( hermon, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) ); ! 1720: ! 1721: /* Identify I/O buffer */ ! 1722: iobuf = wq->iobufs[ wqe_idx & wqe_idx_mask ]; ! 1723: if ( ! iobuf ) { ! 1724: DBGC ( hermon, "Hermon %p CQN %#lx QPN %#lx empty %s WQE " ! 1725: "%#lx\n", hermon, cq->cqn, qp->qpn, ! 1726: ( is_send ? "send" : "recv" ), wqe_idx ); ! 1727: return -EIO; ! 1728: } ! 1729: wq->iobufs[ wqe_idx & wqe_idx_mask ] = NULL; ! 1730: ! 1731: if ( is_send ) { ! 1732: /* Hand off to completion handler */ ! 1733: ib_complete_send ( ibdev, qp, iobuf, rc ); ! 1734: } else { ! 1735: /* Set received length */ ! 1736: len = MLX_GET ( &cqe->normal, byte_cnt ); ! 1737: assert ( len <= iob_tailroom ( iobuf ) ); ! 1738: iob_put ( iobuf, len ); ! 1739: memset ( &recv_av, 0, sizeof ( recv_av ) ); ! 1740: switch ( qp->type ) { ! 1741: case IB_QPT_SMI: ! 1742: case IB_QPT_GSI: ! 1743: case IB_QPT_UD: ! 1744: assert ( iob_len ( iobuf ) >= sizeof ( *grh ) ); ! 1745: grh = iobuf->data; ! 1746: iob_pull ( iobuf, sizeof ( *grh ) ); ! 1747: /* Construct address vector */ ! 1748: av = &recv_av; ! 1749: av->qpn = MLX_GET ( &cqe->normal, srq_rqpn ); ! 1750: av->lid = MLX_GET ( &cqe->normal, slid_smac47_32 ); ! 1751: av->sl = MLX_GET ( &cqe->normal, sl ); ! 1752: av->gid_present = MLX_GET ( &cqe->normal, g ); ! 1753: memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) ); ! 1754: break; ! 1755: case IB_QPT_RC: ! 1756: av = &qp->av; ! 1757: break; ! 1758: case IB_QPT_ETH: ! 1759: /* Construct address vector */ ! 1760: av = &recv_av; ! 1761: av->vlan_present = MLX_GET ( &cqe->normal, vlan ); ! 1762: av->vlan = MLX_GET ( &cqe->normal, vid ); ! 1763: break; ! 1764: default: ! 1765: assert ( 0 ); ! 1766: return -EINVAL; ! 1767: } ! 1768: /* Hand off to completion handler */ ! 1769: ib_complete_recv ( ibdev, qp, av, iobuf, rc ); ! 1770: } ! 1771: ! 1772: return rc; ! 1773: } ! 1774: ! 1775: /** ! 1776: * Poll completion queue ! 1777: * ! 1778: * @v ibdev Infiniband device ! 1779: * @v cq Completion queue ! 1780: */ ! 1781: static void hermon_poll_cq ( struct ib_device *ibdev, ! 1782: struct ib_completion_queue *cq ) { ! 1783: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1784: struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq ); ! 1785: union hermonprm_completion_entry *cqe; ! 1786: unsigned int cqe_idx_mask; ! 1787: int rc; ! 1788: ! 1789: while ( 1 ) { ! 1790: /* Look for completion entry */ ! 1791: cqe_idx_mask = ( cq->num_cqes - 1 ); ! 1792: cqe = &hermon_cq->cqe[cq->next_idx & cqe_idx_mask]; ! 1793: if ( MLX_GET ( &cqe->normal, owner ) ^ ! 1794: ( ( cq->next_idx & cq->num_cqes ) ? 1 : 0 ) ) { ! 1795: /* Entry still owned by hardware; end of poll */ ! 1796: break; ! 1797: } ! 1798: ! 1799: /* Handle completion */ ! 1800: if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) { ! 1801: DBGC ( hermon, "Hermon %p CQN %#lx failed to complete:" ! 1802: " %s\n", hermon, cq->cqn, strerror ( rc ) ); ! 1803: DBGC_HDA ( hermon, virt_to_phys ( cqe ), ! 1804: cqe, sizeof ( *cqe ) ); ! 1805: } ! 1806: ! 1807: /* Update completion queue's index */ ! 1808: cq->next_idx++; ! 1809: ! 1810: /* Update doorbell record */ ! 1811: MLX_FILL_1 ( hermon_cq->doorbell, 0, update_ci, ! 1812: ( cq->next_idx & 0x00ffffffUL ) ); ! 1813: } ! 1814: } ! 1815: ! 1816: /*************************************************************************** ! 1817: * ! 1818: * Event queues ! 1819: * ! 1820: *************************************************************************** ! 1821: */ ! 1822: ! 1823: /** ! 1824: * Create event queue ! 1825: * ! 1826: * @v hermon Hermon device ! 1827: * @ret rc Return status code ! 1828: */ ! 1829: static int hermon_create_eq ( struct hermon *hermon ) { ! 1830: struct hermon_event_queue *hermon_eq = &hermon->eq; ! 1831: struct hermonprm_eqc eqctx; ! 1832: struct hermonprm_event_mask mask; ! 1833: unsigned int i; ! 1834: int rc; ! 1835: ! 1836: /* Select event queue number */ ! 1837: hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars ); ! 1838: if ( hermon_eq->eqn < hermon->cap.reserved_eqs ) ! 1839: hermon_eq->eqn = hermon->cap.reserved_eqs; ! 1840: ! 1841: /* Calculate doorbell address */ ! 1842: hermon_eq->doorbell = ! 1843: ( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) ); ! 1844: ! 1845: /* Allocate event queue itself */ ! 1846: hermon_eq->eqe_size = ! 1847: ( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) ); ! 1848: hermon_eq->eqe = malloc_dma ( hermon_eq->eqe_size, ! 1849: sizeof ( hermon_eq->eqe[0] ) ); ! 1850: if ( ! hermon_eq->eqe ) { ! 1851: rc = -ENOMEM; ! 1852: goto err_eqe; ! 1853: } ! 1854: memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size ); ! 1855: for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) { ! 1856: MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 ); ! 1857: } ! 1858: barrier(); ! 1859: ! 1860: /* Allocate MTT entries */ ! 1861: if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe, ! 1862: hermon_eq->eqe_size, ! 1863: &hermon_eq->mtt ) ) != 0 ) ! 1864: goto err_alloc_mtt; ! 1865: ! 1866: /* Hand queue over to hardware */ ! 1867: memset ( &eqctx, 0, sizeof ( eqctx ) ); ! 1868: MLX_FILL_2 ( &eqctx, 0, ! 1869: st, 0xa /* "Fired" */, ! 1870: oi, 1 ); ! 1871: MLX_FILL_1 ( &eqctx, 2, ! 1872: page_offset, ( hermon_eq->mtt.page_offset >> 5 ) ); ! 1873: MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) ); ! 1874: MLX_FILL_H ( &eqctx, 6, mtt_base_addr_h, ! 1875: hermon_eq->mtt.mtt_base_addr ); ! 1876: MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l, ! 1877: ( hermon_eq->mtt.mtt_base_addr >> 3 ) ); ! 1878: if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn, ! 1879: &eqctx ) ) != 0 ) { ! 1880: DBGC ( hermon, "Hermon %p EQN %#lx SW2HW_EQ failed: %s\n", ! 1881: hermon, hermon_eq->eqn, strerror ( rc ) ); ! 1882: goto err_sw2hw_eq; ! 1883: } ! 1884: ! 1885: /* Map all events to this event queue */ ! 1886: memset ( &mask, 0xff, sizeof ( mask ) ); ! 1887: if ( ( rc = hermon_cmd_map_eq ( hermon, ! 1888: ( HERMON_MAP_EQ | hermon_eq->eqn ), ! 1889: &mask ) ) != 0 ) { ! 1890: DBGC ( hermon, "Hermon %p EQN %#lx MAP_EQ failed: %s\n", ! 1891: hermon, hermon_eq->eqn, strerror ( rc ) ); ! 1892: goto err_map_eq; ! 1893: } ! 1894: ! 1895: DBGC ( hermon, "Hermon %p EQN %#lx ring [%08lx,%08lx), doorbell " ! 1896: "%08lx\n", hermon, hermon_eq->eqn, ! 1897: virt_to_phys ( hermon_eq->eqe ), ! 1898: ( virt_to_phys ( hermon_eq->eqe ) + hermon_eq->eqe_size ), ! 1899: virt_to_phys ( hermon_eq->doorbell ) ); ! 1900: return 0; ! 1901: ! 1902: err_map_eq: ! 1903: hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx ); ! 1904: err_sw2hw_eq: ! 1905: hermon_free_mtt ( hermon, &hermon_eq->mtt ); ! 1906: err_alloc_mtt: ! 1907: free_dma ( hermon_eq->eqe, hermon_eq->eqe_size ); ! 1908: err_eqe: ! 1909: memset ( hermon_eq, 0, sizeof ( *hermon_eq ) ); ! 1910: return rc; ! 1911: } ! 1912: ! 1913: /** ! 1914: * Destroy event queue ! 1915: * ! 1916: * @v hermon Hermon device ! 1917: */ ! 1918: static void hermon_destroy_eq ( struct hermon *hermon ) { ! 1919: struct hermon_event_queue *hermon_eq = &hermon->eq; ! 1920: struct hermonprm_eqc eqctx; ! 1921: struct hermonprm_event_mask mask; ! 1922: int rc; ! 1923: ! 1924: /* Unmap events from event queue */ ! 1925: memset ( &mask, 0xff, sizeof ( mask ) ); ! 1926: if ( ( rc = hermon_cmd_map_eq ( hermon, ! 1927: ( HERMON_UNMAP_EQ | hermon_eq->eqn ), ! 1928: &mask ) ) != 0 ) { ! 1929: DBGC ( hermon, "Hermon %p EQN %#lx FATAL MAP_EQ failed to " ! 1930: "unmap: %s\n", hermon, hermon_eq->eqn, strerror ( rc ) ); ! 1931: /* Continue; HCA may die but system should survive */ ! 1932: } ! 1933: ! 1934: /* Take ownership back from hardware */ ! 1935: if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, ! 1936: &eqctx ) ) != 0 ) { ! 1937: DBGC ( hermon, "Hermon %p EQN %#lx FATAL HW2SW_EQ failed: %s\n", ! 1938: hermon, hermon_eq->eqn, strerror ( rc ) ); ! 1939: /* Leak memory and return; at least we avoid corruption */ ! 1940: return; ! 1941: } ! 1942: ! 1943: /* Free MTT entries */ ! 1944: hermon_free_mtt ( hermon, &hermon_eq->mtt ); ! 1945: ! 1946: /* Free memory */ ! 1947: free_dma ( hermon_eq->eqe, hermon_eq->eqe_size ); ! 1948: memset ( hermon_eq, 0, sizeof ( *hermon_eq ) ); ! 1949: } ! 1950: ! 1951: /** ! 1952: * Handle port state event ! 1953: * ! 1954: * @v hermon Hermon device ! 1955: * @v eqe Port state change event queue entry ! 1956: */ ! 1957: static void hermon_event_port_state_change ( struct hermon *hermon, ! 1958: union hermonprm_event_entry *eqe){ ! 1959: unsigned int port; ! 1960: int link_up; ! 1961: ! 1962: /* Get port and link status */ ! 1963: port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 ); ! 1964: link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 ); ! 1965: DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ), ! 1966: ( link_up ? "up" : "down" ) ); ! 1967: ! 1968: /* Sanity check */ ! 1969: if ( port >= hermon->cap.num_ports ) { ! 1970: DBGC ( hermon, "Hermon %p port %d does not exist!\n", ! 1971: hermon, ( port + 1 ) ); ! 1972: return; ! 1973: } ! 1974: ! 1975: /* Notify device of port state change */ ! 1976: hermon->port[port].type->state_change ( hermon, &hermon->port[port], ! 1977: link_up ); ! 1978: } ! 1979: ! 1980: /** ! 1981: * Poll event queue ! 1982: * ! 1983: * @v ibdev Infiniband device ! 1984: */ ! 1985: static void hermon_poll_eq ( struct ib_device *ibdev ) { ! 1986: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 1987: struct hermon_event_queue *hermon_eq = &hermon->eq; ! 1988: union hermonprm_event_entry *eqe; ! 1989: union hermonprm_doorbell_register db_reg; ! 1990: unsigned int eqe_idx_mask; ! 1991: unsigned int event_type; ! 1992: ! 1993: /* No event is generated upon reaching INIT, so we must poll ! 1994: * separately for link state changes while we remain DOWN. ! 1995: */ ! 1996: if ( ib_is_open ( ibdev ) && ! 1997: ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) { ! 1998: ib_smc_update ( ibdev, hermon_mad ); ! 1999: } ! 2000: ! 2001: /* Poll event queue */ ! 2002: while ( 1 ) { ! 2003: /* Look for event entry */ ! 2004: eqe_idx_mask = ( HERMON_NUM_EQES - 1 ); ! 2005: eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask]; ! 2006: if ( MLX_GET ( &eqe->generic, owner ) ^ ! 2007: ( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) { ! 2008: /* Entry still owned by hardware; end of poll */ ! 2009: break; ! 2010: } ! 2011: DBGCP ( hermon, "Hermon %p EQN %#lx event:\n", ! 2012: hermon, hermon_eq->eqn ); ! 2013: DBGCP_HDA ( hermon, virt_to_phys ( eqe ), ! 2014: eqe, sizeof ( *eqe ) ); ! 2015: ! 2016: /* Handle event */ ! 2017: event_type = MLX_GET ( &eqe->generic, event_type ); ! 2018: switch ( event_type ) { ! 2019: case HERMON_EV_PORT_STATE_CHANGE: ! 2020: hermon_event_port_state_change ( hermon, eqe ); ! 2021: break; ! 2022: default: ! 2023: DBGC ( hermon, "Hermon %p EQN %#lx unrecognised event " ! 2024: "type %#x:\n", ! 2025: hermon, hermon_eq->eqn, event_type ); ! 2026: DBGC_HDA ( hermon, virt_to_phys ( eqe ), ! 2027: eqe, sizeof ( *eqe ) ); ! 2028: break; ! 2029: } ! 2030: ! 2031: /* Update event queue's index */ ! 2032: hermon_eq->next_idx++; ! 2033: ! 2034: /* Ring doorbell */ ! 2035: MLX_FILL_1 ( &db_reg.event, 0, ! 2036: ci, ( hermon_eq->next_idx & 0x00ffffffUL ) ); ! 2037: writel ( db_reg.dword[0], hermon_eq->doorbell ); ! 2038: } ! 2039: } ! 2040: ! 2041: /*************************************************************************** ! 2042: * ! 2043: * Infiniband link-layer operations ! 2044: * ! 2045: *************************************************************************** ! 2046: */ ! 2047: ! 2048: /** ! 2049: * Initialise Infiniband link ! 2050: * ! 2051: * @v ibdev Infiniband device ! 2052: * @ret rc Return status code ! 2053: */ ! 2054: static int hermon_open ( struct ib_device *ibdev ) { ! 2055: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2056: union hermonprm_set_port set_port; ! 2057: int rc; ! 2058: ! 2059: /* Set port parameters */ ! 2060: memset ( &set_port, 0, sizeof ( set_port ) ); ! 2061: MLX_FILL_8 ( &set_port.ib, 0, ! 2062: mmc, 1, ! 2063: mvc, 1, ! 2064: mp, 1, ! 2065: mg, 1, ! 2066: mtu_cap, IB_MTU_2048, ! 2067: vl_cap, IB_VL_0, ! 2068: rcm, 1, ! 2069: lss, 1 ); ! 2070: MLX_FILL_2 ( &set_port.ib, 10, ! 2071: max_pkey, 1, ! 2072: max_gid, 1 ); ! 2073: MLX_FILL_1 ( &set_port.ib, 28, ! 2074: link_speed_supported, 1 ); ! 2075: if ( ( rc = hermon_cmd_set_port ( hermon, 0, ibdev->port, ! 2076: &set_port ) ) != 0 ) { ! 2077: DBGC ( hermon, "Hermon %p port %d could not set port: %s\n", ! 2078: hermon, ibdev->port, strerror ( rc ) ); ! 2079: return rc; ! 2080: } ! 2081: ! 2082: /* Initialise port */ ! 2083: if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) { ! 2084: DBGC ( hermon, "Hermon %p port %d could not initialise port: " ! 2085: "%s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2086: return rc; ! 2087: } ! 2088: ! 2089: /* Update MAD parameters */ ! 2090: ib_smc_update ( ibdev, hermon_mad ); ! 2091: ! 2092: return 0; ! 2093: } ! 2094: ! 2095: /** ! 2096: * Close Infiniband link ! 2097: * ! 2098: * @v ibdev Infiniband device ! 2099: */ ! 2100: static void hermon_close ( struct ib_device *ibdev ) { ! 2101: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2102: int rc; ! 2103: ! 2104: if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) { ! 2105: DBGC ( hermon, "Hermon %p port %d could not close port: %s\n", ! 2106: hermon, ibdev->port, strerror ( rc ) ); ! 2107: /* Nothing we can do about this */ ! 2108: } ! 2109: } ! 2110: ! 2111: /** ! 2112: * Inform embedded subnet management agent of a received MAD ! 2113: * ! 2114: * @v ibdev Infiniband device ! 2115: * @v mad MAD ! 2116: * @ret rc Return status code ! 2117: */ ! 2118: static int hermon_inform_sma ( struct ib_device *ibdev, ! 2119: union ib_mad *mad ) { ! 2120: int rc; ! 2121: ! 2122: /* Send the MAD to the embedded SMA */ ! 2123: if ( ( rc = hermon_mad ( ibdev, mad ) ) != 0 ) ! 2124: return rc; ! 2125: ! 2126: /* Update parameters held in software */ ! 2127: ib_smc_update ( ibdev, hermon_mad ); ! 2128: ! 2129: return 0; ! 2130: } ! 2131: ! 2132: /*************************************************************************** ! 2133: * ! 2134: * Multicast group operations ! 2135: * ! 2136: *************************************************************************** ! 2137: */ ! 2138: ! 2139: /** ! 2140: * Attach to multicast group ! 2141: * ! 2142: * @v ibdev Infiniband device ! 2143: * @v qp Queue pair ! 2144: * @v gid Multicast GID ! 2145: * @ret rc Return status code ! 2146: */ ! 2147: static int hermon_mcast_attach ( struct ib_device *ibdev, ! 2148: struct ib_queue_pair *qp, ! 2149: union ib_gid *gid ) { ! 2150: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2151: struct hermonprm_mgm_hash hash; ! 2152: struct hermonprm_mcg_entry mcg; ! 2153: unsigned int index; ! 2154: int rc; ! 2155: ! 2156: /* Generate hash table index */ ! 2157: if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) { ! 2158: DBGC ( hermon, "Hermon %p could not hash GID: %s\n", ! 2159: hermon, strerror ( rc ) ); ! 2160: return rc; ! 2161: } ! 2162: index = MLX_GET ( &hash, hash ); ! 2163: ! 2164: /* Check for existing hash table entry */ ! 2165: if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) { ! 2166: DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n", ! 2167: hermon, index, strerror ( rc ) ); ! 2168: return rc; ! 2169: } ! 2170: if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) { ! 2171: /* FIXME: this implementation allows only a single QP ! 2172: * per multicast group, and doesn't handle hash ! 2173: * collisions. Sufficient for IPoIB but may need to ! 2174: * be extended in future. ! 2175: */ ! 2176: DBGC ( hermon, "Hermon %p MGID index %#x already in use\n", ! 2177: hermon, index ); ! 2178: return -EBUSY; ! 2179: } ! 2180: ! 2181: /* Update hash table entry */ ! 2182: MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 ); ! 2183: MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn ); ! 2184: memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) ); ! 2185: if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) { ! 2186: DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n", ! 2187: hermon, index, strerror ( rc ) ); ! 2188: return rc; ! 2189: } ! 2190: ! 2191: return 0; ! 2192: } ! 2193: ! 2194: /** ! 2195: * Detach from multicast group ! 2196: * ! 2197: * @v ibdev Infiniband device ! 2198: * @v qp Queue pair ! 2199: * @v gid Multicast GID ! 2200: */ ! 2201: static void hermon_mcast_detach ( struct ib_device *ibdev, ! 2202: struct ib_queue_pair *qp __unused, ! 2203: union ib_gid *gid ) { ! 2204: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2205: struct hermonprm_mgm_hash hash; ! 2206: struct hermonprm_mcg_entry mcg; ! 2207: unsigned int index; ! 2208: int rc; ! 2209: ! 2210: /* Generate hash table index */ ! 2211: if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) { ! 2212: DBGC ( hermon, "Hermon %p could not hash GID: %s\n", ! 2213: hermon, strerror ( rc ) ); ! 2214: return; ! 2215: } ! 2216: index = MLX_GET ( &hash, hash ); ! 2217: ! 2218: /* Clear hash table entry */ ! 2219: memset ( &mcg, 0, sizeof ( mcg ) ); ! 2220: if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) { ! 2221: DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n", ! 2222: hermon, index, strerror ( rc ) ); ! 2223: return; ! 2224: } ! 2225: } ! 2226: ! 2227: /** Hermon Infiniband operations */ ! 2228: static struct ib_device_operations hermon_ib_operations = { ! 2229: .create_cq = hermon_create_cq, ! 2230: .destroy_cq = hermon_destroy_cq, ! 2231: .create_qp = hermon_create_qp, ! 2232: .modify_qp = hermon_modify_qp, ! 2233: .destroy_qp = hermon_destroy_qp, ! 2234: .post_send = hermon_post_send, ! 2235: .post_recv = hermon_post_recv, ! 2236: .poll_cq = hermon_poll_cq, ! 2237: .poll_eq = hermon_poll_eq, ! 2238: .open = hermon_open, ! 2239: .close = hermon_close, ! 2240: .mcast_attach = hermon_mcast_attach, ! 2241: .mcast_detach = hermon_mcast_detach, ! 2242: .set_port_info = hermon_inform_sma, ! 2243: .set_pkey_table = hermon_inform_sma, ! 2244: }; ! 2245: ! 2246: /** ! 2247: * Register Hermon Infiniband device ! 2248: * ! 2249: * @v hermon Hermon device ! 2250: * @v port Hermon port ! 2251: * @ret rc Return status code ! 2252: */ ! 2253: static int hermon_register_ibdev ( struct hermon *hermon, ! 2254: struct hermon_port *port ) { ! 2255: struct ib_device *ibdev = port->ibdev; ! 2256: int rc; ! 2257: ! 2258: /* Initialise parameters using SMC */ ! 2259: ib_smc_init ( ibdev, hermon_mad ); ! 2260: ! 2261: /* Register Infiniband device */ ! 2262: if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) { ! 2263: DBGC ( hermon, "Hermon %p port %d could not register IB " ! 2264: "device: %s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2265: return rc; ! 2266: } ! 2267: ! 2268: return 0; ! 2269: } ! 2270: ! 2271: /** ! 2272: * Handle Hermon Infiniband device port state change ! 2273: * ! 2274: * @v hermon Hermon device ! 2275: * @v port Hermon port ! 2276: * @v link_up Link is up ! 2277: */ ! 2278: static void hermon_state_change_ibdev ( struct hermon *hermon __unused, ! 2279: struct hermon_port *port, ! 2280: int link_up __unused ) { ! 2281: struct ib_device *ibdev = port->ibdev; ! 2282: ! 2283: /* Update MAD parameters */ ! 2284: ib_smc_update ( ibdev, hermon_mad ); ! 2285: } ! 2286: ! 2287: /** ! 2288: * Unregister Hermon Infiniband device ! 2289: * ! 2290: * @v hermon Hermon device ! 2291: * @v port Hermon port ! 2292: */ ! 2293: static void hermon_unregister_ibdev ( struct hermon *hermon __unused, ! 2294: struct hermon_port *port ) { ! 2295: struct ib_device *ibdev = port->ibdev; ! 2296: ! 2297: unregister_ibdev ( ibdev ); ! 2298: } ! 2299: ! 2300: /** Hermon Infiniband port type */ ! 2301: static struct hermon_port_type hermon_port_type_ib = { ! 2302: .register_dev = hermon_register_ibdev, ! 2303: .state_change = hermon_state_change_ibdev, ! 2304: .unregister_dev = hermon_unregister_ibdev, ! 2305: }; ! 2306: ! 2307: /*************************************************************************** ! 2308: * ! 2309: * Ethernet operation ! 2310: * ! 2311: *************************************************************************** ! 2312: */ ! 2313: ! 2314: /** Number of Hermon Ethernet send work queue entries */ ! 2315: #define HERMON_ETH_NUM_SEND_WQES 2 ! 2316: ! 2317: /** Number of Hermon Ethernet receive work queue entries */ ! 2318: #define HERMON_ETH_NUM_RECV_WQES 4 ! 2319: ! 2320: /** Number of Hermon Ethernet completion entries */ ! 2321: #define HERMON_ETH_NUM_CQES 8 ! 2322: ! 2323: /** ! 2324: * Transmit packet via Hermon Ethernet device ! 2325: * ! 2326: * @v netdev Network device ! 2327: * @v iobuf I/O buffer ! 2328: * @ret rc Return status code ! 2329: */ ! 2330: static int hermon_eth_transmit ( struct net_device *netdev, ! 2331: struct io_buffer *iobuf ) { ! 2332: struct hermon_port *port = netdev->priv; ! 2333: struct ib_device *ibdev = port->ibdev; ! 2334: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2335: int rc; ! 2336: ! 2337: /* Transmit packet */ ! 2338: if ( ( rc = ib_post_send ( ibdev, port->eth_qp, NULL, ! 2339: iobuf ) ) != 0 ) { ! 2340: DBGC ( hermon, "Hermon %p port %d could not transmit: %s\n", ! 2341: hermon, ibdev->port, strerror ( rc ) ); ! 2342: return rc; ! 2343: } ! 2344: ! 2345: return 0; ! 2346: } ! 2347: ! 2348: /** ! 2349: * Handle Hermon Ethernet device send completion ! 2350: * ! 2351: * @v ibdev Infiniband device ! 2352: * @v qp Queue pair ! 2353: * @v iobuf I/O buffer ! 2354: * @v rc Completion status code ! 2355: */ ! 2356: static void hermon_eth_complete_send ( struct ib_device *ibdev __unused, ! 2357: struct ib_queue_pair *qp, ! 2358: struct io_buffer *iobuf, int rc ) { ! 2359: struct net_device *netdev = ib_qp_get_ownerdata ( qp ); ! 2360: ! 2361: netdev_tx_complete_err ( netdev, iobuf, rc ); ! 2362: } ! 2363: ! 2364: /** ! 2365: * Handle Hermon Ethernet device receive completion ! 2366: * ! 2367: * @v ibdev Infiniband device ! 2368: * @v qp Queue pair ! 2369: * @v av Address vector, or NULL ! 2370: * @v iobuf I/O buffer ! 2371: * @v rc Completion status code ! 2372: */ ! 2373: static void hermon_eth_complete_recv ( struct ib_device *ibdev __unused, ! 2374: struct ib_queue_pair *qp, ! 2375: struct ib_address_vector *av, ! 2376: struct io_buffer *iobuf, int rc ) { ! 2377: struct net_device *netdev = ib_qp_get_ownerdata ( qp ); ! 2378: struct net_device *vlan; ! 2379: ! 2380: /* Find VLAN device, if applicable */ ! 2381: if ( av->vlan_present ) { ! 2382: if ( ( vlan = vlan_find ( netdev, av->vlan ) ) != NULL ) { ! 2383: netdev = vlan; ! 2384: } else if ( rc == 0 ) { ! 2385: rc = -ENODEV; ! 2386: } ! 2387: } ! 2388: ! 2389: /* Hand off to network layer */ ! 2390: if ( rc == 0 ) { ! 2391: netdev_rx ( netdev, iobuf ); ! 2392: } else { ! 2393: netdev_rx_err ( netdev, iobuf, rc ); ! 2394: } ! 2395: } ! 2396: ! 2397: /** Hermon Ethernet device completion operations */ ! 2398: static struct ib_completion_queue_operations hermon_eth_cq_op = { ! 2399: .complete_send = hermon_eth_complete_send, ! 2400: .complete_recv = hermon_eth_complete_recv, ! 2401: }; ! 2402: ! 2403: /** ! 2404: * Poll Hermon Ethernet device ! 2405: * ! 2406: * @v netdev Network device ! 2407: */ ! 2408: static void hermon_eth_poll ( struct net_device *netdev ) { ! 2409: struct hermon_port *port = netdev->priv; ! 2410: struct ib_device *ibdev = port->ibdev; ! 2411: ! 2412: ib_poll_eq ( ibdev ); ! 2413: } ! 2414: ! 2415: /** ! 2416: * Open Hermon Ethernet device ! 2417: * ! 2418: * @v netdev Network device ! 2419: * @ret rc Return status code ! 2420: */ ! 2421: static int hermon_eth_open ( struct net_device *netdev ) { ! 2422: struct hermon_port *port = netdev->priv; ! 2423: struct ib_device *ibdev = port->ibdev; ! 2424: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2425: union hermonprm_set_port set_port; ! 2426: int rc; ! 2427: ! 2428: /* Allocate completion queue */ ! 2429: port->eth_cq = ib_create_cq ( ibdev, HERMON_ETH_NUM_CQES, ! 2430: &hermon_eth_cq_op ); ! 2431: if ( ! port->eth_cq ) { ! 2432: DBGC ( hermon, "Hermon %p port %d could not create completion " ! 2433: "queue\n", hermon, ibdev->port ); ! 2434: rc = -ENOMEM; ! 2435: goto err_create_cq; ! 2436: } ! 2437: ! 2438: /* Allocate queue pair */ ! 2439: port->eth_qp = ib_create_qp ( ibdev, IB_QPT_ETH, ! 2440: HERMON_ETH_NUM_SEND_WQES, port->eth_cq, ! 2441: HERMON_ETH_NUM_RECV_WQES, port->eth_cq ); ! 2442: if ( ! port->eth_qp ) { ! 2443: DBGC ( hermon, "Hermon %p port %d could not create queue " ! 2444: "pair\n", hermon, ibdev->port ); ! 2445: rc = -ENOMEM; ! 2446: goto err_create_qp; ! 2447: } ! 2448: ib_qp_set_ownerdata ( port->eth_qp, netdev ); ! 2449: ! 2450: /* Activate queue pair */ ! 2451: if ( ( rc = ib_modify_qp ( ibdev, port->eth_qp ) ) != 0 ) { ! 2452: DBGC ( hermon, "Hermon %p port %d could not modify queue " ! 2453: "pair: %s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2454: goto err_modify_qp; ! 2455: } ! 2456: ! 2457: /* Fill receive rings */ ! 2458: ib_refill_recv ( ibdev, port->eth_qp ); ! 2459: ! 2460: /* Set port general parameters */ ! 2461: memset ( &set_port, 0, sizeof ( set_port ) ); ! 2462: MLX_FILL_3 ( &set_port.general, 0, ! 2463: v_mtu, 1, ! 2464: v_pprx, 1, ! 2465: v_pptx, 1 ); ! 2466: MLX_FILL_1 ( &set_port.general, 1, ! 2467: mtu, ( ETH_FRAME_LEN + 40 /* Used by card */ ) ); ! 2468: MLX_FILL_1 ( &set_port.general, 2, ! 2469: pfctx, ( 1 << FCOE_VLAN_PRIORITY ) ); ! 2470: MLX_FILL_1 ( &set_port.general, 3, ! 2471: pfcrx, ( 1 << FCOE_VLAN_PRIORITY ) ); ! 2472: if ( ( rc = hermon_cmd_set_port ( hermon, 1, ! 2473: ( HERMON_SET_PORT_GENERAL_PARAM | ! 2474: ibdev->port ), ! 2475: &set_port ) ) != 0 ) { ! 2476: DBGC ( hermon, "Hermon %p port %d could not set port general " ! 2477: "parameters: %s\n", ! 2478: hermon, ibdev->port, strerror ( rc ) ); ! 2479: goto err_set_port_general_params; ! 2480: } ! 2481: ! 2482: /* Set port receive QP */ ! 2483: memset ( &set_port, 0, sizeof ( set_port ) ); ! 2484: MLX_FILL_1 ( &set_port.rqp_calc, 0, base_qpn, port->eth_qp->qpn ); ! 2485: MLX_FILL_1 ( &set_port.rqp_calc, 2, ! 2486: mac_miss_index, 128 /* MAC misses go to promisc QP */ ); ! 2487: MLX_FILL_2 ( &set_port.rqp_calc, 3, ! 2488: vlan_miss_index, 127 /* VLAN misses go to promisc QP */, ! 2489: no_vlan_index, 126 /* VLAN-free go to promisc QP */ ); ! 2490: MLX_FILL_2 ( &set_port.rqp_calc, 5, ! 2491: promisc_qpn, port->eth_qp->qpn, ! 2492: en_uc_promisc, 1 ); ! 2493: MLX_FILL_2 ( &set_port.rqp_calc, 6, ! 2494: def_mcast_qpn, port->eth_qp->qpn, ! 2495: mc_promisc_mode, 2 /* Receive all multicasts */ ); ! 2496: if ( ( rc = hermon_cmd_set_port ( hermon, 1, ! 2497: ( HERMON_SET_PORT_RECEIVE_QP | ! 2498: ibdev->port ), ! 2499: &set_port ) ) != 0 ) { ! 2500: DBGC ( hermon, "Hermon %p port %d could not set port receive " ! 2501: "QP: %s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2502: goto err_set_port_receive_qp; ! 2503: } ! 2504: ! 2505: /* Initialise port */ ! 2506: if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) { ! 2507: DBGC ( hermon, "Hermon %p port %d could not initialise port: " ! 2508: "%s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2509: goto err_init_port; ! 2510: } ! 2511: ! 2512: return 0; ! 2513: ! 2514: err_init_port: ! 2515: err_set_port_receive_qp: ! 2516: err_set_port_general_params: ! 2517: err_modify_qp: ! 2518: ib_destroy_qp ( ibdev, port->eth_qp ); ! 2519: err_create_qp: ! 2520: ib_destroy_cq ( ibdev, port->eth_cq ); ! 2521: err_create_cq: ! 2522: return rc; ! 2523: } ! 2524: ! 2525: /** ! 2526: * Close Hermon Ethernet device ! 2527: * ! 2528: * @v netdev Network device ! 2529: */ ! 2530: static void hermon_eth_close ( struct net_device *netdev ) { ! 2531: struct hermon_port *port = netdev->priv; ! 2532: struct ib_device *ibdev = port->ibdev; ! 2533: struct hermon *hermon = ib_get_drvdata ( ibdev ); ! 2534: int rc; ! 2535: ! 2536: /* Close port */ ! 2537: if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) { ! 2538: DBGC ( hermon, "Hermon %p port %d could not close port: %s\n", ! 2539: hermon, ibdev->port, strerror ( rc ) ); ! 2540: /* Nothing we can do about this */ ! 2541: } ! 2542: ! 2543: /* Tear down the queues */ ! 2544: ib_destroy_qp ( ibdev, port->eth_qp ); ! 2545: ib_destroy_cq ( ibdev, port->eth_cq ); ! 2546: } ! 2547: ! 2548: /** Hermon Ethernet network device operations */ ! 2549: static struct net_device_operations hermon_eth_operations = { ! 2550: .open = hermon_eth_open, ! 2551: .close = hermon_eth_close, ! 2552: .transmit = hermon_eth_transmit, ! 2553: .poll = hermon_eth_poll, ! 2554: }; ! 2555: ! 2556: /** ! 2557: * Register Hermon Ethernet device ! 2558: * ! 2559: * @v hermon Hermon device ! 2560: * @v port Hermon port ! 2561: * @ret rc Return status code ! 2562: */ ! 2563: static int hermon_register_netdev ( struct hermon *hermon, ! 2564: struct hermon_port *port ) { ! 2565: struct net_device *netdev = port->netdev; ! 2566: struct ib_device *ibdev = port->ibdev; ! 2567: struct hermonprm_query_port_cap query_port; ! 2568: union { ! 2569: uint8_t bytes[8]; ! 2570: uint32_t dwords[2]; ! 2571: } mac; ! 2572: int rc; ! 2573: ! 2574: /* Retrieve MAC address */ ! 2575: if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port, ! 2576: &query_port ) ) != 0 ) { ! 2577: DBGC ( hermon, "Hermon %p port %d could not query port: %s\n", ! 2578: hermon, ibdev->port, strerror ( rc ) ); ! 2579: return rc; ! 2580: } ! 2581: mac.dwords[0] = htonl ( MLX_GET ( &query_port, mac_47_32 ) ); ! 2582: mac.dwords[1] = htonl ( MLX_GET ( &query_port, mac_31_0 ) ); ! 2583: memcpy ( netdev->hw_addr, ! 2584: &mac.bytes[ sizeof ( mac.bytes ) - ETH_ALEN ], ETH_ALEN ); ! 2585: ! 2586: /* Register network device */ ! 2587: if ( ( rc = register_netdev ( netdev ) ) != 0 ) { ! 2588: DBGC ( hermon, "Hermon %p port %d could not register network " ! 2589: "device: %s\n", hermon, ibdev->port, strerror ( rc ) ); ! 2590: return rc; ! 2591: } ! 2592: ! 2593: return 0; ! 2594: } ! 2595: ! 2596: /** ! 2597: * Handle Hermon Ethernet device port state change ! 2598: * ! 2599: * @v hermon Hermon device ! 2600: * @v port Hermon port ! 2601: * @v link_up Link is up ! 2602: */ ! 2603: static void hermon_state_change_netdev ( struct hermon *hermon __unused, ! 2604: struct hermon_port *port, ! 2605: int link_up ) { ! 2606: struct net_device *netdev = port->netdev; ! 2607: ! 2608: if ( link_up ) { ! 2609: netdev_link_up ( netdev ); ! 2610: } else { ! 2611: netdev_link_down ( netdev ); ! 2612: } ! 2613: } ! 2614: ! 2615: /** ! 2616: * Unregister Hermon Ethernet device ! 2617: * ! 2618: * @v hermon Hermon device ! 2619: * @v port Hermon port ! 2620: */ ! 2621: static void hermon_unregister_netdev ( struct hermon *hermon __unused, ! 2622: struct hermon_port *port ) { ! 2623: struct net_device *netdev = port->netdev; ! 2624: ! 2625: unregister_netdev ( netdev ); ! 2626: } ! 2627: ! 2628: /** Hermon Ethernet port type */ ! 2629: static struct hermon_port_type hermon_port_type_eth = { ! 2630: .register_dev = hermon_register_netdev, ! 2631: .state_change = hermon_state_change_netdev, ! 2632: .unregister_dev = hermon_unregister_netdev, ! 2633: }; ! 2634: ! 2635: /*************************************************************************** ! 2636: * ! 2637: * Port type detection ! 2638: * ! 2639: *************************************************************************** ! 2640: */ ! 2641: ! 2642: /** Timeout for port sensing */ ! 2643: #define HERMON_SENSE_PORT_TIMEOUT ( TICKS_PER_SEC / 2 ) ! 2644: ! 2645: /** ! 2646: * Name port type ! 2647: * ! 2648: * @v port_type Port type ! 2649: * @v port_type_name Port type name ! 2650: */ ! 2651: static inline const char * hermon_name_port_type ( unsigned int port_type ) { ! 2652: switch ( port_type ) { ! 2653: case HERMON_PORT_TYPE_UNKNOWN: return "unknown"; ! 2654: case HERMON_PORT_TYPE_IB: return "Infiniband"; ! 2655: case HERMON_PORT_TYPE_ETH: return "Ethernet"; ! 2656: default: return "INVALID"; ! 2657: } ! 2658: } ! 2659: ! 2660: /** ! 2661: * Sense port type ! 2662: * ! 2663: * @v hermon Hermon device ! 2664: * @v port Hermon port ! 2665: * @ret port_type Port type, or negative error ! 2666: */ ! 2667: static int hermon_sense_port_type ( struct hermon *hermon, ! 2668: struct hermon_port *port ) { ! 2669: struct ib_device *ibdev = port->ibdev; ! 2670: struct hermonprm_sense_port sense_port; ! 2671: int port_type; ! 2672: int rc; ! 2673: ! 2674: /* If DPDP is not supported, always assume Infiniband */ ! 2675: if ( ! hermon->cap.dpdp ) { ! 2676: port_type = HERMON_PORT_TYPE_IB; ! 2677: DBGC ( hermon, "Hermon %p port %d does not support DPDP; " ! 2678: "assuming an %s network\n", hermon, ibdev->port, ! 2679: hermon_name_port_type ( port_type ) ); ! 2680: return port_type; ! 2681: } ! 2682: ! 2683: /* Sense the port type */ ! 2684: if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port, ! 2685: &sense_port ) ) != 0 ) { ! 2686: DBGC ( hermon, "Hermon %p port %d sense failed: %s\n", ! 2687: hermon, ibdev->port, strerror ( rc ) ); ! 2688: return rc; ! 2689: } ! 2690: port_type = MLX_GET ( &sense_port, port_type ); ! 2691: ! 2692: DBGC ( hermon, "Hermon %p port %d sensed an %s network\n", ! 2693: hermon, ibdev->port, hermon_name_port_type ( port_type ) ); ! 2694: return port_type; ! 2695: } ! 2696: ! 2697: /** ! 2698: * Set port type ! 2699: * ! 2700: * @v hermon Hermon device ! 2701: * @v port Hermon port ! 2702: * @ret rc Return status code ! 2703: */ ! 2704: static int hermon_set_port_type ( struct hermon *hermon, ! 2705: struct hermon_port *port ) { ! 2706: struct ib_device *ibdev = port->ibdev; ! 2707: struct hermonprm_query_port_cap query_port; ! 2708: int ib_supported; ! 2709: int eth_supported; ! 2710: int port_type; ! 2711: unsigned long start; ! 2712: unsigned long elapsed; ! 2713: int rc; ! 2714: ! 2715: /* Check to see which types are supported */ ! 2716: if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port, ! 2717: &query_port ) ) != 0 ) { ! 2718: DBGC ( hermon, "Hermon %p port %d could not query port: %s\n", ! 2719: hermon, ibdev->port, strerror ( rc ) ); ! 2720: return rc; ! 2721: } ! 2722: ib_supported = MLX_GET ( &query_port, ib ); ! 2723: eth_supported = MLX_GET ( &query_port, eth ); ! 2724: DBGC ( hermon, "Hermon %p port %d supports%s%s%s\n", ! 2725: hermon, ibdev->port, ( ib_supported ? " Infiniband" : "" ), ! 2726: ( ( ib_supported && eth_supported ) ? " and" : "" ), ! 2727: ( eth_supported ? " Ethernet" : "" ) ); ! 2728: ! 2729: /* Sense network, if applicable */ ! 2730: if ( ib_supported && eth_supported ) { ! 2731: ! 2732: /* Both types are supported; try sensing network */ ! 2733: start = currticks(); ! 2734: do { ! 2735: /* Try sensing port */ ! 2736: port_type = hermon_sense_port_type ( hermon, port ); ! 2737: if ( port_type < 0 ) { ! 2738: rc = port_type; ! 2739: return rc; ! 2740: } ! 2741: } while ( ( port_type == HERMON_PORT_TYPE_UNKNOWN ) && ! 2742: ( ( elapsed = ( currticks() - start ) ) < ! 2743: HERMON_SENSE_PORT_TIMEOUT ) ); ! 2744: ! 2745: /* Set port type based on sensed network, defaulting ! 2746: * to Infiniband if nothing was sensed. ! 2747: */ ! 2748: switch ( port_type ) { ! 2749: case HERMON_PORT_TYPE_ETH: ! 2750: port->type = &hermon_port_type_eth; ! 2751: break; ! 2752: case HERMON_PORT_TYPE_IB: ! 2753: case HERMON_PORT_TYPE_UNKNOWN: ! 2754: port->type = &hermon_port_type_ib; ! 2755: break; ! 2756: default: ! 2757: return -EINVAL; ! 2758: } ! 2759: ! 2760: } else if ( eth_supported ) { ! 2761: port->type = &hermon_port_type_eth; ! 2762: } else { ! 2763: port->type = &hermon_port_type_ib; ! 2764: } ! 2765: ! 2766: assert ( port->type != NULL ); ! 2767: return 0; ! 2768: } ! 2769: ! 2770: /*************************************************************************** ! 2771: * ! 2772: * Firmware control ! 2773: * ! 2774: *************************************************************************** ! 2775: */ ! 2776: ! 2777: /** ! 2778: * Map virtual to physical address for firmware usage ! 2779: * ! 2780: * @v hermon Hermon device ! 2781: * @v map Mapping function ! 2782: * @v va Virtual address ! 2783: * @v pa Physical address ! 2784: * @v len Length of region ! 2785: * @ret rc Return status code ! 2786: */ ! 2787: static int hermon_map_vpm ( struct hermon *hermon, ! 2788: int ( *map ) ( struct hermon *hermon, ! 2789: const struct hermonprm_virtual_physical_mapping* ), ! 2790: uint64_t va, physaddr_t pa, size_t len ) { ! 2791: struct hermonprm_virtual_physical_mapping mapping; ! 2792: physaddr_t start; ! 2793: physaddr_t low; ! 2794: physaddr_t high; ! 2795: physaddr_t end; ! 2796: size_t size; ! 2797: int rc; ! 2798: ! 2799: /* Sanity checks */ ! 2800: assert ( ( va & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); ! 2801: assert ( ( pa & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); ! 2802: assert ( ( len & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); ! 2803: ! 2804: /* Calculate starting points */ ! 2805: start = pa; ! 2806: end = ( start + len ); ! 2807: size = ( 1UL << ( fls ( start ^ end ) - 1 ) ); ! 2808: low = high = ( end & ~( size - 1 ) ); ! 2809: assert ( start < low ); ! 2810: assert ( high <= end ); ! 2811: ! 2812: /* These mappings tend to generate huge volumes of ! 2813: * uninteresting debug data, which basically makes it ! 2814: * impossible to use debugging otherwise. ! 2815: */ ! 2816: DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); ! 2817: ! 2818: /* Map blocks in descending order of size */ ! 2819: while ( size >= HERMON_PAGE_SIZE ) { ! 2820: ! 2821: /* Find the next candidate block */ ! 2822: if ( ( low - size ) >= start ) { ! 2823: low -= size; ! 2824: pa = low; ! 2825: } else if ( ( high + size ) <= end ) { ! 2826: pa = high; ! 2827: high += size; ! 2828: } else { ! 2829: size >>= 1; ! 2830: continue; ! 2831: } ! 2832: assert ( ( va & ( size - 1 ) ) == 0 ); ! 2833: assert ( ( pa & ( size - 1 ) ) == 0 ); ! 2834: ! 2835: /* Map this block */ ! 2836: memset ( &mapping, 0, sizeof ( mapping ) ); ! 2837: MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) ); ! 2838: MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) ); ! 2839: MLX_FILL_H ( &mapping, 2, pa_h, pa ); ! 2840: MLX_FILL_2 ( &mapping, 3, ! 2841: log2size, ( ( fls ( size ) - 1 ) - 12 ), ! 2842: pa_l, ( pa >> 12 ) ); ! 2843: if ( ( rc = map ( hermon, &mapping ) ) != 0 ) { ! 2844: DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); ! 2845: DBGC ( hermon, "Hermon %p could not map %08llx+%zx to " ! 2846: "%08lx: %s\n", ! 2847: hermon, va, size, pa, strerror ( rc ) ); ! 2848: return rc; ! 2849: } ! 2850: va += size; ! 2851: } ! 2852: assert ( low == start ); ! 2853: assert ( high == end ); ! 2854: ! 2855: DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); ! 2856: return 0; ! 2857: } ! 2858: ! 2859: /** ! 2860: * Start firmware running ! 2861: * ! 2862: * @v hermon Hermon device ! 2863: * @ret rc Return status code ! 2864: */ ! 2865: static int hermon_start_firmware ( struct hermon *hermon ) { ! 2866: struct hermonprm_query_fw fw; ! 2867: unsigned int fw_pages; ! 2868: size_t fw_size; ! 2869: physaddr_t fw_base; ! 2870: int rc; ! 2871: ! 2872: /* Get firmware parameters */ ! 2873: if ( ( rc = hermon_cmd_query_fw ( hermon, &fw ) ) != 0 ) { ! 2874: DBGC ( hermon, "Hermon %p could not query firmware: %s\n", ! 2875: hermon, strerror ( rc ) ); ! 2876: goto err_query_fw; ! 2877: } ! 2878: DBGC ( hermon, "Hermon %p firmware version %d.%d.%d\n", hermon, ! 2879: MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ), ! 2880: MLX_GET ( &fw, fw_rev_subminor ) ); ! 2881: fw_pages = MLX_GET ( &fw, fw_pages ); ! 2882: DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n", ! 2883: hermon, fw_pages, ( fw_pages * 4 ) ); ! 2884: ! 2885: /* Allocate firmware pages and map firmware area */ ! 2886: fw_size = ( fw_pages * HERMON_PAGE_SIZE ); ! 2887: hermon->firmware_area = umalloc ( fw_size ); ! 2888: if ( ! hermon->firmware_area ) { ! 2889: rc = -ENOMEM; ! 2890: goto err_alloc_fa; ! 2891: } ! 2892: fw_base = user_to_phys ( hermon->firmware_area, 0 ); ! 2893: DBGC ( hermon, "Hermon %p firmware area at physical [%08lx,%08lx)\n", ! 2894: hermon, fw_base, ( fw_base + fw_size ) ); ! 2895: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_fa, ! 2896: 0, fw_base, fw_size ) ) != 0 ) { ! 2897: DBGC ( hermon, "Hermon %p could not map firmware: %s\n", ! 2898: hermon, strerror ( rc ) ); ! 2899: goto err_map_fa; ! 2900: } ! 2901: ! 2902: /* Start firmware */ ! 2903: if ( ( rc = hermon_cmd_run_fw ( hermon ) ) != 0 ) { ! 2904: DBGC ( hermon, "Hermon %p could not run firmware: %s\n", ! 2905: hermon, strerror ( rc ) ); ! 2906: goto err_run_fw; ! 2907: } ! 2908: ! 2909: DBGC ( hermon, "Hermon %p firmware started\n", hermon ); ! 2910: return 0; ! 2911: ! 2912: err_run_fw: ! 2913: err_map_fa: ! 2914: hermon_cmd_unmap_fa ( hermon ); ! 2915: ufree ( hermon->firmware_area ); ! 2916: hermon->firmware_area = UNULL; ! 2917: err_alloc_fa: ! 2918: err_query_fw: ! 2919: return rc; ! 2920: } ! 2921: ! 2922: /** ! 2923: * Stop firmware running ! 2924: * ! 2925: * @v hermon Hermon device ! 2926: */ ! 2927: static void hermon_stop_firmware ( struct hermon *hermon ) { ! 2928: int rc; ! 2929: ! 2930: if ( ( rc = hermon_cmd_unmap_fa ( hermon ) ) != 0 ) { ! 2931: DBGC ( hermon, "Hermon %p FATAL could not stop firmware: %s\n", ! 2932: hermon, strerror ( rc ) ); ! 2933: /* Leak memory and return; at least we avoid corruption */ ! 2934: return; ! 2935: } ! 2936: ufree ( hermon->firmware_area ); ! 2937: hermon->firmware_area = UNULL; ! 2938: } ! 2939: ! 2940: /*************************************************************************** ! 2941: * ! 2942: * Infinihost Context Memory management ! 2943: * ! 2944: *************************************************************************** ! 2945: */ ! 2946: ! 2947: /** ! 2948: * Get device limits ! 2949: * ! 2950: * @v hermon Hermon device ! 2951: * @ret rc Return status code ! 2952: */ ! 2953: static int hermon_get_cap ( struct hermon *hermon ) { ! 2954: struct hermonprm_query_dev_cap dev_cap; ! 2955: int rc; ! 2956: ! 2957: if ( ( rc = hermon_cmd_query_dev_cap ( hermon, &dev_cap ) ) != 0 ) { ! 2958: DBGC ( hermon, "Hermon %p could not get device limits: %s\n", ! 2959: hermon, strerror ( rc ) ); ! 2960: return rc; ! 2961: } ! 2962: ! 2963: hermon->cap.cmpt_entry_size = MLX_GET ( &dev_cap, c_mpt_entry_sz ); ! 2964: hermon->cap.reserved_qps = ! 2965: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_qps ) ); ! 2966: hermon->cap.qpc_entry_size = MLX_GET ( &dev_cap, qpc_entry_sz ); ! 2967: hermon->cap.altc_entry_size = MLX_GET ( &dev_cap, altc_entry_sz ); ! 2968: hermon->cap.auxc_entry_size = MLX_GET ( &dev_cap, aux_entry_sz ); ! 2969: hermon->cap.reserved_srqs = ! 2970: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_srqs ) ); ! 2971: hermon->cap.srqc_entry_size = MLX_GET ( &dev_cap, srq_entry_sz ); ! 2972: hermon->cap.reserved_cqs = ! 2973: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_cqs ) ); ! 2974: hermon->cap.cqc_entry_size = MLX_GET ( &dev_cap, cqc_entry_sz ); ! 2975: hermon->cap.reserved_eqs = MLX_GET ( &dev_cap, num_rsvd_eqs ); ! 2976: if ( hermon->cap.reserved_eqs == 0 ) { ! 2977: /* Backward compatibility */ ! 2978: hermon->cap.reserved_eqs = ! 2979: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_eqs ) ); ! 2980: } ! 2981: hermon->cap.eqc_entry_size = MLX_GET ( &dev_cap, eqc_entry_sz ); ! 2982: hermon->cap.reserved_mtts = ! 2983: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mtts ) ); ! 2984: hermon->cap.mtt_entry_size = MLX_GET ( &dev_cap, mtt_entry_sz ); ! 2985: hermon->cap.reserved_mrws = ! 2986: ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) ); ! 2987: hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz ); ! 2988: hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars ); ! 2989: hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports ); ! 2990: hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp ); ! 2991: ! 2992: /* Sanity check */ ! 2993: if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) { ! 2994: DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n", ! 2995: hermon, hermon->cap.num_ports, HERMON_MAX_PORTS ); ! 2996: hermon->cap.num_ports = HERMON_MAX_PORTS; ! 2997: } ! 2998: ! 2999: return 0; ! 3000: } ! 3001: ! 3002: /** ! 3003: * Align ICM table ! 3004: * ! 3005: * @v icm_offset Current ICM offset ! 3006: * @v len ICM table length ! 3007: * @ret icm_offset ICM offset ! 3008: */ ! 3009: static uint64_t icm_align ( uint64_t icm_offset, size_t len ) { ! 3010: ! 3011: /* Round up to a multiple of the table size */ ! 3012: assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) ); ! 3013: return ( ( icm_offset + len - 1 ) & ~( ( ( uint64_t ) len ) - 1 ) ); ! 3014: } ! 3015: ! 3016: /** ! 3017: * Allocate ICM ! 3018: * ! 3019: * @v hermon Hermon device ! 3020: * @v init_hca INIT_HCA structure to fill in ! 3021: * @ret rc Return status code ! 3022: */ ! 3023: static int hermon_alloc_icm ( struct hermon *hermon, ! 3024: struct hermonprm_init_hca *init_hca ) { ! 3025: struct hermonprm_scalar_parameter icm_size; ! 3026: struct hermonprm_scalar_parameter icm_aux_size; ! 3027: uint64_t icm_offset = 0; ! 3028: unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs; ! 3029: unsigned int log_num_mtts, log_num_mpts, log_num_mcs; ! 3030: size_t cmpt_max_len; ! 3031: size_t icm_len, icm_aux_len; ! 3032: size_t len; ! 3033: physaddr_t icm_phys; ! 3034: int i; ! 3035: int rc; ! 3036: ! 3037: /* ! 3038: * Start by carving up the ICM virtual address space ! 3039: * ! 3040: */ ! 3041: ! 3042: /* Calculate number of each object type within ICM */ ! 3043: log_num_qps = fls ( hermon->cap.reserved_qps + ! 3044: HERMON_RSVD_SPECIAL_QPS + HERMON_MAX_QPS - 1 ); ! 3045: log_num_srqs = fls ( hermon->cap.reserved_srqs - 1 ); ! 3046: log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 ); ! 3047: log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 ); ! 3048: log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 ); ! 3049: log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 ); ! 3050: log_num_mcs = HERMON_LOG_MULTICAST_HASH_SIZE; ! 3051: ! 3052: /* ICM starts with the cMPT tables, which are sparse */ ! 3053: cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES * ! 3054: ( ( uint64_t ) hermon->cap.cmpt_entry_size ) ); ! 3055: len = ( ( ( ( 1 << log_num_qps ) * hermon->cap.cmpt_entry_size ) + ! 3056: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) ); ! 3057: hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset; ! 3058: hermon->icm_map[HERMON_ICM_QP_CMPT].len = len; ! 3059: icm_offset += cmpt_max_len; ! 3060: len = ( ( ( ( 1 << log_num_srqs ) * hermon->cap.cmpt_entry_size ) + ! 3061: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) ); ! 3062: hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset; ! 3063: hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = len; ! 3064: icm_offset += cmpt_max_len; ! 3065: len = ( ( ( ( 1 << log_num_cqs ) * hermon->cap.cmpt_entry_size ) + ! 3066: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) ); ! 3067: hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset; ! 3068: hermon->icm_map[HERMON_ICM_CQ_CMPT].len = len; ! 3069: icm_offset += cmpt_max_len; ! 3070: len = ( ( ( ( 1 << log_num_eqs ) * hermon->cap.cmpt_entry_size ) + ! 3071: HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) ); ! 3072: hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset; ! 3073: hermon->icm_map[HERMON_ICM_EQ_CMPT].len = len; ! 3074: icm_offset += cmpt_max_len; ! 3075: ! 3076: hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset; ! 3077: ! 3078: /* Queue pair contexts */ ! 3079: len = ( ( 1 << log_num_qps ) * hermon->cap.qpc_entry_size ); ! 3080: icm_offset = icm_align ( icm_offset, len ); ! 3081: MLX_FILL_1 ( init_hca, 12, ! 3082: qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h, ! 3083: ( icm_offset >> 32 ) ); ! 3084: MLX_FILL_2 ( init_hca, 13, ! 3085: qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l, ! 3086: ( icm_offset >> 5 ), ! 3087: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp, ! 3088: log_num_qps ); ! 3089: DBGC ( hermon, "Hermon %p ICM QPC is %d x %#zx at [%08llx,%08llx)\n", ! 3090: hermon, ( 1 << log_num_qps ), hermon->cap.qpc_entry_size, ! 3091: icm_offset, ( icm_offset + len ) ); ! 3092: icm_offset += len; ! 3093: ! 3094: /* Extended alternate path contexts */ ! 3095: len = ( ( 1 << log_num_qps ) * hermon->cap.altc_entry_size ); ! 3096: icm_offset = icm_align ( icm_offset, len ); ! 3097: MLX_FILL_1 ( init_hca, 24, ! 3098: qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h, ! 3099: ( icm_offset >> 32 ) ); ! 3100: MLX_FILL_1 ( init_hca, 25, ! 3101: qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l, ! 3102: icm_offset ); ! 3103: DBGC ( hermon, "Hermon %p ICM ALTC is %d x %#zx at [%08llx,%08llx)\n", ! 3104: hermon, ( 1 << log_num_qps ), hermon->cap.altc_entry_size, ! 3105: icm_offset, ( icm_offset + len ) ); ! 3106: icm_offset += len; ! 3107: ! 3108: /* Extended auxiliary contexts */ ! 3109: len = ( ( 1 << log_num_qps ) * hermon->cap.auxc_entry_size ); ! 3110: icm_offset = icm_align ( icm_offset, len ); ! 3111: MLX_FILL_1 ( init_hca, 28, ! 3112: qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h, ! 3113: ( icm_offset >> 32 ) ); ! 3114: MLX_FILL_1 ( init_hca, 29, ! 3115: qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l, ! 3116: icm_offset ); ! 3117: DBGC ( hermon, "Hermon %p ICM AUXC is %d x %#zx at [%08llx,%08llx)\n", ! 3118: hermon, ( 1 << log_num_qps ), hermon->cap.auxc_entry_size, ! 3119: icm_offset, ( icm_offset + len ) ); ! 3120: icm_offset += len; ! 3121: ! 3122: /* Shared receive queue contexts */ ! 3123: len = ( ( 1 << log_num_srqs ) * hermon->cap.srqc_entry_size ); ! 3124: icm_offset = icm_align ( icm_offset, len ); ! 3125: MLX_FILL_1 ( init_hca, 18, ! 3126: qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h, ! 3127: ( icm_offset >> 32 ) ); ! 3128: MLX_FILL_2 ( init_hca, 19, ! 3129: qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l, ! 3130: ( icm_offset >> 5 ), ! 3131: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq, ! 3132: log_num_srqs ); ! 3133: DBGC ( hermon, "Hermon %p ICM SRQC is %d x %#zx at [%08llx,%08llx)\n", ! 3134: hermon, ( 1 << log_num_srqs ), hermon->cap.srqc_entry_size, ! 3135: icm_offset, ( icm_offset + len ) ); ! 3136: icm_offset += len; ! 3137: ! 3138: /* Completion queue contexts */ ! 3139: len = ( ( 1 << log_num_cqs ) * hermon->cap.cqc_entry_size ); ! 3140: icm_offset = icm_align ( icm_offset, len ); ! 3141: MLX_FILL_1 ( init_hca, 20, ! 3142: qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h, ! 3143: ( icm_offset >> 32 ) ); ! 3144: MLX_FILL_2 ( init_hca, 21, ! 3145: qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l, ! 3146: ( icm_offset >> 5 ), ! 3147: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq, ! 3148: log_num_cqs ); ! 3149: DBGC ( hermon, "Hermon %p ICM CQC is %d x %#zx at [%08llx,%08llx)\n", ! 3150: hermon, ( 1 << log_num_cqs ), hermon->cap.cqc_entry_size, ! 3151: icm_offset, ( icm_offset + len ) ); ! 3152: icm_offset += len; ! 3153: ! 3154: /* Event queue contexts */ ! 3155: len = ( ( 1 << log_num_eqs ) * hermon->cap.eqc_entry_size ); ! 3156: icm_offset = icm_align ( icm_offset, len ); ! 3157: MLX_FILL_1 ( init_hca, 32, ! 3158: qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h, ! 3159: ( icm_offset >> 32 ) ); ! 3160: MLX_FILL_2 ( init_hca, 33, ! 3161: qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l, ! 3162: ( icm_offset >> 5 ), ! 3163: qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq, ! 3164: log_num_eqs ); ! 3165: DBGC ( hermon, "Hermon %p ICM EQC is %d x %#zx at [%08llx,%08llx)\n", ! 3166: hermon, ( 1 << log_num_eqs ), hermon->cap.eqc_entry_size, ! 3167: icm_offset, ( icm_offset + len ) ); ! 3168: icm_offset += len; ! 3169: ! 3170: /* Memory translation table */ ! 3171: len = ( ( 1 << log_num_mtts ) * hermon->cap.mtt_entry_size ); ! 3172: icm_offset = icm_align ( icm_offset, len ); ! 3173: MLX_FILL_1 ( init_hca, 64, ! 3174: tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) ); ! 3175: MLX_FILL_1 ( init_hca, 65, ! 3176: tpt_parameters.mtt_base_addr_l, icm_offset ); ! 3177: DBGC ( hermon, "Hermon %p ICM MTT is %d x %#zx at [%08llx,%08llx)\n", ! 3178: hermon, ( 1 << log_num_mtts ), hermon->cap.mtt_entry_size, ! 3179: icm_offset, ( icm_offset + len ) ); ! 3180: icm_offset += len; ! 3181: ! 3182: /* Memory protection table */ ! 3183: len = ( ( 1 << log_num_mpts ) * hermon->cap.dmpt_entry_size ); ! 3184: icm_offset = icm_align ( icm_offset, len ); ! 3185: MLX_FILL_1 ( init_hca, 60, ! 3186: tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) ); ! 3187: MLX_FILL_1 ( init_hca, 61, ! 3188: tpt_parameters.dmpt_base_adr_l, icm_offset ); ! 3189: MLX_FILL_1 ( init_hca, 62, ! 3190: tpt_parameters.log_dmpt_sz, log_num_mpts ); ! 3191: DBGC ( hermon, "Hermon %p ICM DMPT is %d x %#zx at [%08llx,%08llx)\n", ! 3192: hermon, ( 1 << log_num_mpts ), hermon->cap.dmpt_entry_size, ! 3193: icm_offset, ( icm_offset + len ) ); ! 3194: icm_offset += len; ! 3195: ! 3196: /* Multicast table */ ! 3197: len = ( ( 1 << log_num_mcs ) * sizeof ( struct hermonprm_mcg_entry ) ); ! 3198: icm_offset = icm_align ( icm_offset, len ); ! 3199: MLX_FILL_1 ( init_hca, 48, ! 3200: multicast_parameters.mc_base_addr_h, ! 3201: ( icm_offset >> 32 ) ); ! 3202: MLX_FILL_1 ( init_hca, 49, ! 3203: multicast_parameters.mc_base_addr_l, icm_offset ); ! 3204: MLX_FILL_1 ( init_hca, 52, ! 3205: multicast_parameters.log_mc_table_entry_sz, ! 3206: fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) ); ! 3207: MLX_FILL_1 ( init_hca, 53, ! 3208: multicast_parameters.log_mc_table_hash_sz, log_num_mcs ); ! 3209: MLX_FILL_1 ( init_hca, 54, ! 3210: multicast_parameters.log_mc_table_sz, log_num_mcs ); ! 3211: DBGC ( hermon, "Hermon %p ICM MC is %d x %#zx at [%08llx,%08llx)\n", ! 3212: hermon, ( 1 << log_num_mcs ), ! 3213: sizeof ( struct hermonprm_mcg_entry ), ! 3214: icm_offset, ( icm_offset + len ) ); ! 3215: icm_offset += len; ! 3216: ! 3217: ! 3218: hermon->icm_map[HERMON_ICM_OTHER].len = ! 3219: ( icm_offset - hermon->icm_map[HERMON_ICM_OTHER].offset ); ! 3220: ! 3221: /* ! 3222: * Allocate and map physical memory for (portions of) ICM ! 3223: * ! 3224: * Map is: ! 3225: * ICM AUX area (aligned to its own size) ! 3226: * cMPT areas ! 3227: * Other areas ! 3228: */ ! 3229: ! 3230: /* Calculate physical memory required for ICM */ ! 3231: icm_len = 0; ! 3232: for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) { ! 3233: icm_len += hermon->icm_map[i].len; ! 3234: } ! 3235: ! 3236: /* Get ICM auxiliary area size */ ! 3237: memset ( &icm_size, 0, sizeof ( icm_size ) ); ! 3238: MLX_FILL_1 ( &icm_size, 0, value_hi, ( icm_offset >> 32 ) ); ! 3239: MLX_FILL_1 ( &icm_size, 1, value, icm_offset ); ! 3240: if ( ( rc = hermon_cmd_set_icm_size ( hermon, &icm_size, ! 3241: &icm_aux_size ) ) != 0 ) { ! 3242: DBGC ( hermon, "Hermon %p could not set ICM size: %s\n", ! 3243: hermon, strerror ( rc ) ); ! 3244: goto err_set_icm_size; ! 3245: } ! 3246: icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * HERMON_PAGE_SIZE ); ! 3247: ! 3248: /* Allocate ICM data and auxiliary area */ ! 3249: DBGC ( hermon, "Hermon %p requires %zd kB ICM and %zd kB AUX ICM\n", ! 3250: hermon, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) ); ! 3251: hermon->icm = umalloc ( icm_aux_len + icm_len ); ! 3252: if ( ! hermon->icm ) { ! 3253: rc = -ENOMEM; ! 3254: goto err_alloc; ! 3255: } ! 3256: icm_phys = user_to_phys ( hermon->icm, 0 ); ! 3257: ! 3258: /* Map ICM auxiliary area */ ! 3259: DBGC ( hermon, "Hermon %p mapping ICM AUX => %08lx\n", ! 3260: hermon, icm_phys ); ! 3261: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm_aux, ! 3262: 0, icm_phys, icm_aux_len ) ) != 0 ) { ! 3263: DBGC ( hermon, "Hermon %p could not map AUX ICM: %s\n", ! 3264: hermon, strerror ( rc ) ); ! 3265: goto err_map_icm_aux; ! 3266: } ! 3267: icm_phys += icm_aux_len; ! 3268: ! 3269: /* MAP ICM area */ ! 3270: for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) { ! 3271: DBGC ( hermon, "Hermon %p mapping ICM %llx+%zx => %08lx\n", ! 3272: hermon, hermon->icm_map[i].offset, ! 3273: hermon->icm_map[i].len, icm_phys ); ! 3274: if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm, ! 3275: hermon->icm_map[i].offset, ! 3276: icm_phys, ! 3277: hermon->icm_map[i].len ) ) != 0 ){ ! 3278: DBGC ( hermon, "Hermon %p could not map ICM: %s\n", ! 3279: hermon, strerror ( rc ) ); ! 3280: goto err_map_icm; ! 3281: } ! 3282: icm_phys += hermon->icm_map[i].len; ! 3283: } ! 3284: ! 3285: return 0; ! 3286: ! 3287: err_map_icm: ! 3288: assert ( i == 0 ); /* We don't handle partial failure at present */ ! 3289: err_map_icm_aux: ! 3290: hermon_cmd_unmap_icm_aux ( hermon ); ! 3291: ufree ( hermon->icm ); ! 3292: hermon->icm = UNULL; ! 3293: err_alloc: ! 3294: err_set_icm_size: ! 3295: return rc; ! 3296: } ! 3297: ! 3298: /** ! 3299: * Free ICM ! 3300: * ! 3301: * @v hermon Hermon device ! 3302: */ ! 3303: static void hermon_free_icm ( struct hermon *hermon ) { ! 3304: struct hermonprm_scalar_parameter unmap_icm; ! 3305: int i; ! 3306: ! 3307: for ( i = ( HERMON_ICM_NUM_REGIONS - 1 ) ; i >= 0 ; i-- ) { ! 3308: memset ( &unmap_icm, 0, sizeof ( unmap_icm ) ); ! 3309: MLX_FILL_1 ( &unmap_icm, 0, value_hi, ! 3310: ( hermon->icm_map[i].offset >> 32 ) ); ! 3311: MLX_FILL_1 ( &unmap_icm, 1, value, ! 3312: hermon->icm_map[i].offset ); ! 3313: hermon_cmd_unmap_icm ( hermon, ! 3314: ( 1 << fls ( ( hermon->icm_map[i].len / ! 3315: HERMON_PAGE_SIZE ) - 1)), ! 3316: &unmap_icm ); ! 3317: } ! 3318: hermon_cmd_unmap_icm_aux ( hermon ); ! 3319: ufree ( hermon->icm ); ! 3320: hermon->icm = UNULL; ! 3321: } ! 3322: ! 3323: /*************************************************************************** ! 3324: * ! 3325: * BOFM interface ! 3326: * ! 3327: *************************************************************************** ! 3328: */ ! 3329: ! 3330: /** ! 3331: * Harvest Ethernet MAC for BOFM ! 3332: * ! 3333: * @v bofm BOFM device ! 3334: * @v mport Multi-port index ! 3335: * @v mac MAC to fill in ! 3336: * @ret rc Return status code ! 3337: */ ! 3338: static int hermon_bofm_harvest ( struct bofm_device *bofm, unsigned int mport, ! 3339: uint8_t *mac ) { ! 3340: struct hermon *hermon = container_of ( bofm, struct hermon, bofm ); ! 3341: struct hermonprm_mod_stat_cfg stat_cfg; ! 3342: union { ! 3343: uint8_t bytes[8]; ! 3344: uint32_t dwords[2]; ! 3345: } buf; ! 3346: int rc; ! 3347: ! 3348: /* Query static configuration */ ! 3349: if ( ( rc = hermon_mod_stat_cfg ( hermon, mport, ! 3350: HERMON_MOD_STAT_CFG_QUERY, ! 3351: HERMON_MOD_STAT_CFG_OFFSET ( mac_m ), ! 3352: &stat_cfg ) ) != 0 ) { ! 3353: DBGC ( hermon, "Hermon %p port %d could not query " ! 3354: "configuration: %s\n", hermon, mport, strerror ( rc ) ); ! 3355: return rc; ! 3356: } ! 3357: ! 3358: /* Retrieve MAC address */ ! 3359: buf.dwords[0] = htonl ( MLX_GET ( &stat_cfg, mac_high ) ); ! 3360: buf.dwords[1] = htonl ( MLX_GET ( &stat_cfg, mac_low ) ); ! 3361: memcpy ( mac, &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ], ! 3362: ETH_ALEN ); ! 3363: ! 3364: DBGC ( hermon, "Hermon %p port %d harvested MAC address %s\n", ! 3365: hermon, mport, eth_ntoa ( mac ) ); ! 3366: ! 3367: return 0; ! 3368: } ! 3369: ! 3370: /** ! 3371: * Update Ethernet MAC for BOFM ! 3372: * ! 3373: * @v bofm BOFM device ! 3374: * @v mport Multi-port index ! 3375: * @v mac MAC to fill in ! 3376: * @ret rc Return status code ! 3377: */ ! 3378: static int hermon_bofm_update ( struct bofm_device *bofm, unsigned int mport, ! 3379: const uint8_t *mac ) { ! 3380: struct hermon *hermon = container_of ( bofm, struct hermon, bofm ); ! 3381: struct hermonprm_mod_stat_cfg stat_cfg; ! 3382: union { ! 3383: uint8_t bytes[8]; ! 3384: uint32_t dwords[2]; ! 3385: uint64_t qword; ! 3386: } buf; ! 3387: uint8_t *mac_copy = &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ]; ! 3388: int rc; ! 3389: ! 3390: /* Prepare MAC address */ ! 3391: memset ( &buf, 0, sizeof ( buf ) ); ! 3392: memcpy ( mac_copy, mac, ETH_ALEN ); ! 3393: ! 3394: /* Current BOFM versions are unable to create entries with ! 3395: * mport>1, which means that only the port 1 MAC address can ! 3396: * be explicitly specified. Work around this by using the ! 3397: * provided MAC address as a base address for all subsequent ! 3398: * ports. For example, if BOFM assigns the address ! 3399: * ! 3400: * 00:1A:64:76:00:09 for port 1 ! 3401: * ! 3402: * then we will assign the addresses ! 3403: * ! 3404: * 00:1A:64:76:00:09 for port 1 ! 3405: * 00:1A:64:76:00:0a for port 2 ! 3406: * ! 3407: * Note that hermon->cap.num_ports is not yet defined at this ! 3408: * point. ! 3409: */ ! 3410: for ( ; mport <= HERMON_MAX_PORTS ; mport++ ) { ! 3411: ! 3412: /* Modify static configuration */ ! 3413: memset ( &stat_cfg, 0, sizeof ( stat_cfg ) ); ! 3414: MLX_FILL_2 ( &stat_cfg, 36, ! 3415: mac_m, 1, ! 3416: mac_high, ntohl ( buf.dwords[0] ) ); ! 3417: MLX_FILL_1 ( &stat_cfg, 37, mac_low, ntohl ( buf.dwords[1] ) ); ! 3418: if ( ( rc = hermon_mod_stat_cfg ( hermon, mport, ! 3419: HERMON_MOD_STAT_CFG_SET, ! 3420: HERMON_MOD_STAT_CFG_OFFSET ( mac_m ), ! 3421: &stat_cfg ) ) != 0 ) { ! 3422: DBGC ( hermon, "Hermon %p port %d could not modify " ! 3423: "configuration: %s\n", ! 3424: hermon, mport, strerror ( rc ) ); ! 3425: return rc; ! 3426: } ! 3427: ! 3428: DBGC ( hermon, "Hermon %p port %d updated MAC address to %s\n", ! 3429: hermon, mport, eth_ntoa ( mac_copy ) ); ! 3430: ! 3431: /* Increment MAC address */ ! 3432: buf.qword = cpu_to_be64 ( be64_to_cpu ( buf.qword ) + 1 ); ! 3433: } ! 3434: ! 3435: return 0; ! 3436: } ! 3437: ! 3438: /** Hermon BOFM operations */ ! 3439: static struct bofm_operations hermon_bofm_operations = { ! 3440: .harvest = hermon_bofm_harvest, ! 3441: .update = hermon_bofm_update, ! 3442: }; ! 3443: ! 3444: /*************************************************************************** ! 3445: * ! 3446: * PCI interface ! 3447: * ! 3448: *************************************************************************** ! 3449: */ ! 3450: ! 3451: /** ! 3452: * Set up memory protection table ! 3453: * ! 3454: * @v hermon Hermon device ! 3455: * @ret rc Return status code ! 3456: */ ! 3457: static int hermon_setup_mpt ( struct hermon *hermon ) { ! 3458: struct hermonprm_mpt mpt; ! 3459: uint32_t key; ! 3460: int rc; ! 3461: ! 3462: /* Derive key */ ! 3463: key = ( hermon->cap.reserved_mrws | HERMON_MKEY_PREFIX ); ! 3464: hermon->lkey = ( ( key << 8 ) | ( key >> 24 ) ); ! 3465: ! 3466: /* Initialise memory protection table */ ! 3467: memset ( &mpt, 0, sizeof ( mpt ) ); ! 3468: MLX_FILL_7 ( &mpt, 0, ! 3469: atomic, 1, ! 3470: rw, 1, ! 3471: rr, 1, ! 3472: lw, 1, ! 3473: lr, 1, ! 3474: pa, 1, ! 3475: r_w, 1 ); ! 3476: MLX_FILL_1 ( &mpt, 2, mem_key, key ); ! 3477: MLX_FILL_1 ( &mpt, 3, ! 3478: pd, HERMON_GLOBAL_PD ); ! 3479: MLX_FILL_1 ( &mpt, 10, len64, 1 ); ! 3480: if ( ( rc = hermon_cmd_sw2hw_mpt ( hermon, ! 3481: hermon->cap.reserved_mrws, ! 3482: &mpt ) ) != 0 ) { ! 3483: DBGC ( hermon, "Hermon %p could not set up MPT: %s\n", ! 3484: hermon, strerror ( rc ) ); ! 3485: return rc; ! 3486: } ! 3487: ! 3488: return 0; ! 3489: } ! 3490: ! 3491: /** ! 3492: * Configure special queue pairs ! 3493: * ! 3494: * @v hermon Hermon device ! 3495: * @ret rc Return status code ! 3496: */ ! 3497: static int hermon_configure_special_qps ( struct hermon *hermon ) { ! 3498: int rc; ! 3499: ! 3500: /* Special QP block must be aligned on its own size */ ! 3501: hermon->special_qpn_base = ( ( hermon->cap.reserved_qps + ! 3502: HERMON_NUM_SPECIAL_QPS - 1 ) ! 3503: & ~( HERMON_NUM_SPECIAL_QPS - 1 ) ); ! 3504: hermon->qpn_base = ( hermon->special_qpn_base + ! 3505: HERMON_NUM_SPECIAL_QPS ); ! 3506: DBGC ( hermon, "Hermon %p special QPs at [%lx,%lx]\n", hermon, ! 3507: hermon->special_qpn_base, ( hermon->qpn_base - 1 ) ); ! 3508: ! 3509: /* Issue command to configure special QPs */ ! 3510: if ( ( rc = hermon_cmd_conf_special_qp ( hermon, 0x00, ! 3511: hermon->special_qpn_base ) ) != 0 ) { ! 3512: DBGC ( hermon, "Hermon %p could not configure special QPs: " ! 3513: "%s\n", hermon, strerror ( rc ) ); ! 3514: return rc; ! 3515: } ! 3516: ! 3517: return 0; ! 3518: } ! 3519: ! 3520: /** ! 3521: * Reset device ! 3522: * ! 3523: * @v hermon Hermon device ! 3524: * @v pci PCI device ! 3525: */ ! 3526: static void hermon_reset ( struct hermon *hermon, ! 3527: struct pci_device *pci ) { ! 3528: struct pci_config_backup backup; ! 3529: static const uint8_t backup_exclude[] = ! 3530: PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c ); ! 3531: ! 3532: pci_backup ( pci, &backup, backup_exclude ); ! 3533: writel ( HERMON_RESET_MAGIC, ! 3534: ( hermon->config + HERMON_RESET_OFFSET ) ); ! 3535: mdelay ( HERMON_RESET_WAIT_TIME_MS ); ! 3536: pci_restore ( pci, &backup, backup_exclude ); ! 3537: } ! 3538: ! 3539: /** ! 3540: * Allocate Hermon device ! 3541: * ! 3542: * @v pci PCI device ! 3543: * @v id PCI ID ! 3544: * @ret rc Return status code ! 3545: */ ! 3546: static struct hermon * hermon_alloc ( void ) { ! 3547: struct hermon *hermon; ! 3548: ! 3549: /* Allocate Hermon device */ ! 3550: hermon = zalloc ( sizeof ( *hermon ) ); ! 3551: if ( ! hermon ) ! 3552: goto err_hermon; ! 3553: ! 3554: /* Allocate space for mailboxes */ ! 3555: hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE, ! 3556: HERMON_MBOX_ALIGN ); ! 3557: if ( ! hermon->mailbox_in ) ! 3558: goto err_mailbox_in; ! 3559: hermon->mailbox_out = malloc_dma ( HERMON_MBOX_SIZE, ! 3560: HERMON_MBOX_ALIGN ); ! 3561: if ( ! hermon->mailbox_out ) ! 3562: goto err_mailbox_out; ! 3563: ! 3564: return hermon; ! 3565: ! 3566: free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE ); ! 3567: err_mailbox_out: ! 3568: free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); ! 3569: err_mailbox_in: ! 3570: free ( hermon ); ! 3571: err_hermon: ! 3572: return NULL; ! 3573: } ! 3574: ! 3575: /** ! 3576: * Free Hermon device ! 3577: * ! 3578: * @v hermon Hermon device ! 3579: */ ! 3580: static void hermon_free ( struct hermon *hermon ) { ! 3581: ! 3582: free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE ); ! 3583: free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); ! 3584: free ( hermon ); ! 3585: } ! 3586: ! 3587: /** ! 3588: * Initialise Hermon PCI parameters ! 3589: * ! 3590: * @v hermon Hermon device ! 3591: * @v pci PCI device ! 3592: */ ! 3593: static void hermon_pci_init ( struct hermon *hermon, struct pci_device *pci ) { ! 3594: ! 3595: /* Fix up PCI device */ ! 3596: adjust_pci_device ( pci ); ! 3597: ! 3598: /* Get PCI BARs */ ! 3599: hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR), ! 3600: HERMON_PCI_CONFIG_BAR_SIZE ); ! 3601: hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ), ! 3602: HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE ); ! 3603: } ! 3604: ! 3605: /** ! 3606: * Probe PCI device ! 3607: * ! 3608: * @v pci PCI device ! 3609: * @v id PCI ID ! 3610: * @ret rc Return status code ! 3611: */ ! 3612: static int hermon_probe ( struct pci_device *pci ) { ! 3613: struct hermon *hermon; ! 3614: struct ib_device *ibdev; ! 3615: struct net_device *netdev; ! 3616: struct hermon_port *port; ! 3617: struct hermonprm_init_hca init_hca; ! 3618: unsigned int i; ! 3619: int rc; ! 3620: ! 3621: /* Allocate Hermon device */ ! 3622: hermon = hermon_alloc(); ! 3623: if ( ! hermon ) { ! 3624: rc = -ENOMEM; ! 3625: goto err_alloc; ! 3626: } ! 3627: pci_set_drvdata ( pci, hermon ); ! 3628: ! 3629: /* Initialise PCI parameters */ ! 3630: hermon_pci_init ( hermon, pci ); ! 3631: ! 3632: /* Reset device */ ! 3633: hermon_reset ( hermon, pci ); ! 3634: ! 3635: /* Start firmware */ ! 3636: if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 ) ! 3637: goto err_start_firmware; ! 3638: ! 3639: /* Get device limits */ ! 3640: if ( ( rc = hermon_get_cap ( hermon ) ) != 0 ) ! 3641: goto err_get_cap; ! 3642: ! 3643: /* Allocate Infiniband devices */ ! 3644: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { ! 3645: ibdev = alloc_ibdev ( 0 ); ! 3646: if ( ! ibdev ) { ! 3647: rc = -ENOMEM; ! 3648: goto err_alloc_ibdev; ! 3649: } ! 3650: hermon->port[i].ibdev = ibdev; ! 3651: ibdev->op = &hermon_ib_operations; ! 3652: ibdev->dev = &pci->dev; ! 3653: ibdev->port = ( HERMON_PORT_BASE + i ); ! 3654: ib_set_drvdata ( ibdev, hermon ); ! 3655: } ! 3656: ! 3657: /* Allocate network devices */ ! 3658: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { ! 3659: netdev = alloc_etherdev ( 0 ); ! 3660: if ( ! netdev ) { ! 3661: rc = -ENOMEM; ! 3662: goto err_alloc_netdev; ! 3663: } ! 3664: hermon->port[i].netdev = netdev; ! 3665: netdev_init ( netdev, &hermon_eth_operations ); ! 3666: netdev->dev = &pci->dev; ! 3667: netdev->priv = &hermon->port[i]; ! 3668: } ! 3669: ! 3670: /* Allocate ICM */ ! 3671: memset ( &init_hca, 0, sizeof ( init_hca ) ); ! 3672: if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 ) ! 3673: goto err_alloc_icm; ! 3674: ! 3675: /* Initialise HCA */ ! 3676: MLX_FILL_1 ( &init_hca, 0, version, 0x02 /* "Must be 0x02" */ ); ! 3677: MLX_FILL_1 ( &init_hca, 5, udp, 1 ); ! 3678: MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 8 ); ! 3679: if ( ( rc = hermon_cmd_init_hca ( hermon, &init_hca ) ) != 0 ) { ! 3680: DBGC ( hermon, "Hermon %p could not initialise HCA: %s\n", ! 3681: hermon, strerror ( rc ) ); ! 3682: goto err_init_hca; ! 3683: } ! 3684: ! 3685: /* Set up memory protection */ ! 3686: if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 ) ! 3687: goto err_setup_mpt; ! 3688: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) ! 3689: hermon->port[i].ibdev->rdma_key = hermon->lkey; ! 3690: ! 3691: /* Set up event queue */ ! 3692: if ( ( rc = hermon_create_eq ( hermon ) ) != 0 ) ! 3693: goto err_create_eq; ! 3694: ! 3695: /* Configure special QPs */ ! 3696: if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 ) ! 3697: goto err_conf_special_qps; ! 3698: ! 3699: /* Determine port types */ ! 3700: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { ! 3701: port = &hermon->port[i]; ! 3702: if ( ( rc = hermon_set_port_type ( hermon, port ) ) != 0 ) ! 3703: goto err_set_port_type; ! 3704: } ! 3705: ! 3706: /* Register devices */ ! 3707: for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { ! 3708: port = &hermon->port[i]; ! 3709: if ( ( rc = port->type->register_dev ( hermon, port ) ) != 0 ) ! 3710: goto err_register; ! 3711: } ! 3712: ! 3713: return 0; ! 3714: ! 3715: i = hermon->cap.num_ports; ! 3716: err_register: ! 3717: for ( i-- ; ( signed int ) i >= 0 ; i-- ) { ! 3718: port = &hermon->port[i]; ! 3719: port->type->unregister_dev ( hermon, port ); ! 3720: } ! 3721: err_set_port_type: ! 3722: err_conf_special_qps: ! 3723: hermon_destroy_eq ( hermon ); ! 3724: err_create_eq: ! 3725: err_setup_mpt: ! 3726: hermon_cmd_close_hca ( hermon ); ! 3727: err_init_hca: ! 3728: hermon_free_icm ( hermon ); ! 3729: err_alloc_icm: ! 3730: i = hermon->cap.num_ports; ! 3731: err_alloc_netdev: ! 3732: for ( i-- ; ( signed int ) i >= 0 ; i-- ) { ! 3733: netdev_nullify ( hermon->port[i].netdev ); ! 3734: netdev_put ( hermon->port[i].netdev ); ! 3735: } ! 3736: i = hermon->cap.num_ports; ! 3737: err_alloc_ibdev: ! 3738: for ( i-- ; ( signed int ) i >= 0 ; i-- ) ! 3739: ibdev_put ( hermon->port[i].ibdev ); ! 3740: err_get_cap: ! 3741: hermon_stop_firmware ( hermon ); ! 3742: err_start_firmware: ! 3743: hermon_free ( hermon ); ! 3744: err_alloc: ! 3745: return rc; ! 3746: } ! 3747: ! 3748: /** ! 3749: * Remove PCI device ! 3750: * ! 3751: * @v pci PCI device ! 3752: */ ! 3753: static void hermon_remove ( struct pci_device *pci ) { ! 3754: struct hermon *hermon = pci_get_drvdata ( pci ); ! 3755: struct hermon_port *port; ! 3756: int i; ! 3757: ! 3758: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) { ! 3759: port = &hermon->port[i]; ! 3760: port->type->unregister_dev ( hermon, port ); ! 3761: } ! 3762: hermon_destroy_eq ( hermon ); ! 3763: hermon_cmd_close_hca ( hermon ); ! 3764: hermon_free_icm ( hermon ); ! 3765: hermon_stop_firmware ( hermon ); ! 3766: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) { ! 3767: netdev_nullify ( hermon->port[i].netdev ); ! 3768: netdev_put ( hermon->port[i].netdev ); ! 3769: } ! 3770: for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) ! 3771: ibdev_put ( hermon->port[i].ibdev ); ! 3772: hermon_free ( hermon ); ! 3773: } ! 3774: ! 3775: /** ! 3776: * Probe PCI device for BOFM ! 3777: * ! 3778: * @v pci PCI device ! 3779: * @v id PCI ID ! 3780: * @ret rc Return status code ! 3781: */ ! 3782: static int hermon_bofm_probe ( struct pci_device *pci ) { ! 3783: struct hermon *hermon; ! 3784: int rc; ! 3785: ! 3786: /* Allocate Hermon device */ ! 3787: hermon = hermon_alloc(); ! 3788: if ( ! hermon ) { ! 3789: rc = -ENOMEM; ! 3790: goto err_alloc; ! 3791: } ! 3792: pci_set_drvdata ( pci, hermon ); ! 3793: ! 3794: /* Initialise PCI parameters */ ! 3795: hermon_pci_init ( hermon, pci ); ! 3796: ! 3797: /* Initialise BOFM device */ ! 3798: bofm_init ( &hermon->bofm, pci, &hermon_bofm_operations ); ! 3799: ! 3800: /* Register BOFM device */ ! 3801: if ( ( rc = bofm_register ( &hermon->bofm ) ) != 0 ) { ! 3802: DBGC ( hermon, "Hermon %p could not register BOFM device: " ! 3803: "%s\n", hermon, strerror ( rc ) ); ! 3804: goto err_bofm_register; ! 3805: } ! 3806: ! 3807: return 0; ! 3808: ! 3809: err_bofm_register: ! 3810: hermon_free ( hermon ); ! 3811: err_alloc: ! 3812: return rc; ! 3813: } ! 3814: ! 3815: /** ! 3816: * Remove PCI device for BOFM ! 3817: * ! 3818: * @v pci PCI device ! 3819: */ ! 3820: static void hermon_bofm_remove ( struct pci_device *pci ) { ! 3821: struct hermon *hermon = pci_get_drvdata ( pci ); ! 3822: ! 3823: bofm_unregister ( &hermon->bofm ); ! 3824: hermon_free ( hermon ); ! 3825: } ! 3826: ! 3827: static struct pci_device_id hermon_nics[] = { ! 3828: PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ), ! 3829: PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ), ! 3830: PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ), ! 3831: PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ), ! 3832: PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ), ! 3833: PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ), ! 3834: PCI_ROM ( 0x15b3, 0x6368, "mt25448", "MT25448 HCA driver", 0 ), ! 3835: PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ), ! 3836: PCI_ROM ( 0x15b3, 0x6372, "mt25458", "MT25458 HCA driver", 0 ), ! 3837: PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ), ! 3838: PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ), ! 3839: PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ), ! 3840: }; ! 3841: ! 3842: struct pci_driver hermon_driver __pci_driver = { ! 3843: .ids = hermon_nics, ! 3844: .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ), ! 3845: .probe = hermon_probe, ! 3846: .remove = hermon_remove, ! 3847: }; ! 3848: ! 3849: struct pci_driver hermon_bofm_driver __bofm_driver = { ! 3850: .ids = hermon_nics, ! 3851: .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ), ! 3852: .probe = hermon_bofm_probe, ! 3853: .remove = hermon_bofm_remove, ! 3854: };
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.