Return to ib_mi.c CVS log

Up to [Qemu by Fabrice Bellard] / qemu / roms / ipxe / src / net / infiniband

Annotation of qemu/roms/ipxe/src/net/infiniband/ib_mi.c, revision 1.1

1.1     ! root        1: /*
        !             2:  * Copyright (C) 2009 Michael Brown <[email protected]>.
        !             3:  *
        !             4:  * This program is free software; you can redistribute it and/or
        !             5:  * modify it under the terms of the GNU General Public License as
        !             6:  * published by the Free Software Foundation; either version 2 of the
        !             7:  * License, or any later version.
        !             8:  *
        !             9:  * This program is distributed in the hope that it will be useful, but
        !            10:  * WITHOUT ANY WARRANTY; without even the implied warranty of
        !            11:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
        !            12:  * General Public License for more details.
        !            13:  *
        !            14:  * You should have received a copy of the GNU General Public License
        !            15:  * along with this program; if not, write to the Free Software
        !            16:  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
        !            17:  */
        !            18: 
        !            19: FILE_LICENCE ( GPL2_OR_LATER );
        !            20: 
        !            21: #include <stdint.h>
        !            22: #include <stdlib.h>
        !            23: #include <string.h>
        !            24: #include <errno.h>
        !            25: #include <stdio.h>
        !            26: #include <unistd.h>
        !            27: #include <byteswap.h>
        !            28: #include <ipxe/infiniband.h>
        !            29: #include <ipxe/iobuf.h>
        !            30: #include <ipxe/ib_mi.h>
        !            31: 
        !            32: /**
        !            33:  * @file
        !            34:  *
        !            35:  * Infiniband management interfaces
        !            36:  *
        !            37:  */
        !            38: 
        !            39: /** Management interface number of send WQEs
        !            40:  *
        !            41:  * This is a policy decision.
        !            42:  */
        !            43: #define IB_MI_NUM_SEND_WQES 4
        !            44: 
        !            45: /** Management interface number of receive WQEs
        !            46:  *
        !            47:  * This is a policy decision.
        !            48:  */
        !            49: #define IB_MI_NUM_RECV_WQES 2
        !            50: 
        !            51: /** Management interface number of completion queue entries
        !            52:  *
        !            53:  * This is a policy decision
        !            54:  */
        !            55: #define IB_MI_NUM_CQES 8
        !            56: 
        !            57: /** TID magic signature */
        !            58: #define IB_MI_TID_MAGIC ( ( 'i' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
        !            59: 
        !            60: /** TID to use for next MAD */
        !            61: static unsigned int next_tid;
        !            62: 
        !            63: /**
        !            64:  * Handle received MAD
        !            65:  *
        !            66:  * @v ibdev            Infiniband device
        !            67:  * @v mi               Management interface
        !            68:  * @v mad              Received MAD
        !            69:  * @v av               Source address vector
        !            70:  * @ret rc             Return status code
        !            71:  */
        !            72: static int ib_mi_handle ( struct ib_device *ibdev,
        !            73:                          struct ib_mad_interface *mi,
        !            74:                          union ib_mad *mad,
        !            75:                          struct ib_address_vector *av ) {
        !            76:        struct ib_mad_hdr *hdr = &mad->hdr;
        !            77:        struct ib_mad_transaction *madx;
        !            78:        struct ib_mad_agent *agent;
        !            79: 
        !            80:        /* Look for a matching transaction by TID */
        !            81:        list_for_each_entry ( madx, &mi->madx, list ) {
        !            82:                if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
        !            83:                              sizeof ( hdr->tid ) ) != 0 )
        !            84:                        continue;
        !            85:                /* Found a matching transaction */
        !            86:                madx->op->complete ( ibdev, mi, madx, 0, mad, av );
        !            87:                return 0;
        !            88:        }
        !            89: 
        !            90:        /* If there is no matching transaction, look for a listening agent */
        !            91:        for_each_table_entry ( agent, IB_MAD_AGENTS ) {
        !            92:                if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
        !            93:                       ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
        !            94:                     ( agent->class_version != hdr->class_version ) ||
        !            95:                     ( agent->attr_id != hdr->attr_id ) )
        !            96:                        continue;
        !            97:                /* Found a matching agent */
        !            98:                agent->handle ( ibdev, mi, mad, av );
        !            99:                return 0;
        !           100:        }
        !           101: 
        !           102:        /* Otherwise, ignore it */
        !           103:        DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
        !           104:               mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
        !           105:        return -ENOTSUP;
        !           106: }
        !           107: 
        !           108: /**
        !           109:  * Complete receive via management interface
        !           110:  *
        !           111:  *
        !           112:  * @v ibdev            Infiniband device
        !           113:  * @v qp               Queue pair
        !           114:  * @v av               Address vector
        !           115:  * @v iobuf            I/O buffer
        !           116:  * @v rc               Completion status code
        !           117:  */
        !           118: static void ib_mi_complete_recv ( struct ib_device *ibdev,
        !           119:                                  struct ib_queue_pair *qp,
        !           120:                                  struct ib_address_vector *av,
        !           121:                                  struct io_buffer *iobuf, int rc ) {
        !           122:        struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
        !           123:        union ib_mad *mad;
        !           124:        struct ib_mad_hdr *hdr;
        !           125: 
        !           126:        /* Ignore errors */
        !           127:        if ( rc != 0 ) {
        !           128:                DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
        !           129:                goto out;
        !           130:        }
        !           131: 
        !           132:        /* Sanity checks */
        !           133:        if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
        !           134:                DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
        !           135:                       mi, iob_len ( iobuf ) );
        !           136:                DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
        !           137:                goto out;
        !           138:        }
        !           139:        mad = iobuf->data;
        !           140:        hdr = &mad->hdr;
        !           141:        if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
        !           142:                DBGC ( mi, "MI %p RX unsupported base version %x\n",
        !           143:                       mi, hdr->base_version );
        !           144:                DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
        !           145:                goto out;
        !           146:        }
        !           147:        DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
        !           148:               "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
        !           149:               hdr->mgmt_class, hdr->class_version, hdr->method,
        !           150:               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        !           151:        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
        !           152: 
        !           153:        /* Handle MAD */
        !           154:        if ( ( rc = ib_mi_handle ( ibdev, mi, mad, av ) ) != 0 )
        !           155:                goto out;
        !           156: 
        !           157:  out:
        !           158:        free_iob ( iobuf );
        !           159: }
        !           160: 
        !           161: /** Management interface completion operations */
        !           162: static struct ib_completion_queue_operations ib_mi_completion_ops = {
        !           163:        .complete_recv = ib_mi_complete_recv,
        !           164: };
        !           165: 
        !           166: /**
        !           167:  * Transmit MAD
        !           168:  *
        !           169:  * @v ibdev            Infiniband device
        !           170:  * @v mi               Management interface
        !           171:  * @v mad              MAD
        !           172:  * @v av               Destination address vector
        !           173:  * @ret rc             Return status code
        !           174:  */
        !           175: int ib_mi_send ( struct ib_device *ibdev, struct ib_mad_interface *mi,
        !           176:                 union ib_mad *mad, struct ib_address_vector *av ) {
        !           177:        struct ib_mad_hdr *hdr = &mad->hdr;
        !           178:        struct io_buffer *iobuf;
        !           179:        int rc;
        !           180: 
        !           181:        /* Set common fields */
        !           182:        hdr->base_version = IB_MGMT_BASE_VERSION;
        !           183:        if ( ( hdr->tid[0] == 0 ) && ( hdr->tid[1] == 0 ) ) {
        !           184:                hdr->tid[0] = htonl ( IB_MI_TID_MAGIC );
        !           185:                hdr->tid[1] = htonl ( ++next_tid );
        !           186:        }
        !           187:        DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
        !           188:               "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
        !           189:               hdr->mgmt_class, hdr->class_version, hdr->method,
        !           190:               ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
        !           191:        DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
        !           192: 
        !           193:        /* Construct directed route portion of response, if necessary */
        !           194:        if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
        !           195:                struct ib_mad_smp *smp = &mad->smp;
        !           196:                unsigned int hop_pointer;
        !           197:                unsigned int hop_count;
        !           198: 
        !           199:                smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
        !           200:                hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
        !           201:                hop_count = smp->mad_hdr.class_specific.smp.hop_count;
        !           202:                assert ( hop_count == hop_pointer );
        !           203:                if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
        !           204:                                     sizeof ( smp->return_path.hops[0] ) ) ) {
        !           205:                        smp->return_path.hops[hop_pointer] = ibdev->port;
        !           206:                } else {
        !           207:                        DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
        !           208:                               "%d\n", mi, ntohl ( hdr->tid[0] ),
        !           209:                               ntohl ( hdr->tid[1] ), hop_pointer );
        !           210:                        return -EINVAL;
        !           211:                }
        !           212:        }
        !           213: 
        !           214:        /* Construct I/O buffer */
        !           215:        iobuf = alloc_iob ( sizeof ( *mad ) );
        !           216:        if ( ! iobuf ) {
        !           217:                DBGC ( mi, "MI %p could not allocate buffer for TID "
        !           218:                       "%08x%08x\n",
        !           219:                       mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
        !           220:                return -ENOMEM;
        !           221:        }
        !           222:        memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );
        !           223: 
        !           224:        /* Send I/O buffer */
        !           225:        if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
        !           226:                DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
        !           227:                       mi,  ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
        !           228:                       strerror ( rc ) );
        !           229:                free_iob ( iobuf );
        !           230:                return rc;
        !           231:        }
        !           232: 
        !           233:        return 0;
        !           234: }
        !           235: 
        !           236: /**
        !           237:  * Handle management transaction timer expiry
        !           238:  *
        !           239:  * @v timer            Retry timer
        !           240:  * @v expired          Failure indicator
        !           241:  */
        !           242: static void ib_mi_timer_expired ( struct retry_timer *timer, int expired ) {
        !           243:        struct ib_mad_transaction *madx =
        !           244:                container_of ( timer, struct ib_mad_transaction, timer );
        !           245:        struct ib_mad_interface *mi = madx->mi;
        !           246:        struct ib_device *ibdev = mi->ibdev;
        !           247:        struct ib_mad_hdr *hdr = &madx->mad.hdr;
        !           248: 
        !           249:        /* Abandon transaction if we have tried too many times */
        !           250:        if ( expired ) {
        !           251:                DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
        !           252:                       mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
        !           253:                madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
        !           254:                return;
        !           255:        }
        !           256: 
        !           257:        /* Restart retransmission timer */
        !           258:        start_timer ( timer );
        !           259: 
        !           260:        /* Resend MAD */
        !           261:        ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
        !           262: }
        !           263: 
        !           264: /**
        !           265:  * Create management transaction
        !           266:  *
        !           267:  * @v ibdev            Infiniband device
        !           268:  * @v mi               Management interface
        !           269:  * @v mad              MAD to send
        !           270:  * @v av               Destination address, or NULL to use SM's GSI
        !           271:  * @v op               Management transaction operations
        !           272:  * @ret madx           Management transaction, or NULL
        !           273:  */
        !           274: struct ib_mad_transaction *
        !           275: ib_create_madx ( struct ib_device *ibdev, struct ib_mad_interface *mi,
        !           276:                 union ib_mad *mad, struct ib_address_vector *av,
        !           277:                 struct ib_mad_transaction_operations *op ) {
        !           278:        struct ib_mad_transaction *madx;
        !           279: 
        !           280:        /* Allocate and initialise structure */
        !           281:        madx = zalloc ( sizeof ( *madx ) );
        !           282:        if ( ! madx )
        !           283:                return NULL;
        !           284:        timer_init ( &madx->timer, ib_mi_timer_expired, NULL );
        !           285:        madx->mi = mi;
        !           286:        madx->op = op;
        !           287: 
        !           288:        /* Determine address vector */
        !           289:        if ( av ) {
        !           290:                memcpy ( &madx->av, av, sizeof ( madx->av ) );
        !           291:        } else {
        !           292:                madx->av.lid = ibdev->sm_lid;
        !           293:                madx->av.sl = ibdev->sm_sl;
        !           294:                madx->av.qpn = IB_QPN_GSI;
        !           295:                madx->av.qkey = IB_QKEY_GSI;
        !           296:        }
        !           297: 
        !           298:        /* Copy MAD */
        !           299:        memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );
        !           300: 
        !           301:        /* Add to list and start timer to send initial MAD */
        !           302:        list_add ( &madx->list, &mi->madx );
        !           303:        start_timer_nodelay ( &madx->timer );
        !           304: 
        !           305:        return madx;
        !           306: }
        !           307: 
        !           308: /**
        !           309:  * Destroy management transaction
        !           310:  *
        !           311:  * @v ibdev            Infiniband device
        !           312:  * @v mi               Management interface
        !           313:  * @v madx             Management transaction
        !           314:  */
        !           315: void ib_destroy_madx ( struct ib_device *ibdev __unused,
        !           316:                       struct ib_mad_interface *mi __unused,
        !           317:                       struct ib_mad_transaction *madx ) {
        !           318: 
        !           319:        /* Stop timer and remove from list */
        !           320:        stop_timer ( &madx->timer );
        !           321:        list_del ( &madx->list );
        !           322: 
        !           323:        /* Free transaction */
        !           324:        free ( madx );
        !           325: }
        !           326: 
        !           327: /**
        !           328:  * Create management interface
        !           329:  *
        !           330:  * @v ibdev            Infiniband device
        !           331:  * @v type             Queue pair type
        !           332:  * @ret mi             Management agent, or NULL
        !           333:  */
        !           334: struct ib_mad_interface * ib_create_mi ( struct ib_device *ibdev,
        !           335:                                         enum ib_queue_pair_type type ) {
        !           336:        struct ib_mad_interface *mi;
        !           337:        int rc;
        !           338: 
        !           339:        /* Allocate and initialise fields */
        !           340:        mi = zalloc ( sizeof ( *mi ) );
        !           341:        if ( ! mi )
        !           342:                goto err_alloc;
        !           343:        mi->ibdev = ibdev;
        !           344:        INIT_LIST_HEAD ( &mi->madx );
        !           345: 
        !           346:        /* Create completion queue */
        !           347:        mi->cq = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops );
        !           348:        if ( ! mi->cq ) {
        !           349:                DBGC ( mi, "MI %p could not allocate completion queue\n", mi );
        !           350:                goto err_create_cq;
        !           351:        }
        !           352: 
        !           353:        /* Create queue pair */
        !           354:        mi->qp = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
        !           355:                                IB_MI_NUM_RECV_WQES, mi->cq );
        !           356:        if ( ! mi->qp ) {
        !           357:                DBGC ( mi, "MI %p could not allocate queue pair\n", mi );
        !           358:                goto err_create_qp;
        !           359:        }
        !           360:        ib_qp_set_ownerdata ( mi->qp, mi );
        !           361:        DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
        !           362:               mi, ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" ), mi->qp->qpn );
        !           363: 
        !           364:        /* Set queue key */
        !           365:        mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
        !           366:        if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
        !           367:                DBGC ( mi, "MI %p could not set queue key: %s\n",
        !           368:                       mi, strerror ( rc ) );
        !           369:                goto err_modify_qp;
        !           370:        }
        !           371: 
        !           372:        /* Fill receive ring */
        !           373:        ib_refill_recv ( ibdev, mi->qp );
        !           374:        return mi;
        !           375: 
        !           376:  err_modify_qp:
        !           377:        ib_destroy_qp ( ibdev, mi->qp );
        !           378:  err_create_qp:
        !           379:        ib_destroy_cq ( ibdev, mi->cq );
        !           380:  err_create_cq:
        !           381:        free ( mi );
        !           382:  err_alloc:
        !           383:        return NULL;
        !           384: }
        !           385: 
        !           386: /**
        !           387:  * Destroy management interface
        !           388:  *
        !           389:  * @v mi               Management interface
        !           390:  */
        !           391: void ib_destroy_mi ( struct ib_device *ibdev, struct ib_mad_interface *mi ) {
        !           392:        struct ib_mad_transaction *madx;
        !           393:        struct ib_mad_transaction *tmp;
        !           394: 
        !           395:        /* Flush any outstanding requests */
        !           396:        list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
        !           397:                DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
        !           398:                       mi, ntohl ( madx->mad.hdr.tid[0] ),
        !           399:                       ntohl ( madx->mad.hdr.tid[1] ) );
        !           400:                madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
        !           401:        }
        !           402: 
        !           403:        ib_destroy_qp ( ibdev, mi->qp );
        !           404:        ib_destroy_cq ( ibdev, mi->cq );
        !           405:        free ( mi );
        !           406: }

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.