From 9a158100115f1f69f0aaf2dd77d2c9a527d98e18 Mon Sep 17 00:00:00 2001 From: Gregory Linschitz Date: Sun, 8 Sep 2024 21:10:32 +0300 Subject: [PATCH] libibmad: Add support SMI & GSI devices (POC) Added internal API for store SMI/GSI pairs - find_ports_record - add_ports_record - remove_port_record_ptr - remove_port_record Chagned rpc functions: - rpc_open - close - _do_madrpc Signed-off-by: Gregory Linschitz --- libibmad/CMakeLists.txt | 2 + libibmad/ext_umad.c | 399 ++++++++++++++++++++++++++++++++++++++++ libibmad/ext_umad.h | 69 +++++++ libibmad/register.c | 21 ++- libibmad/rpc.c | 32 ++-- libibmad/serv.c | 55 +++++- libibmad/smi_gsi.c | 135 ++++++++++++++ libibmad/smi_gsi.h | 106 +++++++++++ 8 files changed, 799 insertions(+), 20 deletions(-) create mode 100755 libibmad/ext_umad.c create mode 100755 libibmad/ext_umad.h create mode 100755 libibmad/smi_gsi.c create mode 100755 libibmad/smi_gsi.h diff --git a/libibmad/CMakeLists.txt b/libibmad/CMakeLists.txt index 43d560a43..659725f0f 100644 --- a/libibmad/CMakeLists.txt +++ b/libibmad/CMakeLists.txt @@ -24,6 +24,8 @@ rdma_library(ibmad libibmad.map serv.c smp.c vendor.c + ext_umad.c + smi_gsi.c ) target_link_libraries(ibmad LINK_PRIVATE ibumad diff --git a/libibmad/ext_umad.c b/libibmad/ext_umad.c new file mode 100755 index 000000000..9ff7c1654 --- /dev/null +++ b/libibmad/ext_umad.c @@ -0,0 +1,399 @@ +/* +* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +*/ + +#include +#include +#include +#include +#include + +#include "ext_umad.h" + +#define CAPMASK_IS_SM_DISABLED 0x400 + +typedef typeof(((struct umad_port *)0)->port_guid) umad_guid_t; + +/** + * @brief struct to save the number of ports with a specific port GUID + */ +struct port_guid_port_count { + umad_guid_t port_guid; + uint8_t count; +}; + +/** + * @brief A mapping between a port GUID, and the extended ca that has ports with this GUID. + * Used to search the correct extended ca for a given port. + */ +struct guid_ext_ca_mapping { + umad_guid_t port_guid; + ext_umad_ca_t *ext_ca; +}; + +/** + * @brief search the 'counts' array for a struct with a given GUID / the first + * empty struct if GUID was not found + * + * @param counts[in] + * @param max - size of counts + * @param port_guid + * @param index[out] + * @return true - a struct was found, 'index' contains it's index + * @return false - a struct was not found, 'index' contains the + first unused index in counts / the last index if counts is full. + */ +static bool find_port_guid_count(struct port_guid_port_count counts[], size_t max, + umad_guid_t port_guid, size_t *index) +{ + size_t i = 0; + for (i = 0; i < max; ++i) { + if (counts[i].port_guid == 0) { + *index = i; + return false; + } + if (counts[i].port_guid == port_guid) { + *index = i; + return true; + } + } + + *index = max; + return false; +} + +/** + * @brief count the number of ports that hold each GUID. + * + * @param legacy_ca_names[in] - ca names given by umad_get_cas_names + * @param num_cas - number of cas returned by umad_get_cas_names + * @param counts[out] - each entry in this array contains a port guid and + the number of ports with that guid. + * @param max - maximum output array size. new GUIDs will be + ignored after the maximum amount was added. + * @return number of guids counted (output array length) + */ +static int count_ports_by_guid(char legacy_ca_names[][UMAD_CA_NAME_LEN], size_t num_cas, + struct port_guid_port_count counts[], size_t max) +{ + // how many unique port GUIDs were added + size_t num_of_guid = 0; + + memset(counts, 0, max * sizeof(struct port_guid_port_count)); + + size_t c_idx = 0; + for (c_idx = 0; c_idx < num_cas; ++c_idx) { + umad_ca_t curr_ca; + + if (umad_get_ca(legacy_ca_names[c_idx], &curr_ca) < 0) + continue; + + size_t p_idx = 1; + for (p_idx = 1; p_idx < (size_t)curr_ca.numports + 1; ++p_idx) { + umad_port_t *p_port = curr_ca.ports[p_idx]; + size_t count_idx = 0; + + if (!p_port) + continue; + + if (find_port_guid_count(counts, max, p_port->port_guid, &count_idx)) { + // port GUID already has a count struct + ++counts[count_idx].count; + } else { + // add a new count struct for this GUID. + // if the maximum amount was already added, do nothing. + if (count_idx != max) { + counts[count_idx].port_guid = p_port->port_guid; + counts[count_idx].count = 1; + ++num_of_guid; + } + } + } + + umad_release_ca(&curr_ca); + } + + return num_of_guid; +} + +/** + * @brief return the amount of ports with the same port GUID as the one given. + * simply searches the counts array for the correct GUID. + * + * @param guid + * @param counts[in] - an array holding each guid and it's count. + * @param max_guids - maximum amount of entries in 'counts' array. + * @return size_t + */ +static uint8_t get_port_guid_count(umad_guid_t guid, const struct port_guid_port_count counts[], + size_t max_guids) +{ + size_t i = 0; + for (i = 0; i < max_guids; ++i) { + if (counts[i].port_guid == guid) + return counts[i].count; + } + + return 0; +} + +static bool is_smi_disabled(umad_port_t *p_port) +{ + return (be32toh(p_port->capmask) & CAPMASK_IS_SM_DISABLED); +} + +/** + * @brief Get a pointer to the device in which a planarized port + * with 'port_guid' should be inserted. + * + * Search the mapping array for the given port_guid. + * if found, return the result pointer. + * if not found, return the first non-initialized 'dev' + * array index (or NULL if the array is full), + * add a new mapping for the given port, and advance 'added' counters. + * + * @param port_guid + * @param mapping[input, output] - search this array for the given port GUID. + * @param map_max - maximum size of the mapping array + * @param map_added - amount of mappings in 'mapping'. + * will be increased if a new mapping is added. + * @param devs[input] - the array from which the index will be returned + * @param devs_max - maximum size of 'devs' array + * @param devs_added - amount of initialized devices in 'devs' array. + * will be changed if a new device is added. + * @return address of the device that corresponds to the given GUID. + * NULL if not found and 'devs' is full. + */ +static ext_umad_ca_t *get_ext_ca_from_arr_by_guid(umad_guid_t port_guid, + struct guid_ext_ca_mapping mapping[], + size_t map_max, size_t *map_added, + ext_umad_ca_t devs[], + size_t devs_max, size_t *devs_added) +{ + ext_umad_ca_t *dev = NULL; + // attempt to find the port guid in the mapping + size_t i = 0; + for (i = 0; i < *map_added; ++i) { + if (mapping[i].port_guid == port_guid) + return mapping[i].ext_ca; + } + + // attempt to add a new mapping/device + if (*map_added >= map_max || *devs_added >= devs_max) + return NULL; + + dev = &devs[*devs_added]; + mapping[*map_added].port_guid = port_guid; + mapping[*map_added].ext_ca = dev; + (*devs_added)++; + (*map_added)++; + + return dev; +} + +/** + * @brief add a new port to a device's port numbers array (zero terminated). + * set the device's name if it doesn't have one. + * + * @param dev[output] - devices the port number should be added to. + * @param p_port[input] - the port whose number will be added to the + * list (and potentially ca name) + */ +static void add_new_port(ext_umad_device_t *dev, umad_port_t *p_port) +{ + if (!dev->name[0]) { + memcpy(dev->name, p_port->ca_name, UMAD_CA_NAME_LEN); + dev->numports = 0; + } + + if (dev->numports < UMAD_CA_MAX_PORTS) + dev->ports[dev->numports++] = p_port->portnum; +} + +int ext_umad_get_cas(ext_umad_ca_t cas[], size_t max) +{ + size_t added_devices = 0, added_mappings = 0; + char legacy_ca_names[UMAD_MAX_DEVICES][UMAD_CA_NAME_LEN] = {}; + struct port_guid_port_count counts[UMAD_MAX_PORTS] = {}; + struct guid_ext_ca_mapping mapping[UMAD_MAX_PORTS] = {}; + + memset(cas, 0, sizeof(ext_umad_ca_t) * max); + int cas_found = umad_get_cas_names(legacy_ca_names, UMAD_MAX_DEVICES); + + if (cas_found < 0) + return 0; + + count_ports_by_guid(legacy_ca_names, cas_found, counts, UMAD_MAX_PORTS); + + size_t c_idx = 0; + for (c_idx = 0; c_idx < (size_t)cas_found; ++c_idx) { + umad_ca_t curr_ca; + + if (umad_get_ca(legacy_ca_names[c_idx], &curr_ca) < 0) + continue; + + size_t p_idx = 1; + for (p_idx = 1; p_idx < (size_t)curr_ca.numports + 1; ++p_idx) { + umad_port_t *p_port = curr_ca.ports[p_idx]; + uint8_t guid_count = 0; + + if (!p_port) + continue; + + guid_count = get_port_guid_count(curr_ca.ports[p_idx]->port_guid, + counts, UMAD_MAX_PORTS); + ext_umad_ca_t *dev = get_ext_ca_from_arr_by_guid(p_port->port_guid, + mapping, UMAD_MAX_PORTS, + &added_mappings, cas, + max, &added_devices); + if (!dev) + continue; + if (guid_count > 1) { + // planarized port + add_new_port(is_smi_disabled(p_port) ? + &dev->gsi : &dev->smi, p_port); + } else if (guid_count == 1) { + if (!is_smi_disabled(p_port)) + add_new_port(&dev->smi, p_port); + + // all ports are GSI ports in legacy HCAs + add_new_port(&dev->gsi, p_port); + } else { + return -1; + } + } + + umad_release_ca(&curr_ca); + } + + return added_devices; +} + +static int ext_umad_check_active(ext_umad_device_t * dev, int prefered) +{ + umad_ca_t ca; + + if (!dev) + return 1; + + // check if candidate is active + if (umad_get_ca(dev->name, &ca) < 0) + return 2; + + int state = ca.ports[prefered]->state; + + umad_release_ca(&ca); + + if (state > 1) { + dev->preferred_port = prefered; + return 0; + } + + return 1; +} + +static int ext_umad_find_active(ext_umad_device_t * dev) { + int i; + + if (!dev) + return 1; + + for (i = 0; i < dev->numports; ++i) + if(!ext_umad_check_active(dev, dev->ports[i])) + return 0; + + return 1; +} + +int ext_umad_get_ca_by_name(const char *name, uint8_t portnum, ext_umad_ca_t *ca) +{ + int rc = 1; + size_t i = 0; + int num_cas = 0; + size_t port_idx = 0; + bool is_gsi = false; + bool found_port = false; + + ext_umad_device_t * dev = NULL; + ext_umad_ca_t ext_cas[UMAD_MAX_PORTS] = {}; + + if (!ca) + return -1; + + memset(ext_cas, 0, sizeof(ext_cas)); + memset(ca, 0, sizeof(*ca)); + + num_cas = ext_umad_get_cas(ext_cas, UMAD_MAX_PORTS); + + if (num_cas <= 0) + return num_cas; + + for (i = 0; i < (size_t)num_cas; ++i) { + if (!ext_cas[i].gsi.name[0] || !ext_cas[i].smi.name[0] || + !ext_cas[i].gsi.numports || !ext_cas[i].smi.numports) + continue; + + if (name) + // name doesn't match - keep searching + if (strncmp(ext_cas[i].gsi.name, name, UMAD_CA_NAME_LEN) + && strncmp(ext_cas[i].smi.name, name, UMAD_CA_NAME_LEN)) { + continue; + } + + // check that the device given by "name" has a port number "portnum" + // (if name doesn't exist, assume SMI port is given) + is_gsi = (name && !strncmp(name, ext_cas[i].gsi.name, UMAD_CA_NAME_LEN)); + + if (portnum) { + dev = is_gsi ? &ext_cas[i].gsi : &ext_cas[i].smi; + + found_port = false; + + for (port_idx = 0; port_idx < dev->numports; ++port_idx) { + + if (!dev->ports[port_idx]) + break; + + if (dev->ports[port_idx] == portnum) + found_port = true; + } + + // couldn't find portnum - keep searching + if (!found_port) + continue; + } + + // fill candidate + *ca = ext_cas[i]; + + if (portnum) + if (is_gsi) + rc = ext_umad_find_active(&ca->smi) + + ext_umad_check_active(&ca->gsi, portnum); + else + rc = ext_umad_check_active(&ca->smi, portnum) + + ext_umad_find_active(&ca->gsi); + else { + rc = ext_umad_find_active(&ca->smi) + + ext_umad_find_active(&ca->gsi); + } + + if (!rc) + break; + } + + if (rc) { + errno = ENODEV; + return -errno; + } + + return rc; +} diff --git a/libibmad/ext_umad.h b/libibmad/ext_umad.h new file mode 100755 index 000000000..58c5c0eb5 --- /dev/null +++ b/libibmad/ext_umad.h @@ -0,0 +1,69 @@ +/* +* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +*/ + + +#ifndef EXT_UMAD_H_ +#define EXT_UMAD_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief represents SMI/GSI HCA device. + * + */ +typedef struct ext_umad_device { + char name[UMAD_CA_NAME_LEN]; + uint32_t ports[UMAD_CA_MAX_PORTS]; + uint32_t numports; + uint32_t preferred_port; +} ext_umad_device_t; + +/** + * @brief represents SMI and GSI pairs on an HCA. + * on legacy CAs both devices should have the same name. + */ +typedef struct ext_umad_ca { + ext_umad_device_t smi; + ext_umad_device_t gsi; +} ext_umad_ca_t; + +/** + * @brief fill a user allocated struct of extended cas according to umad data. + * + * @param cas[out] + * @param max - maximum amount of devices to fill. + * + * @return the number of devices filled, -1 on error. + */ +int ext_umad_get_cas(ext_umad_ca_t cas[], size_t max); + +/** + * @brief get extended ca and port nums based on device name and portnum. + * the given portnum addresses a port in the given device name (might be smi or gsi). + * + * @param devname[input] - ca (smi, gsi or legacy) name to search, or NULL to find the first one. + * @param out[output] - extended ca found (optional) + * @param portnum - a port number of a port in 'devname'. + * 0 for the first SMI and GSI with identical GUIDs. + * @return 0 if a device and ports were found, 1 otherwise + */ +int ext_umad_get_ca_by_name(const char *devname, uint8_t portnum, ext_umad_ca_t *ca); + +#ifdef __cplusplus +} +#endif + +#endif /* EXT_UMAD_H_ */ diff --git a/libibmad/register.c b/libibmad/register.c index 5092f0dbf..c70e4a6c2 100644 --- a/libibmad/register.c +++ b/libibmad/register.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. + * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -40,6 +41,7 @@ #include #include "mad_internal.h" +#include "smi_gsi.h" #undef DEBUG #define DEBUG if (ibdebug) IBWARN @@ -87,6 +89,11 @@ int mad_register_port_client(int port_id, int mgmt, uint8_t rmpp_version) return -1; } + port_id = smi_gsi_port_by_class(port_id, mgmt); + + if (port_id < 0) + IBWARN("Couldn't resolve SMI/GSI device for port_id %d.", port_id); + agent = umad_register(port_id, mgmt, vers, rmpp_version, NULL); if (agent < 0) DEBUG("Can't register agent for class %d", mgmt); @@ -129,7 +136,7 @@ int mad_register_server_via(int mgmt, uint8_t rmpp_version, { long class_method_mask[16 / sizeof(long)]; uint8_t oui[3]; - int agent, vers; + int agent, vers, port_id; if (method_mask) memcpy(class_method_mask, method_mask, @@ -149,20 +156,26 @@ int mad_register_server_via(int mgmt, uint8_t rmpp_version, DEBUG("Unknown class 0x%x mgmt_class", mgmt); return -1; } + + port_id = smi_gsi_port_by_class(srcport->port_id, mgmt); + + if (port_id < 0) + IBWARN("Couldn't resolve SMI/GSI device for port_id %d.", srcport->port_id); + if (mgmt >= IB_VENDOR_RANGE2_START_CLASS && mgmt <= IB_VENDOR_RANGE2_END_CLASS) { oui[0] = (class_oui >> 16) & 0xff; oui[1] = (class_oui >> 8) & 0xff; oui[2] = class_oui & 0xff; if ((agent = - umad_register_oui(srcport->port_id, mgmt, rmpp_version, - oui, class_method_mask)) < 0) { + umad_register_oui(port_id, mgmt, rmpp_version, + oui, class_method_mask)) < 0) { DEBUG("Can't register agent for class %d", mgmt); return -1; } } else if ((agent = - umad_register(srcport->port_id, mgmt, vers, rmpp_version, + umad_register(port_id, mgmt, vers, rmpp_version, class_method_mask)) < 0) { DEBUG("Can't register agent for class %d", mgmt); return -1; diff --git a/libibmad/rpc.c b/libibmad/rpc.c index 2a7801767..50d7d8ced 100644 --- a/libibmad/rpc.c +++ b/libibmad/rpc.c @@ -2,6 +2,7 @@ * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -42,6 +43,8 @@ #include #include "mad_internal.h" +#include "ext_umad.h" +#include "smi_gsi.h" int ibdebug; @@ -145,6 +148,12 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, trid = (uint32_t) mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F); + port_id = smi_gsi_port_by_class(port_id, + mad_get_field(umad_get_mad(sndbuf), 0, IB_MAD_MGMTCLASS_F)); + + if (port_id < 0) + IBWARN("Couldn't resolve SMI/GSI device for port_id %d.", port_id); + for (retries = 0; retries < max_retries; retries++) { if (retries) ERRS("retry %d (timeout %d ms)", retries, timeout); @@ -344,19 +353,21 @@ void *madrpc_rmpp(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes) { - int fd; + int port_id; if (umad_init() < 0) IBPANIC("can't init UMAD library"); - if ((fd = umad_open_port(dev_name, dev_port)) < 0) - IBPANIC("can't open UMAD port (%s:%d)", - dev_name ? dev_name : "(nil)", dev_port); + port_id = smi_gsi_port_open(dev_name, dev_port); + + if (port_id < 0) { + return; + } if (num_classes >= MAX_CLASS) IBPANIC("too many classes %d requested", num_classes); - ibmp->port_id = fd; + ibmp->port_id = port_id; memset(ibmp->class_agents, 0xff, sizeof ibmp->class_agents); while (num_classes--) { uint8_t rmpp_version = 0; @@ -401,10 +412,9 @@ struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, } memset(p, 0, sizeof(*p)); - if ((port_id = umad_open_port(dev_name, dev_port)) < 0) { - IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port); - if (!errno) - errno = EIO; + port_id = smi_gsi_port_open(dev_name, dev_port); + + if (port_id < 0) { free(p); return NULL; } @@ -422,7 +432,7 @@ struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, IBWARN("client_register for mgmt %d failed", mgmt); if (!errno) errno = EINVAL; - umad_close_port(port_id); + smi_gsi_port_close(port_id); free(p); return NULL; } @@ -433,6 +443,6 @@ struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, void mad_rpc_close_port(struct ibmad_port *port) { - umad_close_port(port->port_id); + smi_gsi_port_close(port->port_id); free(port); } diff --git a/libibmad/serv.c b/libibmad/serv.c index 040bb62b6..40f0625d8 100644 --- a/libibmad/serv.c +++ b/libibmad/serv.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. + * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -35,10 +36,12 @@ #include #include #include +#include #include #include +#include "smi_gsi.h" #include "mad_internal.h" #undef DEBUG @@ -55,6 +58,7 @@ int mad_send_via(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, { uint8_t pktbuf[1024]; void *umad = pktbuf; + int port_id; memset(pktbuf, 0, umad_size() + IB_MAD_SIZE); @@ -69,7 +73,11 @@ int mad_send_via(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, (char *)umad_get_mad(umad) + rpc->dataoffs, rpc->datasz); } - if (umad_send(srcport->port_id, srcport->class_agents[rpc->mgtclass & 0xff], + port_id = smi_gsi_port_by_class(srcport->port_id, + mad_get_field(umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F)); + + + if (umad_send(port_id, srcport->class_agents[rpc->mgtclass & 0xff], umad, IB_MAD_SIZE, mad_get_timeout(srcport, rpc->timeout), 0) < 0) { IBWARN("send failed; %s", strerror(errno)); @@ -170,14 +178,51 @@ void *mad_receive(void *umad, int timeout) void *mad_receive_via(void *umad, int timeout, struct ibmad_port *srcport) { - void *mad = umad ? umad : umad_alloc(1, umad_size() + IB_MAD_SIZE); - int agent; + void *mad; + int rc; int length = IB_MAD_SIZE; - if ((agent = umad_recv(srcport->port_id, mad, &length, - mad_get_timeout(srcport, timeout))) < 0) { + struct pollfd fds[2]; + ports_record_t * x = smi_gsi_record_find(srcport->port_id); + + if (!x) { + IBWARN("Couldn't resolve SMI/GSI_device for %d.", srcport->port_id); + } + + fds[0].fd = umad_get_fd(x->smi_port_id); + fds[0].events = POLLIN; + fds[0].revents = 0; + + fds[1].fd = umad_get_fd(x->gsi_port_id); + fds[1].events = POLLIN; + fds[1].revents = 0; + + rc = poll(fds, 2, timeout); + + if (rc < 0) { + IBWARN("Call poll failed for %d with error: %s", + srcport->port_id, strerror(errno)); + return NULL; + } + + mad = umad ? umad : umad_alloc(1, umad_size() + IB_MAD_SIZE); + + if (fds[0].revents & POLLIN) { + rc = umad_recv(x->smi_port_id, mad, &length, 0); + } else if (fds[1].revents & POLLIN) { + rc = umad_recv(x->gsi_port_id, mad, &length, 0); + } else { + IBWARN("Call poll failed for %d with error: %s", + srcport->port_id, strerror(errno)); + + rc = -1; + } + + if (rc < 0) { + if (!umad) umad_free(mad); + DEBUG("recv failed: %s", strerror(errno)); return NULL; } diff --git a/libibmad/smi_gsi.c b/libibmad/smi_gsi.c new file mode 100755 index 000000000..02404a189 --- /dev/null +++ b/libibmad/smi_gsi.c @@ -0,0 +1,135 @@ +/* +* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +*/ + + +#include +#include +#include + +#include + +#include "ext_umad.h" +#include "smi_gsi.h" + +ports_record_t * ports_list_head = NULL; + +ports_record_t * smi_gsi_record_find(int port_id) +{ + ports_record_t * x = ports_list_head; + + while (x) { + if (x->smi_port_id == port_id) + break; + x = x->next; + } + + return x; +} + +ports_record_t * smi_gsi_record_add(int smi_port_id, int gsi_port_id) +{ + ports_record_t * x = (ports_record_t*) calloc(1, sizeof(ports_record_t)); + + x->smi_port_id = smi_port_id; + x->gsi_port_id = gsi_port_id; + + x->next = ports_list_head; + ports_list_head = x; + + return x; +} + +void smi_gsi_record_ptr_remove(ports_record_t * x) +{ + if (x) { + if (x->prev) + x->prev->next = x->next; + else + ports_list_head = x->next; + + if (x->next) + x->next->prev = x->prev; + + free(x); + } +} + +void smi_gsi_record_remove(int id) +{ + smi_gsi_record_ptr_remove(smi_gsi_record_find(id)); +} + +int smi_gsi_port_by_class(int port_id, int mgmt) +{ + if (mgmt != IB_SMI_CLASS && mgmt != IB_SMI_DIRECT_CLASS) { + ports_record_t * x = smi_gsi_record_find(port_id); + + if (!x) { + IBWARN("Couldn't resolve SMI/GSI device for port_id %d.", port_id); + return -1; + } + + port_id = x->gsi_port_id; + } + + return port_id; +} + +int smi_gsi_port_open(char *ca_name, int portnum) +{ + ext_umad_ca_t ext_ca; + int smi_port_id; + int gsi_port_id; + int rc; + + if ((rc = ext_umad_get_ca_by_name(ca_name, portnum, &ext_ca)) < 0) { + IBWARN("Can't open UMAD port (%s) (%s:%d)", strerror(-rc), ca_name, portnum); + return rc; + } + + if ((smi_port_id = umad_open_port(ext_ca.smi.name, ext_ca.smi.preferred_port)) < 0) { + IBWARN("Can't open SMI UMAD port (%s) (%s:%d)", strerror(-smi_port_id), ext_ca.smi.name, ext_ca.smi.preferred_port); + return smi_port_id; + } + + if ((gsi_port_id = umad_open_port(ext_ca.gsi.name, ext_ca.gsi.preferred_port)) < 0) { + IBWARN("Can't open GSI UMAD port (%s) (%s:%d)", strerror(-gsi_port_id), ext_ca.gsi.name, ext_ca.gsi.preferred_port); + umad_close_port(smi_port_id); + return gsi_port_id; + } + + if (smi_gsi_record_add(smi_port_id, gsi_port_id) < 0) { + IBWARN("Failed to allocate memory for SMI/GSI mapping"); + umad_close_port(smi_port_id); + umad_close_port(gsi_port_id); + errno = ENOMEM; + return -errno; + } + + return smi_port_id; +} + +void smi_gsi_port_close(int port_id) +{ + if (port_id > 0) { + ports_record_t * x = smi_gsi_record_find(port_id); + if (x) { + umad_close_port(x->smi_port_id); + umad_close_port(x->smi_port_id); + smi_gsi_record_ptr_remove(x); + } + else { + umad_close_port(port_id); + IBWARN("Couldn't resolve SMI/GSI device for port_id %d.", port_id); + } + } +} diff --git a/libibmad/smi_gsi.h b/libibmad/smi_gsi.h new file mode 100755 index 000000000..8cd3db4a2 --- /dev/null +++ b/libibmad/smi_gsi.h @@ -0,0 +1,106 @@ +/* +* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +*/ + + +#ifndef SMI_GSI_DB_H_ +#define SMI_GSI_DB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief represents pair of SMI and GSI IDs as double linked list in db. + * + */ +typedef struct _ports_record { + int smi_port_id; + int gsi_port_id; + struct _ports_record* next; + struct _ports_record* prev; +} ports_record_t; + +/** + * @brief find pair of SMI and GSI IDs in DB + * + * @param port_id - ID of SMI port + * + * @return pointer of ports record. + */ +ports_record_t * smi_gsi_record_find(int port_id); + + +/** + * @brief add new record of pair SMI and GSI IDs into DB + * + * @param smi_port_id - SMI port ID + * @param gsi_port_id - GSI port ID + * + * @return pointer of ports record. + */ +ports_record_t * smi_gsi_record_add(int smi_port_id, int gsi_port_id); + + +/** + * @brief remove current record from DB + * + * @param x - pointer to the ports record + * + * @return void + */ +void smi_gsi_record_ptr_remove(ports_record_t * x); + + +/** + * @brief remove current record from DB + * + * @param port_id - SMI port ID + * + * @return void + */ +void smi_gsi_record_remove(int id); + +/** + * @brief remove current record from DB + * + * @param port_id - SMI port ID + * @param class - Management class + * + * @return port_id + */ +int smi_gsi_port_by_class(int port_id, int mgmt); + +/** + * @brief Open SMI & GSI ports pair + * + * @param ca_name - Device name (SMI or GSI) + * @param portnum - Device port number + * + * @return SMI port_id + */ +int smi_gsi_port_open(char *ca_name, int portnum); + +/** + * @brief Close SMI & GSI ports pair + * + * @param port_id - SMI port ID + * + * @return void + */ +void smi_gsi_port_close(int port_id); + + +#ifdef __cplusplus +} +#endif + +#endif /* SMI_GSI_DB_H_ */