in tensorflow_networking/verbs/rdma.cc [267:315]
uint8_t set_gid(uint8_t port_num, ibv_context* context) {
ibv_port_attr port_attr;
string gid_str;
int rc, i, gids_num = 0, v2_ip_num = 0;
union ibv_gid gid;
uint8_t gid_index = 0;
rc = ibv_query_port(context, port_num, &port_attr);
CHECK(!rc) << "Failed to query the port" << port_num;
for (i = 0; i < port_attr.gid_tbl_len; i++) {
rc = ibv_query_gid(context, port_num, i, &gid);
CHECK(!rc) << "Failed to query gid to port " << (int)port_num << " index "
<< i;
if (gid.global.interface_id) {
gids_num++;
if (gid.global.subnet_prefix == 0 &&
is_gid_type_roce_v2(context, port_num, i)) {
if (v2_ip_num == 0) {
// can be overwritten by RDMA_GID_INDEX later
gid_index = i;
}
v2_ip_num++;
}
}
}
switch (port_attr.link_layer) {
case (IBV_LINK_LAYER_ETHERNET):
gid_str = get_env_var("RDMA_GID_INDEX");
if (!gid_str.empty()) {
gid_index = stoi(gid_str);
CHECK(gid_index < gids_num)
<< "RDMA_GID_INDEX should be less than GIDs amount" << gids_num;
} else {
CHECK(v2_ip_num <= 1)
<< "More than one IP is available, please specify GID_INDEX";
}
break;
case (IBV_LINK_LAYER_INFINIBAND): // no need in GID index
break;
default:
LOG(INFO) << "Unknown port link layer. Currently supporting Ethernet and "
"InfiniBand only. ";
}
if (!is_gid_type_roce_v2(context, port_num, gid_index)) {
LOG(INFO) << "RoCE v2 is not configured for GID_INDEX " << (int)gid_index;
}
return gid_index;
}