in prov/bgq/src/fi_bgq_ep.c [973:1379]
static int fi_bgq_ep_rx_init(struct fi_bgq_ep *bgq_ep)
{
assert(FI_SHARED_CONTEXT != bgq_ep->rx.index);
struct fi_bgq_domain * bgq_domain = bgq_ep->domain;
BG_CoordinateMapping_t my_coords = bgq_domain->my_coords;
const uint32_t fifo_map =
fi_bgq_mu_calculate_fifo_map_single(my_coords, my_coords);
const MUHWI_Destination_t destination =
fi_bgq_spi_coordinates_to_destination(my_coords);
const uint32_t rx =
fi_bgq_addr_calculate_base_rx(my_coords.t, Kernel_ProcessCount()) + bgq_ep->rx.index;
bgq_ep->rx.self.fi = fi_bgq_addr_create(destination, fifo_map, rx);
/* assign the mu reception fifos - all potential
* reception fifos were allocated at domain initialization */
if (NULL == bgq_domain->rx.rfifo[fi_bgq_uid_get_rx(bgq_ep->rx.self.uid.fi)]) {
assert(0);
goto err;
}
if (NULL != bgq_ep->rx.poll.muspi_recfifo) {
assert(0);
goto err;
}
bgq_ep->rx.poll.muspi_recfifo = bgq_domain->rx.rfifo[fi_bgq_uid_get_rx(bgq_ep->rx.self.uid.fi)];
#ifdef FI_BGQ_TRACE
fprintf(stderr,"fi_bgq_ep_rx_init recfifo set to %u created addr:\n",fi_bgq_uid_get_rx(bgq_ep->rx.self.uid.fi));
FI_BGQ_ADDR_DUMP(&bgq_ep->rx.self.fi);
#endif
bgq_ep->rx.poll.bat = bgq_domain->bat;
/* **** acquire the mu lock (node scoped) **** */
l2atomic_lock_acquire(&bgq_domain->mu.lock);
/* create an injection fifo for rendezvous and ack messages */
const int num_fifos_to_allocate = 1;
if (num_fifos_to_allocate !=
fi_bgq_spi_injfifo_init(&bgq_ep->rx.poll.injfifo,
&bgq_ep->rx.poll.injfifo_subgroup,
num_fifos_to_allocate,
FI_BGQ_RX_SIZE,
sizeof(union fi_bgq_mu_packet_payload),
0 /* is_remote_get */,
1 /* is_top_down */)) {
assert(0);
goto err;
}
/*
* fi_atomic*() descriptor models
*/
{
MUHWI_Descriptor_t * desc = NULL;
/*
* fi_atomic*() direct-put fetch response model
*/
desc = &bgq_ep->rx.poll.atomic_dput_model;
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_PUT;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Pacing =
MUHWI_PACKET_DIRECT_PUT_IS_NOT_PACED;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Counter_Base_Address_Id =
FI_BGQ_MU_BAT_ID_GLOBAL;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Counter_Offset =
fi_bgq_node_bat_read(&bgq_domain->fabric->node,
FI_BGQ_MU_BAT_ID_COUNTER);
/* specified at injection time */
desc->Torus_FIFO_Map = -1;
desc->Pa_Payload = 0;
desc->Message_Length = 0;
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = -1;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Payload_Base_Address_Id = -1;
MUSPI_SetRecPayloadBaseAddressInfo(desc, FI_BGQ_MU_BAT_ID_GLOBAL, 0);
/*
* fi_atomic*() direct-put fi_cntr completion model
*/
desc = &bgq_ep->rx.poll.atomic_cntr_update_model[0]; /* intranode .. TODO - use an enum */
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_PUT;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Pacing =
MUHWI_PACKET_DIRECT_PUT_IS_NOT_PACED;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Counter_Offset = 0;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Counter_Base_Address_Id =
FI_BGQ_MU_BAT_ID_COUNTER;
desc->Pa_Payload = bgq_ep->tx.read.global_one_paddr;
desc->Message_Length = 8;
desc->Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EP;
MUSPI_SetRecPayloadBaseAddressInfo(desc,
FI_BGQ_MU_BAT_ID_GLOBAL, /* the bat id will be updated at injection time */
MUSPI_GetAtomicAddress(0,
MUHWI_ATOMIC_OPCODE_STORE_ADD));
/* specified at injection time */
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = -1;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Payload_Base_Address_Id = -1;
/* initialize the "intranode" version of the descriptor model */
bgq_ep->rx.poll.atomic_cntr_update_model[1] = *desc; /* internode .. TODO - use an enum */
bgq_ep->rx.poll.atomic_cntr_update_model[1].Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL0 |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL1;
}
/*
* initialize the remote-get descriptor models - used for
* the "rendezvous" protocol
*/
{
/* initialize the "internode" version of the descriptor model */
MUHWI_Descriptor_t * desc = &bgq_ep->rx.poll.rzv.rget_model[0]; /* TODO - use an enum */
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EP;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_GET;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
/* specified at injection time */
desc->Pa_Payload = 0;
desc->Message_Length = sizeof(MUHWI_Descriptor_t);
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = -1;
desc->PacketHeader.messageUnitHeader.Packet_Types.Remote_Get.Rget_Inj_FIFO_Id = -1;
/* initialize the "intranode" version of the descriptor model */
bgq_ep->rx.poll.rzv.rget_model[1] = *desc; /* TODO - use an enum */
bgq_ep->rx.poll.rzv.rget_model[1].Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL0 |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL1;
}
/*
* initialize the direct-put descriptor models - used
* to transfer the application data in the "rendezvous"
* protocol
*/
{
/* initialize the "internode" version of the descriptor model */
MUHWI_Descriptor_t * desc = &bgq_ep->rx.poll.rzv.dput_model[0]; /* TODO - use an enum */
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EP;
desc->PacketHeader.NetworkHeader.pt2pt.Destination =
fi_bgq_uid_get_destination(bgq_ep->rx.self.uid.fi);
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_PUT;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Payload_Base_Address_Id =
FI_BGQ_MU_BAT_ID_GLOBAL;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Pacing =
MUHWI_PACKET_DIRECT_PUT_IS_NOT_PACED;
/* specified at injection time */
desc->Pa_Payload = 0;
desc->Message_Length = 0;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Put_Offset_MSB = 0;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Put_Offset_LSB = 0;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Counter_Base_Address_Id = -1;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Counter_Offset = 0;
/* initialize the "intranode" version of the descriptor model */
bgq_ep->rx.poll.rzv.dput_model[1] = *desc; /* TODO - use an enum */
bgq_ep->rx.poll.rzv.dput_model[1].Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL0 |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL1;
}
/*
* initialize the 'local completion' direct-put
* descriptor model - used to zero the byte counter
* of the send operation on the origin for the
* "rendezvous" protocol
*
* see also -> fi_bgq_cq::local_completion_model
*/
{
MUHWI_Descriptor_t * desc = &bgq_ep->rx.poll.rzv.dput_completion_model;
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only = MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt = MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->Message_Length = sizeof(uint64_t);
desc->Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL0 |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL1;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_PUT;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Payload_Base_Address_Id =
FI_BGQ_MU_BAT_ID_COUNTER;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Pacing =
MUHWI_PACKET_DIRECT_PUT_IS_NOT_PACED;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Counter_Base_Address_Id =
FI_BGQ_MU_BAT_ID_COUNTER;
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = 0;
/* specified at injection time */
desc->Pa_Payload = 0;
}
{
MUHWI_Descriptor_t * desc = &bgq_ep->rx.poll.rzv.multi_recv_ack_model;
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->Message_Length = 0;
desc->Pa_Payload = 0;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_FIFO;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.NetworkHeader.pt2pt.Destination =
fi_bgq_uid_get_destination(bgq_ep->rx.self.uid.fi);
desc->PacketHeader.messageUnitHeader.Packet_Types.Memory_FIFO.Rec_FIFO_Id =
fi_bgq_uid_get_rx(bgq_ep->rx.self.uid.fi);
union fi_bgq_mu_packet_hdr * hdr = (union fi_bgq_mu_packet_hdr *) &desc->PacketHeader;
fi_bgq_mu_packet_type_set(hdr, FI_BGQ_MU_PACKET_TYPE_ACK);
/* specified at injection time */
desc->Torus_FIFO_Map = 0;
hdr->ack.context = 0;
}
/*
* initialize the direct-put descriptor models used to zero an arbitrary
* 8 byte variable - used to implement FI_DELIVERY_COMPLETE
*/
{
/* initialize the "internode" version of the descriptor model */
MUHWI_Descriptor_t * desc = &bgq_ep->rx.poll.ack_model[0]; /* TODO - use an enum */
MUSPI_DescriptorZeroOut(desc);
desc->Half_Word0.Prefetch_Only =
MUHWI_DESCRIPTOR_PRE_FETCH_ONLY_NO;
desc->Half_Word1.Interrupt =
MUHWI_DESCRIPTOR_DO_NOT_INTERRUPT_ON_PACKET_ARRIVAL;
desc->Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_BP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_CP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_DP |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EM |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_EP;
desc->PacketHeader.NetworkHeader.pt2pt.Data_Packet_Type =
MUHWI_PT2PT_DATA_PACKET_TYPE;
desc->PacketHeader.NetworkHeader.pt2pt.Byte3.Byte3 =
MUHWI_PACKET_VIRTUAL_CHANNEL_DETERMINISTIC;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Byte8 =
MUHWI_PACKET_TYPE_PUT;
desc->PacketHeader.NetworkHeader.pt2pt.Byte8.Size = 16;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Pacing =
MUHWI_PACKET_DIRECT_PUT_IS_NOT_PACED;
desc->Pa_Payload = bgq_domain->zero.paddr;
desc->Message_Length = sizeof(uint64_t);
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Rec_Counter_Base_Address_Id =
FI_BGQ_MU_BAT_ID_COUNTER;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Counter_Offset = 0;
/* specified at injection time - not used for local transfers */
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = -1;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Put_Offset_MSB = 0;
desc->PacketHeader.messageUnitHeader.Packet_Types.Direct_Put.Put_Offset_LSB = 0;
desc->PacketHeader.NetworkHeader.pt2pt.Destination.Destination.Destination = -1;
/* initialize the "intranode" version of the descriptor model */
bgq_ep->rx.poll.ack_model[1] = *desc; /* TODO - use an enum */
bgq_ep->rx.poll.ack_model[1].Torus_FIFO_Map =
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL0 |
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_LOCAL1;
}
/* allocate the l2atomic fifos for match information and control information */
{
struct l2atomic_fifo_data * memptr = NULL;
size_t bytes = (sizeof(struct l2atomic_fifo_data) + sizeof(uint64_t) * bgq_ep->recv_cq->size) * 2;
bytes += sizeof(struct l2atomic_fifo_data) + sizeof(uint64_t) * FI_BGQ_L2FIFO_CTL_SIZE;
if (posix_memalign((void **)&memptr, 32, bytes)) {
errno = FI_ENOMEM;
goto err;
}
memset((void*)memptr, 0, bytes);
bgq_ep->rx.l2atomic_memptr = (void*)memptr;
l2atomic_fifo_initialize(&bgq_ep->rx.poll.rfifo[IS_TAG].match,
&bgq_ep->rx.post.match[IS_TAG],
memptr, bgq_ep->recv_cq->size);
memptr = (struct l2atomic_fifo_data *)((uintptr_t)memptr + sizeof(struct l2atomic_fifo_data) + sizeof(uint64_t) * bgq_ep->recv_cq->size);
l2atomic_fifo_initialize(&bgq_ep->rx.poll.rfifo[IS_MSG].match,
&bgq_ep->rx.post.match[IS_MSG],
memptr, bgq_ep->recv_cq->size);
memptr = (struct l2atomic_fifo_data *)((uintptr_t)memptr + sizeof(struct l2atomic_fifo_data) + sizeof(uint64_t) * bgq_ep->recv_cq->size);
l2atomic_fifo_initialize(&bgq_ep->rx.poll.control,
&bgq_ep->rx.post.control,
memptr, FI_BGQ_L2FIFO_CTL_SIZE);
}
/* **** release the mu lock (node scoped) **** */
l2atomic_lock_release(&bgq_domain->mu.lock);
bgq_ep->rx.state = FI_BGQ_EP_INITITALIZED_ENABLED;
return 0;
err:
return 1;
}