[Pkg-ofed-commits] [infinipath-psm] 07/11: Fix hang with 64K rank support when using Intel MPI. During MPI_Finalize with Intel MPI, many processes would hang in ips_proto_mq_isend. This problem was only noticed at a larger amount of ranks and nodes. This bug was introduced in 2b70a0f379c1a72b924bc4d5cf6ea19749c0bd57. There were some bit shift and mask errors with the change. The commidx base id was not properly used correctly in ips_epstate functions, causing some messages to never get received properly.

Ana Beatriz Guerrero López ana at moszumanska.debian.org
Sun Apr 3 20:04:11 UTC 2016


This is an automated email from the git hooks/post-receive script.

ana pushed a commit to branch master
in repository infinipath-psm.

commit 67c0807c74e9d445900d5541358f0f575f22a630
Author: Henry Estela <henry.r.estela at intel.com>
Date:   Tue Nov 17 15:24:44 2015 -0600

    Fix hang with 64K rank support when using Intel MPI.
    During MPI_Finalize with Intel MPI, many processes would hang in
    ips_proto_mq_isend.
    This problem was only noticed at a larger amount of ranks and nodes.
    This bug was introduced in 2b70a0f379c1a72b924bc4d5cf6ea19749c0bd57.
    There were some bit shift and mask errors with the change.
    The commidx base id was not properly used correctly in ips_epstate functions,
    causing some messages to never get received properly.
---
 include/ipath_common.h   | 8 ++++----
 ptl_ips/ips_epstate.c    | 7 ++++---
 ptl_ips/ips_epstate.h    | 7 +++++--
 ptl_ips/ips_proto.c      | 7 ++-----
 ptl_ips/ips_proto_help.h | 9 +++------
 5 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/include/ipath_common.h b/include/ipath_common.h
index 3ddd8e1..8bf9986 100644
--- a/include/ipath_common.h
+++ b/include/ipath_common.h
@@ -674,11 +674,11 @@ struct ipath_flash {
 #define INFINIPATH_KPF_INTR 0x1
 #define INFINIPATH_KPF_HDRSUPP 0x2
 #define INFINIPATH_KPF_INTR_HDRSUPP_MASK 0x3
-#define INFINIPATH_KPF_COMMIDX_MASK 0xF
+#define INFINIPATH_KPF_COMMIDX_MASK 0x003C
 #define INFINIPATH_KPF_COMMIDX_SHIFT 2
-#define INFINIPATH_KPF_RESERVED_BITS(pkt_flags)            \
-  ((__le16_to_cpu(pkt_flags) << IPS_EPSTATE_COMMIDX_SHIFT) \
-  & INFINIPATH_KPF_COMMIDX_MASK)                           \
+#define INFINIPATH_KPF_RESERVED_BITS(pktflags)            \
+  ((__le16_to_cpu(pktflags) & INFINIPATH_KPF_COMMIDX_MASK) \
+    << IPS_EPSTATE_COMMIDX_SHIFT) \
 
 #define INFINIPATH_MAX_SUBCONTEXT	4
 
diff --git a/ptl_ips/ips_epstate.c b/ptl_ips/ips_epstate.c
index 2ee7888..43c81ba 100644
--- a/ptl_ips/ips_epstate.c
+++ b/ptl_ips/ips_epstate.c
@@ -49,7 +49,8 @@ ips_epstate_init(struct ips_epstate *eps, const psmi_context_t *context)
 {
     memset(eps, 0, sizeof(*eps));
     eps->context = context;
-    eps->eps_base_idx = 0;
+    eps->eps_base_idx = ((ips_epstate_idx)get_cycles()) &
+      (IPS_EPSTATE_COMMIDX_MAX-1);
     return PSM_OK;
 }
 
@@ -102,7 +103,7 @@ ips_epstate_add(struct ips_epstate *eps, struct ptl_epaddr *ipsaddr,
 	}
     }
     psmi_assert_always(i != eps->eps_tabsize);
-    commidx = j & (IPS_EPSTATE_COMMIDX_MAX - 1);
+    commidx = (j - eps->eps_base_idx) & (IPS_EPSTATE_COMMIDX_MAX-1);
     _IPATH_VDBG("node %s gets commidx=%d (table idx %d)\n", 
 	    psmi_epaddr_get_name(ipsaddr->epaddr->epid), commidx, j);
     eps->eps_tab[j].epid = 
@@ -124,7 +125,7 @@ ips_epstate_del(struct ips_epstate *eps, ips_epstate_idx commidx)
 {
     ips_epstate_idx idx;
     /* actual table index */
-    idx = commidx & (IPS_EPSTATE_COMMIDX_MAX - 1);
+    idx = (commidx + eps->eps_base_idx) & (IPS_EPSTATE_COMMIDX_MAX-1);
     psmi_assert_always(idx < eps->eps_tabsize);
     _IPATH_VDBG("commidx=%d, table_idx=%d\n", commidx, idx);
     eps->eps_tab[idx].epid = 0;
diff --git a/ptl_ips/ips_epstate.h b/ptl_ips/ips_epstate.h
index 0a6c0bc..b6aca57 100644
--- a/ptl_ips/ips_epstate.h
+++ b/ptl_ips/ips_epstate.h
@@ -38,8 +38,11 @@
 
 typedef uint32_t ips_epstate_idx;
 #define IPS_EPSTATE_COMMIDX_MAX (1<<20)
-#define IPS_EPSTATE_COMMIDX_MASK 0x3C
+#define IPS_EPSTATE_COMMIDX_MASK 0xF0000
 #define IPS_EPSTATE_COMMIDX_SHIFT 14
+#define IPS_EPSTATE_COMMIDX_PACK(ipscommidx) \
+  ((ipscommidx & IPS_EPSTATE_COMMIDX_MASK) \
+    >> IPS_EPSTATE_COMMIDX_SHIFT)
 
 struct ptl_epaddr;
 
@@ -70,7 +73,7 @@ PSMI_INLINE(
 struct ips_epstate_entry *
 ips_epstate_lookup(const struct ips_epstate *eps, ips_epstate_idx idx))
 {
-    idx = idx & (IPS_EPSTATE_COMMIDX_MAX - 1);
+  idx = (idx + eps->eps_base_idx) & (IPS_EPSTATE_COMMIDX_MAX-1);
     if (idx < eps->eps_tabsize)
 	return &eps->eps_tab[idx];
     else
diff --git a/ptl_ips/ips_proto.c b/ptl_ips/ips_proto.c
index cb991ec..e9715e4 100644
--- a/ptl_ips/ips_proto.c
+++ b/ptl_ips/ips_proto.c
@@ -762,7 +762,7 @@ _build_ctrl_message(struct ips_proto *proto,
     uint32_t tot_paywords = sizeof(struct ips_message_header) >> 2;
     struct ips_epinfo *epinfo = &proto->epinfo;
     struct ips_epinfo_remote *epr = &ipsaddr->epr;
-    uint16_t pkt_flags = 0;
+    uint16_t pkt_flags = IPS_EPSTATE_COMMIDX_PACK(epr->epr_commidx_to);
     struct ips_message_header *p_hdr = &msg->pbc_hdr.hdr;
     ips_path_rec_t *ctrl_path = ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][ipsaddr->epr.epr_hpp_index];
     int paylen = 0;
@@ -1021,10 +1021,7 @@ _build_ctrl_message(struct ips_proto *proto,
         (IPS_PROTO_VERSION << INFINIPATH_I_VERS_SHIFT) +
         (epr->epr_pkt_context << INFINIPATH_I_CONTEXT_SHIFT) +
         (IPATH_EAGER_TID_ID << INFINIPATH_I_TID_SHIFT));
-    p_hdr->iph.pkt_flags = __cpu_to_le16(
-        (pkt_flags & INFINIPATH_KPF_INTR_HDRSUPP_MASK) |
-        ((epr->epr_commidx_to >> IPS_EPSTATE_COMMIDX_SHIFT)
-        & IPS_EPSTATE_COMMIDX_MASK));
+    p_hdr->iph.pkt_flags = __cpu_to_le16(pkt_flags);
     
     ips_kdeth_cksum(p_hdr);  // Generate KDETH  checksum
     
diff --git a/ptl_ips/ips_proto_help.h b/ptl_ips/ips_proto_help.h
index 3d83a62..96aa509 100644
--- a/ptl_ips/ips_proto_help.h
+++ b/ptl_ips/ips_proto_help.h
@@ -316,8 +316,7 @@ void ips_proto_hdr(ips_scb_t *scb,
 		(scb->offset >> 2)); // convert from byte to word offset
 
 	p_hdr->lrh[2] = __cpu_to_be16(paywords + SIZE_OF_CRC);
-	p_hdr->iph.pkt_flags |= __cpu_to_le16(
-    (kpf_flags & INFINIPATH_KPF_INTR_HDRSUPP_MASK));
+	p_hdr->iph.pkt_flags = __cpu_to_le16(kpf_flags);
 
 	ips_kdeth_cksum(p_hdr); // Generate KDETH checksum
 
@@ -350,9 +349,7 @@ void ips_proto_hdr(ips_scb_t *scb,
         (epr->epr_pkt_context << INFINIPATH_I_CONTEXT_SHIFT) +
         (scb->tid << INFINIPATH_I_TID_SHIFT) +
         (scb->offset >> 2)); // convert from byte to word offset
-    p_hdr->iph.pkt_flags = __cpu_to_le16(
-      kpf_flags | ((epr->epr_commidx_to >> IPS_EPSTATE_COMMIDX_SHIFT) &
-      IPS_EPSTATE_COMMIDX_MASK));
+    p_hdr->iph.pkt_flags = __cpu_to_le16(kpf_flags);
     
     ips_kdeth_cksum(p_hdr); // Generate KDETH checksum
 
@@ -387,7 +384,7 @@ ips_scb_prepare_flow_inner(ips_scb_t *scb,
 {
     uint32_t extra_bytes;
     uint32_t tot_paywords;
-    uint16_t pkt_flags = 0;
+    uint16_t pkt_flags = IPS_EPSTATE_COMMIDX_PACK(epr->epr_commidx_to);
     
     extra_bytes = scb->payload_size & 3;
     if (extra_bytes) {

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ofed/infinipath-psm.git



More information about the Pkg-ofed-commits mailing list