[Pkg-ofed-commits] r741 - /branches/ofed-1.5.1lenny/perftest/branches/upstream/c...
gmpc-guest at alioth.debian.org
gmpc-guest at alioth.debian.org
Wed May 5 19:05:13 UTC 2010
Author: gmpc-guest
Date: Wed May 5 19:05:11 2010
New Revision: 741
URL: http://svn.debian.org/wsvn/pkg-ofed/?sc=1&rev=741
Log:
[svn-upgrade] Integrating new upstream version, perftest (1.2.3-0.10.g90b10d8)
Added:
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c (with props)
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h (with props)
Modified:
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/Makefile
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/README
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/perftest.spec
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/read_bw.c
branches/ofed-1.5.1lenny/perftest/branches/upstream/current/send_bw.c
Modified: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/Makefile
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/Makefile?rev=741&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/Makefile (original)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/Makefile Wed May 5 19:05:11 2010
@@ -1,26 +1,33 @@
RDMACM_TESTS = rdma_lat rdma_bw
-TESTS = write_bw_postlist send_lat send_bw write_lat write_bw read_lat read_bw
+MCAST_TESTS = send_bw
+TESTS = write_bw_postlist send_lat write_lat write_bw read_lat read_bw
UTILS = clock_test
-all: ${RDMACM_TESTS} ${TESTS} ${UTILS}
+all: ${RDMACM_TESTS} ${MCAST_TESTS} ${TESTS} ${UTILS}
CFLAGS += -Wall -g -D_GNU_SOURCE -O2
EXTRA_FILES = get_clock.c
-EXTRA_HEADERS = get_clock.h
+MCAST_FILES = multicast_resources.c
+EXTRA_HEADERS = get_clock.h
+MCAST_HEADERS = multicast_resources.h
#The following seems to help GNU make on some platforms
LOADLIBES +=
LDFLAGS +=
${RDMACM_TESTS}: LOADLIBES += -libverbs -lrdmacm
+${MCAST_TESTS}: LOADLIBES += -libverbs -lpthread -libumad
${TESTS} ${UTILS}: LOADLIBES += -libverbs
${RDMACM_TESTS}: %: %.c ${EXTRA_FILES} ${EXTRA_HEADERS}
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} $(LOADLIBES) $(LDLIBS) -o $@
+${MCAST_TESTS}: %: %.c ${EXTRA_FILES} ${MCAST_FILES} ${EXTRA_HEADERS} ${MCAST_HEADERS}
+ $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} ${MCAST_FILES} $(LOADLIBES) $(LDLIBS) -o ib_$@
${TESTS} ${UTILS}: %: %.c ${EXTRA_FILES} ${EXTRA_HEADERS}
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} $(LOADLIBES) $(LDLIBS) -o ib_$@
clean:
$(foreach fname,${RDMACM_TESTS}, rm -f ${fname})
+ $(foreach fname,${MCAST_TESTS}, rm -f ib_${fname})
$(foreach fname,${TESTS} ${UTILS}, rm -f ib_${fname})
.DELETE_ON_ERROR:
.PHONY: all clean
Modified: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/README
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/README?rev=741&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/README (original)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/README Wed May 5 19:05:11 2010
@@ -1,7 +1,7 @@
Open Fabrics Enterprise Distribution (OFED)
- Performance Tests README for OFED 1.3
+ Performance Tests README for OFED 1.5
- March 2008
+ December 2009
@@ -9,7 +9,7 @@
Table of Contents
===============================================================================
1. Overview
-2. Notes on Testing Method
+2. Notes on Testing Methodology
3. Test Descriptions
4. Running Tests
@@ -18,30 +18,42 @@
===============================================================================
This is a collection of tests written over uverbs intended for use as a
performance micro-benchmark. As an example, the tests can be used for
-hardware or software tuning and/or functional testing.
+HW or SW tuning and/or functional testing.
-Please post results and observations to the openib-general mailing list.
+The collection conatains a set of BW and latency benchmark such as :
+
+ * Read - ib_read_bw and ib_read_lat.
+ * Write - ib_write_bw and ib_wriet_lat.
+ * Send - ib_send_bw and ib_send_lat.
+ * RDMA - rdma_bw and rdma_lat.
+ * In additional : ib_write_bw_postlist and ib_clock_test.
+
+Please post results/observations to the openib-general mailing list.
See "Contact Us" at http://openib.org/mailman/listinfo/openib-general and
http://www.openib.org.
===============================================================================
-2. Notes on Testing Method
+2. Notes on Testing Methodology
===============================================================================
-- The benchmark uses the CPU cycle counter to get time stamps without a context
- switch. Some CPU architectures (e.g., Intel's 80486 or older PPC) do NOT have
- such capability.
+- The benchmark used the CPU cycle counter to get time stamps without context
+ switch. Some CPU architectures (e.g., Intel's 80486 or older PPC) do NOT
+ have such capability.
- The benchmark measures round-trip time but reports half of that as one-way
latency. This means that it may not be sufficiently accurate for asymmetrical
configurations.
-- Min/Median/Max results are reported.
- The Median (vs average) is less sensitive to extreme scores.
- Typically, the Max value is the first value measured.
+- On Bw benchmarks , We calculate the BW on send side only, as he calculates
+ the Bw after collecting completion from the receive side.
+ In case we use the bidirectional flag , BW is calculated on both sides
-- Larger samples only help marginally. The default (1000) is very satisfactory.
- Note that an array of cycles_t (typically an unsigned long) is allocated
+- Min/Median/Max result is reported.
+ The median (vs average) is less sensitive to extreme scores.
+ Typically, the "Max" value is the first value measured.
+
+- Larger samples help marginally only. The default (1000) is pretty good.
+ Note that an array of cycles_t (typically unsigned long) is allocated
once to collect samples and again to store the difference between them.
Really big sample sizes (e.g., 1 million) might expose other problems
with the program.
@@ -54,32 +66,29 @@
===============================================================================
-3. Test Descriptions
+4. Test Descriptions
===============================================================================
+rdma_lat.c latency test with RDMA write transactions
+rdma_bw.c streaming BW test with RDMA write transactions
-The following tests are mainly useful for hardware/software benchmarking.
+The following tests are mainly useful for HW/SW benchmarking.
+They are not intended as actual usage examples.
+send_lat.c latency test with send transactions
+send_bw.c BW test with send transactions
write_lat.c latency test with RDMA write transactions
-write_bw.c bandwidth test with RDMA write transactions
-send_lat.c latency test with send transactions
-send_bw.c bandwidth test with send transactions
+write_bw.c BW test with RDMA write transactions
read_lat.c latency test with RDMA read transactions
-read_bw.c bandwidth test with RDMA read transactions
+read_bw.c BW test with RDMA read transactions
+The executable name of each test starts with the general prefix "ib_",
+e.g., ib_write_lat , exept of those of RDMA tests , in their case
+their excutable have the same name except of the .c.
-Legacy tests: (To be removed in the next release)
-rdma_lat.c latency test with RDMA write transactions
-rdma_bw.c streaming bandwidth test with RDMA write transactions
-
-The executable name of each test starts with the general prefix "ib_";
-for example, ib_write_lat.
-
-
-===============================================================================
-4. Running Tests
-===============================================================================
+Running Tests
+-------------
Prerequisites:
kernel 2.6
@@ -90,25 +99,6 @@
Client: ./<test name> <options> <server IP address>
o <server address> is IPv4 or IPv6 address. You can use the IPoIB
-diags_release_notes.txt
-mpi-selector_release_notes.txt
-rdma_cm_release_notes.txt
-MSTFLINT_README.txt
-open_mpi_release_notes.txt RDS_README.txt
-ib-bonding.txt
-mthca_release_notes.txt
-opensm_release_notes.txt
-rds_release_notes.txt
-ibutils_release_notes.txt*
-mvapich_release_notes.txt
-PERF_TEST_README.txt
-sdp_release_notes.txt
-ipoib_release_notes.txt
-srp_release_notes.txt
-QoS_in_OFED.txt
-SRPT_README.txt
-mlx4_release_notes.txt
-QoS_management_in_OpenSM.
address if IPoIB is configured.
o --help lists the available <options>
@@ -116,30 +106,38 @@
Common Options to all tests:
- -p, --port=<port> listen on/connect to port <port> (default: 18515)
- -m, --mtu=<mtu> mtu size (default: 1024)
- -d, --ib-dev=<dev> use IB device <dev> (default: first device found)
- -i, --ib-port=<port> use port <port> of IB device (default: 1)
- -s, --size=<size> size of message to exchange (default: 1)
- -a, --all run sizes from 2 till 2^23
- -t, --tx-depth=<dep> size of tx queue (default: 50)
- -n, --iters=<iters> number of exchanges (at least 100, default: 1000)
- -S, --sl=<sl> SL (default 0)
- -C, --report-cycles report times in cpu cycle units
- (default: microseconds)
- -H, --report-histogram print out all results
- (default: print summary only)
- -U, --report-unsorted (implies -H) print out unsorted results
- (default: sorted)
- -V, --version display version number
+ -p, --port=<port> Listen on/connect to port <port> (default: 18515).
+ -m, --mtu=<mtu> Mtu size (default: 1024).
+ -d, --ib-dev=<dev> Use IB device <dev> (default: first device found).
+ -i, --ib-port=<port> Use port <port> of IB device (default: 1).
+ -s, --size=<size> Size of message to exchange (default: 1).
+ -a, --all Run sizes from 2 till 2^23.
+ -t, --tx-depth=<dep> Size of tx queue (default: 50).
+ -r, --rx-depth=<dep> Make rx queue bigger than tx (default 600).
+ -n, --iters=<iters> Number of exchanges (at least 100, default: 1000).
+ -I, --inline_size=<size> Max size of message to be sent in inline mode.
+ On Bw tests default is 1,latency tests is 400.
+ -C, --report-cycles Report times in cpu cycle units.
+ -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec*2 ^(timeout).
+ Default is 14.
+ -S, --sl=<sl> SL (default 0).
+ -H, --report-histogram Print out all results (Default: summary only).
+ Only on Latnecy tests.
+ -x, --gid-index=<index> Test uses GID with GID index taken from command
+ Line (for RDMAoE index should be 0).
+ -b, --bidirectional Measure bidirectional bandwidth (default uni).
+ On BW tests only (Implicit on latency tests).
+ -V, --version Display version number.
+ -e, --events Sleep on CQ events (default poll).
+ -N, --no peak-bw Cancel peak-bw calculation (default with peak-bw)
+ -F, --CPU-freq Do not fail even if cpufreq_ondemand module.
*** IMPORTANT NOTE: You need to be running a Subnet Manager on the switch or
on one of the nodes in your fabric.
Example:
-Run "ib_write_lat -a" on the server side.
-Then run "ib_write_lat -a <server IP address>" on the client side.
+Run "ib_rdma_lat -C" on the server side.
+Then run "ib_rdma_lat -C <server IP address>" on the client.
-ib_write_lat will exit on both server and client after printing results.
+ib_rdma_lat will exit on both server and client after printing results.
-
Added: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c?rev=741&op=file
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c (added)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c Wed May 5 19:05:11 2010
@@ -1,0 +1,780 @@
+
+/************************************************************************
+ * Files and Modules included for work. *
+ ************************************************************************/
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <limits.h>
+#include <byteswap.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <netdb.h>
+#include <malloc.h>
+#include <getopt.h>
+#include <time.h>
+#include <errno.h>
+#include <pthread.h>
+
+#include "multicast_resources.h"
+
+
+/************************************************************************
+ *
+ ************************************************************************/
+
+struct mcast_package {
+ struct mcast_parameters *ptr_to_params;
+ struct mcast_group *ptr_to_mcast;
+ pthread_barrier_t *our_barrier;
+};
+
+/************************************************************************
+ *
+ * Umad registration methods.
+ *
+ ************************************************************************/
+
+/****************************************
+ * Function: prepare_mcast_mad
+ ****************************************/
+static void prepare_mcast_mad(u_int8_t method,
+ struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct sa_mad_packet_t *samad_packet) {
+
+ u_int8_t *ptr;
+ uint64_t comp_mask;
+
+ memset(samad_packet,0,sizeof(*samad_packet));
+
+ /* prepare the MAD header. according to Table 145 in IB spec 1.2.1 */
+ ptr = samad_packet->mad_header_buf;
+ ptr[0] = 0x01; /* BaseVersion */
+ ptr[1] = MANAGMENT_CLASS_SUBN_ADM; /* MgmtClass */
+ ptr[2] = 0x02; /* ClassVersion */
+ ptr[3] = INSERTF(ptr[3], 0, method, 0, 7); /* Method */
+ (*(u_int64_t *)(ptr + 8)) = htonll((u_int64_t)DEF_TRANS_ID); /* TransactionID */
+ (*(u_int16_t *)(ptr + 16)) = htons(SUBN_ADM_ATTR_MC_MEMBER_RECORD); /* AttributeID */
+
+ ptr = samad_packet->SubnetAdminData;
+
+ memcpy(&ptr[0],mcast_manager->mgid.raw, 16);
+ memcpy(&ptr[16],params->port_gid.raw, 16);
+
+ (*(u_int32_t *)(ptr + 32)) = htonl(DEF_QKEY);
+ (*(u_int16_t *)(ptr + 40)) = htons(params->pkey);
+ ptr[39] = DEF_TCLASS;
+ ptr[44] = INSERTF(ptr[44], 4, DEF_SL, 0, 4);
+ ptr[44] = INSERTF(ptr[44], 0, DEF_FLOW_LABLE, 16, 4);
+ ptr[45] = INSERTF(ptr[45], 0, DEF_FLOW_LABLE, 8, 8);
+ ptr[46] = INSERTF(ptr[46], 0, DEF_FLOW_LABLE, 0, 8);
+ ptr[48] = INSERTF(ptr[48], 0, MCMEMBER_JOINSTATE_FULL_MEMBER, 0, 4);
+
+ comp_mask = SUBN_ADM_COMPMASK_MGID | SUBN_ADM_COMPMASK_PORT_GID | SUBN_ADM_COMPMASK_Q_KEY |
+ SUBN_ADM_COMPMASK_P_KEY | SUBN_ADM_COMPMASK_TCLASS | SUBN_ADM_COMPMASK_SL |
+ SUBN_ADM_COMPMASK_FLOW_LABEL | SUBN_ADM_COMPMASK_JOIN_STATE;
+
+ samad_packet->ComponentMask = htonll(comp_mask);
+}
+
+/*****************************************
+* Function: check_mad_status
+*****************************************/
+static int check_mad_status(struct sa_mad_packet_t *samad_packet) {
+
+ u_int8_t *ptr;
+ u_int32_t user_trans_id;
+ u_int16_t mad_header_status;
+
+ ptr = samad_packet->mad_header_buf;
+
+ // the upper 32 bits of TransactionID were set by the kernel
+ user_trans_id = ntohl(*(u_int32_t *)(ptr + 12));
+
+ // check the TransactionID to make sure this is the response
+ // for the join/leave multicast group request we posted
+ if (user_trans_id != DEF_TRANS_ID) {
+ fprintf(stderr, "received a mad with TransactionID 0x%x, when expecting 0x%x\n",
+ (unsigned int)user_trans_id, (unsigned int)DEF_TRANS_ID);;
+ return 1;
+ }
+
+ mad_header_status = 0x0;
+ mad_header_status = INSERTF(mad_header_status, 8, ptr[4], 0, 7);
+ mad_header_status = INSERTF(mad_header_status, 0, ptr[5], 0, 8);
+
+ if (mad_header_status) {
+ fprintf(stderr,"received UMAD with an error: 0x%x\n", mad_header_status);
+ return 1;
+ }
+
+ return 0;
+}
+
+
+/*****************************************
+* Function: get_mlid_from_mad
+*****************************************/
+static void get_mlid_from_mad(struct sa_mad_packet_t *samad_packet,
+ uint16_t *mlid) {
+
+ u_int8_t *ptr;
+
+ ptr = samad_packet->SubnetAdminData;
+ *mlid = ntohs(*(u_int16_t *)(ptr + 36));
+}
+
+/****************************************
+ * Function: join_multicast_group
+ ****************************************/
+static int join_multicast_group(subn_adm_method method,
+ struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+ int portid = -1;
+ int agentid = -1;
+ void *umad_buff = NULL;
+ void *mad = NULL;
+ struct ibv_port_attr port_attr;
+ int length = MAD_SIZE;
+ int test_result = 0;
+
+ /* use casting to loose the "const char0 *" */
+ portid = umad_open_port((char*)ibv_get_device_name(params->ib_dev),params->ib_port);
+ if (portid < 0) {
+ fprintf(stderr,"failed to open UMAD port %d\n",params->ib_port);
+ goto cleanup;
+ }
+
+ agentid = umad_register(portid,MANAGMENT_CLASS_SUBN_ADM, 2, 0, 0);
+ if (agentid < 0) {
+ fprintf(stderr,"failed to register UMAD agent for MADs\n");
+ goto cleanup;
+ }
+
+ umad_buff = umad_alloc(1, umad_size() + MAD_SIZE);
+ if (!umad_buff) {
+ fprintf(stderr, "failed to allocate MAD buffer\n");
+ goto cleanup;
+ }
+
+ mad = umad_get_mad(umad_buff);
+ prepare_mcast_mad(method,mcast_manager,params,(struct sa_mad_packet_t *)mad);
+
+ if (ibv_query_port(params->ctx,params->ib_port,&port_attr)) {
+ fprintf(stderr,"ibv_query_port failed\n");
+ return 1;
+ }
+
+ if (umad_set_addr(umad_buff,port_attr.sm_lid,1,port_attr.sm_sl,QP1_WELL_KNOWN_Q_KEY) < 0) {
+ fprintf(stderr, "failed to set the destination address of the SMP\n");
+ goto cleanup;
+ }
+
+ if (umad_send(portid,agentid,umad_buff,MAD_SIZE,100,5) < 0) {
+ fprintf(stderr, "failed to send MAD\n");
+ goto cleanup;
+ }
+
+ if (umad_recv(portid,umad_buff,&length,5000) < 0) {
+ fprintf(stderr, "failed to receive MAD response\n");
+ goto cleanup;
+ }
+
+ if (check_mad_status((struct sa_mad_packet_t*)mad)) {
+ fprintf(stderr, "failed to get mlid from MAD\n");
+ goto cleanup;
+ }
+
+ // "Join multicast group" message was sent
+ if (method == SUBN_ADM_METHOD_SET) {
+ get_mlid_from_mad((struct sa_mad_packet_t*)mad,&mcast_manager->mlid);
+ mcast_manager->mcast_state |= MCAST_IS_JOINED;
+
+ // "Leave multicast group" message was sent
+ } else {
+ mcast_manager->mcast_state &= ~MCAST_IS_JOINED;
+ }
+
+cleanup:
+ if (umad_buff)
+ umad_free(umad_buff);
+
+ if (portid >= 0) {
+ if (agentid >= 0) {
+ if (umad_unregister(portid, agentid)) {
+ fprintf(stderr, "failed to deregister UMAD agent for MADs\n");
+ test_result = 1;
+ }
+ }
+
+ if (umad_close_port(portid)) {
+ fprintf(stderr, "failed to close UMAD portid\n");
+ test_result = 1;
+ }
+ }
+
+ return test_result;
+}
+
+
+/************************************************************************
+ * Internal structures and static functions.
+ ************************************************************************/
+
+static int set_multicast_gid(struct mcast_group *mcast_manager,struct mcast_parameters *params) {
+
+ uint8_t mcg_gid[16] = MCG_GID;
+ const char *pstr = params->user_mgid;
+ char *term = NULL;
+ char tmp[20];
+ int i;
+
+ if (params->is_user_mgid == TRUE) {
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr+1);
+ tmp[term - pstr] = 0;
+ mcg_gid[0] = (unsigned char)strtoll(tmp, NULL, 0);
+ for (i = 1; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr+1);
+ tmp[term - pstr] = 0;
+ mcg_gid[i] = (unsigned char)strtoll(tmp, NULL, 0);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ mcg_gid[15] = (unsigned char)strtoll(tmp, NULL, 0);
+ }
+
+ memcpy(mcast_manager[0].mgid.raw,mcg_gid,16);
+ for (i=1; i < params->num_of_groups; i++) {
+ mcg_gid[15]++;
+ memcpy(mcast_manager[i].mgid.raw,mcg_gid,16);
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+static int run_iter_client(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+ int ne,scnt,ccnt;
+ struct ibv_wc wc;
+ struct ibv_send_wr *bad_wr;
+
+ scnt = ccnt = 0;
+ while (scnt < params->iterations || ccnt < params->iterations) {
+ while (scnt < params->iterations && (scnt -ccnt) < params->tx_depth) {
+ mcast_manager->posted[scnt++] = get_cycles();
+ if (ibv_post_send(mcast_manager->send_qp,mcast_manager->wr,&bad_wr)) {
+ fprintf(stderr, "Couldn't post send: scnt=%d\n",scnt);
+ return 1;
+ }
+ }
+ if (ccnt < params->iterations) {
+ while (1) {
+ ne = ibv_poll_cq(mcast_manager->mcg_cq,1,&wc);
+ if (ne <= 0) break;
+ if (wc.status != IBV_WC_SUCCESS) {
+ printf("Failed to poll CQ successfully \n");
+ return 1;
+ }
+ mcast_manager->completed[ccnt++] = get_cycles();
+ }
+ if (ne < 0) {
+ fprintf(stderr, "poll CQ failed %d\n", ne);
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+static int run_iter_server(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+ int i,j,ne,num_of_iters;
+ int rcnt = 0;
+ unsigned long scnt = 0;
+ struct ibv_wc wc;
+ struct ibv_recv_wr *bad_wr;
+ cycles_t t;
+
+ // Initilizing values of the time stamp arryas.
+ for (i=0; i < params->num_qps_on_group; i++) {
+ mcast_manager->recv_mcg[i].package_counter = 0;
+ }
+ num_of_iters = params->iterations*params->num_qps_on_group;
+ while (rcnt < num_of_iters && scnt < MAX_POLL_ITERATION_TIMEOUT) {
+ ne = ibv_poll_cq(mcast_manager->mcg_cq,1,&wc);
+ if (ne > 0) {
+ rcnt ++;
+ scnt = 0;
+ t = get_cycles();
+ for (j=0; j < params->num_qps_on_group; j++) {
+ if (wc.qp_num == mcast_manager->recv_mcg[j].qp->qp_num) {
+ mcast_manager->recv_mcg[j].completion_array[mcast_manager->recv_mcg[j].package_counter++] = t;
+ if (ibv_post_recv(mcast_manager->recv_mcg[j].qp,params->rwr,&bad_wr)) {
+ fprintf(stderr, "Couldn't post recv: rcnt=%d\n",rcnt);
+ return 1;
+ }
+ break;
+ }
+ }
+ } else if (ne == 0) {
+ scnt++;
+ } else {
+ fprintf(stderr, "Poll Recieve CQ failed\n");
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+static void* run_iter(void *arguments) {
+
+ struct mcast_package *mcast = arguments;
+
+ pthread_barrier_wait(mcast->our_barrier);
+ if (mcast->ptr_to_params->is_client) {
+ if (run_iter_client(mcast->ptr_to_mcast,mcast->ptr_to_params)) {
+ fprintf(stderr,"Unable to send data\n");
+ }
+ }
+
+ else {
+ if (run_iter_server(mcast->ptr_to_mcast,mcast->ptr_to_params)) {
+ fprintf(stderr,"Unable to send receive data\n");
+ }
+ }
+ pthread_exit(NULL);
+}
+
+
+
+
+/************************************************************************
+ *
+ * Multicast Resources methods implementation.
+ *
+ ************************************************************************/
+struct mcast_group* mcast_create_resources(struct mcast_parameters *params) {
+
+ int i,j;
+ struct mcast_group *new_mcast;
+
+ ALLOCATE(new_mcast,struct mcast_group,params->num_of_groups);
+
+ for (i=0; i < params->num_of_groups; i++) {
+ // Client Side.
+ if (params->is_client) {
+ ALLOCATE(new_mcast[i].posted,cycles_t,params->iterations);
+ ALLOCATE(new_mcast[i].completed,cycles_t,params->iterations);
+
+ // Server Side.
+ } else {
+ ALLOCATE(new_mcast[i].recv_mcg,struct mcg_qp,params->num_qps_on_group);
+ new_mcast[i].ah = NULL;
+ new_mcast[i].wr = NULL;
+ new_mcast[i].posted = new_mcast[i].completed = NULL;
+ for (j=0; j < params->num_qps_on_group; j++) {
+ ALLOCATE(new_mcast[i].recv_mcg[j].completion_array,cycles_t,params->iterations);
+ new_mcast[i].recv_mcg[j].package_counter = 0;
+ }
+ }
+ }
+ return new_mcast;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_init_resources(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_attr *attr,int flags) {
+
+ int i,j;
+
+ if (set_multicast_gid(mcast_manager,params)) {
+ fprintf(stderr,"Unable to set mcast gids\n");
+ return 1;
+ }
+
+ // mlid will be assigned to the new LID after the join
+ if (umad_init() < 0) {
+ fprintf(stderr, "failed to init the UMAD library\n");
+ return 1;
+ }
+
+ for (i=0; i < params->num_of_groups; i++) {
+
+ if (join_multicast_group(SUBN_ADM_METHOD_SET,&mcast_manager[i],params)) {
+ fprintf(stderr, "Failed to join mcast group no %d\n",i);
+ return 1;
+ }
+
+ // Client side.
+ if ( params->is_client) {
+ if (ibv_modify_qp(mcast_manager[i].send_qp,attr,flags)) {
+ fprintf(stderr, "Failed to modify UD Mcast QP to INIT\n");
+ return 1;
+ }
+ }
+ // Server side.
+ else {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ if (ibv_modify_qp(mcast_manager[i].recv_mcg[j].qp,attr,flags)) {
+ fprintf(stderr, "Failed to modify UD Mcast QP to INIT\n");
+ return 1;
+ }
+ if (ibv_attach_mcast(mcast_manager[i].recv_mcg[j].qp,&mcast_manager[i].mgid,mcast_manager[i].mlid)) {
+ fprintf(stderr, "Couldn't attach QP to Multi group No. %d\n",i+1);
+ return 1;
+ }
+ mcast_manager[i].mcast_state |= MCAST_IS_ATTACHED;
+ }
+ }
+ }
+ return 0;
+}
+
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_create_qps(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_init_attr *attr) {
+
+ int i,j;
+
+ for (i=0; i < params->num_of_groups; i++) {
+ attr->send_cq = mcast_manager[i].mcg_cq;
+ attr->recv_cq = mcast_manager[i].mcg_cq;
+
+ // Client Side.
+ if (params->is_client) {
+ mcast_manager[i].send_qp = ibv_create_qp(params->pd,attr);
+ if (!mcast_manager[i].send_qp) {
+ fprintf(stderr, "Couldn't create QP\n");
+ return 1;
+ }
+ }
+ // Server side.
+ else {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ mcast_manager[i].recv_mcg[j].qp = ibv_create_qp(params->pd,attr);
+ if (!mcast_manager[i].recv_mcg[j].qp) {
+ fprintf(stderr, "Couldn't create QP\n");
+ return 1;
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_modify_qp_to_rtr(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_attr *attr) {
+ int i,j;
+
+ attr->dest_qp_num = QPNUM_MCAST;
+ attr->ah_attr.grh.hop_limit = 1;
+ attr->ah_attr.sl = 0;
+ attr->ah_attr.is_global = 1;
+ attr->ah_attr.grh.sgid_index = 0;
+ for (i=0; i < params->num_of_groups; i++) {
+ // Client side.
+ if (params->is_client) {
+
+ attr->ah_attr.dlid = mcast_manager[i].mlid;
+ memcpy(attr->ah_attr.grh.dgid.raw,mcast_manager[i].mgid.raw,16);
+ if (ibv_modify_qp(mcast_manager[i].send_qp,attr,IBV_QP_STATE)) {
+ fprintf(stderr, "Failed to modify UD QP to RTR\n");
+ return 1;
+ }
+ mcast_manager[i].ah = ibv_create_ah(params->pd,&attr->ah_attr);
+ if (!mcast_manager[i].ah) {
+ fprintf(stderr, "Failed to create AH for UD\n");
+ return 1;
+ }
+ }
+ // Server side.
+ else {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ if (ibv_modify_qp(mcast_manager[i].recv_mcg[j].qp,attr,IBV_QP_STATE)) {
+ printf("Failed to modify Multicast QP to RTR %d\n",j+1);
+ return 1;
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+
+
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_post_receive(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+ int i,j,k;
+ struct ibv_recv_wr *bad_wr_recv;
+
+ // Server side.
+ if (!params->is_client) {
+ for (i=0; i < params->num_of_groups; i++) {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ params->rwr->wr_id = j;
+ for (k=0; k < params->rx_depth; k++) {
+ if (ibv_post_recv(mcast_manager[i].recv_mcg[j].qp,params->rwr,&bad_wr_recv)) {
+ fprintf(stderr, "Couldn't post recv: counter=%d\n", j);
+ return 14;
+ }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_post_send(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_sge *sge_list) {
+
+ int i;
+
+ if (params->is_client) {
+ for (i=0; i < params->num_of_groups; i++) {
+ ALLOCATE(mcast_manager[i].wr,struct ibv_send_wr,1);
+ mcast_manager[i].wr->wr_id = PINGPONG_SEND_WRID + i;
+ mcast_manager[i].wr->sg_list = sge_list;
+ mcast_manager[i].wr->num_sge = 1;
+ mcast_manager[i].wr->opcode = IBV_WR_SEND;
+ mcast_manager[i].wr->next = NULL;
+ mcast_manager[i].wr->wr.ud.ah = mcast_manager[i].ah;
+ mcast_manager[i].wr->wr.ud.remote_qkey = 0x11111111;
+ mcast_manager[i].wr->wr.ud.remote_qpn = QPNUM_MCAST;
+ if (params->size > params->inline_size) {
+ mcast_manager[i].wr->send_flags = IBV_SEND_SIGNALED;
+ }
+ }
+ }
+ return 0;
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_run_iter_uni(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+ int i;
+ pthread_barrier_t barrier;
+ pthread_t *thread_array = NULL;
+ struct mcast_package *current_mcast = NULL;
+ void* (*ptr_to_func)(void*) = run_iter;
+
+ ALLOCATE(thread_array,pthread_t,params->num_of_groups);
+ ALLOCATE(current_mcast,struct mcast_package,params->num_of_groups);
+
+ if (pthread_barrier_init(&barrier,NULL,params->num_of_groups+1)) {
+ fprintf(stderr,"Couldn't create pthread barrier\n");
+ return 1;
+ }
+ for (i=0; i < params->num_of_groups; i++) {
+ current_mcast[i].ptr_to_params = params;
+ current_mcast[i].ptr_to_mcast = &(mcast_manager[i]);
+ current_mcast[i].our_barrier = &barrier;
+ if (pthread_create(&(thread_array[i]),NULL,ptr_to_func,¤t_mcast[i])) {
+ printf("Error trying to create the new thread\n");
+ return 1;
+ }
+ }
+ pthread_barrier_wait(&barrier);
+ for (i=0; i < params->num_of_groups; i++) {
+ if (pthread_join(thread_array[i],NULL)) {
+ printf("Problem using Pthreads\n");
+ return 1;
+ }
+ }
+ if (pthread_barrier_destroy(&barrier)) {
+ fprintf(stderr,"Couldn't destroy the phtread barrier\n");
+ return 1;
+ }
+ free(thread_array);
+ free(current_mcast);
+ return 0;
+}
+
+
+/************************************************************************
+ *
+ ************************************************************************/
+void mcast_print_server(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ int no_cpu_freq_fail) {
+
+ int i,j;
+ unsigned int pkg_cntr,lower;
+ unsigned long packets_sum = 0;
+ double cycles_to_units;
+ struct mcg_qp current;
+ cycles_t max_first, min_last;
+
+ current = mcast_manager[0].recv_mcg[0];
+ pkg_cntr = current.package_counter ? current.package_counter - 1 : 0;
+ cycles_to_units = get_cpu_mhz(no_cpu_freq_fail) * 1000000;
+
+ max_first = current.completion_array[0];
+ min_last = current.completion_array[pkg_cntr];
+
+ for (i=0; i < params->num_of_groups; i++) {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ current = mcast_manager[i].recv_mcg[j];
+ if (current.completion_array[0] > max_first) {
+ max_first = current.completion_array[0];
+ }
+ pkg_cntr = current.package_counter ? current.package_counter - 1: 0;
+ if (current.completion_array[pkg_cntr] < min_last) {
+ min_last = current.completion_array[pkg_cntr];
+ }
+ }
+ }
+ for (i=0; i < params->num_of_groups; i++) {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ current = mcast_manager[i].recv_mcg[j];
+ pkg_cntr = current.package_counter ? current.package_counter - 1: 0;
+ lower = 0;
+ while (current.completion_array[lower] < max_first && lower < pkg_cntr) {
+ lower++;
+ }
+ while (pkg_cntr > 0 && current.completion_array[pkg_cntr] > min_last && pkg_cntr + 1 > lower) {
+ pkg_cntr--;
+ }
+ packets_sum += (pkg_cntr - lower + 1);
+ }
+ }
+ printf(" %d %6d %14.2lf %21.2f\n",params->size,params->iterations,
+ (double)packets_sum/(params->iterations*params->num_of_groups*params->num_qps_on_group),
+ packets_sum*params->size*cycles_to_units/(min_last - max_first) / 0x100000);
+}
+
+/************************************************************************
+ *
+ ************************************************************************/
+void mcast_print_client(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ int no_cpu_freq_fail) {
+
+ double cycles_to_units;
+ cycles_t max_first_posted,min_last_completed;
+ int packets_sum = 0;
+ int bottom,last_packet,i;
+
+ last_packet = params->iterations - 1;
+ max_first_posted = mcast_manager[0].posted[0];
+ min_last_completed = mcast_manager[0].completed[last_packet];
+
+ for (i=1; i < params->num_of_groups; i++) {
+ if (mcast_manager[i].posted[0] > max_first_posted) {
+ max_first_posted = mcast_manager[i].posted[0];
+ }
+ if (mcast_manager[i].completed[last_packet] < min_last_completed) {
+ min_last_completed = mcast_manager[i].completed[last_packet];
+ }
+ }
+ for (i=0; i < params->num_of_groups; i++) {
+ bottom = 0;
+ last_packet = params->iterations - 1;
+ while (mcast_manager[i].posted[bottom] < max_first_posted &&
+ bottom < last_packet) bottom++;
+ while (mcast_manager[i].completed[last_packet] > min_last_completed &&
+ last_packet > bottom + 1) last_packet--;
+ packets_sum += last_packet - bottom + 1;
+ }
+ cycles_to_units = get_cpu_mhz(no_cpu_freq_fail) * 1000000;
+ printf(" %d %6d %14.2lf %21.2f\n",params->size,params->iterations,
+ (double)packets_sum/(params->iterations*params->num_of_groups),
+ packets_sum*params->size*cycles_to_units/(min_last_completed - max_first_posted)/ 0x100000);
+}
+
+
+/************************************************************************
+ *
+ ************************************************************************/
+int mcast_destroy_resources(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params) {
+
+
+ int i,j;
+ int test_result = 0;
+
+ for (i=0; i < params->num_of_groups; i++) {
+ if (params->is_client) {
+ if (ibv_destroy_ah(mcast_manager[i].ah)) {
+ fprintf(stderr, "failed to destroy AH\n");
+ test_result = 1;
+ }
+ free(mcast_manager[i].wr);
+ if (ibv_destroy_qp(mcast_manager[i].send_qp)) {
+ fprintf(stderr, "failed to destroy QP\n");
+ test_result = 1;
+ }
+ } else {
+ for (j=0; j < params->num_qps_on_group; j++) {
+ if (ibv_detach_mcast(mcast_manager[i].recv_mcg[j].qp,&mcast_manager[i].mgid,mcast_manager[i].mlid)) {
+ fprintf(stderr, "failed to deatttached QP\n");
+ test_result = 1;
+ }
+ if (ibv_destroy_qp(mcast_manager[i].recv_mcg[j].qp)) {
+ fprintf(stderr, "failed to destroy QP\n");
+ test_result = 1;
+ }
+ free(mcast_manager[i].recv_mcg[j].completion_array);
+ mcast_manager[i].mcast_state &= ~MCAST_IS_ATTACHED;
+ }
+ free(mcast_manager[i].recv_mcg);
+ }
+
+ // leave the multicast group
+ if (mcast_manager[i].mcast_state & MCAST_IS_JOINED) {
+ if (join_multicast_group(SUBN_ADM_METHOD_DELETE,&mcast_manager[i],params)) {
+ fprintf(stderr, "failed to leave the mcast group\n");
+ test_result = 1;
+ }
+ }
+
+ if (ibv_destroy_cq(mcast_manager[i].mcg_cq)) {
+ fprintf(stderr, "failed to destroy CQ\n");
+ test_result = 1;
+ }
+ }
+ free(mcast_manager);
+ return test_result;
+}
Propchange: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c
------------------------------------------------------------------------------
svn:executable = *
Added: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h?rev=741&op=file
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h (added)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h Wed May 5 19:05:11 2010
@@ -1,0 +1,270 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Ido Shamay <idos at dev.mellanox.co.il>
+ */
+
+
+#ifndef MULTICAST_RESOURCES_H
+#define MULTICAST_RESOURCES_H
+
+ /* Multicast Module for perftest.
+ *
+ * Description :
+ *
+ * This file contains the structures and methods for implementing a multiple
+ * multicast groups in user space enviroment.
+ * The module is in use in "send_bw" and "send_lat" ,but can be used on other
+ * applications and can generate more methods and serve more benchmarks.
+ * The Module uses only the structire defined here , enabling generic use of it.
+ *
+ * Defined Types :
+ *
+ * mcast_parameters - Contains all the parameters needed for this module.
+ * mcast_group - The multicast group entitiy itself.
+ * mcg_qp - Is a QP structure that is attahced to the group.
+ *
+ */
+
+
+/************************************************************************
+ * Macros , Defines and Files included for work. *
+ ************************************************************************/
+
+#include <infiniband/verbs.h>
+#include <infiniband/umad.h>
+#include "get_clock.h"
+
+
+#define MCG_LID 0xc00f
+#define PINGPONG_SEND_WRID 1
+#define DEF_QKEY 0x11111111
+#define QPNUM_MCAST 0xffffff
+#define DEF_PKEY_IDX 0
+#define DEF_SL 0
+#define MAX_POLL_ITERATION_TIMEOUT 1000000
+#define MCG_GID {255,1,0,0,0,2,201,133,0,0,0,0,0,0,0,0}
+
+// Definitions section for MADs
+#define SUBN_ADM_ATTR_MC_MEMBER_RECORD 0x38
+#define MANAGMENT_CLASS_SUBN_ADM 0x03 /* Subnet Administration class */
+#define MCMEMBER_JOINSTATE_FULL_MEMBER 0x1
+#define MAD_SIZE 256 /* The size of a MAD is 256 bytes */
+#define QP1_WELL_KNOWN_Q_KEY 0x80010000 /* Q_Key value of QP1 */
+#define DEF_TRANS_ID 0x12345678 /* TransactionID */
+#define DEF_TCLASS 0
+#define DEF_FLOW_LABLE 0
+
+
+// Macro for allocating.
+#define ALLOCATE(var,type,size) \
+ { if((var = (type*)malloc(sizeof(type)*(size))) == NULL) \
+ { fprintf(stderr, "Cannot Allocate\n"); exit(1);}}
+
+// Macro for 64 bit variables to switch to from net
+#define ntohll(x) (((u_int64_t)(ntohl((int)((x << 32) >> 32))) << 32) | (unsigned int)ntohl(((int)(x >> 32))))
+#define htonll(x) ntohll(x)
+
+// generate a bit mask S bits width
+#define MASK32(S) ( ((u_int32_t) ~0L) >> (32-(S)) )
+
+// generate a bit mask with bits O+S..O set (assumes 32 bit integer).
+#define BITS32(O,S) ( MASK32(S) << (O) )
+
+// extract S bits from (u_int32_t)W with offset O and shifts them O places to the right
+#define EXTRACT32(W,O,S) ( ((W)>>(O)) & MASK32(S) )
+
+// insert S bits with offset O from field F into word W (u_int32_t)
+#define INSERT32(W,F,O,S) (/*(W)=*/ ( ((W) & (~BITS32(O,S)) ) | (((F) & MASK32(S))<<(O)) ))
+
+#ifndef INSERTF
+ #define INSERTF(W,O1,F,O2,S) (INSERT32(W, EXTRACT32(F, O2, S), O1, S) )
+#endif
+
+
+// according to Table 187 in the IB spec 1.2.1
+typedef enum {
+ SUBN_ADM_METHOD_SET = 0x2,
+ SUBN_ADM_METHOD_DELETE = 0x15
+} subn_adm_method;
+
+// Utilities for Umad Usage.
+typedef enum {
+ SUBN_ADM_COMPMASK_MGID = (1ULL << 0),
+ SUBN_ADM_COMPMASK_PORT_GID = (1ULL << 1),
+ SUBN_ADM_COMPMASK_Q_KEY = (1ULL << 2),
+ SUBN_ADM_COMPMASK_P_KEY = (1ULL << 7),
+ SUBN_ADM_COMPMASK_TCLASS = (1ULL << 6),
+ SUBN_ADM_COMPMASK_SL = (1ULL << 12),
+ SUBN_ADM_COMPMASK_FLOW_LABEL = (1ULL << 13),
+ SUBN_ADM_COMPMASK_JOIN_STATE = (1ULL << 16),
+} subn_adm_component_mask;
+
+typedef enum {
+ MCAST_IS_JOINED = 1,
+ MCAST_IS_ATTACHED = (1 << 1)
+} mcast_state;
+
+// Using the Bool to avoid program compilation with -C99 flag.
+typedef enum { FALSE , TRUE } Bool;
+
+
+/************************************************************************
+ * Multicast data structures. *
+ ************************************************************************/
+
+// Needed parameters for creating a multiple multicast group entity.
+struct mcast_parameters {
+ struct ibv_pd *pd;
+ struct ibv_recv_wr *rwr;
+ struct ibv_device *ib_dev;
+ struct ibv_context *ctx;
+ int num_of_groups;
+ int num_qps_on_group;
+ int iterations;
+ int tx_depth;
+ int rx_depth;
+ int inline_size;
+ int ib_port;
+ unsigned size;
+ const char *user_mgid;
+ Bool is_client;
+ Bool is_user_mgid;
+ uint16_t pkey;
+ union ibv_gid port_gid;
+};
+
+// Basic structure of the Multicast entity.
+struct mcast_group {
+ struct ibv_qp *send_qp;
+ struct ibv_cq *mcg_cq;
+ struct ibv_send_wr *wr;
+ struct ibv_ah *ah;
+ struct mcg_qp *recv_mcg;
+ cycles_t *posted;
+ cycles_t *completed;
+ uint16_t mlid; /* the multicast group's mlid */
+ union ibv_gid mgid;
+ int mcast_state;
+};
+
+// A single QP that is attached.
+struct mcg_qp {
+ struct ibv_qp *qp;
+ unsigned int package_counter;
+ cycles_t *completion_array;
+};
+
+// according to Table 195 in the IB spec 1.2.1
+struct sa_mad_packet_t {
+ u_int8_t mad_header_buf[24];
+ u_int8_t rmpp_header_buf[12];
+ u_int64_t SM_Key;
+ u_int16_t AttributeOffset;
+ u_int16_t Reserved1;
+ u_int64_t ComponentMask;
+ u_int8_t SubnetAdminData[200];
+}__attribute__((packed));
+
+/************************************************************************
+ * Multicast resources methods. *
+ ************************************************************************/
+
+
+/*
+ *
+ */
+struct mcast_group* mcast_create_resources(struct mcast_parameters *params);
+
+/*
+ *
+ */
+int mcast_init_resources(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_attr *attr,int flags);
+
+/*
+ *
+ */
+int mcast_create_qps(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_init_attr *attr);
+
+/*
+ *
+ */
+int mcast_modify_qp_to_rtr(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_qp_attr *attr);
+
+
+/*
+ *
+ */
+int mcast_post_receive(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params);
+
+/*
+ *
+ */
+int mcast_post_send(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ struct ibv_sge *sge_list);
+
+/*
+ *
+ */
+int mcast_run_iter_uni(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params);
+
+
+/*
+ *
+ */
+void mcast_print_server(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ int no_cpu_freq_fail);
+
+/*
+ *
+ */
+void mcast_print_client(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params,
+ int no_cpu_freq_fail);
+
+/*
+ *
+ */
+int mcast_destroy_resources(struct mcast_group *mcast_manager,
+ struct mcast_parameters *params);
+
+
+#endif /* MULTICAST_RESOURCES_H */
Propchange: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h
------------------------------------------------------------------------------
svn:executable = *
Modified: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/perftest.spec
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/perftest.spec?rev=741&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/perftest.spec (original)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/perftest.spec Wed May 5 19:05:11 2010
@@ -1,10 +1,10 @@
Name: perftest
Summary: IB Performance tests
Version: 1.2.3
-Release: 0.8.g196d994
+Release: 0.10.g90b10d8
License: BSD 3-Clause, GPL v2 or later
Group: Productivity/Networking/Diagnostic
-Source: http://www.openfabrics.org/downloads/perftest-1.2.3-0.8.g196d994.tar.gz
+Source: http://www.openfabrics.org/downloads/perftest-1.2.3-0.10.g90b10d8.tar.gz
Url: http://www.openfabrics.org
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildRequires: libibverbs-devel librdmacm-devel
Modified: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/read_bw.c
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/read_bw.c?rev=741&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/read_bw.c (original)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/read_bw.c Wed May 5 19:05:11 2010
@@ -56,20 +56,22 @@
#include "get_clock.h"
-#define MAX_OUTSTANDING_READ 16
+#define MAX_OUT_READ_HERMON 16
+#define MAX_OUT_READ 4
#define PINGPONG_READ_WRID 1
-#define VERSION 1.2
+#define VERSION 1.3
#define ALL 1
#define RC 0
struct user_parameters {
- const char *servername;
+ const char *servername;
int connection_type;
int mtu;
int all; /* run all msg size */
int iters;
int tx_depth;
int max_out_read;
+ int use_out_read;
int use_event;
int qp_timeout;
int gid_index; /* if value not negative, we use gid AND gid_index=value */
@@ -103,6 +105,37 @@
};
+/*
+ *
+ */
+static int set_max_out_read(struct user_parameters *param,struct ibv_device_attr *attr) {
+
+ int is_hermon = 0;
+ int max_out_reads;
+
+ // Checks the devide type for setting the max outstanding reads.
+ if (attr->vendor_part_id == 25408 || attr->vendor_part_id == 25418 ||
+ attr->vendor_part_id == 25448 || attr->vendor_part_id == 26418 ||
+ attr->vendor_part_id == 26428 || attr->vendor_part_id == 26438 ||
+ attr->vendor_part_id == 26448 || attr->vendor_part_id == 26458 ||
+ attr->vendor_part_id == 26468 || attr->vendor_part_id == 26478) {
+ is_hermon = 1;
+ }
+
+ max_out_reads = (is_hermon == 1) ? MAX_OUT_READ_HERMON : MAX_OUT_READ;
+
+ if (param->use_out_read) {
+ if (param->max_out_read <= 0 || param->max_out_read > max_out_reads) {
+ fprintf(stderr,"Max outstanding reads for this device is %d\n",max_out_reads);
+ return 1;
+ }
+ }
+ else {
+ param->max_out_read = max_out_reads;
+ }
+ return 0;
+}
+
static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
{
struct ibv_port_attr attr;
@@ -417,11 +450,18 @@
ibv_get_device_name(ib_dev));
return NULL;
}
- if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
- if (ibv_query_device(ctx->context, &device_attr)) {
- fprintf(stderr, "Failed to query device props");
- return NULL;
- }
+
+ if (ibv_query_device(ctx->context, &device_attr)) {
+ fprintf(stderr, "Failed to query device props");
+ return NULL;
+ }
+
+ if (set_max_out_read(user_parm,&device_attr)) {
+ fprintf(stderr, "Failed to set the number of outstanding reads\n");
+ return NULL;
+ }
+
+ if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1)
user_parm->mtu = 1024;
else
@@ -578,7 +618,7 @@
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
printf(" -m, --mtu=<mtu> mtu size (256 - 4096. default for hermon is 2048)\n");
- printf(" -o, --outs=<num> num of outstanding read/atom(default 16)\n");
+ printf(" -o, --outs=<num> num of outstanding read/atom(default for hermon 16 (others 4)\n");
printf(" -s, --size=<size> size of message to exchange (default 65536)\n");
printf(" -a, --all Run sizes from 2 till 2^23\n");
printf(" -t, --tx-depth=<dep> size of tx queue (default 100)\n");
@@ -726,11 +766,11 @@
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
user_param.mtu = 0;
+ user_param.use_out_read = 0;
user_param.iters = 1000;
user_param.tx_depth = 100;
user_param.servername = NULL;
user_param.use_event = 0;
- user_param.max_out_read = MAX_OUTSTANDING_READ; /* the device capability on gen2 */
user_param.qp_timeout = 14;
user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
@@ -781,6 +821,7 @@
break;
case 'o':
user_param.max_out_read = strtol(optarg, NULL, 0);
+ user_param.use_out_read = 1;
break;
case 'a':
user_param.all = ALL;
Modified: branches/ofed-1.5.1lenny/perftest/branches/upstream/current/send_bw.c
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/branches/upstream/current/send_bw.c?rev=741&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/branches/upstream/current/send_bw.c (original)
+++ branches/ofed-1.5.1lenny/perftest/branches/upstream/current/send_bw.c Wed May 5 19:05:11 2010
@@ -57,6 +57,7 @@
#include <infiniband/verbs.h>
#include "get_clock.h"
+#include "multicast_resources.h"
#define PINGPONG_SEND_WRID 1
#define PINGPONG_RECV_WRID 2
@@ -69,11 +70,6 @@
#define SIGNAL 1
#define MAX_INLINE 400
#define ALL 1
-#define MAX_POLL_ITERATION_TIMEOUT 100000000
-#define QPNUM_MCAST 0xffffff
-#define DEF_QKEY 0x11111111
-#define MCG_LID 0xc001
-#define MCG_GID {255,1,0,0,0,2,201,133,0,0,1,1,0,0,0,0}
struct user_parameters {
const char *servername;
@@ -87,9 +83,13 @@
int rx_depth;
int duplex;
int use_event;
+ int ib_port;
+ // Ido
+ int num_of_mcg_group;
+ int num_of_qp_on_group;
+ // End Ido
int use_mcg;
int use_user_mgid;
- int num_of_clients_mcg;
int inline_size;
int qp_timeout;
int gid_index; /* if value not negative, we use gid AND gid_index=value */
@@ -102,23 +102,26 @@
int post_recv;
struct pingpong_context {
- struct ibv_context *context;
+ struct ibv_context *context;
struct ibv_comp_channel *channel;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- struct mcg_qp *qp_mcg;
- void *buf;
- unsigned size;
- int tx_depth;
- int rx_depth;
- struct ibv_sge list;
- struct ibv_sge recv_list;
- struct ibv_send_wr wr;
- struct ibv_recv_wr rwr;
- struct ibv_ah *ah;
- union ibv_gid dgid;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ // Ido
+ struct mcast_group *mcg_group;
+ struct mcast_parameters *mcg_params;
+ //End Ido
+ void *buf;
+ unsigned size;
+ int tx_depth;
+ int rx_depth;
+ struct ibv_sge list;
+ struct ibv_sge recv_list;
+ struct ibv_send_wr wr;
+ struct ibv_recv_wr rwr;
+ struct ibv_ah *ah;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -130,53 +133,87 @@
union ibv_gid dgid;
};
-struct mcg_qp {
-
- struct ibv_qp *qp;
- unsigned int package_counter;
- cycles_t *completion_array;
-};
-
/*
*
*/
-static int set_multicast_gid(struct pingpong_context *ctx,struct user_parameters *param) {
-
- uint8_t mcg_gid[16] = MCG_GID;
- const char *pstr = param->user_mgid;
- char *term = NULL;
- char tmp[20];
- int i;
-
- if (param->use_user_mgid) {
- term = strpbrk(pstr, ":");
- memcpy(tmp, pstr, term - pstr+1);
- tmp[term - pstr] = 0;
- mcg_gid[0] = (unsigned char)strtoll(tmp, NULL, 0);
- for (i = 1; i < 15; ++i) {
- pstr += term - pstr + 1;
- term = strpbrk(pstr, ":");
- memcpy(tmp, pstr, term - pstr+1);
- tmp[term - pstr] = 0;
- mcg_gid[i] = (unsigned char)strtoll(tmp, NULL, 0);
- }
- pstr += term - pstr + 1;
- strcpy(tmp, pstr);
- mcg_gid[15] = (unsigned char)strtoll(tmp, NULL, 0);
- }
- memcpy(ctx->dgid.raw,mcg_gid,16);
- return 0;
+static int create_mcast(struct pingpong_context *ctx,
+ struct user_parameters *param,
+ struct ibv_device *ib_dev) {
+
+
+ ALLOCATE(ctx->mcg_params,struct mcast_parameters,1);
+ ctx->mcg_params->num_of_groups = param->num_of_mcg_group;
+ ctx->mcg_params->num_qps_on_group = param->num_of_qp_on_group;
+ ctx->mcg_params->iterations = param->iters;
+ ctx->mcg_params->tx_depth = ctx->tx_depth;
+ ctx->mcg_params->rx_depth = ctx->rx_depth;
+ ctx->mcg_params->pd = ctx->pd;
+ ctx->mcg_params->ctx = ctx->context;
+ ctx->mcg_params->ib_dev = ib_dev;
+ ctx->mcg_params->ib_port = param->ib_port;
+ ctx->mcg_params->is_client = param->servername ? TRUE : FALSE;
+ ctx->mcg_params->is_user_mgid = param->use_user_mgid ? TRUE : FALSE;
+ ctx->mcg_params->inline_size = param->inline_size;
+ ctx->mcg_params->user_mgid = param->user_mgid;
+
+ // Query pkey.
+ if (ibv_query_pkey(ctx->context,param->ib_port,DEF_PKEY_IDX,&ctx->mcg_params->pkey)) {
+ fprintf(stderr, "failed to query PKey table of port %d\n",param->ib_port);
+ return 1;
+ }
+
+ // Query port gid.
+ if( ibv_query_gid(ctx->context,param->ib_port,0,&ctx->mcg_params->port_gid)) {
+ fprintf(stderr, "failed to query GID table of port %d.\n",param->ib_port);
+ return 1;
+ }
+
+ // Create the Mcast entities.
+ ctx->mcg_group = mcast_create_resources(ctx->mcg_params);
+ if (ctx->mcg_group == NULL) {
+ fprintf(stderr,"Couldn't create Multicast resources\n");
+ return 1;
+ }
+ return 0;
}
+
/*
*
*/
-static int
-qp_create(struct pingpong_context *ctx, struct user_parameters *param) {
+static int cq_create(struct pingpong_context *ctx,struct user_parameters *param) {
+
+ int i,rx_depth;
+
+ rx_depth = ctx->rx_depth;
+
+ if (param->use_mcg) {
+ rx_depth *= param->num_of_qp_on_group;
+ for (i=0; i < param->num_of_mcg_group; i++) {
+ ctx->mcg_group[i].mcg_cq = ibv_create_cq(ctx->context,rx_depth,NULL,ctx->channel,0);
+ if (!ctx->mcg_group[i].mcg_cq) {
+ fprintf(stderr, "Couldn't create CQ\n");
+ return 1;
+ }
+ }
+ }
+ else {
+ ctx->cq = ibv_create_cq(ctx->context,rx_depth,NULL,ctx->channel,0);
+ if (!ctx->cq) {
+ fprintf(stderr, "Couldn't create CQ --- \n");
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+ *
+ */
+static int qp_create(struct pingpong_context *ctx, struct user_parameters *param) {
struct ibv_qp_init_attr attr;
- int i;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.send_cq = ctx->cq;
@@ -200,21 +237,15 @@
default:
fprintf(stderr, "Unknown connection type \n");
return 1;
- }
- if (param->use_mcg && !param->servername) {
- ctx->qp_mcg = malloc(sizeof(struct mcg_qp)*param->num_of_clients_mcg);
- if(ctx->qp_mcg == NULL) {
- fprintf(stdout, "Couldn't create QP\n");
- return 1;
- }
- for (i=0; i < param->num_of_clients_mcg; i++ ) {
- ctx->qp_mcg[i].qp = ibv_create_qp(ctx->pd, &attr);
- if (!ctx->qp_mcg[i].qp) {
- fprintf(stderr, "Couldn't create QP\n");
- return 1;
- }
- }
- }
+ }
+ // Ido
+ if (param->use_mcg) {
+ if (mcast_create_qps(ctx->mcg_group,ctx->mcg_params,&attr)){
+ fprintf(stderr,"Couldn't create QPs for Multicast group\n");
+ return 1;
+ }
+ }
+ // End Ido
else {
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
@@ -225,95 +256,94 @@
return 0;
}
-/*
+/*
*
*/
-static int modify_qp_to_init(struct ibv_qp *qp,int port,int connection_type) {
-
- struct ibv_qp_attr attr;
- int flags;
-
- memset(&attr, 0, sizeof(attr));
- attr.qp_state = IBV_QPS_INIT;
- attr.pkey_index = 0;
- attr.port_num = port;
-
- if (connection_type == UD) {
-
+static int modify_qp_to_init(struct pingpong_context *ctx,struct user_parameters *param) {
+
+ struct ibv_qp_attr attr;
+ int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
+
+ memset(&attr, 0, sizeof(struct ibv_qp_attr));
+ attr.qp_state = IBV_QPS_INIT;
+ attr.pkey_index = 0;
+ attr.port_num = param->ib_port;
+
+ if (param->connection_type == UD) {
attr.qkey = DEF_QKEY;
- flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY;
- }
- else {
-
+ flags |= IBV_QP_QKEY;
+ } else {
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
- flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS;
- }
- if (ibv_modify_qp(qp,&attr,flags)) {
-
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return 1;
+ flags |= IBV_QP_ACCESS_FLAGS;
+ }
+
+ if (param->use_mcg) {
+ if (mcast_init_resources(ctx->mcg_group,ctx->mcg_params,&attr,flags)){
+ fprintf(stderr,"Couldn't init Multicast resources\n");
+ return 1;
+ }
+ }
+ else {
+ if (ibv_modify_qp(ctx->qp,&attr,flags)) {
+ fprintf(stderr, "Failed to modify UD QP to INIT\n");
+ return 1;
+ }
}
return 0;
}
-/*
- *
- */
+
static int destroy_ctx_resources(struct pingpong_context *ctx,
struct user_parameters *param) {
- int i;
int test_result = 0;
- if (ctx->ah) {
- if (ibv_destroy_ah(ctx->ah)) {
- fprintf(stderr, "failed to destroy AH\n");
+
+ if (param->use_mcg) {
+ if (mcast_destroy_resources(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr, "failed to destroy MultiCast resources\n");
test_result = 1;
}
- }
- if (param->use_mcg && !param->servername) {
- for(i=0; i < param->num_of_clients_mcg; i++) {
-
- if (ibv_detach_mcast(ctx->qp_mcg[i].qp, &ctx->dgid,MCG_LID)) {
- fprintf(stderr, "failed to deatttached QP\n");
+ free(ctx->mcg_params);
+ }
+ else {
+ if (ctx->ah) {
+ if (ibv_destroy_ah(ctx->ah)) {
+ fprintf(stderr, "failed to destroy AH\n");
test_result = 1;
}
- if (ibv_destroy_qp(ctx->qp_mcg[i].qp)) {
- fprintf(stderr, "failed to destroy QP\n");
- test_result = 1;
- }
- free(ctx->qp_mcg[i].completion_array);
- }
- free(ctx->qp_mcg);
+ }
- } else {
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
- }
+
+ if (ctx->cq) {
+ if (ibv_destroy_cq(ctx->cq)) {
+ fprintf(stderr, "failed to destroy CQ\n");
+ test_result = 1;
+ }
+ }
+ }
+
if (ctx->mr) {
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
}
-
+
if (ctx->buf)
free(ctx->buf);
-
- if (ctx->cq) {
- if (ibv_destroy_cq(ctx->cq)) {
- fprintf(stderr, "failed to destroy CQ\n");
- test_result = 1;
- }
- }
+
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "failed to destroy channel \n");
test_result = 1;
}
}
+
if (ctx->pd) {
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
@@ -326,32 +356,22 @@
test_result = 1;
}
}
+
if(!param->use_mcg || param->servername) {
free(tposted);
free(tcompleted);
}
-
return test_result;
}
-/*
- *
- */
+
static int
alloc_time_res(struct pingpong_context *ctx, struct user_parameters *param ) {
- int i;
-
- if(param->use_mcg && !param->servername ) {
- for(i=0; i < param->num_of_clients_mcg; i++){
- ctx->qp_mcg[i].completion_array = malloc(sizeof(cycles_t)*param->iters);
- if(ctx->qp_mcg[i].completion_array == NULL) {
- perror("malloc");
- return 1;
- }
- }
- }
- else {
+ if(param->use_mcg) {
+ ;
+ }
+ else if (param->servername || param->duplex){
tposted = malloc(sizeof(cycles_t)*param->iters);
if (!tposted) {
perror("malloc");
@@ -660,12 +680,10 @@
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
unsigned size,int tx_depth,
- int rx_depth, int port,
- struct user_parameters *user_parm)
+ int rx_depth,struct user_parameters *user_parm)
{
struct pingpong_context *ctx;
struct ibv_device_attr device_attr;
- int i,cq_rx_depth;
ctx = malloc(sizeof *ctx);
if (!ctx)
@@ -675,8 +693,7 @@
ctx->channel = NULL;
ctx->size = size;
ctx->tx_depth = tx_depth;
- ctx->rx_depth = rx_depth + tx_depth;
- cq_rx_depth = ctx->rx_depth;
+ ctx->rx_depth = tx_depth + rx_depth;
/* in case of UD need space for the GRH */
if (user_parm->connection_type==UD) {
ctx->buf = memalign(page_size, ( size + 40 ) * 2);
@@ -743,49 +760,36 @@
return NULL;
}
}
- if (user_parm->use_mcg && !user_parm->servername) {
- cq_rx_depth *= user_parm->num_of_clients_mcg;
- }
- ctx->cq = ibv_create_cq(ctx->context,cq_rx_depth, NULL, ctx->channel, 0);
- if (!ctx->cq) {
- fprintf(stderr, "Couldn't create CQ\n");
+
+ // Ido.
+ if (user_parm->use_mcg) {
+ if (create_mcast(ctx,user_parm,ib_dev)) {
+ return NULL;
+ }
+ }
+ // Ido.
+ if (cq_create(ctx,user_parm)) {
+ fprintf(stderr, "Couldn't Create CQ as requested\n");
return NULL;
}
+ // Ido.
if (qp_create(ctx,user_parm)) {
fprintf(stderr, "Couldn't create QP\n");
return NULL;
}
-
- if (user_parm->use_mcg && !user_parm->servername) {
- if (set_multicast_gid(ctx,user_parm)) {
- fprintf(stderr, "Couldn't Set the Multicast M_GID\n");
- return NULL;
- }
-
- for(i=0; i < user_parm->num_of_clients_mcg; i++) {
- if (modify_qp_to_init(ctx->qp_mcg[i].qp,port,user_parm->connection_type)) {
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return NULL;
- }
- if (ibv_attach_mcast(ctx->qp_mcg[i].qp,&ctx->dgid,MCG_LID)) {
- fprintf(stderr, "Couldn't attach QP to mcg\n");
- return NULL;
- }
- }
- printf(" Successfully created and attached %d QPs to a Mcast group\n\n",i);
- } else {
- if (modify_qp_to_init(ctx->qp,port,user_parm->connection_type)) {
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return NULL;
- }
- }
+ // Ido.
+ if (modify_qp_to_init(ctx,user_parm)) {
+ fprintf(stderr, "Couldn't modify to init\n");
+ return NULL;
+ }
+ // End Ido.
return ctx;
}
/*
*
*/
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+static int pp_connect_ctx(struct pingpong_context *ctx,int my_psn,
struct pingpong_dest *dest,
struct user_parameters *user_parm)
{
@@ -811,9 +815,8 @@
attr.path_mtu = IBV_MTU_4096;
break;
}
- printf(" Mtu : %d\n", user_parm->mtu);
- attr.dest_qp_num = (user_parm->use_mcg && user_parm->servername) ?
- QPNUM_MCAST : dest->qpn;
+ printf("Mtu : %d\n", user_parm->mtu);
+ attr.dest_qp_num = dest->qpn;
attr.rq_psn = dest->psn;
if (user_parm->connection_type == RC) {
attr.max_dest_rd_atomic = 1;
@@ -830,16 +833,8 @@
attr.ah_attr.sl = 0;
}
attr.ah_attr.src_path_bits = 0;
- attr.ah_attr.port_num = port;
- if (user_parm->use_mcg && user_parm->servername) {
- attr.ah_attr.grh.hop_limit = 1;
- attr.ah_attr.sl = 0;
- attr.ah_attr.dlid = MCG_LID;
- attr.ah_attr.is_global = 1;
- attr.ah_attr.grh.sgid_index = 0;
- memcpy(attr.ah_attr.grh.dgid.raw,dest->dgid.raw, 16);
- }
-
+ attr.ah_attr.port_num = user_parm->ib_port;
+
if (user_parm->connection_type == RC) {
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
@@ -865,29 +860,27 @@
fprintf(stderr, "Failed to modify UC QP to RTR\n");
return 1;
}
- }
+ }
+ // Ido
+ else if (user_parm->use_mcg) {
+ if (mcast_modify_qp_to_rtr(ctx->mcg_group,ctx->mcg_params,&attr)) {
+ fprintf(stderr,"Couldn't modify Multicast Qps to RTR\n");
+ return 1;
+ }
+ }
+ // End Ido.
else {
-
- if (user_parm->use_mcg && !user_parm->servername) {
- for(i=0; i < user_parm->num_of_clients_mcg; i++) {
- if (ibv_modify_qp(ctx->qp_mcg[i].qp, &attr, IBV_QP_STATE )) {
- printf("Failed to modify UD QP to RTR\n");
- return 1;
- }
- }
- }
- else {
- if (ibv_modify_qp(ctx->qp,&attr, IBV_QP_STATE )) {
- fprintf(stderr, "Failed to modify UD QP to RTR\n");
- return 1;
- }
- }
- ctx->ah = ibv_create_ah(ctx->pd, &attr.ah_attr);
- if (!ctx->ah) {
- fprintf(stderr, "Failed to create AH for UD\n");
- return 1;
- }
- }
+ if (ibv_modify_qp(ctx->qp,&attr, IBV_QP_STATE )) {
+ fprintf(stderr, "Failed to modify UD QP to RTR\n");
+ return 1;
+ }
+ ctx->ah = ibv_create_ah(ctx->pd, &attr.ah_attr);
+ if (!ctx->ah) {
+ fprintf(stderr, "Failed to create AH for UD\n");
+ return 1;
+ }
+ }
+
attr.qp_state = IBV_QPS_RTS;
attr.sq_psn = my_psn;
attr.max_rd_atomic = 1;
@@ -903,17 +896,25 @@
fprintf(stderr, "Failed to modify RC QP to RTS\n");
return 1;
}
- } else { /*both UC and UD */
- if(!user_parm->use_mcg || user_parm->servername) {
- if(ibv_modify_qp(ctx->qp, &attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
+ }
+ else if (!user_parm->use_mcg) {
+ if(ibv_modify_qp(ctx->qp, &attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
fprintf(stderr, "Failed to modify UC QP to RTS\n");
return 1;
- }
- }
- }
+ }
+ }
+ else if (user_parm->servername) {
+ for(i=0; i < user_parm->num_of_mcg_group; i++) {
+ if (ibv_modify_qp(ctx->mcg_group[i].send_qp,&attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
+ printf("Failed to modify UD QP to RTS\n");
+ return 1;
+ }
+ }
+ }
+
/* post recieve max msg size*/
{
- int i,j;
+ int i;
struct ibv_recv_wr *bad_wr_recv;
//recieve
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
@@ -927,16 +928,15 @@
ctx->recv_list.length = ctx->size;
}
ctx->recv_list.lkey = ctx->mr->lkey;
- if ( user_parm->use_mcg && !user_parm->servername) {
- for (i=0; i < user_parm->num_of_clients_mcg; i++) {
- for (j = 0; j < ctx->rx_depth; ++j) {
- if (ibv_post_recv(ctx->qp_mcg[i].qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: counter=%d\n", j);
- return 14;
- }
- }
- }
- } else {
+
+ if (user_parm->use_mcg) {
+ ctx->mcg_params->rwr = &ctx->rwr;
+ if (mcast_post_receive(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr,"Couldn't post receive\n");
+ return 1;
+ }
+ }
+ else {
for (i = 0; i < ctx->rx_depth; ++i) {
if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
fprintf(stderr, "Couldn't post recv: counter=%d\n", i);
@@ -964,7 +964,8 @@
printf(" -s, --size=<size> Size of message to exchange (default 65536)\n");
printf(" -a, --all Run sizes from 2 till 2^23\n");
printf(" -t, --tx-depth=<dep> Size of tx queue (default 300)\n");
- printf(" -g, --mcg=<num_of_qps> Send messages to multicast group with <num_of_qps> qps attached to it.\n");
+ printf(" -g, --mcg=<num_of_mcg> Send messages to <num_of_mcg> multicast groups .\n");
+ printf(" -q, --qp_num=<num_of_qps> Attach <num_of_qps> QPs to each multicast group (default 1).\n");
printf(" -r, --rx-depth=<dep> Make rx queue bigger than tx (default 600)\n");
printf(" -n, --iters=<iters> Number of exchanges (at least 2, default 1000)\n");
printf(" -I, --inline_size=<size> Max size of message to be sent in inline mode (default 0)\n");
@@ -1019,52 +1020,7 @@
tsize * iters * cycles_to_units /(tcompleted[iters - 1] - tposted[0]) / 0x100000);
}
-/*
- *
- */
-static void print_mcg_report(struct pingpong_context *ctx,
- struct user_parameters *param,
- unsigned size,
- int no_cpu_freq_fail) {
-
- int i,packets_loss = 0;
- unsigned long packets_sum = 0;
- unsigned int lower , upper;
- int last_pos;
- double cycles_to_units;
- cycles_t max_first_comp, min_last_comp;
-
- cycles_to_units = get_cpu_mhz(no_cpu_freq_fail) * 1000000;
-
- last_pos = (ctx->qp_mcg[0].package_counter) ? ctx->qp_mcg[0].package_counter -1 : 0;
- max_first_comp = ctx->qp_mcg[0].completion_array[0];
- min_last_comp = ctx->qp_mcg[0].completion_array[last_pos];
-
- for (i=0; i < param->num_of_clients_mcg; i++) {
- if (ctx->qp_mcg[i].completion_array[0] > max_first_comp) {
- max_first_comp = ctx->qp_mcg[i].completion_array[0];
- }
- last_pos = (ctx->qp_mcg[i].package_counter) ? ctx->qp_mcg[i].package_counter -1 : 0;
- if (ctx->qp_mcg[i].completion_array[last_pos] < min_last_comp) {
- min_last_comp = ctx->qp_mcg[i].completion_array[last_pos];
- }
- }
- for (i=0; i < param->num_of_clients_mcg; i++ ) {
-
- upper = lower = 0;
- last_pos = (ctx->qp_mcg[i].package_counter) ? ctx->qp_mcg[i].package_counter -1 : 0;
- while (ctx->qp_mcg[i].completion_array[lower] < max_first_comp) {
- lower++;
- }
- while (ctx->qp_mcg[i].completion_array[last_pos] > min_last_comp) {
- last_pos++;
- }
- packets_sum += ctx->qp_mcg[i].package_counter - upper -lower;
- packets_loss += (param->iters - ctx->qp_mcg[i].package_counter);
- }
- printf("%7d %d %d %7.2f\n",size,param->iters,packets_loss,
- packets_sum*size*cycles_to_units/(min_last_comp - max_first_comp) / 0x100000);
-}
+
/*
*
@@ -1201,11 +1157,9 @@
struct pingpong_dest *rem_dest,
int size)
{
- int i,ne,num_of_iters;
int scnt, ccnt, rcnt;
struct ibv_recv_wr *bad_wr_recv;
struct ibv_send_wr *bad_wr;
- struct ibv_wc wc;
if (user_param->connection_type == UD) {
@@ -1232,18 +1186,30 @@
scnt = 0;
ccnt = 0;
rcnt = 0;
- if (user_param->use_mcg && !user_param->servername ) {
- for (i=0; i < user_param->num_of_clients_mcg; i++) {
- ctx->qp_mcg[i].package_counter = 0;
- }
- }
+ // Ido
+ if (user_param->use_mcg) {
+
+ ctx->mcg_params->rwr = &ctx->rwr;
+ ctx->mcg_params->size = size;
+ if (user_param->servername) {
+ if(mcast_post_send(ctx->mcg_group,ctx->mcg_params,&ctx->list)) {
+ fprintf(stderr,"Cannot post send\n");
+ return 1;
+ }
+ }
+ // Need to add more things.
+ if (mcast_run_iter_uni(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr,"Cannot run the benchmark iteration\n");
+ return 1;
+ }
+ }
+ else {
// Server side.
if (!user_param->servername) {
-
- num_of_iters = user_param->use_mcg ?
- user_param->iters*user_param->num_of_clients_mcg : user_param->iters;
- while (rcnt < num_of_iters && scnt < MAX_POLL_ITERATION_TIMEOUT) {
-
+ while (rcnt < user_param->iters) {
+ int ne;
+ struct ibv_wc wc;
+ /*Server is polling on recieve first */
if (user_param->use_event) {
struct ibv_cq *ev_cq;
void *ev_ctx;
@@ -1261,44 +1227,25 @@
}
}
do {
- ne = ibv_poll_cq(ctx->cq, 1 , &wc);
- if (ne > 0) {
+ ne = ibv_poll_cq(ctx->cq, 1, &wc);
+ if (ne) {
+ // tcompleted[ccnt] = get_cycles();
if (wc.status != IBV_WC_SUCCESS) {
- fprintf(stderr, "Completion wth error at server :\n");
+ fprintf(stderr, "Completion wth error at %s:\n",
+ user_param->servername ? "client" : "server");
fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n",
- wc.status, (int) wc.wr_id, wc.vendor_err);
+ wc.status, (int) wc.wr_id, wc.vendor_err);
fprintf(stderr, "scnt=%d, ccnt=%d\n",
- scnt, ccnt);
+ scnt, ccnt);
return 1;
}
++rcnt;
- scnt = 0;
- if (user_param->use_mcg) {
- for (i=0; i < user_param->num_of_clients_mcg; i++) {
- if ( wc.qp_num == ctx->qp_mcg[i].qp->qp_num) {
-
- ccnt = ctx->qp_mcg[i].package_counter;
- ctx->qp_mcg[i].completion_array[ccnt] = get_cycles();
- ctx->qp_mcg[i].package_counter++;
-
- if (ibv_post_recv(ctx->qp_mcg[i].qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: rcnt=%d\n",rcnt);
- return 15;
- }
- break;
- }
- }
+ if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
+ fprintf(stderr, "Couldn't post recv: rcnt=%d\n",
+ rcnt);
+ return 15;
}
- else {
- if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: rcnt=%d\n",rcnt);
- return 15;
- }
- }
- }
- else {
- scnt++;
- //deal with stat ne = 0
+
}
} while (ne > 0 );
@@ -1307,8 +1254,7 @@
return 12;
}
}
- }
- else {
+ } else {
/* client is posting and not receiving. */
while (scnt < user_param->iters || ccnt < user_param->iters) {
while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) {
@@ -1364,21 +1310,21 @@
}
}
}
+ }
return 0;
}
int main(int argc, char *argv[])
{
- struct ibv_device **dev_list;
- struct ibv_device *ib_dev;
+ struct ibv_device **dev_list;
+ struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
struct pingpong_dest *rem_dest;
struct user_parameters user_param;
- struct ibv_device_attr device_attribute;
+ struct ibv_device_attr device_attribute;
char *ib_devname = NULL;
int port = 18515;
- int ib_port = 1;
long long size = 65536;
int sockfd;
int i = 0;
@@ -1392,9 +1338,11 @@
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
user_param.mtu = 0;
- user_param.num_of_clients_mcg = 1;
+ user_param.ib_port = 1;
+ user_param.num_of_qp_on_group = 1;
user_param.iters = 1000;
user_param.tx_depth = 300;
+ user_param.rx_depth = 300;
user_param.servername = NULL;
user_param.user_mgid = NULL;
user_param.use_event = 0;
@@ -1421,6 +1369,7 @@
{ .name = "sl", .has_arg = 1, .val = 'S' },
{ .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "mcg", .has_arg = 1, .val = 'g' },
+ { .name = "qp_num", .has_arg = 1, .val = 'q' },
{ .name = "MGID", .has_arg = 1, .val = 'M' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
@@ -1431,7 +1380,7 @@
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:g:M:ebaVNF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:g:q:M:ebaVNF", long_options, NULL);
if (c == -1)
break;
@@ -1448,8 +1397,15 @@
break;
case 'g':
++user_param.use_mcg;
- user_param.num_of_clients_mcg = strtol(optarg, NULL, 0);
- if (user_param.num_of_clients_mcg < 0) {
+ user_param.num_of_mcg_group = strtol(optarg, NULL, 0);
+ if (user_param.num_of_mcg_group <= 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'q':
+ user_param.num_of_qp_on_group = strtol(optarg, NULL, 0);
+ if (user_param.num_of_mcg_group <= 0) {
usage(argv[0]);
return 1;
}
@@ -1478,8 +1434,8 @@
return 0;
break;
case 'i':
- ib_port = strtol(optarg, NULL, 0);
- if (ib_port < 0) {
+ user_param.ib_port = strtol(optarg, NULL, 0);
+ if (user_param.ib_port < 0) {
usage(argv[0]);
return 1;
}
@@ -1517,9 +1473,9 @@
case 'r':
errno = 0;
- user_param.rx_depth = strtol(optarg, NULL, 0);
- if (errno) { usage(argv[0]); return 1; }
- break;
+ user_param.rx_depth = strtol(optarg, NULL, 0);
+ if (errno) { usage(argv[0]); return 1; }
+ break;
case 'n':
user_param.iters = strtol(optarg, NULL, 0);
@@ -1641,23 +1597,22 @@
user_param.inline_size = 0;
}
- printf("Inline data is used up to %d bytes message\n\n", user_param.inline_size);
-
- ctx = pp_init_ctx(ib_dev, size, user_param.tx_depth, user_param.rx_depth,ib_port, &user_param);
+ printf("Inline data is used up to %d bytes message\n", user_param.inline_size);
+
+ ctx = pp_init_ctx(ib_dev, size, user_param.tx_depth, user_param.rx_depth, &user_param);
if (!ctx)
return 1;
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
- my_dest.lid = pp_get_local_lid(ctx, ib_port);
+ my_dest.lid = pp_get_local_lid(ctx, user_param.ib_port);
my_dest.psn = lrand48() & 0xffffff;
- my_dest.qpn = (user_param.use_mcg && !user_param.servername) ?
- ctx->qp_mcg[0].qp->qp_num : ctx->qp->qp_num;
+ my_dest.qpn = (user_param.use_mcg) ? QPNUM_MCAST : ctx->qp->qp_num;
if (user_param.gid_index != -1) {
int err=0;
- err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);
+ err = ibv_query_gid (ctx->context,user_param.ib_port, user_param.gid_index, &gid);
if (err) {
return -1;
}
@@ -1670,19 +1625,21 @@
return 1;
}
}
+ if (user_param.use_mcg) {
+ ctx->dgid = ctx->mcg_group[0].mgid;
+ }
my_dest.dgid = ctx->dgid;
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + size;
- printf(" local address: LID %#04x, PSN %#06x " ,my_dest.lid, my_dest.psn);
+ printf(" local address: LID %#04x, QPN %#06x, PSN %#06x\n" ,my_dest.lid,my_dest.qpn,my_dest.psn);
if (user_param.use_mcg && !user_param.servername) {
- printf("\n M_gid adress: M_ ");
- } else {
- printf(" QPN %#06x\n" ,my_dest.qpn);
- }
+ printf(" M_");
+ }
+
if (user_param.gid_index > -1 || (user_param.use_mcg && !user_param.servername) ) {
- printf("GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ printf("GID %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n",
my_dest.dgid.raw[0],my_dest.dgid.raw[1],
my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
@@ -1706,17 +1663,13 @@
if (!rem_dest)
return 1;
- printf(" remote address: LID %#04x, PSN %#06x ", rem_dest->lid, rem_dest->psn);
+ printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x\n",rem_dest->lid,rem_dest->qpn,rem_dest->psn);
if (user_param.use_mcg && user_param.servername) {
- printf("\n M_gid adress: M_");
- }
- else {
- printf(" QPN %#06x\n",rem_dest->qpn);
- }
-
+ printf(" M_");
+ }
if (user_param.gid_index > -1 || (user_param.use_mcg && user_param.servername) ) {
- printf("GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ printf("GID %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n",
rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
rem_dest->dgid.raw[5],rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
rem_dest->dgid.raw[10],rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
@@ -1724,7 +1677,7 @@
}
- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))
+ if (pp_connect_ctx(ctx,my_dest.psn, rem_dest, &user_param))
return 1;
/* An additional handshake is required *after* moving qp to RTR.
@@ -1742,18 +1695,18 @@
return 1;
}
}
- printf("--------------------------------------------------------------------------\n");
- if (user_param.servername || user_param.duplex) {
- printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n");
- }
- else if (user_param.use_mcg) {
- printf(" #bytes #iterations Packets Loss Aggregated BW [MB/s] \n");
+ printf("------------------------------------------------------------------\n");
+ if (user_param.use_mcg) {
+ printf(" #bytes #iterations #Pcg Success BW Aggreg.[MB/sec]\n");
+ }
+ else if (user_param.servername || user_param.duplex) {
+ printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n");
}
else {
printf(" Results are in the send side , Have a good day \n");
}
-
+
if(alloc_time_res(ctx,&user_param)) {
fprintf(stderr, "Couldn't allocate the resources for the tests\n");
return 1;
@@ -1773,12 +1726,18 @@
ctx->list.addr = (uintptr_t)ctx->buf;
}
- ctx->list.lkey = ctx->mr->lkey;
+ ctx->list.lkey = ctx->mr->lkey;
ctx->wr.wr_id = PINGPONG_SEND_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
ctx->wr.opcode = IBV_WR_SEND;
ctx->wr.next = NULL;
+ if (user_param.use_mcg) {
+ if (mcast_post_send(ctx->mcg_group,ctx->mcg_params,&ctx->list)) {
+ fprintf(stderr,"Couldn't create Sending Wqe\n");
+ return 1;
+ }
+ }
/* recieve */
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
@@ -1787,6 +1746,10 @@
ctx->rwr.next = NULL;
ctx->recv_list.addr = (uintptr_t) ctx->buf;
ctx->recv_list.lkey = ctx->mr->lkey;
+ if (user_param.use_mcg) {
+ ctx->mcg_params->rwr = &ctx->rwr;
+ }
+
if (user_param.all == ALL) {
if (user_param.connection_type == UD) {
@@ -1806,14 +1769,18 @@
if(run_iter_uni(ctx, &user_param, rem_dest, size))
return 17;
}
- if (user_param.servername) {
+ if (user_param.use_mcg) {
+ if (user_param.servername) {
+ mcast_print_client(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ else {
+ mcast_print_server(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ }
+ else if (user_param.servername) {
print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
}
- else {
- if (user_param.use_mcg) {
- print_mcg_report(ctx,&user_param,size,no_cpu_freq_fail);
- }
- }
+
if (user_param.servername) {
rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else {
@@ -1831,12 +1798,17 @@
return 18;
}
- if (user_param.use_mcg && !user_param.servername) {
- print_mcg_report(ctx,&user_param,size,no_cpu_freq_fail);
- }
-
- if (user_param.servername || user_param.duplex)
- print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
+ if (user_param.use_mcg) {
+ if (user_param.servername) {
+ mcast_print_client(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ else {
+ mcast_print_server(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ }
+ else if (user_param.servername) {
+ print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
+ }
}
/* close sockets */
More information about the Pkg-ofed-commits
mailing list