[Pkg-ofed-commits] [perftest] 04/12: New upstream version 3.0+3.2.g8ade541
Benjamin Drung
bdrung at moszumanska.debian.org
Mon Aug 22 09:17:55 UTC 2016
This is an automated email from the git hooks/post-receive script.
bdrung pushed a commit to branch master
in repository perftest.
commit e0e1a4858b6ac2db18abda09fc1bffc11715a46c
Author: Benjamin Drung <benjamin.drung at profitbricks.com>
Date: Mon Aug 22 10:36:50 2016 +0200
New upstream version 3.0+3.2.g8ade541
---
perftest.spec | 4 +-
src/perftest_parameters.c | 140 ++++++++++++++++++++--------
src/perftest_parameters.h | 5 +-
src/perftest_resources.c | 134 +++++++++++++++-----------
src/raw_ethernet_resources.c | 38 ++++----
src/raw_ethernet_send_bw.c | 217 ++++++++++++++++++++++---------------------
6 files changed, 318 insertions(+), 220 deletions(-)
diff --git a/perftest.spec b/perftest.spec
index 1b13315..c4eb173 100644
--- a/perftest.spec
+++ b/perftest.spec
@@ -1,10 +1,10 @@
Name: perftest
Summary: IB Performance tests
Version: 3.0
-Release: 3.1.gb36a595
+Release: 3.2.g8ade541
License: BSD 3-Clause, GPL v2 or later
Group: Productivity/Networking/Diagnostic
-Source: http://www.openfabrics.org/downloads/perftest-3.0-3.1.gb36a595.tar.gz
+Source: http://www.openfabrics.org/downloads/perftest-3.0-3.2.g8ade541.tar.gz
Url: http://www.openfabrics.org
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildRequires: libibverbs-devel librdmacm-devel libibumad-devel
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index e316143..d5964c6 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -9,7 +9,7 @@
#include <sys/socket.h>
#endif
#include "perftest_parameters.h"
-
+#include<math.h>
#define MAC_LEN (17)
#define ETHERTYPE_LEN (6)
#define MAC_ARR_LEN (6)
@@ -393,6 +393,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
printf(" --output=<units>");
printf(" Set verbosity output level: bandwidth , message_rate, latency \n");
+ printf(" Latency measurement is Average calculation \n");
printf(" --pkey_index=<pkey index> PKey index to use for QP\n");
@@ -507,10 +508,11 @@ void usage_raw_ethernet(TestType tst)
printf(" Disable Scatter FCS feature. (Scatter FCS is enabled by default when using --use_exp flag). \n");
#endif
- if (tst == LAT) {
- printf(" --flows");
- printf(" set number of TCP/UDP flows, starting from <src_port, dst_port>. \n");
- }
+ printf(" --flows");
+ printf(" set number of TCP/UDP flows, starting from <src_port, dst_port>. \n");
+
+ printf(" --flows_burst");
+ printf(" set number of burst size per TCP/UDP flow. \n");
printf(" --promiscuous");
printf(" run promiscuous mode.\n");
@@ -624,6 +626,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
user_param->traffic_class = 0;
user_param->disable_fcs = 0;
user_param->flows = DEF_FLOWS;
+ user_param->flows_burst = 1;
}
/******************************************************************************
@@ -749,6 +752,38 @@ void get_gbps_str_by_ibv_rate(char *rate_input_value, int *rate)
/******************************************************************************
*
******************************************************************************/
+void flow_rules_force_dependecies(struct perftest_parameters *user_param)
+{
+ int min_iter_req = 0;
+ if (user_param->flows != DEF_FLOWS) {
+ if (user_param->is_server_port == OFF) {
+ fprintf(stderr, " Flows feature works with UDP/TCP packets only for now\n");
+ exit(1);
+ }
+ if (user_param->test_type == ITERATIONS) {
+ min_iter_req = user_param->flows * user_param->flows_burst;
+ if (user_param->iters / min_iter_req < 1) {
+ fprintf(stderr, " Current iteration number will not complete full cycle on all flows, it need to be multiple of the product between flows and flows_burst\n");
+ fprintf(stderr, " Set N*%d Iterations \n", user_param->flows * user_param->flows_burst);
+ exit(1);
+ }
+ }
+ if (user_param->duplex) {
+ fprintf(stderr, " Flows is currently designed to work with unidir tests only\n");
+ exit(1);
+ }
+ } else {
+ if (user_param->flows_burst > 1) {
+ fprintf(stderr, " Flows burst is designed to work with more then single flow\n");
+ exit(1);
+ }
+ }
+ return;
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
static void force_dependecies(struct perftest_parameters *user_param)
{
/*Additional configuration and assignments.*/
@@ -917,19 +952,7 @@ static void force_dependecies(struct perftest_parameters *user_param)
exit(1);
}
- if (user_param->flows != DEF_FLOWS) {
-
- if (user_param->tst != LAT) {
- fprintf(stderr, " Flows feature works with Latency test only\n");
- exit(1);
- }
-
- /* UDP/TCP must have server_port, so this check is enough */
- if (user_param->is_server_port == OFF) {
- fprintf(stderr, " Flows feature works with UDP/TCP packets only for now\n");
- exit(1);
- }
- }
+ flow_rules_force_dependecies(user_param);
}
if (user_param->use_mcg && user_param->gid_index == -1) {
@@ -1079,7 +1102,7 @@ static void force_dependecies(struct perftest_parameters *user_param)
}
} else if (user_param->rate_limit_type == HW_RATE_LIMIT) {
if (user_param->use_rdma_cm == ON || user_param->work_rdma_cm == ON) {
- fprintf(stderr," HW rate limit isn't supported yet with rdma_cm flows\n");
+ fprintf(stderr," HW rate limit isn't supported yet with rdma_cm scenarios\n");
exit(1);
}
double rate_limit_gbps = 0;
@@ -1224,7 +1247,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
return;
}
-
/******************************************************************************
*
******************************************************************************/
@@ -1505,6 +1527,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
static int wait_destroy_flag = 0;
static int disable_fcs_flag = 0;
static int flows_flag = 0;
+ static int flows_burst_flag = 0;
init_perftest_params(user_param);
@@ -1602,6 +1625,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
{ .name = "disable_fcs", .has_arg = 0, .flag = &disable_fcs_flag, .val = 1},
#endif
{ .name = "flows", .has_arg = 1, .flag = &flows_flag, .val = 1},
+ { .name = "flows_burst", .has_arg = 1, .flag = &flows_burst_flag, .val = 1},
{ 0 }
};
c = getopt_long(argc,argv,"w:y:p:d:i:m:s:n:t:u:S:x:c:q:I:o:M:r:Q:A:l:D:f:B:T:E:J:j:K:k:aFegzRvhbNVCHUOZP",long_options,NULL);
@@ -1940,12 +1964,21 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
}
if (flows_flag) {
user_param->flows = (uint16_t)strtol(optarg, NULL, 0);
- if (user_param->flows <= 0) {
+ if (user_param->flows == 0) {
fprintf(stderr, "Invalid flows value. Please set a positive number\n");
return FAILURE;
}
flows_flag = 0;
}
+ if (flows_burst_flag) {
+ user_param->flows_burst = (uint16_t)strtol(optarg, NULL, 0);
+ if (user_param->flows_burst == 0) {
+ fprintf(stderr, "Invalid burst flow value. Please set a positive number\n");
+ return FAILURE;
+ }
+ flows_burst_flag = 0;
+
+ }
break;
default:
@@ -2485,62 +2518,93 @@ static int cycles_compare(const void *aptr, const void *bptr)
/******************************************************************************
*
******************************************************************************/
+#define cycles_to_usec(sample, cycles_to_units, factor) ((((sample) / (cycles_to_units)) / (factor)))
+
void print_report_lat (struct perftest_parameters *user_param)
{
int i;
int rtt_factor;
- double cycles_to_units;
- cycles_t median;
+ double cycles_to_units, temp_var, pow_var;
+ cycles_t median ;
cycles_t *delta = NULL;
const char* units;
- double latency;
+ double latency, stdev, average_sum = 0 , average, stdev_sum = 0;
+ int iters_99,iters_99_9;
rtt_factor = (user_param->verb == READ || user_param->verb == ATOMIC) ? 1 : 2;
ALLOCATE(delta,cycles_t,user_param->iters - 1);
- for (i = 0; i < user_param->iters - 1; ++i)
- delta[i] = user_param->tposted[i + 1] - user_param->tposted[i];
-
if (user_param->r_flag->cycles) {
cycles_to_units = 1;
units = "cycles";
-
} else {
cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f);
units = "usec";
}
+ for (i = 0; i < user_param->iters - 1; ++i)
+ delta[i] = user_param->tposted[i + 1] - user_param->tposted[i];
+
if (user_param->r_flag->unsorted) {
printf("#, %s\n", units);
for (i = 0; i < user_param->iters - 1; ++i)
- printf("%d, %g\n", i + 1, delta[i] / cycles_to_units / rtt_factor);
+ printf("%d, %g\n", i + 1, cycles_to_usec(delta[i], cycles_to_units, rtt_factor));
}
qsort(delta, user_param->iters - 1, sizeof *delta, cycles_compare);
+ median = get_median(user_param->iters - 1, delta);
+
+ iters_99 = ceil((user_param->iters - 1 )*0.99);
+ iters_99_9 = ceil((user_param->iters - 1)*0.999);
+
+ /* calcualte average sum on sorted array*/
+ for (i = 0; i < user_param->iters - 1; ++i)
+ average_sum += cycles_to_usec(delta[i], cycles_to_units, rtt_factor);
+
+ average = average_sum / (user_param->iters - 1);
+
+ /* Calculate stdev by variance*/
+ for (i = 0; i < user_param->iters - 1; ++i) {
+ temp_var = average - cycles_to_usec(delta[i], cycles_to_units, rtt_factor);
+ pow_var = pow(temp_var, 2 );
+ stdev_sum += pow_var;
+ }
+ stdev = sqrt(stdev_sum / (user_param->iters));
+
if (user_param->r_flag->histogram) {
printf("#, %s\n", units);
for (i = 0; i < user_param->iters - 1; ++i)
- printf("%d, %g\n", i + 1, delta[i] / cycles_to_units / rtt_factor);
+ printf("%d, %g\n", i + 1, cycles_to_usec(delta[i], cycles_to_units, rtt_factor));
}
- median = get_median(user_param->iters - 1, delta);
+ if (user_param->r_flag->unsorted || user_param->r_flag->histogram) {
+ if (user_param->output == FULL_VERBOSITY) {
+ printf(RESULT_LINE);
+ printf("%s",(user_param->test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+ printf((user_param->cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+ }
+ }
- latency = median / cycles_to_units / rtt_factor;
+ latency = cycles_to_usec(median, cycles_to_units, rtt_factor);
- if (user_param->output == OUTPUT_LAT) {
- printf("%lf\n",latency);
- }
+ if (user_param->output == OUTPUT_LAT)
+ printf("%lf\n",average);
else {
printf(REPORT_FMT_LAT,
(unsigned long)user_param->size,
user_param->iters,
- delta[0] / cycles_to_units / rtt_factor,
- delta[user_param->iters - 2] / cycles_to_units / rtt_factor,
- latency);
+ cycles_to_usec(delta[0], cycles_to_units, rtt_factor),
+ cycles_to_usec(delta[user_param->iters - 2], cycles_to_units, rtt_factor),
+ latency,
+ average,
+ stdev,
+ cycles_to_usec(delta[iters_99], cycles_to_units, rtt_factor),
+ cycles_to_usec(delta[iters_99_9], cycles_to_units, rtt_factor));
printf( user_param->cpu_util_data.enable ? REPORT_EXT_CPU_UTIL : REPORT_EXT , calc_cpu_util(user_param));
}
+
free(delta);
}
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index 1122792..faea8c3 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -181,7 +181,7 @@
#define RESULT_FMT_G_QOS " #bytes #sl #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]"
-#define RESULT_FMT_LAT " #bytes #iterations t_min[usec] t_max[usec] t_typical[usec]"
+#define RESULT_FMT_LAT " #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99""%"" percentile[usec] 99.9""%"" percentile[usec] "
#define RESULT_FMT_LAT_DUR " #bytes #iterations t_avg[usec] "
@@ -203,7 +203,7 @@
#define REPORT_FMT_QOS " %-7lu %d %lu %-7.2lf %-7.2lf %-7.6lf\n"
/* Result print format for latency tests. */
-#define REPORT_FMT_LAT " %-7lu %d %-7.2f %-7.2f %-7.2f"
+#define REPORT_FMT_LAT " %-7lu %d %-7.2f %-7.2f %-7.2f %-7.2f %-7.2f %-7.2f %-7.2f"
#define REPORT_FMT_LAT_DUR " %-7lu %d %-7.2f"
@@ -434,6 +434,7 @@ struct perftest_parameters {
uint32_t wait_destroy;
int disable_fcs;
int flows;
+ int flows_burst;
};
struct report_options {
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index f23dabb..afae5f2 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2254,7 +2254,7 @@ int ctx_connect(struct pingpong_context *ctx,
if (user_param->rate_limit_type == HW_RATE_LIMIT) {
struct ibv_qp_attr qp_attr;
struct ibv_qp_init_attr init_attr;
- int err, qp_static_rate;
+ int err, qp_static_rate=0;
memset(&qp_attr,0,sizeof(struct ibv_qp_attr));
memset(&init_attr,0,sizeof(struct ibv_qp_init_attr));
@@ -2271,7 +2271,7 @@ int ctx_connect(struct pingpong_context *ctx,
user_param->rate_limit_type = SW_RATE_LIMIT;
fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit, providing SW rate limit \x1b[0m\n");
} else {
- fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit \x1b[0m\n");
+ fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit \x1b[0m\n");
return FAILURE;
}
@@ -2880,6 +2880,10 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
int cpu_mhz = 0;
int return_value = 0;
int wc_id;
+ int send_flows_index = 0;
+ uintptr_t primary_send_addr = ctx->sge_list[0].addr;
+ int flows_burst_iter = 0;
+ int address_offset = 0;
ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
@@ -2944,7 +2948,6 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
/* main loop to run over all the qps and post each time n messages */
for (index =0 ; index < num_of_qps ; index++) {
-
if (user_param->rate_limit_type == SW_RATE_LIMIT && is_sending_burst == 0) {
if (gap_deadline > get_cycles()) {
/* Go right to cq polling until gap time is over. */
@@ -2957,6 +2960,7 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
while ((ctx->scnt[index] < user_param->iters || user_param->test_type == DURATION) && (ctx->scnt[index] - ctx->ccnt[index]) < (user_param->tx_depth) &&
!((user_param->rate_limit_type == SW_RATE_LIMIT ) && is_sending_burst == 0)) {
+
if (ctx->send_rcredit) {
uint32_t swindow = ctx->scnt[index] + user_param->post_list - ctx->credit_buf[index];
if (swindow >= user_param->rx_depth)
@@ -2993,14 +2997,14 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
for (pl_index = 0; pl_index < user_param->post_list; pl_index++) {
sg_l = ctx->exp_wr[index*user_param->post_list + pl_index].sg_list;
ctx->qp_burst_family[index]->send_pending(ctx->qp[index], sg_l->addr, sg_l->length, sg_l->lkey,
- ctx->exp_wr[index*user_param->post_list + pl_index].exp_send_flags);
+ ctx->exp_wr[index*user_param->post_list + pl_index].exp_send_flags);
}
ctx->qp_burst_family[index]->send_flush(ctx->qp[index]);
} else {
#endif
if (user_param->use_exp == 1) {
err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],
- &ctx->exp_wr[index*user_param->post_list],&bad_exp_wr);
+ &ctx->exp_wr[index*user_param->post_list],&bad_exp_wr);
}
else {
err = (ctx->post_send_func_pointer)(ctx->qp[index],
@@ -3018,15 +3022,27 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
goto cleaning;
}
+ if (user_param->flows != DEF_FLOWS) {
+ /*we inc the address after we post_send , maybe need to skip next If*/
+ if(++flows_burst_iter == user_param->flows_burst) {
+ flows_burst_iter = 0;
+ if (++send_flows_index == user_param->flows)
+ send_flows_index = 0;
+ address_offset = send_flows_index * ctx->cycle_buffer;
+ ctx->sge_list[0].addr = primary_send_addr + address_offset;
+ }
+ }
+
+ /* in multiple flow scenarios we will go to next cycle buffer address in the main buffer*/
if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
#ifdef HAVE_VERBS_EXP
if (user_param->use_exp == 1)
increase_loc_addr(ctx->exp_wr[index].sg_list,user_param->size,
- ctx->scnt[index],ctx->my_addr[index],0,ctx->cache_line_size,ctx->cycle_buffer);
+ ctx->scnt[index],ctx->my_addr[index] + address_offset,0,ctx->cache_line_size,ctx->cycle_buffer);
else
#endif
increase_loc_addr(ctx->wr[index].sg_list,user_param->size,ctx->scnt[index],
- ctx->my_addr[index],0,ctx->cache_line_size,ctx->cycle_buffer);
+ ctx->my_addr[index] + address_offset ,0,ctx->cache_line_size,ctx->cycle_buffer);
if (user_param->verb != SEND) {
#ifdef HAVE_VERBS_EXP
@@ -3044,7 +3060,6 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
ctx->scnt[index] += user_param->post_list;
totscnt += user_param->post_list;
-
/* ask for completion on this wr */
if (user_param->post_list == 1 &&
(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
@@ -3074,63 +3089,60 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
}
}
}
-
if (totccnt < tot_iters || (user_param->test_type == DURATION && totccnt < totscnt)) {
- if (user_param->use_event) {
- if (ctx_notify_events(ctx->channel)) {
- fprintf(stderr, "Couldn't request CQ notification\n");
- return_value = 1;
- goto cleaning;
+ if (user_param->use_event) {
+ if (ctx_notify_events(ctx->channel)) {
+ fprintf(stderr, "Couldn't request CQ notification\n");
+ return_value = 1;
+ goto cleaning;
+ }
}
- }
- #ifdef HAVE_ACCL_VERBS
- if (user_param->verb_type == ACCL_INTF)
- ne = ctx->send_cq_family->poll_cnt(ctx->send_cq, CTX_POLL_BATCH);
- else
- #endif
- ne = ibv_poll_cq(ctx->send_cq,CTX_POLL_BATCH,wc);
+ #ifdef HAVE_ACCL_VERBS
+ if (user_param->verb_type == ACCL_INTF)
+ ne = ctx->send_cq_family->poll_cnt(ctx->send_cq, CTX_POLL_BATCH);
+ else
+ #endif
+ ne = ibv_poll_cq(ctx->send_cq,CTX_POLL_BATCH,wc);
- if (ne > 0) {
- for (i = 0; i < ne; i++) {
- wc_id = (user_param->verb_type == ACCL_INTF) ?
- 0 : (int)wc[i].wr_id;
+ if (ne > 0) {
+ for (i = 0; i < ne; i++) {
+ wc_id = (user_param->verb_type == ACCL_INTF) ?
+ 0 : (int)wc[i].wr_id;
- if (user_param->verb_type != ACCL_INTF) {
- if (wc[i].status != IBV_WC_SUCCESS) {
- NOTIFY_COMP_ERROR_SEND(wc[i],totscnt,totccnt);
- return_value = 1;
- goto cleaning;
+ if (user_param->verb_type != ACCL_INTF) {
+ if (wc[i].status != IBV_WC_SUCCESS) {
+ NOTIFY_COMP_ERROR_SEND(wc[i],totscnt,totccnt);
+ return_value = 1;
+ goto cleaning;
+ }
}
- }
-
- ctx->ccnt[wc_id] += user_param->cq_mod;
- totccnt += user_param->cq_mod;
- if (user_param->noPeak == OFF) {
+ ctx->ccnt[wc_id] += user_param->cq_mod;
+ totccnt += user_param->cq_mod;
+ if (user_param->noPeak == OFF) {
- if (totccnt >= tot_iters - 1)
- user_param->tcompleted[user_param->iters*num_of_qps - 1] = get_cycles();
- else
- user_param->tcompleted[totccnt-1] = get_cycles();
- }
+ if (totccnt >= tot_iters - 1)
+ user_param->tcompleted[user_param->iters*num_of_qps - 1] = get_cycles();
+ else
+ user_param->tcompleted[totccnt-1] = get_cycles();
+ }
- if (user_param->test_type==DURATION && user_param->state == SAMPLE_STATE) {
- if (user_param->report_per_port) {
- user_param->iters_per_port[user_param->port_by_qp[wc_id]] += user_param->cq_mod;
+ if (user_param->test_type==DURATION && user_param->state == SAMPLE_STATE) {
+ if (user_param->report_per_port) {
+ user_param->iters_per_port[user_param->port_by_qp[wc_id]] += user_param->cq_mod;
+ }
+ user_param->iters += user_param->cq_mod;
}
- user_param->iters += user_param->cq_mod;
}
- }
- } else if (ne < 0) {
- fprintf(stderr, "poll CQ failed %d\n",ne);
- return_value = 1;
- goto cleaning;
- }
+ } else if (ne < 0) {
+ fprintf(stderr, "poll CQ failed %d\n",ne);
+ return_value = 1;
+ goto cleaning;
+ }
}
}
-
if (user_param->noPeak == ON && user_param->test_type == ITERATIONS)
user_param->tcompleted[0] = get_cycles();
@@ -3180,6 +3192,10 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
user_param->rx_depth/user_param->num_of_qps : user_param->rx_depth;
int return_value = 0;
int wc_id;
+ int recv_flows_index = 0;
+ uintptr_t primary_recv_addr = ctx->recv_sge_list[0].addr;
+ int recv_flows_burst = 0;
+ int address_flows_offset =0;
ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
ALLOCATE(swc ,struct ibv_wc ,user_param->tx_depth);
@@ -3203,7 +3219,7 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
check_alive_data.g_total_iters = tot_iters;
- while (rcnt < tot_iters || (user_param->test_type == DURATION && user_param->state != END_STATE)) {
+ while (rcnt < tot_iters || (user_param->test_type == DURATION && user_param->state != END_STATE)) {
if (user_param->use_event) {
if (ctx_notify_events(ctx->channel)) {
@@ -3282,6 +3298,16 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
}
}
+
+ if (user_param->flows != DEF_FLOWS) {
+ if (++recv_flows_burst == user_param->flows_burst) {
+ recv_flows_burst = 0;
+ if (++recv_flows_index == user_param->flows)
+ recv_flows_index = 0;
+ address_flows_offset = recv_flows_index * ctx->cycle_buffer;
+ ctx->recv_sge_list[0].addr = primary_recv_addr + address_flows_offset;
+ }
+ }
#ifdef HAVE_ACCL_VERBS
}
#endif
@@ -3289,7 +3315,7 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
increase_loc_addr(ctx->rwr[wc_id].sg_list,
user_param->size,
rcnt_for_qp[wc_id] + size_per_qp,
- ctx->rx_buffer_addr[wc_id],
+ ctx->rx_buffer_addr[wc_id] + address_flows_offset,
user_param->connection_type,ctx->cache_line_size,ctx->cycle_buffer);
}
}
@@ -4462,6 +4488,8 @@ int check_masked_atomics_support(struct pingpong_context *ctx)
MASK_IS_SET(IBV_EXP_DEVICE_EXT_ATOMICS, attr.exp_device_cap_flags);
}
#endif
+
+
/******************************************************************************
* End
******************************************************************************/
diff --git a/src/raw_ethernet_resources.c b/src/raw_ethernet_resources.c
index 3b1516e..be3b607 100755
--- a/src/raw_ethernet_resources.c
+++ b/src/raw_ethernet_resources.c
@@ -442,7 +442,7 @@ void create_raw_eth_pkt( struct perftest_parameters *user_param,
struct raw_ethernet_info *rem_dest_info)
{
int offset = 0;
- int i;
+ int i, print_flag = 0;
struct ETH_header* eth_header;
uint16_t ip_next_protocol = 0;
uint16_t eth_type = user_param->is_ethertype ? user_param->ethertype :
@@ -454,30 +454,31 @@ void create_raw_eth_pkt( struct perftest_parameters *user_param,
eth_header = (void*)ctx->buf[0];
- /* build single packet on ctx buffer */
- build_pkt_on_buffer(eth_header, my_dest_info, rem_dest_info, user_param,
- eth_type, ip_next_protocol, PRINT_ON,
- ctx->size - RAWETH_ADDITION, 0);
-
if (user_param->tst == BW) {
- /* fill ctx buffer with same packets */
- if (ctx->size <= (ctx->cycle_buffer / 2)) {
- while (offset < ctx->cycle_buffer-INC(ctx->size,ctx->cache_line_size)) {
- offset += INC(ctx->size, ctx->cache_line_size);
- eth_header = (void*)ctx->buf[0] + offset;
+ /* fill ctx buffer with different packets according to flows_offset */
+ for (i = 0; i < user_param->flows; i++) {
+ print_flag = PRINT_ON;
+ offset = (ctx->cycle_buffer) * i; /* update the offset to next flow */
+ eth_header = (void*)ctx->buf[0] + offset;/* update the eth_header to next flow */
+ /* fill ctx buffer with same packets */
+ while (offset-(ctx->cycle_buffer * i) < ctx->cycle_buffer-INC(ctx->size,ctx->cache_line_size)) {
build_pkt_on_buffer(eth_header, my_dest_info, rem_dest_info,
user_param, eth_type, ip_next_protocol,
- PRINT_OFF ,ctx->size - RAWETH_ADDITION, 0);
+ print_flag , ctx->size - RAWETH_ADDITION, i);
+ print_flag = PRINT_OFF;
+ offset += INC(ctx->size, ctx->cache_line_size);/* update the offset to next packet in same flow */
+ eth_header = (void*)ctx->buf[0] + offset;/* update the eth_header to next packet in same flow */
}
}
- } else if (user_param->tst == LAT && user_param->flows != DEF_FLOWS) {
+ } else if (user_param->tst == LAT) {
/* fill ctx buffer with different packets according to flows_offset */
- for (i = 1; i < user_param->flows; i++) {
- offset += INC(ctx->size, ctx->cache_line_size);
- eth_header = (void*)ctx->buf[0] + offset;
+ for (i = 0; i < user_param->flows; i++) {
build_pkt_on_buffer(eth_header, my_dest_info, rem_dest_info,
user_param, eth_type, ip_next_protocol,
PRINT_ON ,ctx->size - RAWETH_ADDITION, i);
+ offset += INC(ctx->size, ctx->cache_line_size);
+ eth_header = (void*)ctx->buf[0] + offset;
+
}
}
@@ -614,8 +615,8 @@ static int set_up_flow_rules(
if(user_param->machine == SERVER) {
- spec_info->tcp_udp.val.dst_port = htons(user_param->server_port);
- spec_info->tcp_udp.val.src_port = htons(user_param->client_port);
+ spec_info->tcp_udp.val.dst_port = htons(user_param->server_port + flows_offset);
+ spec_info->tcp_udp.val.src_port = htons(user_param->client_port + flows_offset);
} else {
spec_info->tcp_udp.val.dst_port = htons(user_param->client_port + flows_offset);
@@ -630,7 +631,6 @@ static int set_up_flow_rules(
spec_info->eth.val.ether_type = htons(user_param->ethertype);
spec_info->eth.mask.ether_type = 0xffff;
}
-
return 0;
}
diff --git a/src/raw_ethernet_send_bw.c b/src/raw_ethernet_send_bw.c
index 89b8ba0..30bd171 100755
--- a/src/raw_ethernet_send_bw.c
+++ b/src/raw_ethernet_send_bw.c
@@ -67,18 +67,18 @@ int main(int argc, char *argv[])
struct perftest_parameters user_param;
#ifdef HAVE_RAW_ETH_EXP
- struct ibv_exp_flow *flow_create_result = NULL;
- struct ibv_exp_flow_attr *flow_rules = NULL;
- struct ibv_exp_flow *flow_promisc = NULL;
+ struct ibv_exp_flow **flow_create_result;
+ struct ibv_exp_flow_attr **flow_rules;
+ struct ibv_exp_flow *flow_promisc = NULL ;
#else
- struct ibv_flow *flow_create_result = NULL;
- struct ibv_flow_attr *flow_rules = NULL;
+ struct ibv_flow **flow_create_result;
+ struct ibv_flow_attr **flow_rules;
#endif
-
+ int i;
union ibv_gid mgid;
/* init default values to user's parameters */
- memset(&ctx, 0,sizeof(struct pingpong_context));
+ memset(&ctx, 0, sizeof(struct pingpong_context));
memset(&user_param, 0 , sizeof(struct perftest_parameters));
memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info));
memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info));
@@ -88,16 +88,24 @@ int main(int argc, char *argv[])
strncpy(user_param.version, VERSION, sizeof(user_param.version));
user_param.connection_type = RawEth;
- ret_parser = parser(&user_param,argv,argc);
+ ret_parser = parser(&user_param, argv, argc);
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) {
- fprintf(stderr," Parser function exited with Error\n");
+ fprintf(stderr, " Parser function exited with Error\n");
}
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE,"<<<<<<%s", __FUNCTION__);
return 1;
}
+ #ifdef HAVE_RAW_ETH_EXP
+ ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows);
+ ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows);
+ #else
+ ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows);
+ ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows);
+ #endif
+
if (user_param.raw_mcast) {
/* Transform IPv4 to Multicast MAC */
user_param.dest_mac[0] = 0x01;
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 1;
}
GET_STRING(user_param.ib_devname, ibv_get_device_name(ib_dev));
@@ -142,23 +150,23 @@ int main(int argc, char *argv[])
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 1;
}
/* See if MTU and link type are valid and supported. */
- if (check_link_and_mtu(ctx.context,&user_param)) {
+ if (check_link_and_mtu(ctx.context, &user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return FAILURE;
}
/* Allocating arrays needed for the test. */
- alloc_ctx(&ctx,&user_param);
+ alloc_ctx(&ctx, &user_param);
/* set mac address by user choose */
- if (send_set_up_connection(&flow_rules,&ctx,&user_param,&my_dest_info,&rem_dest_info)) {
- fprintf(stderr," Unable to set up socket connection\n");
+ if (send_set_up_connection(flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) {
+ fprintf(stderr, " Unable to set up socket connection\n");
return 1;
}
@@ -166,32 +174,33 @@ int main(int argc, char *argv[])
ctx_print_test_info(&user_param);
if ( !user_param.raw_mcast && (user_param.machine == SERVER || user_param.duplex)) {
- print_spec(flow_rules,&user_param);
+ for (i = 0; i < user_param.flows; i++)
+ print_spec(flow_rules[i], &user_param);
}
/* Create (if necessary) the rdma_cm ids and channel. */
if (user_param.work_rdma_cm == ON) {
- if (create_rdma_resources(&ctx,&user_param)) {
- fprintf(stderr," Unable to create the rdma_resources\n");
+ if (create_rdma_resources(&ctx, &user_param)) {
+ fprintf(stderr, " Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
- if (rdma_client_connect(&ctx,&user_param)) {
- fprintf(stderr,"Unable to perform rdma_client function\n");
+ if (rdma_client_connect(&ctx, &user_param)) {
+ fprintf(stderr, "Unable to perform rdma_client function\n");
return FAILURE;
}
- } else if (rdma_server_connect(&ctx,&user_param)) {
- fprintf(stderr,"Unable to perform rdma_client function\n");
+ } else if (rdma_server_connect(&ctx, &user_param)) {
+ fprintf(stderr, "Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
- if (ctx_init(&ctx,&user_param)) {
+ if (ctx_init(&ctx, &user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
@@ -199,14 +208,48 @@ int main(int argc, char *argv[])
/* build raw Ethernet packets on ctx buffer */
if((user_param.machine == CLIENT || user_param.duplex) && !user_param.mac_fwd) {
- create_raw_eth_pkt(&user_param,&ctx, &my_dest_info , &rem_dest_info);
+ create_raw_eth_pkt(&user_param, &ctx, &my_dest_info, &rem_dest_info);
}
+ /* create flow rules for servers/duplex clients , that not test raw_mcast */
+ if ( !user_param.raw_mcast && (user_param.machine == SERVER || user_param.duplex)) {
+
+ /* attaching the qp to the spec */
+ for (i = 0; i < user_param.flows; i++) {
+ #ifdef HAVE_RAW_ETH_EXP
+ flow_create_result[i] = ibv_exp_create_flow(ctx.qp[0], flow_rules[i]);
+ #else
+ flow_create_result[i] = ibv_create_flow(ctx.qp[0], flow_rules[i]);
+ #endif
+
+ if (!flow_create_result[i]){
+ perror("error");
+ fprintf(stderr, "Couldn't attach QP\n");
+ return FAILURE;
+ }
+ }
+
+ #ifdef HAVE_RAW_ETH_EXP
+ if (user_param.use_promiscuous) {
+ struct ibv_exp_flow_attr attr = {
+ .type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
+ .num_of_specs = 0,
+ .port = user_param.ib_port,
+ .flags = 0
+ };
+
+ if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
+ perror("error");
+ fprintf(stderr, "Couldn't attach promiscous rule QP\n");
+ }
+ }
+ #endif
+ }
/* Prepare IB resources for rtr/rts. */
if (user_param.work_rdma_cm == OFF) {
- if (ctx_connect(&ctx,NULL,&user_param,NULL)) {
- fprintf(stderr," Unable to Connect the HCA's through the link\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ if (ctx_connect(&ctx, NULL, &user_param, NULL)) {
+ fprintf(stderr, " Unable to Connect the HCA's through the link\n");
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 1;
}
}
@@ -215,7 +258,7 @@ int main(int argc, char *argv[])
if (user_param.machine == SERVER) {
/* join Multicast group by MGID */
ibv_attach_mcast(ctx.qp[0], &mgid, 0);
- printf(PERF_RAW_MGID_FMT,"MGID",
+ printf(PERF_RAW_MGID_FMT, "MGID",
mgid.raw[0], mgid.raw[1],
mgid.raw[2], mgid.raw[3],
mgid.raw[4], mgid.raw[5],
@@ -226,38 +269,6 @@ int main(int argc, char *argv[])
mgid.raw[14],mgid.raw[15]);
}
}
- else {
- /* attaching the qp to the spec */
- if(user_param.machine == SERVER || user_param.duplex) {
- #ifdef HAVE_RAW_ETH_EXP
- flow_create_result = ibv_exp_create_flow(ctx.qp[0], flow_rules);
- #else
- flow_create_result = ibv_create_flow(ctx.qp[0], flow_rules);
- #endif
- if (!flow_create_result){
- perror("error");
- fprintf(stderr, "Couldn't attach QP\n");
- return FAILURE;
- }
-
- #ifdef HAVE_RAW_ETH_EXP
- if (user_param.use_promiscuous) {
- struct ibv_exp_flow_attr attr = {
- .type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
- .num_of_specs = 0,
- .port = user_param.ib_port,
- .flags = 0
- };
-
- if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
- perror("error");
- fprintf(stderr, "Couldn't attach promiscous rule QP\n");
- }
- }
- #endif
-
- }
- }
if (user_param.output == FULL_VERBOSITY) {
printf(RESULT_LINE);
@@ -270,122 +281,116 @@ int main(int argc, char *argv[])
if (user_param.test_method == RUN_REGULAR) {
if (user_param.machine == CLIENT || user_param.duplex) {
- ctx_set_send_wqes(&ctx,&user_param,NULL);
+ ctx_set_send_wqes(&ctx, &user_param, NULL);
}
if (user_param.machine == SERVER || user_param.duplex) {
- if (ctx_set_recv_wqes(&ctx,&user_param)) {
+ if (ctx_set_recv_wqes(&ctx, &user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 1;
}
}
if (user_param.mac_fwd) {
- if(run_iter_fw(&ctx,&user_param)) {
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ if(run_iter_fw(&ctx, &user_param)) {
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return FAILURE;
}
} else if (user_param.duplex) {
- if(run_iter_bi(&ctx,&user_param)) {
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ if(run_iter_bi(&ctx, &user_param)) {
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return FAILURE;
}
} else if (user_param.machine == CLIENT) {
- if(run_iter_bw(&ctx,&user_param)) {
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ if(run_iter_bw(&ctx, &user_param)) {
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return FAILURE;
}
} else {
- if(run_iter_bw_server(&ctx,&user_param)) {
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ if(run_iter_bw_server(&ctx, &user_param)) {
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 17;
}
}
- print_report_bw(&user_param,NULL);
+ print_report_bw(&user_param, NULL);
} else if (user_param.test_method == RUN_INFINITELY) {
if (user_param.machine == CLIENT)
- ctx_set_send_wqes(&ctx,&user_param,NULL);
+ ctx_set_send_wqes(&ctx, &user_param, NULL);
else if (user_param.machine == SERVER) {
- if (ctx_set_recv_wqes(&ctx,&user_param)) {
- fprintf(stderr," Failed to post receive recv_wqes\n");
+ if (ctx_set_recv_wqes(&ctx, &user_param)) {
+ fprintf(stderr, "Failed to post receive recv_wqes\n");
return 1;
}
}
if (user_param.machine == CLIENT) {
- if(run_iter_bw_infinitely(&ctx,&user_param)) {
- fprintf(stderr," Error occured while running infinitely! aborting ...\n");
+ if(run_iter_bw_infinitely(&ctx, &user_param)) {
+ fprintf(stderr, " Error occured while running infinitely! aborting ...\n");
return 1;
}
} else if (user_param.machine == SERVER) {
- if(run_iter_bw_infinitely_server(&ctx,&user_param)) {
- fprintf(stderr," Error occured while running infinitely on server! aborting ...\n");
+ if(run_iter_bw_infinitely_server(&ctx, &user_param)) {
+ fprintf(stderr, " Error occured while running infinitely on server! aborting ...\n");
return 1;
}
}
}
-
if(user_param.machine == SERVER || user_param.duplex) {
-
- if (user_param.raw_mcast) {
- if (ibv_detach_mcast(ctx.qp[0], &mgid,0)) {
- perror("error");
- fprintf(stderr,"Couldn't leave multicast group\n");
- }
- } else {
+ /* destroy flow */
+ for (i = 0; i < user_param.flows; i++) {
#ifdef HAVE_RAW_ETH_EXP
- if (user_param.use_promiscuous) {
- if (ibv_exp_destroy_flow(flow_promisc)) {
- perror("error");
- fprintf(stderr, "Couldn't Destory promisc flow\n");
- return FAILURE;
- }
- }
- #endif
-
- #ifdef HAVE_RAW_ETH_EXP
- if (ibv_exp_destroy_flow(flow_create_result)) {
+ if (ibv_exp_destroy_flow(flow_create_result[i])) {
#else
- if (ibv_destroy_flow(flow_create_result)) {
+ if (ibv_destroy_flow(flow_create_result[i])) {
#endif
perror("error");
fprintf(stderr, "Couldn't Destory flow\n");
return FAILURE;
}
- free(flow_rules);
+
+ free(flow_rules[i]);
+ }
+ }
+ if(user_param.machine == SERVER || user_param.duplex) {
+
+ if (user_param.raw_mcast) {
+ if (ibv_detach_mcast(ctx.qp[0], &mgid, 0)) {
+ perror("error");
+ fprintf(stderr, "Couldn't leave multicast group\n");
+ }
}
}
if (destroy_ctx(&ctx, &user_param)) {
- fprintf(stderr,"Failed to destroy_ctx\n");
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ fprintf(stderr, "Failed to destroy_ctx\n");
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 1;
}
/* limit verifier */
if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
- fprintf(stderr,"Error: BW result is below bw limit\n");
+ fprintf(stderr, "Error: BW result is below bw limit\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY)
printf(RESULT_LINE);
- DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+ DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
return 0;
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ofed/perftest.git
More information about the Pkg-ofed-commits
mailing list