1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
From 6d8f1feed063e04d0419c0d895e919a24c7c24d4 Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Tue, 8 Nov 2022 20:20:44 +0800
Subject: Perftest: replace rand() with getrandom() during MR buffer
initialization
rand() has very poor performance in some OS.
ib_send_bw will spend a lot of time during MR initialization when
testing large packects in above scenario.
test has been done:
"""
\#define HUGE_MR_SIZE 2147483647
int main(int argc, char *argv[])
{
char *a = malloc(HUGE_MR_SIZE * sizeof(char));
unsigned int i;
char *tmp = a;
int ret;
srand(time(NULL));
if (a == NULL)
exit(1);
if (argc <= 1)
goto fall_back;
for (i = HUGE_MR_SIZE; i > 0;) {
ret = getrandom(tmp, i, 0);
if (ret < 0)
goto fall_back;
tmp += ret;
i -= ret;
}
goto out;
fall_back:
for(i = 0; i < HUGE_MR_SIZE; i++)
a[i] = (char)rand();
out:
free(a);
return 0;
}
time ./a.out
real 5m35.033s
user 5m33.546s
sys 0m0.918s
time ./a.out 1
real 0m6.454s
user 0m0.000s
sys 0m6.449s
"""
As shown in the test above, getrandom() has a much better performance,
so replace rand() with it.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
configure.ac | 1 +
src/perftest_resources.c | 31 ++++++++++++++++++++++++++-----
2 files changed, 27 insertions(+), 5 deletions(-)
diff --git a/configure.ac b/configure.ac
index 21a17bc..2bbc7fc 100755
--- a/configure.ac
+++ b/configure.ac
@@ -60,6 +60,7 @@ AC_PROG_LIBTOOL
AC_PROG_RANLIB
AC_HEADER_STDC
AC_CHECK_HEADERS([infiniband/verbs.h],,[AC_MSG_ERROR([ibverbs header files not found])])
+AC_CHECK_HEADERS([sys/random.h],,)
AC_CHECK_LIB([ibverbs], [ibv_get_device_list], [], [AC_MSG_ERROR([libibverbs not found])])
AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-devel not found]))
AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found]))
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 33db58e..23c31d1 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -22,6 +22,9 @@
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
+#ifdef HAVE_SYS_RANDOM_H
+#include <sys/random.h>
+#endif
#ifdef HAVE_SRD
#include <infiniband/efadv.h>
#endif
@@ -1542,12 +1545,33 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa
return ret;
}
+static void random_data(char *buf, int buff_size)
+{
+ int i;
+#ifdef HAVE_SYS_RANDOM_H
+ char *tmp = buf;
+ int ret;
+
+ for(i = buff_size; i > 0;) {
+ ret = getrandom(tmp, i, 0);
+ if(ret < 0)
+ goto fall_back;
+ tmp += ret;
+ i -= ret;
+ }
+ return;
+fall_back:
+#endif
+ srand(time(NULL));
+ for (i = 0; i < buff_size; i++)
+ buf[i] = (char)rand();
+}
+
/******************************************************************************
*
******************************************************************************/
int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param, int qp_index)
{
- int i;
int flags = IBV_ACCESS_LOCAL_WRITE;
@@ -1686,13 +1710,10 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
#ifdef HAVE_CUDA
if (!user_param->use_cuda) {
#endif
- srand(time(NULL));
if (user_param->verb == WRITE && user_param->tst == LAT) {
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else {
- for (i = 0; i < ctx->buff_size; i++) {
- ((char*)ctx->buf[qp_index])[i] = (char)rand();
- }
+ random_data(ctx->buf[qp_index], ctx->buff_size);
}
#ifdef HAVE_CUDA
}
--
2.34.1
|
