Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2020 Intel Corporation.
3 : : * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
4 : : * All rights reserved.
5 : : */
6 : :
7 : : #include "spdk/stdinc.h"
8 : : #include "spdk/thread.h"
9 : : #include "spdk/env.h"
10 : : #include "spdk/event.h"
11 : : #include "spdk/log.h"
12 : : #include "spdk/string.h"
13 : : #include "spdk/accel.h"
14 : : #include "spdk/crc32.h"
15 : : #include "spdk/util.h"
16 : : #include "spdk/xor.h"
17 : : #include "spdk/dif.h"
18 : :
19 : : #define DATA_PATTERN 0x5a
20 : : #define ALIGN_4K 0x1000
21 : : #define COMP_BUF_PAD_PERCENTAGE 1.1L
22 : :
23 : : static uint64_t g_tsc_rate;
24 : : static uint64_t g_tsc_end;
25 : : static int g_rc;
26 : : static int g_xfer_size_bytes = 4096;
27 : : static int g_block_size_bytes = 512;
28 : : static int g_md_size_bytes = 8;
29 : : static int g_queue_depth = 32;
30 : : /* g_allocate_depth indicates how many tasks we allocate per worker. It will
31 : : * be at least as much as the queue depth.
32 : : */
33 : : static int g_allocate_depth = 0;
34 : : static int g_threads_per_core = 1;
35 : : static int g_time_in_sec = 5;
36 : : static uint32_t g_crc32c_seed = 0;
37 : : static uint32_t g_chained_count = 1;
38 : : static int g_fail_percent_goal = 0;
39 : : static uint8_t g_fill_pattern = 255;
40 : : static uint32_t g_xor_src_count = 2;
41 : : static bool g_verify = false;
42 : : static const char *g_workload_type = NULL;
43 : : static enum spdk_accel_opcode g_workload_selection = SPDK_ACCEL_OPC_LAST;
44 : : static const char *g_module_name = NULL;
45 : : static struct worker_thread *g_workers = NULL;
46 : : static int g_num_workers = 0;
47 : : static char *g_cd_file_in_name = NULL;
48 : : static pthread_mutex_t g_workers_lock = PTHREAD_MUTEX_INITIALIZER;
49 : : static struct spdk_app_opts g_opts = {};
50 : :
51 : : struct ap_compress_seg {
52 : : void *uncompressed_data;
53 : : uint32_t uncompressed_len;
54 : : struct iovec *uncompressed_iovs;
55 : : uint32_t uncompressed_iovcnt;
56 : :
57 : : void *compressed_data;
58 : : uint32_t compressed_len;
59 : : uint32_t compressed_len_padded;
60 : : struct iovec *compressed_iovs;
61 : : uint32_t compressed_iovcnt;
62 : :
63 : : STAILQ_ENTRY(ap_compress_seg) link;
64 : : };
65 : :
66 : : static STAILQ_HEAD(, ap_compress_seg) g_compress_segs = STAILQ_HEAD_INITIALIZER(g_compress_segs);
67 : :
68 : : struct worker_thread;
69 : : static void accel_done(void *ref, int status);
70 : :
71 : : struct display_info {
72 : : int core;
73 : : int thread;
74 : : };
75 : :
76 : : struct ap_task {
77 : : void *src;
78 : : struct iovec *src_iovs;
79 : : uint32_t src_iovcnt;
80 : : void **sources;
81 : : struct iovec *dst_iovs;
82 : : uint32_t dst_iovcnt;
83 : : struct iovec md_iov;
84 : : void *dst;
85 : : void *dst2;
86 : : uint32_t *crc_dst;
87 : : uint32_t compressed_sz;
88 : : struct ap_compress_seg *cur_seg;
89 : : struct worker_thread *worker;
90 : : int expected_status; /* used for the compare operation */
91 : : uint32_t num_blocks; /* used for the DIF related operations */
92 : : struct spdk_dif_ctx dif_ctx;
93 : : struct spdk_dif_error dif_err;
94 : : TAILQ_ENTRY(ap_task) link;
95 : : };
96 : :
97 : : struct worker_thread {
98 : : struct spdk_io_channel *ch;
99 : : struct spdk_accel_opcode_stats stats;
100 : : uint64_t xfer_failed;
101 : : uint64_t injected_miscompares;
102 : : uint64_t current_queue_depth;
103 : : TAILQ_HEAD(, ap_task) tasks_pool;
104 : : struct worker_thread *next;
105 : : unsigned core;
106 : : struct spdk_thread *thread;
107 : : bool is_draining;
108 : : struct spdk_poller *is_draining_poller;
109 : : struct spdk_poller *stop_poller;
110 : : void *task_base;
111 : : struct display_info display;
112 : : enum spdk_accel_opcode workload;
113 : : };
114 : :
115 : : static void
116 : 73 : dump_user_config(void)
117 : : {
118 : 73 : const char *module_name = NULL;
119 : : int rc;
120 : :
121 : 73 : rc = spdk_accel_get_opc_module_name(g_workload_selection, &module_name);
122 [ - + ]: 73 : if (rc) {
123 [ # # ]: 0 : printf("error getting module name (%d)\n", rc);
124 : 0 : }
125 : :
126 [ - + ]: 73 : printf("\nSPDK Configuration:\n");
127 [ - + ]: 73 : printf("Core mask: %s\n\n", g_opts.reactor_mask);
128 [ - + ]: 73 : printf("Accel Perf Configuration:\n");
129 [ - + ]: 73 : printf("Workload Type: %s\n", g_workload_type);
130 [ + + # # ]: 73 : if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
131 [ + + ]: 67 : g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
132 [ - + ]: 12 : printf("CRC-32C seed: %u\n", g_crc32c_seed);
133 [ + + ]: 61 : } else if (g_workload_selection == SPDK_ACCEL_OPC_FILL) {
134 [ - + ]: 3 : printf("Fill pattern: 0x%x\n", g_fill_pattern);
135 [ + + - + ]: 58 : } else if ((g_workload_selection == SPDK_ACCEL_OPC_COMPARE) && g_fail_percent_goal > 0) {
136 [ # # ]: 0 : printf("Failure inject: %u percent\n", g_fail_percent_goal);
137 [ + + ]: 58 : } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
138 [ - + ]: 6 : printf("Source buffers: %u\n", g_xor_src_count);
139 : 0 : }
140 [ + + # # ]: 73 : if (g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
141 [ + + ]: 67 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
142 [ + + ]: 64 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
143 [ + + ]: 58 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
144 [ + - ]: 55 : g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
145 [ + + ]: 55 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY) {
146 [ - + ]: 21 : printf("Vector size: %u bytes\n", g_xfer_size_bytes);
147 [ - + ]: 21 : printf("Transfer size: %u bytes\n", g_xfer_size_bytes * g_chained_count);
148 : 0 : } else {
149 [ - + ]: 52 : printf("Transfer size: %u bytes\n", g_xfer_size_bytes);
150 : : }
151 [ + + # # ]: 73 : if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
152 [ + + ]: 67 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
153 [ + - ]: 64 : g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
154 [ + + ]: 64 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
155 [ - + ]: 12 : printf("Block size: %u bytes\n", g_block_size_bytes);
156 [ - + ]: 12 : printf("Metadata size: %u bytes\n", g_md_size_bytes);
157 : 0 : }
158 [ - + ]: 73 : printf("Vector count %u\n", g_chained_count);
159 [ - + ]: 73 : printf("Module: %s\n", module_name);
160 [ + + # # ]: 73 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS ||
161 [ + + ]: 69 : g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
162 [ - + ]: 28 : printf("File Name: %s\n", g_cd_file_in_name);
163 : 0 : }
164 [ - + ]: 73 : printf("Queue depth: %u\n", g_queue_depth);
165 [ - + ]: 73 : printf("Allocate depth: %u\n", g_allocate_depth);
166 [ - + ]: 73 : printf("# threads/core: %u\n", g_threads_per_core);
167 [ - + ]: 73 : printf("Run time: %u seconds\n", g_time_in_sec);
168 [ - + + + : 73 : printf("Verify: %s\n\n", g_verify ? "Yes" : "No");
- + ]
169 : 73 : }
170 : :
171 : : static void
172 : 9 : usage(void)
173 : : {
174 [ - + ]: 9 : printf("accel_perf options:\n");
175 [ - + ]: 9 : printf("\t[-h help message]\n");
176 [ - + ]: 9 : printf("\t[-q queue depth per core]\n");
177 [ - + ]: 9 : printf("\t[-C for supported workloads, use this value to configure the io vector size to test (default 1)\n");
178 [ - + ]: 9 : printf("\t[-T number of threads per core\n");
179 [ - + ]: 9 : printf("\t[-o transfer size in bytes (default: 4KiB. For compress/decompress, 0 means the input file size)]\n");
180 [ - + ]: 9 : printf("\t[-t time in seconds]\n");
181 [ - + ]: 9 : printf("\t[-w workload type must be one of these: copy, fill, crc32c, copy_crc32c, compare, compress, decompress, dualcast, xor,\n");
182 [ - + ]: 9 : printf("\t[ dif_verify, dif_verify_copy, dif_generate, dif_generate_copy, dix_generate, dix_verify\n");
183 [ - + ]: 9 : printf("\t[-M assign module to the operation, not compatible with accel_assign_opc RPC\n");
184 [ - + ]: 9 : printf("\t[-l for compress/decompress workloads, name of uncompressed input file\n");
185 [ - + ]: 9 : printf("\t[-S for crc32c workload, use this seed value (default 0)\n");
186 [ - + ]: 9 : printf("\t[-P for compare workload, percentage of operations that should miscompare (percent, default 0)\n");
187 [ - + ]: 9 : printf("\t[-f for fill workload, use this BYTE value (default 255)\n");
188 [ - + ]: 9 : printf("\t[-x for xor workload, use this number of source buffers (default, minimum: 2)]\n");
189 [ - + ]: 9 : printf("\t[-y verify result if this switch is on]\n");
190 [ - + ]: 9 : printf("\t[-a tasks to allocate per core (default: same value as -q)]\n");
191 [ - + ]: 9 : printf("\t\tCan be used to spread operations across a wider range of memory.\n");
192 : 9 : }
193 : :
194 : : static int
195 : 305 : parse_args(int ch, char *arg)
196 : : {
197 : 305 : int argval = 0;
198 : :
199 [ + + ]: 305 : switch (ch) {
200 : 129 : case 'a':
201 : : case 'C':
202 : : case 'f':
203 : : case 'T':
204 : : case 'o':
205 : : case 'P':
206 : : case 'q':
207 : : case 'S':
208 : : case 't':
209 : : case 'x':
210 : 129 : argval = spdk_strtol(optarg, 10);
211 [ + + ]: 129 : if (argval < 0) {
212 [ - + ]: 3 : fprintf(stderr, "-%c option must be non-negative.\n", ch);
213 : 3 : usage();
214 : 3 : return 1;
215 : : }
216 : 126 : break;
217 : 176 : default:
218 : 176 : break;
219 : : };
220 : :
221 [ + + + + : 302 : switch (ch) {
+ + - + +
+ + + + -
- ]
222 : 3 : case 'a':
223 : 3 : g_allocate_depth = argval;
224 : 3 : break;
225 : 6 : case 'C':
226 : 6 : g_chained_count = argval;
227 : 6 : break;
228 : 31 : case 'l':
229 : 31 : g_cd_file_in_name = optarg;
230 : 31 : break;
231 : 3 : case 'f':
232 : 3 : g_fill_pattern = (uint8_t)argval;
233 : 3 : break;
234 : 8 : case 'T':
235 : 8 : g_threads_per_core = argval;
236 : 8 : break;
237 : 12 : case 'o':
238 : 12 : g_xfer_size_bytes = argval;
239 : 12 : break;
240 : 0 : case 'P':
241 : 0 : g_fail_percent_goal = argval;
242 : 0 : break;
243 : 3 : case 'q':
244 : 3 : g_queue_depth = argval;
245 : 3 : break;
246 : 3 : case 'S':
247 : 3 : g_crc32c_seed = argval;
248 : 3 : break;
249 : 85 : case 't':
250 : 85 : g_time_in_sec = argval;
251 : 85 : break;
252 : 3 : case 'x':
253 : 3 : g_xor_src_count = argval;
254 : 3 : break;
255 : 60 : case 'y':
256 : 60 : g_verify = true;
257 : 60 : break;
258 : 85 : case 'w':
259 : 85 : g_workload_type = optarg;
260 [ - + + + : 85 : if (!strcmp(g_workload_type, "copy")) {
# # ]
261 : 3 : g_workload_selection = SPDK_ACCEL_OPC_COPY;
262 [ - + + + : 82 : } else if (!strcmp(g_workload_type, "fill")) {
# # ]
263 : 3 : g_workload_selection = SPDK_ACCEL_OPC_FILL;
264 [ - + + + : 79 : } else if (!strcmp(g_workload_type, "crc32c")) {
# # ]
265 : 6 : g_workload_selection = SPDK_ACCEL_OPC_CRC32C;
266 [ - + + + : 73 : } else if (!strcmp(g_workload_type, "copy_crc32c")) {
# # ]
267 : 6 : g_workload_selection = SPDK_ACCEL_OPC_COPY_CRC32C;
268 [ - + + + : 67 : } else if (!strcmp(g_workload_type, "compare")) {
# # ]
269 : 3 : g_workload_selection = SPDK_ACCEL_OPC_COMPARE;
270 [ - + + + : 64 : } else if (!strcmp(g_workload_type, "dualcast")) {
# # ]
271 : 3 : g_workload_selection = SPDK_ACCEL_OPC_DUALCAST;
272 [ - + + + : 61 : } else if (!strcmp(g_workload_type, "compress")) {
# # ]
273 : 10 : g_workload_selection = SPDK_ACCEL_OPC_COMPRESS;
274 [ - + + + : 51 : } else if (!strcmp(g_workload_type, "decompress")) {
# # ]
275 : 24 : g_workload_selection = SPDK_ACCEL_OPC_DECOMPRESS;
276 [ - + + + : 27 : } else if (!strcmp(g_workload_type, "xor")) {
# # ]
277 : 9 : g_workload_selection = SPDK_ACCEL_OPC_XOR;
278 [ - + + + : 18 : } else if (!strcmp(g_workload_type, "dif_verify")) {
# # ]
279 : 3 : g_workload_selection = SPDK_ACCEL_OPC_DIF_VERIFY;
280 [ - + - + : 15 : } else if (!strcmp(g_workload_type, "dif_verify_copy")) {
# # ]
281 : 0 : g_workload_selection = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
282 [ - + + + : 15 : } else if (!strcmp(g_workload_type, "dif_generate")) {
# # ]
283 : 6 : g_workload_selection = SPDK_ACCEL_OPC_DIF_GENERATE;
284 [ - + + + : 9 : } else if (!strcmp(g_workload_type, "dif_generate_copy")) {
# # ]
285 : 3 : g_workload_selection = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
286 [ - + + + : 6 : } else if (!strcmp(g_workload_type, "dix_verify")) {
# # ]
287 : 3 : g_workload_selection = SPDK_ACCEL_OPC_DIX_VERIFY;
288 [ - + - + : 3 : } else if (!strcmp(g_workload_type, "dix_generate")) {
# # ]
289 : 0 : g_workload_selection = SPDK_ACCEL_OPC_DIX_GENERATE;
290 : 0 : } else {
291 [ - + ]: 3 : fprintf(stderr, "Unsupported workload type: %s\n", optarg);
292 : 3 : usage();
293 : 3 : return 1;
294 : : }
295 : 82 : break;
296 : 0 : case 'M':
297 : 0 : g_module_name = optarg;
298 : 0 : break;
299 : :
300 : 0 : default:
301 : 0 : usage();
302 : 0 : return 1;
303 : : }
304 : :
305 : 299 : return 0;
306 : 0 : }
307 : :
308 : : static int dump_result(void);
309 : : static void
310 : 105 : unregister_worker(void *arg1)
311 : : {
312 : 105 : struct worker_thread *worker = arg1;
313 : :
314 [ + - # # : 105 : if (worker->ch) {
# # ]
315 [ # # # # : 105 : spdk_accel_get_opcode_stats(worker->ch, worker->workload,
# # # # ]
316 [ # # ]: 0 : &worker->stats, sizeof(worker->stats));
317 [ # # # # ]: 105 : spdk_put_io_channel(worker->ch);
318 [ # # # # ]: 105 : worker->ch = NULL;
319 : 0 : }
320 [ # # # # ]: 105 : free(worker->task_base);
321 : 105 : spdk_thread_exit(spdk_get_thread());
322 [ - + ]: 105 : pthread_mutex_lock(&g_workers_lock);
323 [ - + # # ]: 105 : assert(g_num_workers >= 1);
324 [ + + # # ]: 105 : if (--g_num_workers == 0) {
325 [ - + ]: 73 : pthread_mutex_unlock(&g_workers_lock);
326 : : /* Only dump results on successful runs */
327 [ + - ]: 73 : if (g_rc == 0) {
328 : 73 : g_rc = dump_result();
329 : 0 : }
330 : 73 : spdk_app_stop(g_rc);
331 : 0 : } else {
332 [ - + ]: 32 : pthread_mutex_unlock(&g_workers_lock);
333 : : }
334 : 105 : }
335 : :
336 : : static void
337 : 2380 : accel_perf_construct_iovs(void *buf, uint64_t sz, struct iovec *iovs, uint32_t iovcnt)
338 : : {
339 : : uint64_t ele_size;
340 : : uint8_t *data;
341 : : uint32_t i;
342 : :
343 : 2380 : ele_size = spdk_divide_round_up(sz, iovcnt);
344 : :
345 : 2380 : data = buf;
346 [ + + ]: 4760 : for (i = 0; i < iovcnt; i++) {
347 [ # # ]: 2380 : ele_size = spdk_min(ele_size, sz);
348 [ - + # # ]: 2380 : assert(ele_size > 0);
349 : :
350 [ # # # # : 2380 : iovs[i].iov_base = data;
# # ]
351 [ # # # # : 2380 : iovs[i].iov_len = ele_size;
# # ]
352 : :
353 [ # # ]: 2380 : data += ele_size;
354 : 2380 : sz -= ele_size;
355 : 0 : }
356 [ - + # # ]: 2380 : assert(sz == 0);
357 : 2380 : }
358 : :
359 : : static int
360 : 3456 : _get_task_data_bufs(struct ap_task *task)
361 : : {
362 : 3456 : uint32_t align = 0;
363 : 3456 : uint32_t i = 0;
364 : 3456 : int src_buff_len = g_xfer_size_bytes;
365 : 3456 : int dst_buff_len = g_xfer_size_bytes;
366 : : int md_buff_len;
367 : 992 : struct spdk_dif_ctx_init_ext_opts dif_opts;
368 : : uint32_t num_blocks, transfer_size_with_md;
369 : : int rc;
370 : :
371 : : /* For dualcast, the DSA HW requires 4K alignment on destination addresses but
372 : : * we do this for all modules to keep it simple.
373 : : */
374 [ + + ]: 3456 : if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST) {
375 : 96 : align = ALIGN_4K;
376 : 0 : }
377 : :
378 [ + + # # ]: 3456 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS ||
379 [ + + ]: 3328 : g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
380 [ # # # # ]: 1920 : task->cur_seg = STAILQ_FIRST(&g_compress_segs);
381 : :
382 [ + + ]: 1920 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
383 [ # # # # : 128 : dst_buff_len = task->cur_seg->compressed_len_padded;
# # # # ]
384 : 0 : }
385 : :
386 [ # # # # ]: 1920 : task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
387 [ - + # # : 1920 : if (task->dst == NULL) {
# # ]
388 [ # # # # ]: 0 : fprintf(stderr, "Unable to alloc dst buffer\n");
389 : 0 : return -ENOMEM;
390 : : }
391 : :
392 [ # # # # ]: 1920 : task->dst_iovs = calloc(g_chained_count, sizeof(struct iovec));
393 [ - + # # : 1920 : if (!task->dst_iovs) {
# # ]
394 [ # # # # ]: 0 : fprintf(stderr, "cannot allocate task->dst_iovs for task=%p\n", task);
395 : 0 : return -ENOMEM;
396 : : }
397 [ # # # # ]: 1920 : task->dst_iovcnt = g_chained_count;
398 [ # # # # : 1920 : accel_perf_construct_iovs(task->dst, dst_buff_len, task->dst_iovs, task->dst_iovcnt);
# # # # #
# # # ]
399 : :
400 : 1920 : return 0;
401 : : }
402 : :
403 [ + + ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY) {
404 [ # # # # ]: 96 : task->dst_iovcnt = g_chained_count;
405 [ # # # # : 96 : task->dst_iovs = calloc(task->dst_iovcnt, sizeof(struct iovec));
# # # # ]
406 [ - + # # : 96 : if (!task->dst_iovs) {
# # ]
407 [ # # # # ]: 0 : fprintf(stderr, "cannot allocate task->dst_iovs for task=%p\n", task);
408 : 0 : return -ENOMEM;
409 : : }
410 : :
411 [ - + ]: 96 : num_blocks = g_xfer_size_bytes / g_block_size_bytes;
412 : : /* Add bytes for each block for metadata */
413 : 96 : transfer_size_with_md = g_xfer_size_bytes + (num_blocks * g_md_size_bytes);
414 [ # # # # ]: 96 : task->num_blocks = num_blocks;
415 : :
416 [ + + # # : 192 : for (i = 0; i < task->dst_iovcnt; i++) {
# # ]
417 [ # # # # : 96 : task->dst_iovs[i].iov_base = spdk_dma_zmalloc(transfer_size_with_md, 0, NULL);
# # # # #
# ]
418 [ - + # # : 96 : if (task->dst_iovs[i].iov_base == NULL) {
# # # # #
# # # ]
419 : 0 : return -ENOMEM;
420 : : }
421 [ # # # # : 96 : task->dst_iovs[i].iov_len = transfer_size_with_md;
# # # # #
# ]
422 : 0 : }
423 : :
424 : 96 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
425 [ # # ]: 96 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
426 : :
427 [ # # ]: 96 : rc = spdk_dif_ctx_init(&task->dif_ctx,
428 [ # # ]: 96 : g_block_size_bytes + g_md_size_bytes,
429 : 0 : g_md_size_bytes, true, true,
430 : : SPDK_DIF_TYPE1,
431 [ # # # # : 0 : SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
# # ]
432 : : 0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
433 [ - + ]: 96 : if (rc != 0) {
434 [ # # # # ]: 0 : fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
435 : 0 : return rc;
436 : : }
437 : 0 : }
438 : :
439 [ - + ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
440 : : /* Allocate source buffers */
441 [ # # # # ]: 0 : task->src_iovcnt = g_chained_count;
442 [ # # # # : 0 : task->src_iovs = calloc(task->src_iovcnt, sizeof(struct iovec));
# # # # ]
443 [ # # # # : 0 : if (!task->src_iovs) {
# # ]
444 [ # # # # ]: 0 : fprintf(stderr, "cannot allocate task->src_iovs for task=%p\n", task);
445 : 0 : return -ENOMEM;
446 : : }
447 : :
448 [ # # ]: 0 : num_blocks = g_xfer_size_bytes / g_block_size_bytes;
449 : : /* Add bytes for each block for metadata */
450 : 0 : transfer_size_with_md = g_xfer_size_bytes + (num_blocks * g_md_size_bytes);
451 [ # # # # ]: 0 : task->num_blocks = num_blocks;
452 : :
453 [ # # # # : 0 : for (i = 0; i < task->src_iovcnt; i++) {
# # ]
454 [ # # # # : 0 : task->src_iovs[i].iov_base = spdk_dma_zmalloc(transfer_size_with_md, 0, NULL);
# # # # #
# ]
455 [ # # # # : 0 : if (task->src_iovs[i].iov_base == NULL) {
# # # # #
# # # ]
456 : 0 : return -ENOMEM;
457 : : }
458 [ # # # # : 0 : memset(task->src_iovs[i].iov_base, DATA_PATTERN, transfer_size_with_md);
# # # # #
# # # ]
459 [ # # # # : 0 : task->src_iovs[i].iov_len = transfer_size_with_md;
# # # # #
# ]
460 : 0 : }
461 : :
462 : : /* Allocate destination buffers */
463 [ # # # # ]: 0 : task->dst_iovcnt = g_chained_count;
464 [ # # # # : 0 : task->dst_iovs = calloc(task->dst_iovcnt, sizeof(struct iovec));
# # # # ]
465 [ # # # # : 0 : if (!task->dst_iovs) {
# # ]
466 [ # # # # ]: 0 : fprintf(stderr, "cannot allocated task->dst_iovs fot task=%p\n", task);
467 : 0 : return -ENOMEM;
468 : : }
469 : :
470 [ # # # # : 0 : for (i = 0; i < task->dst_iovcnt; i++) {
# # ]
471 [ # # # # : 0 : task->dst_iovs[i].iov_base = spdk_dma_zmalloc(dst_buff_len, 0, NULL);
# # # # #
# ]
472 [ # # # # : 0 : if (task->dst_iovs[i].iov_base == NULL) {
# # # # #
# # # ]
473 : 0 : return -ENOMEM;
474 : : }
475 [ # # # # : 0 : task->dst_iovs[i].iov_len = dst_buff_len;
# # # # #
# ]
476 : 0 : }
477 : :
478 : 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
479 [ # # ]: 0 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
480 : :
481 : : /* Init DIF ctx */
482 [ # # ]: 0 : rc = spdk_dif_ctx_init(&task->dif_ctx,
483 [ # # ]: 0 : g_block_size_bytes + g_md_size_bytes,
484 : 0 : g_md_size_bytes, true, true,
485 : : SPDK_DIF_TYPE1,
486 [ # # # # : 0 : SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
# # ]
487 : : 0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
488 [ # # ]: 0 : if (rc != 0) {
489 [ # # # # ]: 0 : fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
490 : 0 : return rc;
491 : : }
492 : :
493 [ # # # # : 0 : rc = spdk_dif_generate(task->src_iovs, task->src_iovcnt, task->num_blocks, &task->dif_ctx);
# # # # #
# # # #
# ]
494 [ # # ]: 0 : if (rc != 0) {
495 [ # # # # ]: 0 : fprintf(stderr, "Generation of DIF failed, error (%d)\n", rc);
496 : 0 : return rc;
497 : : }
498 : 0 : }
499 : :
500 [ + + # # ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
501 [ + + ]: 1344 : g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
502 [ # # # # ]: 384 : task->crc_dst = spdk_dma_zmalloc(sizeof(*task->crc_dst), 0, NULL);
503 : 0 : }
504 : :
505 [ + + # # ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
506 [ + + ]: 1344 : g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
507 [ + + ]: 1152 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
508 [ + + ]: 1056 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
509 [ + + ]: 864 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY ||
510 [ + + ]: 768 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
511 [ - + ]: 672 : g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) {
512 [ - + # # ]: 864 : assert(g_chained_count > 0);
513 [ # # # # ]: 864 : task->src_iovcnt = g_chained_count;
514 [ # # # # : 864 : task->src_iovs = calloc(task->src_iovcnt, sizeof(struct iovec));
# # # # ]
515 [ - + # # : 864 : if (!task->src_iovs) {
# # ]
516 [ # # # # ]: 0 : fprintf(stderr, "cannot allocated task->src_iovs fot task=%p\n", task);
517 : 0 : return -ENOMEM;
518 : : }
519 : :
520 [ + + ]: 864 : if (g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
521 : 192 : dst_buff_len = g_xfer_size_bytes * g_chained_count;
522 : 0 : }
523 : :
524 [ + + # # ]: 864 : if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
525 [ + + ]: 672 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY) {
526 [ - + # # : 288 : src_buff_len += (g_xfer_size_bytes / g_block_size_bytes) * g_md_size_bytes;
# # ]
527 : 0 : }
528 : :
529 [ + + # # : 1920 : for (i = 0; i < task->src_iovcnt; i++) {
# # ]
530 [ # # # # : 1056 : task->src_iovs[i].iov_base = spdk_dma_zmalloc(src_buff_len, 0, NULL);
# # # # #
# ]
531 [ - + # # : 1056 : if (task->src_iovs[i].iov_base == NULL) {
# # # # #
# # # ]
532 : 0 : return -ENOMEM;
533 : : }
534 [ - + # # : 1056 : memset(task->src_iovs[i].iov_base, DATA_PATTERN, src_buff_len);
# # # # #
# # # ]
535 [ # # # # : 1056 : task->src_iovs[i].iov_len = src_buff_len;
# # # # #
# ]
536 : 0 : }
537 [ + - # # ]: 864 : if (g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
538 [ + + ]: 864 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
539 [ - + # # ]: 96 : md_buff_len = (g_xfer_size_bytes / g_block_size_bytes) * g_md_size_bytes *
540 : 0 : g_chained_count;
541 [ # # # # : 96 : task->md_iov.iov_base = spdk_dma_zmalloc(md_buff_len, 0, NULL);
# # ]
542 [ - + # # : 96 : if (task->md_iov.iov_base == NULL) {
# # # # ]
543 : 0 : return -ENOMEM;
544 : : }
545 [ # # # # : 96 : task->md_iov.iov_len = md_buff_len;
# # ]
546 : 0 : }
547 [ + + ]: 672 : } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
548 [ - + # # ]: 192 : assert(g_xor_src_count > 1);
549 [ # # # # ]: 192 : task->sources = calloc(g_xor_src_count, sizeof(*task->sources));
550 [ - + # # : 192 : if (!task->sources) {
# # ]
551 : 0 : return -ENOMEM;
552 : : }
553 : :
554 [ + + ]: 672 : for (i = 0; i < g_xor_src_count; i++) {
555 [ # # # # : 480 : task->sources[i] = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
# # # # ]
556 [ - + # # : 480 : if (!task->sources[i]) {
# # # # #
# ]
557 : 0 : return -ENOMEM;
558 : : }
559 [ - + # # : 480 : memset(task->sources[i], DATA_PATTERN, g_xfer_size_bytes);
# # # # #
# ]
560 : 0 : }
561 : 0 : } else {
562 [ # # # # ]: 480 : task->src = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
563 [ - + # # : 480 : if (task->src == NULL) {
# # ]
564 [ # # # # ]: 0 : fprintf(stderr, "Unable to alloc src buffer\n");
565 : 0 : return -ENOMEM;
566 : : }
567 : :
568 : : /* For fill, set the entire src buffer so we can check if verify is enabled. */
569 [ + + ]: 480 : if (g_workload_selection == SPDK_ACCEL_OPC_FILL) {
570 [ - + # # : 192 : memset(task->src, g_fill_pattern, g_xfer_size_bytes);
# # ]
571 : 0 : } else {
572 [ - + # # : 288 : memset(task->src, DATA_PATTERN, g_xfer_size_bytes);
# # ]
573 : : }
574 : : }
575 : :
576 [ + + # # ]: 1536 : if (g_workload_selection != SPDK_ACCEL_OPC_CRC32C &&
577 [ + + ]: 1344 : g_workload_selection != SPDK_ACCEL_OPC_DIF_VERIFY &&
578 [ + + ]: 1248 : g_workload_selection != SPDK_ACCEL_OPC_DIF_GENERATE &&
579 [ + + ]: 1056 : g_workload_selection != SPDK_ACCEL_OPC_DIF_GENERATE_COPY &&
580 [ + - ]: 960 : g_workload_selection != SPDK_ACCEL_OPC_DIF_VERIFY_COPY &&
581 [ + + ]: 960 : g_workload_selection != SPDK_ACCEL_OPC_DIX_VERIFY &&
582 [ + - ]: 864 : g_workload_selection != SPDK_ACCEL_OPC_DIX_GENERATE) {
583 [ # # # # ]: 864 : task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
584 [ - + # # : 864 : if (task->dst == NULL) {
# # ]
585 [ # # # # ]: 0 : fprintf(stderr, "Unable to alloc dst buffer\n");
586 : 0 : return -ENOMEM;
587 : : }
588 : :
589 : : /* For compare we want the buffers to match, otherwise not. */
590 [ + + ]: 864 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPARE) {
591 [ - + # # : 96 : memset(task->dst, DATA_PATTERN, dst_buff_len);
# # ]
592 : 0 : } else {
593 [ - + # # : 768 : memset(task->dst, ~DATA_PATTERN, dst_buff_len);
# # ]
594 : : }
595 : 0 : }
596 : :
597 : : /* For dualcast 2 buffers are needed for the operation. */
598 [ + + # # ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST ||
599 [ + + - + : 1440 : (g_workload_selection == SPDK_ACCEL_OPC_XOR && g_verify)) {
+ - ]
600 [ # # # # ]: 288 : task->dst2 = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
601 [ - + # # : 288 : if (task->dst2 == NULL) {
# # ]
602 [ # # # # ]: 0 : fprintf(stderr, "Unable to alloc dst buffer\n");
603 : 0 : return -ENOMEM;
604 : : }
605 [ - + # # : 288 : memset(task->dst2, ~DATA_PATTERN, g_xfer_size_bytes);
# # ]
606 : 0 : }
607 : :
608 [ + + # # ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
609 [ + + ]: 1344 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
610 [ - + ]: 1248 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
611 : 288 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
612 [ # # ]: 288 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
613 : :
614 [ - + # # : 288 : task->num_blocks = (g_xfer_size_bytes * g_chained_count) / g_block_size_bytes;
# # ]
615 : :
616 [ # # ]: 288 : rc = spdk_dif_ctx_init(&task->dif_ctx,
617 [ # # ]: 288 : g_block_size_bytes + g_md_size_bytes,
618 : 0 : g_md_size_bytes, true, true,
619 : : SPDK_DIF_TYPE1,
620 [ # # # # : 0 : SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
# # ]
621 : : 16, 0xFFFF, 10, 0, 0, &dif_opts);
622 [ - + ]: 288 : if (rc != 0) {
623 [ # # # # ]: 0 : fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
624 : 0 : return rc;
625 : : }
626 : :
627 [ + + # # ]: 288 : if ((g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY) ||
628 [ - + ]: 192 : (g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY)) {
629 [ # # # # : 96 : rc = spdk_dif_generate(task->src_iovs, task->src_iovcnt, task->num_blocks, &task->dif_ctx);
# # # # #
# # # #
# ]
630 [ - + ]: 96 : if (rc != 0) {
631 [ # # # # ]: 0 : fprintf(stderr, "Generation of DIF failed, error (%d)\n", rc);
632 : 0 : return rc;
633 : : }
634 : 0 : }
635 : 0 : }
636 [ + - # # ]: 1536 : if (g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
637 [ + + ]: 1536 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
638 : 96 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
639 [ # # ]: 96 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
640 : :
641 [ - + # # : 96 : task->num_blocks = (g_xfer_size_bytes * g_chained_count) / g_block_size_bytes;
# # ]
642 : :
643 [ # # ]: 96 : rc = spdk_dif_ctx_init(&task->dif_ctx,
644 : 0 : g_block_size_bytes,
645 : 0 : g_md_size_bytes, false, true,
646 : : SPDK_DIF_TYPE1,
647 [ # # # # ]: 0 : SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK |
648 [ # # ]: 0 : SPDK_DIF_FLAGS_REFTAG_CHECK,
649 : : 0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
650 [ - + ]: 96 : if (rc != 0) {
651 [ # # # # ]: 0 : fprintf(stderr, "Initialization of DIX context failed, error (%d)\n", rc);
652 : 0 : return rc;
653 : : }
654 [ + - ]: 96 : if (g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
655 [ # # # # : 96 : rc = spdk_dix_generate(task->src_iovs, task->src_iovcnt, &task->md_iov,
# # # # #
# ]
656 [ # # # # : 96 : task->num_blocks, &task->dif_ctx);
# # ]
657 [ - + ]: 96 : if (rc != 0) {
658 [ # # # # ]: 0 : fprintf(stderr, "Generation of DIX failed, error (%d)\n", rc);
659 : 0 : return rc;
660 : : }
661 : 0 : }
662 : :
663 : 0 : }
664 : :
665 : 1536 : return 0;
666 : 0 : }
667 : :
668 : : inline static struct ap_task *
669 : 14546974 : _get_task(struct worker_thread *worker)
670 : : {
671 : : struct ap_task *task;
672 : :
673 [ + - # # : 14546974 : if (!TAILQ_EMPTY(&worker->tasks_pool)) {
# # # # ]
674 [ # # # # : 14546974 : task = TAILQ_FIRST(&worker->tasks_pool);
# # ]
675 [ + + # # : 14546974 : TAILQ_REMOVE(&worker->tasks_pool, task, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
676 : 0 : } else {
677 [ # # # # ]: 0 : fprintf(stderr, "Unable to get ap_task\n");
678 : 0 : return NULL;
679 : : }
680 : :
681 : 14546974 : return task;
682 : 0 : }
683 : :
684 : : /* Submit one operation using the same ap task that just completed. */
685 : : static void
686 : 14546974 : _submit_single(struct worker_thread *worker, struct ap_task *task)
687 : : {
688 : : int random_num;
689 : 14546974 : int rc = 0;
690 : :
691 [ - + # # ]: 14546974 : assert(worker);
692 : :
693 [ + + + + : 14546974 : switch (worker->workload) {
+ + + + +
+ + + - -
+ - # # #
# ]
694 : 950688 : case SPDK_ACCEL_OPC_COPY:
695 [ # # # # : 950688 : rc = spdk_accel_submit_copy(worker->ch, task->dst, task->src,
# # # # #
# # # ]
696 : 0 : g_xfer_size_bytes, accel_done, task);
697 : 950688 : break;
698 : 1373824 : case SPDK_ACCEL_OPC_FILL:
699 : : /* For fill use the first byte of the task->dst buffer */
700 [ # # # # : 1373824 : rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
# # # # #
# # # #
# ]
701 : 0 : g_xfer_size_bytes, accel_done, task);
702 : 1373824 : break;
703 : 2437824 : case SPDK_ACCEL_OPC_CRC32C:
704 [ # # # # : 2437824 : rc = spdk_accel_submit_crc32cv(worker->ch, task->crc_dst,
# # # # ]
705 [ # # # # : 0 : task->src_iovs, task->src_iovcnt, g_crc32c_seed,
# # # # ]
706 : 0 : accel_done, task);
707 : 2437824 : break;
708 : 1316672 : case SPDK_ACCEL_OPC_COPY_CRC32C:
709 [ # # # # : 1316672 : rc = spdk_accel_submit_copy_crc32cv(worker->ch, task->dst, task->src_iovs, task->src_iovcnt,
# # # # #
# # # # #
# # ]
710 [ # # # # ]: 0 : task->crc_dst, g_crc32c_seed, accel_done, task);
711 : 1316672 : break;
712 : 1381376 : case SPDK_ACCEL_OPC_COMPARE:
713 [ # # ]: 1381376 : random_num = rand() % 100;
714 [ - + ]: 1381376 : if (random_num < g_fail_percent_goal) {
715 [ # # # # ]: 0 : task->expected_status = -EILSEQ;
716 [ # # # # : 0 : *(uint8_t *)task->dst = ~DATA_PATTERN;
# # ]
717 : 0 : } else {
718 [ # # # # ]: 1381376 : task->expected_status = 0;
719 [ # # # # : 1381376 : *(uint8_t *)task->dst = DATA_PATTERN;
# # ]
720 : : }
721 [ # # # # : 1381376 : rc = spdk_accel_submit_compare(worker->ch, task->dst, task->src,
# # # # #
# # # ]
722 : 0 : g_xfer_size_bytes, accel_done, task);
723 : 1381376 : break;
724 : 1139776 : case SPDK_ACCEL_OPC_DUALCAST:
725 [ # # # # : 1139776 : rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
# # # # #
# # # ]
726 [ # # # # ]: 0 : task->src, g_xfer_size_bytes, accel_done, task);
727 : 1139776 : break;
728 : 315364 : case SPDK_ACCEL_OPC_COMPRESS:
729 [ # # # # : 315364 : task->src_iovs = task->cur_seg->uncompressed_iovs;
# # # # #
# # # ]
730 [ # # # # : 315364 : task->src_iovcnt = task->cur_seg->uncompressed_iovcnt;
# # # # #
# # # ]
731 [ # # # # : 630728 : rc = spdk_accel_submit_compress(worker->ch, task->dst, task->cur_seg->compressed_len_padded,
# # # # #
# # # # #
# # ]
732 [ # # # # ]: 0 : task->src_iovs,
733 [ # # # # : 315364 : task->src_iovcnt, &task->compressed_sz, accel_done, task);
# # ]
734 : 315364 : break;
735 : 1921626 : case SPDK_ACCEL_OPC_DECOMPRESS:
736 [ # # # # : 1921626 : task->src_iovs = task->cur_seg->compressed_iovs;
# # # # #
# # # ]
737 [ # # # # : 1921626 : task->src_iovcnt = task->cur_seg->compressed_iovcnt;
# # # # #
# # # ]
738 [ # # # # : 1921626 : rc = spdk_accel_submit_decompress(worker->ch, task->dst_iovs, task->dst_iovcnt, task->src_iovs,
# # # # #
# # # # #
# # ]
739 [ # # # # ]: 1921626 : task->src_iovcnt, NULL, accel_done, task);
740 : 1921626 : break;
741 : 2107424 : case SPDK_ACCEL_OPC_XOR:
742 [ # # # # : 2107424 : rc = spdk_accel_submit_xor(worker->ch, task->dst, task->sources, g_xor_src_count,
# # # # #
# # # ]
743 : 0 : g_xfer_size_bytes, accel_done, task);
744 : 2107424 : break;
745 : 312864 : case SPDK_ACCEL_OPC_DIF_VERIFY:
746 [ # # # # : 625728 : rc = spdk_accel_submit_dif_verify(worker->ch, task->src_iovs, task->src_iovcnt, task->num_blocks,
# # # # #
# # # # #
# # ]
747 [ # # # # ]: 312864 : &task->dif_ctx, &task->dif_err, accel_done, task);
748 : 312864 : break;
749 : 744096 : case SPDK_ACCEL_OPC_DIF_GENERATE:
750 [ # # # # : 744096 : rc = spdk_accel_submit_dif_generate(worker->ch, task->src_iovs, task->src_iovcnt, task->num_blocks,
# # # # #
# # # # #
# # ]
751 [ # # ]: 744096 : &task->dif_ctx, accel_done, task);
752 : 744096 : break;
753 : 285920 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
754 [ # # # # : 571840 : rc = spdk_accel_submit_dif_generate_copy(worker->ch, task->dst_iovs, task->dst_iovcnt,
# # # # #
# # # ]
755 [ # # # # : 285920 : task->src_iovs, task->src_iovcnt,
# # # # ]
756 [ # # # # : 285920 : task->num_blocks, &task->dif_ctx, accel_done, task);
# # ]
757 : 285920 : break;
758 : 0 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
759 [ # # # # : 0 : rc = spdk_accel_submit_dif_verify_copy(worker->ch, task->dst_iovs, task->dst_iovcnt,
# # # # #
# # # ]
760 [ # # # # : 0 : task->src_iovs, task->src_iovcnt, task->num_blocks,
# # # # #
# # # ]
761 [ # # # # ]: 0 : &task->dif_ctx, &task->dif_err, accel_done, task);
762 : 0 : break;
763 : 0 : case SPDK_ACCEL_OPC_DIX_GENERATE:
764 [ # # # # : 0 : rc = spdk_accel_submit_dix_generate(worker->ch, task->src_iovs, task->src_iovcnt,
# # # # #
# # # ]
765 [ # # # # : 0 : &task->md_iov, task->num_blocks,
# # ]
766 [ # # ]: 0 : &task->dif_ctx, accel_done, task);
767 : 0 : break;
768 : 259520 : case SPDK_ACCEL_OPC_DIX_VERIFY:
769 [ # # # # : 519040 : rc = spdk_accel_submit_dix_verify(worker->ch, task->src_iovs, task->src_iovcnt,
# # # # #
# # # ]
770 [ # # # # : 0 : &task->md_iov, task->num_blocks,
# # ]
771 [ # # # # ]: 259520 : &task->dif_ctx, &task->dif_err, accel_done, task);
772 : 259520 : break;
773 : 0 : default:
774 [ # # ]: 0 : assert(false);
775 : : break;
776 : :
777 : : }
778 : :
779 [ # # ]: 14546974 : worker->current_queue_depth++;
780 [ - + ]: 14546974 : if (rc) {
781 : 0 : accel_done(task, rc);
782 : 0 : }
783 : 14546974 : }
784 : :
785 : : static void
786 : 3456 : _free_task_buffers(struct ap_task *task)
787 : : {
788 : : uint32_t i;
789 : :
790 [ + + # # ]: 3456 : if (g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS ||
791 [ + + ]: 1664 : g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
792 [ # # # # ]: 1920 : free(task->dst_iovs);
793 [ + + # # ]: 1536 : } else if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
794 [ + + ]: 1344 : g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
795 [ + + ]: 1152 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
796 [ + + ]: 1056 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
797 [ + + ]: 864 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY ||
798 [ + - ]: 768 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY ||
799 [ + + ]: 768 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
800 [ - + ]: 672 : g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) {
801 [ + + # # : 864 : if (task->crc_dst) {
# # ]
802 [ # # # # ]: 384 : spdk_dma_free(task->crc_dst);
803 : 0 : }
804 [ + - # # : 864 : if (task->src_iovs) {
# # ]
805 [ + + # # : 1920 : for (i = 0; i < task->src_iovcnt; i++) {
# # ]
806 [ + - # # : 1056 : if (task->src_iovs[i].iov_base) {
# # # # #
# # # ]
807 [ # # # # : 1056 : spdk_dma_free(task->src_iovs[i].iov_base);
# # # # #
# ]
808 : 0 : }
809 : 0 : }
810 [ # # # # ]: 864 : free(task->src_iovs);
811 : 0 : }
812 [ + + # # : 864 : if (task->dst_iovs) {
# # ]
813 [ + + # # : 192 : for (i = 0; i < task->dst_iovcnt; i++) {
# # ]
814 [ + - # # : 96 : if (task->dst_iovs[i].iov_base) {
# # # # #
# # # ]
815 [ # # # # : 96 : spdk_dma_free(task->dst_iovs[i].iov_base);
# # # # #
# ]
816 : 0 : }
817 : 0 : }
818 [ # # # # ]: 96 : free(task->dst_iovs);
819 : 0 : }
820 [ + + # # : 864 : if (task->md_iov.iov_base) {
# # # # ]
821 [ # # # # : 96 : spdk_dma_free(task->md_iov.iov_base);
# # ]
822 : 0 : }
823 [ + + ]: 672 : } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
824 [ + - # # : 192 : if (task->sources) {
# # ]
825 [ + + ]: 672 : for (i = 0; i < g_xor_src_count; i++) {
826 [ # # # # : 480 : spdk_dma_free(task->sources[i]);
# # # # ]
827 : 0 : }
828 [ # # # # ]: 192 : free(task->sources);
829 : 0 : }
830 : 0 : } else {
831 [ # # # # ]: 480 : spdk_dma_free(task->src);
832 : : }
833 : :
834 [ # # # # ]: 3456 : spdk_dma_free(task->dst);
835 [ + + + + ]: 3456 : if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST || g_workload_selection == SPDK_ACCEL_OPC_XOR) {
836 [ # # # # ]: 288 : spdk_dma_free(task->dst2);
837 : 0 : }
838 : 3456 : }
839 : :
840 : : static int
841 : 1316672 : _vector_memcmp(void *_dst, struct iovec *src_src_iovs, uint32_t iovcnt)
842 : : {
843 : : uint32_t i;
844 : 1316672 : uint32_t ttl_len = 0;
845 : 1316672 : uint8_t *dst = (uint8_t *)_dst;
846 : :
847 [ + + ]: 3179712 : for (i = 0; i < iovcnt; i++) {
848 [ - + - + : 1863040 : if (memcmp(dst, src_src_iovs[i].iov_base, src_src_iovs[i].iov_len)) {
- + # # #
# # # # #
# # # # ]
849 : 0 : return -1;
850 : : }
851 [ # # # # : 1863040 : dst += src_src_iovs[i].iov_len;
# # # # ]
852 [ # # # # : 1863040 : ttl_len += src_src_iovs[i].iov_len;
# # ]
853 : 0 : }
854 : :
855 [ - + ]: 1316672 : if (ttl_len != iovcnt * g_xfer_size_bytes) {
856 : 0 : return -1;
857 : : }
858 : :
859 : 1316672 : return 0;
860 : 0 : }
861 : :
862 : : static int _worker_stop(void *arg);
863 : :
864 : : static void
865 : 14546974 : accel_done(void *arg1, int status)
866 : : {
867 : 14546974 : struct ap_task *task = arg1;
868 [ # # # # ]: 14546974 : struct worker_thread *worker = task->worker;
869 : : uint32_t sw_crc32c;
870 : 4191936 : struct spdk_dif_error err_blk;
871 : :
872 [ - + # # ]: 14546974 : assert(worker);
873 [ - + # # : 14546974 : assert(worker->current_queue_depth > 0);
# # # # ]
874 : :
875 [ - + + + : 14546974 : if (g_verify && status == 0) {
+ - ]
876 [ + + + + : 12629210 : switch (worker->workload) {
+ + - + +
- - - - -
- - # # #
# ]
877 : 1316672 : case SPDK_ACCEL_OPC_COPY_CRC32C:
878 [ # # # # : 1316672 : sw_crc32c = spdk_crc32c_iov_update(task->src_iovs, task->src_iovcnt, ~g_crc32c_seed);
# # # # ]
879 [ - + # # : 1316672 : if (*task->crc_dst != sw_crc32c) {
# # # # ]
880 : 0 : SPDK_NOTICELOG("CRC-32C miscompare\n");
881 [ # # ]: 0 : worker->xfer_failed++;
882 : 0 : }
883 [ - + # # : 1316672 : if (_vector_memcmp(task->dst, task->src_iovs, task->src_iovcnt)) {
# # # # #
# # # #
# ]
884 : 0 : SPDK_NOTICELOG("Data miscompare\n");
885 [ # # ]: 0 : worker->xfer_failed++;
886 : 0 : }
887 : 1316672 : break;
888 : 2437824 : case SPDK_ACCEL_OPC_CRC32C:
889 [ # # # # : 2437824 : sw_crc32c = spdk_crc32c_iov_update(task->src_iovs, task->src_iovcnt, ~g_crc32c_seed);
# # # # ]
890 [ - + # # : 2437824 : if (*task->crc_dst != sw_crc32c) {
# # # # ]
891 : 0 : SPDK_NOTICELOG("CRC-32C miscompare\n");
892 [ # # ]: 0 : worker->xfer_failed++;
893 : 0 : }
894 : 2437824 : break;
895 : 950688 : case SPDK_ACCEL_OPC_COPY:
896 [ - + - + : 950688 : if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
- + # # #
# # # #
# ]
897 : 0 : SPDK_NOTICELOG("Data miscompare\n");
898 [ # # ]: 0 : worker->xfer_failed++;
899 : 0 : }
900 : 950688 : break;
901 : 1139776 : case SPDK_ACCEL_OPC_DUALCAST:
902 [ - + - + : 1139776 : if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
- + # # #
# # # #
# ]
903 : 0 : SPDK_NOTICELOG("Data miscompare, first destination\n");
904 [ # # ]: 0 : worker->xfer_failed++;
905 : 0 : }
906 [ - + - + : 1139776 : if (memcmp(task->src, task->dst2, g_xfer_size_bytes)) {
- + # # #
# # # #
# ]
907 : 0 : SPDK_NOTICELOG("Data miscompare, second destination\n");
908 [ # # ]: 0 : worker->xfer_failed++;
909 : 0 : }
910 : 1139776 : break;
911 : 1373824 : case SPDK_ACCEL_OPC_FILL:
912 [ - + - + : 1373824 : if (memcmp(task->dst, task->src, g_xfer_size_bytes)) {
- + # # #
# # # #
# ]
913 : 0 : SPDK_NOTICELOG("Data miscompare\n");
914 [ # # ]: 0 : worker->xfer_failed++;
915 : 0 : }
916 : 1373824 : break;
917 : 1381376 : case SPDK_ACCEL_OPC_COMPARE:
918 : 1381376 : break;
919 : 0 : case SPDK_ACCEL_OPC_COMPRESS:
920 : 0 : break;
921 : 1921626 : case SPDK_ACCEL_OPC_DECOMPRESS:
922 [ - + - + : 1921626 : if (memcmp(task->dst, task->cur_seg->uncompressed_data, task->cur_seg->uncompressed_len)) {
- + # # #
# # # # #
# # # # #
# # # # #
# # ]
923 : 0 : SPDK_NOTICELOG("Data miscompare on decompression\n");
924 [ # # ]: 0 : worker->xfer_failed++;
925 : 0 : }
926 : 1921626 : break;
927 : 2107424 : case SPDK_ACCEL_OPC_XOR:
928 [ - + # # : 2107424 : if (spdk_xor_gen(task->dst2, task->sources, g_xor_src_count,
# # # # #
# # # #
# ]
929 : 0 : g_xfer_size_bytes) != 0) {
930 : 0 : SPDK_ERRLOG("Failed to generate xor for verification\n");
931 [ - + - + : 2107424 : } else if (memcmp(task->dst, task->dst2, g_xfer_size_bytes)) {
- + # # #
# # # #
# ]
932 : 0 : SPDK_NOTICELOG("Data miscompare\n");
933 [ # # ]: 0 : worker->xfer_failed++;
934 : 0 : }
935 : 2107424 : break;
936 : 0 : case SPDK_ACCEL_OPC_DIF_VERIFY:
937 : 0 : break;
938 : 0 : case SPDK_ACCEL_OPC_DIF_GENERATE:
939 [ # # # # : 0 : if (spdk_dif_verify(task->src_iovs, task->src_iovcnt, task->num_blocks,
# # # # #
# # # # #
# # ]
940 [ # # ]: 0 : &task->dif_ctx, &err_blk) != 0) {
941 [ # # # # : 0 : SPDK_NOTICELOG("Data miscompare, "
# # ]
942 : : "err_type %u, expected %lu, actual %lu, err_offset %u\n",
943 : : err_blk.err_type, err_blk.expected,
944 : : err_blk.actual, err_blk.err_offset);
945 [ # # ]: 0 : worker->xfer_failed++;
946 : 0 : }
947 : 0 : break;
948 : 0 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
949 [ # # # # : 0 : if (spdk_dif_verify(task->dst_iovs, task->dst_iovcnt, task->num_blocks,
# # # # #
# # # # #
# # ]
950 [ # # ]: 0 : &task->dif_ctx, &err_blk) != 0) {
951 [ # # # # : 0 : SPDK_NOTICELOG("Data miscompare, "
# # ]
952 : : "err_type %u, expected %lu, actual %lu, err_offset %u\n",
953 : : err_blk.err_type, err_blk.expected,
954 : : err_blk.actual, err_blk.err_offset);
955 [ # # ]: 0 : worker->xfer_failed++;
956 : 0 : }
957 : 0 : break;
958 : 0 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
959 : 0 : break;
960 : 0 : case SPDK_ACCEL_OPC_DIX_GENERATE:
961 [ # # # # : 0 : if (spdk_dix_verify(task->src_iovs, task->src_iovcnt, &task->md_iov,
# # # # #
# # # #
# ]
962 [ # # # # : 0 : task->num_blocks, &task->dif_ctx, &err_blk) != 0) {
# # ]
963 [ # # # # : 0 : SPDK_NOTICELOG("Data miscompare, "
# # ]
964 : : "err_type %u, expected %lu, actual %lu, err_offset %u\n",
965 : : err_blk.err_type, err_blk.expected,
966 : : err_blk.actual, err_blk.err_offset);
967 [ # # ]: 0 : worker->xfer_failed++;
968 : 0 : }
969 : 0 : break;
970 : 0 : case SPDK_ACCEL_OPC_DIX_VERIFY:
971 : 0 : break;
972 : 0 : default:
973 [ # # ]: 0 : assert(false);
974 : : break;
975 : : }
976 : 0 : }
977 : :
978 [ + + # # : 14546974 : if (worker->workload == SPDK_ACCEL_OPC_COMPRESS ||
# # # # ]
979 [ + + ]: 14231610 : g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
980 : : /* Advance the task to the next segment */
981 [ # # # # : 2236990 : task->cur_seg = STAILQ_NEXT(task->cur_seg, link);
# # # # #
# # # #
# ]
982 [ + + # # : 2236990 : if (task->cur_seg == NULL) {
# # ]
983 [ # # # # ]: 312174 : task->cur_seg = STAILQ_FIRST(&g_compress_segs);
984 : 0 : }
985 : 0 : }
986 : :
987 [ - + # # : 14546974 : if (task->expected_status == -EILSEQ) {
# # ]
988 [ # # # # ]: 0 : assert(status != 0);
989 [ # # ]: 0 : worker->injected_miscompares++;
990 : 0 : status = 0;
991 [ - + ]: 14546974 : } else if (status) {
992 : : /* Expected to pass but the accel module reported an error (ex: COMPARE operation). */
993 [ # # ]: 0 : worker->xfer_failed++;
994 : 0 : }
995 : :
996 [ # # ]: 14546974 : worker->current_queue_depth--;
997 : :
998 [ - + + + : 14546974 : if (!worker->is_draining && status == 0) {
+ - # # #
# ]
999 [ # # # # : 14543518 : TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1000 : 14543518 : task = _get_task(worker);
1001 : 14543518 : _submit_single(worker, task);
1002 : 0 : } else {
1003 [ # # # # : 3456 : TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1004 : : }
1005 : 14546974 : }
1006 : :
1007 : : static int
1008 : 73 : dump_result(void)
1009 : : {
1010 : 73 : uint64_t total_completed = 0;
1011 : 73 : uint64_t total_failed = 0;
1012 : 73 : uint64_t total_miscompared = 0;
1013 : 73 : uint64_t total_xfer_per_sec, total_bw_in_MiBps = 0;
1014 : 73 : struct worker_thread *worker = g_workers;
1015 : 22 : char tmp[64];
1016 : :
1017 [ - + ]: 73 : printf("\n%-12s %20s %16s %16s %16s\n",
1018 : : "Core,Thread", "Transfers", "Bandwidth", "Failed", "Miscompares");
1019 [ - + ]: 73 : printf("------------------------------------------------------------------------------------\n");
1020 [ + + ]: 178 : while (worker != NULL) {
1021 : :
1022 [ - + # # : 105 : uint64_t xfer_per_sec = worker->stats.executed / g_time_in_sec;
# # # # ]
1023 [ - + # # : 105 : uint64_t bw_in_MiBps = worker->stats.num_bytes /
# # # # ]
1024 [ # # # # ]: 105 : (g_time_in_sec * 1024 * 1024);
1025 : :
1026 [ # # # # : 105 : total_completed += worker->stats.executed;
# # ]
1027 [ # # # # ]: 105 : total_failed += worker->xfer_failed;
1028 [ # # # # ]: 105 : total_miscompared += worker->injected_miscompares;
1029 : 105 : total_bw_in_MiBps += bw_in_MiBps;
1030 : :
1031 [ - + # # : 105 : snprintf(tmp, sizeof(tmp), "%u,%u", worker->display.core, worker->display.thread);
# # # # #
# # # ]
1032 [ + - ]: 105 : if (xfer_per_sec) {
1033 [ - + ]: 105 : printf("%-12s %18" PRIu64 "/s %10" PRIu64 " MiB/s %16"PRIu64 " %16" PRIu64 "\n",
1034 [ # # # # ]: 0 : tmp, xfer_per_sec, bw_in_MiBps, worker->xfer_failed,
1035 [ # # # # ]: 0 : worker->injected_miscompares);
1036 : 0 : }
1037 : :
1038 [ # # # # ]: 105 : worker = worker->next;
1039 : : }
1040 : :
1041 [ - + ]: 73 : total_xfer_per_sec = total_completed / g_time_in_sec;
1042 : :
1043 [ - + ]: 73 : printf("====================================================================================\n");
1044 [ - + ]: 73 : printf("%-12s %18" PRIu64 "/s %10" PRIu64 " MiB/s %16"PRIu64 " %16" PRIu64 "\n",
1045 : 0 : "Total", total_xfer_per_sec, total_bw_in_MiBps, total_failed, total_miscompared);
1046 : :
1047 : 73 : return total_failed ? 1 : 0;
1048 : : }
1049 : :
1050 : : static inline void
1051 : 105 : _free_task_buffers_in_pool(struct worker_thread *worker)
1052 : : {
1053 : : struct ap_task *task;
1054 : :
1055 [ - + # # ]: 105 : assert(worker);
1056 [ + + # # : 3561 : while ((task = TAILQ_FIRST(&worker->tasks_pool))) {
# # # # ]
1057 [ + + # # : 3456 : TAILQ_REMOVE(&worker->tasks_pool, task, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1058 : 3456 : _free_task_buffers(task);
1059 : : }
1060 : 105 : }
1061 : :
1062 : : static int
1063 : 988 : _check_draining(void *arg)
1064 : : {
1065 : 988 : struct worker_thread *worker = arg;
1066 : :
1067 [ - + # # ]: 988 : assert(worker);
1068 : :
1069 [ + + # # : 988 : if (worker->current_queue_depth == 0) {
# # ]
1070 : 105 : _free_task_buffers_in_pool(worker);
1071 [ # # ]: 105 : spdk_poller_unregister(&worker->is_draining_poller);
1072 : 105 : unregister_worker(worker);
1073 : 0 : }
1074 : :
1075 : 988 : return SPDK_POLLER_BUSY;
1076 : : }
1077 : :
1078 : : static int
1079 : 105 : _worker_stop(void *arg)
1080 : : {
1081 : 105 : struct worker_thread *worker = arg;
1082 : :
1083 [ - + # # ]: 105 : assert(worker);
1084 : :
1085 [ # # ]: 105 : spdk_poller_unregister(&worker->stop_poller);
1086 : :
1087 : : /* now let the worker drain and check it's outstanding IO with a poller */
1088 [ # # # # ]: 105 : worker->is_draining = true;
1089 [ # # # # ]: 105 : worker->is_draining_poller = SPDK_POLLER_REGISTER(_check_draining, worker, 0);
1090 : :
1091 : 105 : return SPDK_POLLER_BUSY;
1092 : : }
1093 : :
1094 : : static void shutdown_cb(void);
1095 : :
1096 : : static void
1097 : 105 : _init_thread(void *arg1)
1098 : : {
1099 : : struct worker_thread *worker;
1100 : : struct ap_task *task;
1101 : 105 : int i, num_tasks = g_allocate_depth;
1102 : 105 : struct display_info *display = arg1;
1103 : :
1104 : 105 : worker = calloc(1, sizeof(*worker));
1105 [ - + ]: 105 : if (worker == NULL) {
1106 [ # # # # ]: 0 : fprintf(stderr, "Unable to allocate worker\n");
1107 : 0 : free(display);
1108 : 0 : spdk_thread_exit(spdk_get_thread());
1109 : 0 : goto no_worker;
1110 : : }
1111 : :
1112 [ # # # # ]: 105 : worker->workload = g_workload_selection;
1113 [ # # # # : 105 : worker->display.core = display->core;
# # # # #
# ]
1114 [ # # # # : 105 : worker->display.thread = display->thread;
# # # # #
# ]
1115 : 105 : free(display);
1116 [ # # # # ]: 105 : worker->core = spdk_env_get_current_core();
1117 [ # # # # ]: 105 : worker->thread = spdk_get_thread();
1118 [ - + ]: 105 : pthread_mutex_lock(&g_workers_lock);
1119 [ # # ]: 105 : g_num_workers++;
1120 [ # # # # ]: 105 : worker->next = g_workers;
1121 : 105 : g_workers = worker;
1122 [ - + ]: 105 : pthread_mutex_unlock(&g_workers_lock);
1123 [ # # # # ]: 105 : worker->ch = spdk_accel_get_io_channel();
1124 [ - + # # : 105 : if (worker->ch == NULL) {
# # ]
1125 [ # # # # ]: 0 : fprintf(stderr, "Unable to get an accel channel\n");
1126 : 0 : goto error;
1127 : : }
1128 : :
1129 [ # # # # : 105 : TAILQ_INIT(&worker->tasks_pool);
# # # # #
# # # # #
# # ]
1130 : :
1131 [ # # # # ]: 105 : worker->task_base = calloc(num_tasks, sizeof(struct ap_task));
1132 [ - + # # : 105 : if (worker->task_base == NULL) {
# # ]
1133 [ # # # # ]: 0 : fprintf(stderr, "Could not allocate task base.\n");
1134 : 0 : goto error;
1135 : : }
1136 : :
1137 [ # # # # ]: 105 : task = worker->task_base;
1138 [ + + # # ]: 3561 : for (i = 0; i < num_tasks; i++) {
1139 [ # # # # : 3456 : TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1140 [ # # # # ]: 3456 : task->worker = worker;
1141 [ - + ]: 3456 : if (_get_task_data_bufs(task)) {
1142 [ # # # # ]: 0 : fprintf(stderr, "Unable to get data bufs\n");
1143 : 0 : goto error;
1144 : : }
1145 [ # # ]: 3456 : task++;
1146 : 0 : }
1147 : :
1148 : : /* Register a poller that will stop the worker at time elapsed */
1149 [ # # # # ]: 105 : worker->stop_poller = SPDK_POLLER_REGISTER(_worker_stop, worker,
1150 : : g_time_in_sec * 1000000ULL);
1151 : :
1152 : : /* Load up queue depth worth of operations. */
1153 [ + + # # ]: 3561 : for (i = 0; i < g_queue_depth; i++) {
1154 : 3456 : task = _get_task(worker);
1155 [ - + ]: 3456 : if (task == NULL) {
1156 : 0 : goto error;
1157 : : }
1158 : :
1159 : 3456 : _submit_single(worker, task);
1160 : 0 : }
1161 : 105 : return;
1162 : 0 : error:
1163 : :
1164 : 0 : _free_task_buffers_in_pool(worker);
1165 [ # # # # ]: 0 : free(worker->task_base);
1166 [ # # # # ]: 0 : worker->task_base = NULL;
1167 : 0 : no_worker:
1168 : 0 : shutdown_cb();
1169 : 0 : g_rc = -1;
1170 : 0 : }
1171 : :
1172 : : static void
1173 : 73 : accel_perf_start(void *arg1)
1174 : : {
1175 : 73 : struct spdk_cpuset tmp_cpumask = {};
1176 : 22 : char thread_name[32];
1177 : : uint32_t i;
1178 : : int j;
1179 : : struct spdk_thread *thread;
1180 : : struct display_info *display;
1181 : :
1182 : 73 : g_tsc_rate = spdk_get_ticks_hz();
1183 : 73 : g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
1184 : :
1185 : 73 : dump_user_config();
1186 : :
1187 [ - + ]: 73 : printf("Running for %d seconds...\n", g_time_in_sec);
1188 : 73 : fflush(stdout);
1189 : :
1190 : : /* Create worker threads for each core that was specified. */
1191 [ + + ]: 170 : SPDK_ENV_FOREACH_CORE(i) {
1192 [ + + # # ]: 202 : for (j = 0; j < g_threads_per_core; j++) {
1193 [ - + ]: 105 : snprintf(thread_name, sizeof(thread_name), "ap_worker_%u_%u", i, j);
1194 : 105 : spdk_cpuset_zero(&tmp_cpumask);
1195 : 105 : spdk_cpuset_set_cpu(&tmp_cpumask, i, true);
1196 : 105 : thread = spdk_thread_create(thread_name, &tmp_cpumask);
1197 : 105 : display = calloc(1, sizeof(*display));
1198 [ - + ]: 105 : if (display == NULL) {
1199 [ # # # # ]: 0 : fprintf(stderr, "Unable to allocate memory\n");
1200 : 0 : spdk_app_stop(-1);
1201 : 0 : return;
1202 : : }
1203 [ # # # # ]: 105 : display->core = i;
1204 [ # # # # ]: 105 : display->thread = j;
1205 : 105 : spdk_thread_send_msg(thread, _init_thread, display);
1206 : 0 : }
1207 : 0 : }
1208 : 0 : }
1209 : :
1210 : : static void
1211 : 79 : accel_perf_free_compress_segs(void)
1212 : : {
1213 : : struct ap_compress_seg *seg, *tmp;
1214 : :
1215 [ + + # # : 539 : STAILQ_FOREACH_SAFE(seg, &g_compress_segs, link, tmp) {
# # # # #
# ]
1216 [ # # # # ]: 460 : free(seg->uncompressed_iovs);
1217 [ # # # # ]: 460 : free(seg->compressed_iovs);
1218 [ # # # # ]: 460 : spdk_dma_free(seg->compressed_data);
1219 [ # # # # ]: 460 : spdk_dma_free(seg->uncompressed_data);
1220 [ + + # # : 460 : STAILQ_REMOVE_HEAD(&g_compress_segs, link);
# # # # #
# ]
1221 : 460 : free(seg);
1222 : 0 : }
1223 : 79 : }
1224 : :
1225 : : struct accel_perf_prep_ctx {
1226 : : FILE *file;
1227 : : long remaining;
1228 : : struct spdk_io_channel *ch;
1229 : : struct ap_compress_seg *cur_seg;
1230 : : };
1231 : :
1232 : : static void accel_perf_prep_process_seg(struct accel_perf_prep_ctx *ctx);
1233 : :
1234 : : static void
1235 : 460 : accel_perf_prep_process_seg_cpl(void *ref, int status)
1236 : : {
1237 : 460 : struct accel_perf_prep_ctx *ctx = ref;
1238 : : struct ap_compress_seg *seg;
1239 : :
1240 [ - + ]: 460 : if (status != 0) {
1241 [ # # # # ]: 0 : fprintf(stderr, "error (%d) on initial compress completion\n", status);
1242 [ # # # # : 0 : spdk_dma_free(ctx->cur_seg->compressed_data);
# # # # ]
1243 [ # # # # : 0 : spdk_dma_free(ctx->cur_seg->uncompressed_data);
# # # # ]
1244 [ # # # # ]: 0 : free(ctx->cur_seg);
1245 [ # # # # ]: 0 : spdk_put_io_channel(ctx->ch);
1246 [ # # # # : 0 : fclose(ctx->file);
# # ]
1247 : 0 : free(ctx);
1248 [ # # ]: 0 : spdk_app_stop(-status);
1249 : 0 : return;
1250 : : }
1251 : :
1252 [ # # # # ]: 460 : seg = ctx->cur_seg;
1253 : :
1254 [ + + ]: 460 : if (g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
1255 [ # # # # ]: 348 : seg->compressed_iovs = calloc(g_chained_count, sizeof(struct iovec));
1256 [ - + # # : 348 : if (seg->compressed_iovs == NULL) {
# # ]
1257 [ # # # # ]: 0 : fprintf(stderr, "unable to allocate iovec\n");
1258 [ # # # # ]: 0 : spdk_dma_free(seg->compressed_data);
1259 [ # # # # ]: 0 : spdk_dma_free(seg->uncompressed_data);
1260 : 0 : free(seg);
1261 [ # # # # ]: 0 : spdk_put_io_channel(ctx->ch);
1262 [ # # # # : 0 : fclose(ctx->file);
# # ]
1263 : 0 : free(ctx);
1264 : 0 : spdk_app_stop(-ENOMEM);
1265 : 0 : return;
1266 : : }
1267 [ # # # # ]: 348 : seg->compressed_iovcnt = g_chained_count;
1268 : :
1269 [ # # # # : 348 : accel_perf_construct_iovs(seg->compressed_data, seg->compressed_len, seg->compressed_iovs,
# # # # #
# # # ]
1270 [ # # # # ]: 0 : seg->compressed_iovcnt);
1271 : 0 : }
1272 : :
1273 [ # # # # : 460 : STAILQ_INSERT_TAIL(&g_compress_segs, seg, link);
# # # # #
# # # # #
# # ]
1274 [ # # # # : 460 : ctx->remaining -= seg->uncompressed_len;
# # # # #
# ]
1275 : :
1276 : 460 : accel_perf_prep_process_seg(ctx);
1277 : 0 : }
1278 : :
1279 : : static void
1280 : 488 : accel_perf_prep_process_seg(struct accel_perf_prep_ctx *ctx)
1281 : : {
1282 : : struct ap_compress_seg *seg;
1283 : : int sz, sz_read, sz_padded;
1284 : : void *ubuf, *cbuf;
1285 : 122 : struct iovec iov[1];
1286 : : int rc;
1287 : :
1288 [ + + # # : 488 : if (ctx->remaining == 0) {
# # ]
1289 [ # # # # ]: 28 : spdk_put_io_channel(ctx->ch);
1290 [ - + # # : 28 : fclose(ctx->file);
# # ]
1291 : 28 : free(ctx);
1292 : 28 : accel_perf_start(NULL);
1293 : 373 : return;
1294 : : }
1295 : :
1296 [ # # # # : 460 : sz = spdk_min(ctx->remaining, g_xfer_size_bytes);
# # # # #
# ]
1297 : : /* Add 10% pad to the compress buffer for incompressible data. Note that a real app
1298 : : * would likely either deal with the failure of not having a large enough buffer
1299 : : * by submitting another operation with a larger one. Or, like the vbdev module
1300 : : * does, just accept the error and use the data uncompressed marking it as such in
1301 : : * its own metadata so that in the future it doesn't try to decompress uncompressed
1302 : : * data, etc.
1303 : : */
1304 [ # # ]: 460 : sz_padded = sz * COMP_BUF_PAD_PERCENTAGE;
1305 : :
1306 : 460 : ubuf = spdk_dma_zmalloc(sz, ALIGN_4K, NULL);
1307 [ - + ]: 460 : if (!ubuf) {
1308 [ # # ]: 0 : fprintf(stderr, "unable to allocate uncompress buffer\n");
1309 : 0 : rc = -ENOMEM;
1310 : 0 : goto error;
1311 : : }
1312 : :
1313 : 460 : cbuf = spdk_dma_malloc(sz_padded, ALIGN_4K, NULL);
1314 [ - + ]: 460 : if (!cbuf) {
1315 [ # # ]: 0 : fprintf(stderr, "unable to allocate compress buffer\n");
1316 : 0 : rc = -ENOMEM;
1317 : 0 : spdk_dma_free(ubuf);
1318 : 0 : goto error;
1319 : : }
1320 : :
1321 : 460 : seg = calloc(1, sizeof(*seg));
1322 [ - + ]: 460 : if (!seg) {
1323 [ # # ]: 0 : fprintf(stderr, "unable to allocate comp/decomp segment\n");
1324 : 0 : spdk_dma_free(ubuf);
1325 : 0 : spdk_dma_free(cbuf);
1326 : 0 : rc = -ENOMEM;
1327 : 0 : goto error;
1328 : : }
1329 : :
1330 [ # # # # ]: 460 : sz_read = fread(ubuf, sizeof(uint8_t), sz, ctx->file);
1331 [ - + ]: 460 : if (sz_read != sz) {
1332 [ # # ]: 0 : fprintf(stderr, "unable to read input file\n");
1333 : 0 : free(seg);
1334 : 0 : spdk_dma_free(ubuf);
1335 : 0 : spdk_dma_free(cbuf);
1336 [ # # # # ]: 0 : rc = -errno;
1337 : 0 : goto error;
1338 : : }
1339 : :
1340 [ + + ]: 460 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
1341 [ # # # # ]: 112 : seg->uncompressed_iovs = calloc(g_chained_count, sizeof(struct iovec));
1342 [ - + # # : 112 : if (seg->uncompressed_iovs == NULL) {
# # ]
1343 [ # # ]: 0 : fprintf(stderr, "unable to allocate iovec\n");
1344 : 0 : free(seg);
1345 : 0 : spdk_dma_free(ubuf);
1346 : 0 : spdk_dma_free(cbuf);
1347 : 0 : rc = -ENOMEM;
1348 : 0 : goto error;
1349 : : }
1350 [ # # # # ]: 112 : seg->uncompressed_iovcnt = g_chained_count;
1351 [ # # # # : 112 : accel_perf_construct_iovs(ubuf, sz, seg->uncompressed_iovs, seg->uncompressed_iovcnt);
# # # # ]
1352 : 0 : }
1353 : :
1354 [ # # # # ]: 460 : seg->uncompressed_data = ubuf;
1355 [ # # # # ]: 460 : seg->uncompressed_len = sz;
1356 [ # # # # ]: 460 : seg->compressed_data = cbuf;
1357 [ # # # # ]: 460 : seg->compressed_len = sz;
1358 [ # # # # ]: 460 : seg->compressed_len_padded = sz_padded;
1359 : :
1360 [ # # # # ]: 460 : ctx->cur_seg = seg;
1361 [ # # # # : 460 : iov[0].iov_base = seg->uncompressed_data;
# # # # ]
1362 [ # # # # : 460 : iov[0].iov_len = seg->uncompressed_len;
# # # # #
# ]
1363 : : /* Note that anytime a call is made to spdk_accel_submit_compress() there's a chance
1364 : : * it will fail with -ENOMEM in the event that the destination buffer is not large enough
1365 : : * to hold the compressed data. This example app simply adds 10% buffer for compressed data
1366 : : * but real applications may want to consider a more sophisticated method.
1367 : : */
1368 [ # # # # : 460 : rc = spdk_accel_submit_compress(ctx->ch, seg->compressed_data, seg->compressed_len_padded, iov, 1,
# # # # #
# # # ]
1369 [ # # ]: 0 : &seg->compressed_len, accel_perf_prep_process_seg_cpl, ctx);
1370 [ - + ]: 460 : if (rc < 0) {
1371 [ # # ]: 0 : fprintf(stderr, "error (%d) on initial compress submission\n", rc);
1372 : 0 : goto error;
1373 : : }
1374 : :
1375 : 460 : return;
1376 : :
1377 : 0 : error:
1378 [ # # # # ]: 0 : spdk_put_io_channel(ctx->ch);
1379 [ # # # # : 0 : fclose(ctx->file);
# # ]
1380 : 0 : free(ctx);
1381 : 0 : spdk_app_stop(rc);
1382 : 0 : }
1383 : :
1384 : : static void
1385 : 79 : accel_perf_prep(void *arg1)
1386 : : {
1387 : : struct accel_perf_prep_ctx *ctx;
1388 : 79 : const char *module_name = NULL;
1389 : 79 : int rc = 0;
1390 : :
1391 [ - + ]: 79 : if (g_module_name) {
1392 : 0 : rc = spdk_accel_get_opc_module_name(g_workload_selection, &module_name);
1393 [ # # # # : 0 : if (rc != 0 || strcmp(g_module_name, module_name) != 0) {
# # # # ]
1394 [ # # ]: 0 : fprintf(stderr, "Module '%s' was assigned via JSON config or RPC, instead of '%s'\n",
1395 : 0 : module_name, g_module_name);
1396 [ # # ]: 0 : fprintf(stderr, "-M option is not compatible with accel_assign_opc RPC\n");
1397 : 0 : rc = -EINVAL;
1398 : 0 : goto error_end;
1399 : : }
1400 : 0 : }
1401 : :
1402 [ + + # # ]: 79 : if (g_workload_selection != SPDK_ACCEL_OPC_COMPRESS &&
1403 [ + + ]: 69 : g_workload_selection != SPDK_ACCEL_OPC_DECOMPRESS) {
1404 : 45 : accel_perf_start(arg1);
1405 : 66 : return;
1406 : : }
1407 : :
1408 [ + + ]: 34 : if (g_cd_file_in_name == NULL) {
1409 [ - + ]: 3 : fprintf(stdout, "A filename is required.\n");
1410 : 3 : rc = -EINVAL;
1411 : 3 : goto error_end;
1412 : : }
1413 : :
1414 [ + + - + : 31 : if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS && g_verify) {
+ + ]
1415 [ - + ]: 3 : fprintf(stdout, "\nCompression does not support the verify option, aborting.\n");
1416 : 3 : rc = -ENOTSUP;
1417 : 3 : goto error_end;
1418 : : }
1419 : :
1420 : 28 : printf("Preparing input file...\n");
1421 : :
1422 : 28 : ctx = calloc(1, sizeof(*ctx));
1423 [ - + ]: 28 : if (ctx == NULL) {
1424 : 0 : rc = -ENOMEM;
1425 : 0 : goto error_end;
1426 : : }
1427 : :
1428 [ # # # # ]: 28 : ctx->file = fopen(g_cd_file_in_name, "r");
1429 [ - + # # : 28 : if (ctx->file == NULL) {
# # ]
1430 [ # # ]: 0 : fprintf(stderr, "Could not open file %s.\n", g_cd_file_in_name);
1431 [ # # # # ]: 0 : rc = -errno;
1432 : 0 : goto error_ctx;
1433 : : }
1434 : :
1435 [ # # # # ]: 28 : fseek(ctx->file, 0L, SEEK_END);
1436 [ # # # # : 28 : ctx->remaining = ftell(ctx->file);
# # # # ]
1437 [ # # # # ]: 28 : fseek(ctx->file, 0L, SEEK_SET);
1438 : :
1439 [ # # # # ]: 28 : ctx->ch = spdk_accel_get_io_channel();
1440 [ - + # # : 28 : if (ctx->ch == NULL) {
# # ]
1441 : 0 : rc = -EAGAIN;
1442 : 0 : goto error_file;
1443 : : }
1444 : :
1445 [ + + ]: 28 : if (g_xfer_size_bytes == 0) {
1446 : : /* size of 0 means "file at a time" */
1447 [ # # # # ]: 12 : g_xfer_size_bytes = ctx->remaining;
1448 : 0 : }
1449 : :
1450 : 28 : accel_perf_prep_process_seg(ctx);
1451 : 28 : return;
1452 : :
1453 : 0 : error_file:
1454 [ # # # # : 0 : fclose(ctx->file);
# # ]
1455 : 0 : error_ctx:
1456 : 0 : free(ctx);
1457 : 6 : error_end:
1458 : 6 : spdk_app_stop(rc);
1459 : 0 : }
1460 : :
1461 : : static void
1462 : 0 : worker_shutdown(void *ctx)
1463 : : {
1464 : 0 : _worker_stop(ctx);
1465 : 0 : }
1466 : :
1467 : : static void
1468 : 0 : shutdown_cb(void)
1469 : : {
1470 : : struct worker_thread *worker;
1471 : :
1472 [ # # ]: 0 : pthread_mutex_lock(&g_workers_lock);
1473 [ # # ]: 0 : if (!g_workers) {
1474 : 0 : spdk_app_stop(1);
1475 : 0 : goto unlock;
1476 : : }
1477 : :
1478 : 0 : worker = g_workers;
1479 [ # # ]: 0 : while (worker) {
1480 [ # # # # ]: 0 : spdk_thread_send_msg(worker->thread, worker_shutdown, worker);
1481 [ # # # # ]: 0 : worker = worker->next;
1482 : : }
1483 : 0 : unlock:
1484 [ # # ]: 0 : pthread_mutex_unlock(&g_workers_lock);
1485 : 0 : }
1486 : :
1487 : : int
1488 : 88 : main(int argc, char **argv)
1489 : : {
1490 : : struct worker_thread *worker, *tmp;
1491 : : int rc;
1492 : :
1493 [ - + ]: 88 : pthread_mutex_init(&g_workers_lock, NULL);
1494 : 88 : spdk_app_opts_init(&g_opts, sizeof(g_opts));
1495 : 88 : g_opts.name = "accel_perf";
1496 : 88 : g_opts.reactor_mask = "0x1";
1497 : 88 : g_opts.shutdown_cb = shutdown_cb;
1498 : 88 : g_opts.rpc_addr = NULL;
1499 : :
1500 : 88 : rc = spdk_app_parse_args(argc, argv, &g_opts, "a:C:o:q:t:yw:M:P:f:T:l:S:x:", NULL,
1501 : : parse_args, usage);
1502 [ + + ]: 88 : if (rc != SPDK_APP_PARSE_ARGS_SUCCESS) {
1503 : 9 : return rc == SPDK_APP_PARSE_ARGS_HELP ? 0 : 1;
1504 : : }
1505 : :
1506 [ - + ]: 79 : if (g_workload_selection == SPDK_ACCEL_OPC_LAST) {
1507 [ # # # # ]: 0 : fprintf(stderr, "Must provide a workload type\n");
1508 : 0 : usage();
1509 : 0 : return -1;
1510 : : }
1511 : :
1512 [ + + - + ]: 79 : if (g_allocate_depth > 0 && g_queue_depth > g_allocate_depth) {
1513 [ # # # # ]: 0 : fprintf(stdout, "allocate depth must be at least as big as queue depth\n");
1514 : 0 : usage();
1515 : 0 : return -1;
1516 : : }
1517 : :
1518 [ + + ]: 79 : if (g_allocate_depth == 0) {
1519 : 76 : g_allocate_depth = g_queue_depth;
1520 : 0 : }
1521 : :
1522 [ + + # # ]: 79 : if ((g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
1523 [ + + ]: 73 : g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
1524 [ + + ]: 67 : g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
1525 [ + + ]: 64 : g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
1526 [ + + ]: 58 : g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
1527 [ - + ]: 55 : g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) &&
1528 [ - + ]: 24 : g_chained_count == 0) {
1529 : 0 : usage();
1530 : 0 : return -1;
1531 : : }
1532 : :
1533 [ + + - + ]: 79 : if (g_workload_selection == SPDK_ACCEL_OPC_XOR && g_xor_src_count < 2) {
1534 : 0 : usage();
1535 : 0 : return -1;
1536 : : }
1537 : :
1538 [ - + - - ]: 79 : if (g_module_name && spdk_accel_assign_opc(g_workload_selection, g_module_name)) {
1539 [ # # # # ]: 0 : fprintf(stderr, "Was not able to assign '%s' module to the workload\n", g_module_name);
1540 : 0 : usage();
1541 : 0 : return -1;
1542 : : }
1543 : :
1544 : 79 : g_rc = spdk_app_start(&g_opts, accel_perf_prep, NULL);
1545 [ + + ]: 79 : if (g_rc) {
1546 : 6 : SPDK_ERRLOG("ERROR starting application\n");
1547 : 0 : }
1548 : :
1549 [ - + ]: 79 : pthread_mutex_destroy(&g_workers_lock);
1550 : :
1551 : 79 : worker = g_workers;
1552 [ + + ]: 184 : while (worker) {
1553 [ # # # # ]: 105 : tmp = worker->next;
1554 : 105 : free(worker);
1555 : 105 : worker = tmp;
1556 : : }
1557 : 79 : accel_perf_free_compress_segs();
1558 : 79 : spdk_app_fini();
1559 : 79 : return g_rc;
1560 : 0 : }
|