LCOV - code coverage report
Current view: top level - spdk/examples/accel/perf - accel_perf.c (source / functions) Hit Total Coverage
Test: Combined Lines: 675 1054 64.0 %
Date: 2024-11-15 11:22:19 Functions: 22 24 91.7 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 478 2957 16.2 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2020 Intel Corporation.
       3                 :            :  *   Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
       4                 :            :  *   All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #include "spdk/stdinc.h"
       8                 :            : #include "spdk/thread.h"
       9                 :            : #include "spdk/env.h"
      10                 :            : #include "spdk/event.h"
      11                 :            : #include "spdk/log.h"
      12                 :            : #include "spdk/string.h"
      13                 :            : #include "spdk/accel.h"
      14                 :            : #include "spdk/crc32.h"
      15                 :            : #include "spdk/util.h"
      16                 :            : #include "spdk/xor.h"
      17                 :            : #include "spdk/dif.h"
      18                 :            : 
      19                 :            : #define DATA_PATTERN 0x5a
      20                 :            : #define ALIGN_4K 0x1000
      21                 :            : #define COMP_BUF_PAD_PERCENTAGE 1.1L
      22                 :            : 
      23                 :            : static uint64_t g_tsc_rate;
      24                 :            : static uint64_t g_tsc_end;
      25                 :            : static int g_rc;
      26                 :            : static int g_xfer_size_bytes = 4096;
      27                 :            : static int g_block_size_bytes = 512;
      28                 :            : static int g_md_size_bytes = 8;
      29                 :            : static int g_queue_depth = 32;
      30                 :            : /* g_allocate_depth indicates how many tasks we allocate per worker. It will
      31                 :            :  * be at least as much as the queue depth.
      32                 :            :  */
      33                 :            : static int g_allocate_depth = 0;
      34                 :            : static int g_threads_per_core = 1;
      35                 :            : static int g_time_in_sec = 5;
      36                 :            : static uint32_t g_crc32c_seed = 0;
      37                 :            : static uint32_t g_chained_count = 1;
      38                 :            : static int g_fail_percent_goal = 0;
      39                 :            : static uint8_t g_fill_pattern = 255;
      40                 :            : static uint32_t g_xor_src_count = 2;
      41                 :            : static bool g_verify = false;
      42                 :            : static const char *g_workload_type = NULL;
      43                 :            : static enum spdk_accel_opcode g_workload_selection = SPDK_ACCEL_OPC_LAST;
      44                 :            : static const char *g_module_name = NULL;
      45                 :            : static struct worker_thread *g_workers = NULL;
      46                 :            : static int g_num_workers = 0;
      47                 :            : static char *g_cd_file_in_name = NULL;
      48                 :            : static pthread_mutex_t g_workers_lock = PTHREAD_MUTEX_INITIALIZER;
      49                 :            : static struct spdk_app_opts g_opts = {};
      50                 :            : 
      51                 :            : struct ap_compress_seg {
      52                 :            :         void            *uncompressed_data;
      53                 :            :         uint32_t        uncompressed_len;
      54                 :            :         struct iovec    *uncompressed_iovs;
      55                 :            :         uint32_t        uncompressed_iovcnt;
      56                 :            : 
      57                 :            :         void            *compressed_data;
      58                 :            :         uint32_t        compressed_len;
      59                 :            :         uint32_t        compressed_len_padded;
      60                 :            :         struct iovec    *compressed_iovs;
      61                 :            :         uint32_t        compressed_iovcnt;
      62                 :            : 
      63                 :            :         STAILQ_ENTRY(ap_compress_seg)   link;
      64                 :            : };
      65                 :            : 
      66                 :            : static STAILQ_HEAD(, ap_compress_seg) g_compress_segs = STAILQ_HEAD_INITIALIZER(g_compress_segs);
      67                 :            : 
      68                 :            : struct worker_thread;
      69                 :            : static void accel_done(void *ref, int status);
      70                 :            : 
      71                 :            : struct display_info {
      72                 :            :         int core;
      73                 :            :         int thread;
      74                 :            : };
      75                 :            : 
      76                 :            : struct ap_task {
      77                 :            :         void                    *src;
      78                 :            :         struct iovec            *src_iovs;
      79                 :            :         uint32_t                src_iovcnt;
      80                 :            :         void                    **sources;
      81                 :            :         struct iovec            *dst_iovs;
      82                 :            :         uint32_t                dst_iovcnt;
      83                 :            :         struct iovec            md_iov;
      84                 :            :         void                    *dst;
      85                 :            :         void                    *dst2;
      86                 :            :         uint32_t                *crc_dst;
      87                 :            :         uint32_t                compressed_sz;
      88                 :            :         struct ap_compress_seg *cur_seg;
      89                 :            :         struct worker_thread    *worker;
      90                 :            :         int                     expected_status; /* used for the compare operation */
      91                 :            :         uint32_t                num_blocks; /* used for the DIF related operations */
      92                 :            :         struct spdk_dif_ctx     dif_ctx;
      93                 :            :         struct spdk_dif_error   dif_err;
      94                 :            :         TAILQ_ENTRY(ap_task)    link;
      95                 :            : };
      96                 :            : 
      97                 :            : struct worker_thread {
      98                 :            :         struct spdk_io_channel          *ch;
      99                 :            :         struct spdk_accel_opcode_stats  stats;
     100                 :            :         uint64_t                        xfer_failed;
     101                 :            :         uint64_t                        injected_miscompares;
     102                 :            :         uint64_t                        current_queue_depth;
     103                 :            :         TAILQ_HEAD(, ap_task)           tasks_pool;
     104                 :            :         struct worker_thread            *next;
     105                 :            :         unsigned                        core;
     106                 :            :         struct spdk_thread              *thread;
     107                 :            :         bool                            is_draining;
     108                 :            :         struct spdk_poller              *is_draining_poller;
     109                 :            :         struct spdk_poller              *stop_poller;
     110                 :            :         void                            *task_base;
     111                 :            :         struct display_info             display;
     112                 :            :         enum spdk_accel_opcode          workload;
     113                 :            : };
     114                 :            : 
     115                 :            : static void
     116                 :         73 : dump_user_config(void)
     117                 :            : {
     118                 :         73 :         const char *module_name = NULL;
     119                 :            :         int rc;
     120                 :            : 
     121                 :         73 :         rc = spdk_accel_get_opc_module_name(g_workload_selection, &module_name);
     122         [ -  + ]:         73 :         if (rc) {
     123         [ #  # ]:          0 :                 printf("error getting module name (%d)\n", rc);
     124                 :          0 :         }
     125                 :            : 
     126         [ -  + ]:         73 :         printf("\nSPDK Configuration:\n");
     127         [ -  + ]:         73 :         printf("Core mask:      %s\n\n", g_opts.reactor_mask);
     128         [ -  + ]:         73 :         printf("Accel Perf Configuration:\n");
     129         [ -  + ]:         73 :         printf("Workload Type:  %s\n", g_workload_type);
     130   [ +  +  #  # ]:         73 :         if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
     131         [ +  + ]:         67 :             g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
     132         [ -  + ]:         12 :                 printf("CRC-32C seed:   %u\n", g_crc32c_seed);
     133         [ +  + ]:         61 :         } else if (g_workload_selection == SPDK_ACCEL_OPC_FILL) {
     134         [ -  + ]:          3 :                 printf("Fill pattern:   0x%x\n", g_fill_pattern);
     135   [ +  +  -  + ]:         58 :         } else if ((g_workload_selection == SPDK_ACCEL_OPC_COMPARE) && g_fail_percent_goal > 0) {
     136         [ #  # ]:          0 :                 printf("Failure inject: %u percent\n", g_fail_percent_goal);
     137         [ +  + ]:         58 :         } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
     138         [ -  + ]:          6 :                 printf("Source buffers: %u\n", g_xor_src_count);
     139                 :          0 :         }
     140   [ +  +  #  # ]:         73 :         if (g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
     141         [ +  + ]:         67 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
     142         [ +  + ]:         64 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     143         [ +  + ]:         58 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
     144         [ +  - ]:         55 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
     145         [ +  + ]:         55 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY) {
     146         [ -  + ]:         21 :                 printf("Vector size:    %u bytes\n", g_xfer_size_bytes);
     147         [ -  + ]:         21 :                 printf("Transfer size:  %u bytes\n", g_xfer_size_bytes * g_chained_count);
     148                 :          0 :         } else {
     149         [ -  + ]:         52 :                 printf("Transfer size:  %u bytes\n", g_xfer_size_bytes);
     150                 :            :         }
     151   [ +  +  #  # ]:         73 :         if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     152         [ +  + ]:         67 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
     153         [ +  - ]:         64 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
     154         [ +  + ]:         64 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
     155         [ -  + ]:         12 :                 printf("Block size:     %u bytes\n", g_block_size_bytes);
     156         [ -  + ]:         12 :                 printf("Metadata size:  %u bytes\n", g_md_size_bytes);
     157                 :          0 :         }
     158         [ -  + ]:         73 :         printf("Vector count    %u\n", g_chained_count);
     159         [ -  + ]:         73 :         printf("Module:         %s\n", module_name);
     160   [ +  +  #  # ]:         73 :         if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS ||
     161         [ +  + ]:         69 :             g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
     162         [ -  + ]:         28 :                 printf("File Name:      %s\n", g_cd_file_in_name);
     163                 :          0 :         }
     164         [ -  + ]:         73 :         printf("Queue depth:    %u\n", g_queue_depth);
     165         [ -  + ]:         73 :         printf("Allocate depth: %u\n", g_allocate_depth);
     166         [ -  + ]:         73 :         printf("# threads/core: %u\n", g_threads_per_core);
     167         [ -  + ]:         73 :         printf("Run time:       %u seconds\n", g_time_in_sec);
     168   [ -  +  +  +  :         73 :         printf("Verify:         %s\n\n", g_verify ? "Yes" : "No");
                   -  + ]
     169                 :         73 : }
     170                 :            : 
     171                 :            : static void
     172                 :          9 : usage(void)
     173                 :            : {
     174         [ -  + ]:          9 :         printf("accel_perf options:\n");
     175         [ -  + ]:          9 :         printf("\t[-h help message]\n");
     176         [ -  + ]:          9 :         printf("\t[-q queue depth per core]\n");
     177         [ -  + ]:          9 :         printf("\t[-C for supported workloads, use this value to configure the io vector size to test (default 1)\n");
     178         [ -  + ]:          9 :         printf("\t[-T number of threads per core\n");
     179         [ -  + ]:          9 :         printf("\t[-o transfer size in bytes (default: 4KiB. For compress/decompress, 0 means the input file size)]\n");
     180         [ -  + ]:          9 :         printf("\t[-t time in seconds]\n");
     181         [ -  + ]:          9 :         printf("\t[-w workload type must be one of these: copy, fill, crc32c, copy_crc32c, compare, compress, decompress, dualcast, xor,\n");
     182         [ -  + ]:          9 :         printf("\t[                                       dif_verify, dif_verify_copy, dif_generate, dif_generate_copy, dix_generate, dix_verify\n");
     183         [ -  + ]:          9 :         printf("\t[-M assign module to the operation, not compatible with accel_assign_opc RPC\n");
     184         [ -  + ]:          9 :         printf("\t[-l for compress/decompress workloads, name of uncompressed input file\n");
     185         [ -  + ]:          9 :         printf("\t[-S for crc32c workload, use this seed value (default 0)\n");
     186         [ -  + ]:          9 :         printf("\t[-P for compare workload, percentage of operations that should miscompare (percent, default 0)\n");
     187         [ -  + ]:          9 :         printf("\t[-f for fill workload, use this BYTE value (default 255)\n");
     188         [ -  + ]:          9 :         printf("\t[-x for xor workload, use this number of source buffers (default, minimum: 2)]\n");
     189         [ -  + ]:          9 :         printf("\t[-y verify result if this switch is on]\n");
     190         [ -  + ]:          9 :         printf("\t[-a tasks to allocate per core (default: same value as -q)]\n");
     191         [ -  + ]:          9 :         printf("\t\tCan be used to spread operations across a wider range of memory.\n");
     192                 :          9 : }
     193                 :            : 
     194                 :            : static int
     195                 :        305 : parse_args(int ch, char *arg)
     196                 :            : {
     197                 :        305 :         int argval = 0;
     198                 :            : 
     199         [ +  + ]:        305 :         switch (ch) {
     200                 :        129 :         case 'a':
     201                 :            :         case 'C':
     202                 :            :         case 'f':
     203                 :            :         case 'T':
     204                 :            :         case 'o':
     205                 :            :         case 'P':
     206                 :            :         case 'q':
     207                 :            :         case 'S':
     208                 :            :         case 't':
     209                 :            :         case 'x':
     210                 :        129 :                 argval = spdk_strtol(optarg, 10);
     211         [ +  + ]:        129 :                 if (argval < 0) {
     212         [ -  + ]:          3 :                         fprintf(stderr, "-%c option must be non-negative.\n", ch);
     213                 :          3 :                         usage();
     214                 :          3 :                         return 1;
     215                 :            :                 }
     216                 :        126 :                 break;
     217                 :        176 :         default:
     218                 :        176 :                 break;
     219                 :            :         };
     220                 :            : 
     221   [ +  +  +  +  :        302 :         switch (ch) {
          +  +  -  +  +  
          +  +  +  +  -  
                      - ]
     222                 :          3 :         case 'a':
     223                 :          3 :                 g_allocate_depth = argval;
     224                 :          3 :                 break;
     225                 :          6 :         case 'C':
     226                 :          6 :                 g_chained_count = argval;
     227                 :          6 :                 break;
     228                 :         31 :         case 'l':
     229                 :         31 :                 g_cd_file_in_name = optarg;
     230                 :         31 :                 break;
     231                 :          3 :         case 'f':
     232                 :          3 :                 g_fill_pattern = (uint8_t)argval;
     233                 :          3 :                 break;
     234                 :          8 :         case 'T':
     235                 :          8 :                 g_threads_per_core = argval;
     236                 :          8 :                 break;
     237                 :         12 :         case 'o':
     238                 :         12 :                 g_xfer_size_bytes = argval;
     239                 :         12 :                 break;
     240                 :          0 :         case 'P':
     241                 :          0 :                 g_fail_percent_goal = argval;
     242                 :          0 :                 break;
     243                 :          3 :         case 'q':
     244                 :          3 :                 g_queue_depth = argval;
     245                 :          3 :                 break;
     246                 :          3 :         case 'S':
     247                 :          3 :                 g_crc32c_seed = argval;
     248                 :          3 :                 break;
     249                 :         85 :         case 't':
     250                 :         85 :                 g_time_in_sec = argval;
     251                 :         85 :                 break;
     252                 :          3 :         case 'x':
     253                 :          3 :                 g_xor_src_count = argval;
     254                 :          3 :                 break;
     255                 :         60 :         case 'y':
     256                 :         60 :                 g_verify = true;
     257                 :         60 :                 break;
     258                 :         85 :         case 'w':
     259                 :         85 :                 g_workload_type = optarg;
     260   [ -  +  +  +  :         85 :                 if (!strcmp(g_workload_type, "copy")) {
                   #  # ]
     261                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_COPY;
     262   [ -  +  +  +  :         82 :                 } else if (!strcmp(g_workload_type, "fill")) {
                   #  # ]
     263                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_FILL;
     264   [ -  +  +  +  :         79 :                 } else if (!strcmp(g_workload_type, "crc32c")) {
                   #  # ]
     265                 :          6 :                         g_workload_selection = SPDK_ACCEL_OPC_CRC32C;
     266   [ -  +  +  +  :         73 :                 } else if (!strcmp(g_workload_type, "copy_crc32c")) {
                   #  # ]
     267                 :          6 :                         g_workload_selection = SPDK_ACCEL_OPC_COPY_CRC32C;
     268   [ -  +  +  +  :         67 :                 } else if (!strcmp(g_workload_type, "compare")) {
                   #  # ]
     269                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_COMPARE;
     270   [ -  +  +  +  :         64 :                 } else if (!strcmp(g_workload_type, "dualcast")) {
                   #  # ]
     271                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_DUALCAST;
     272   [ -  +  +  +  :         61 :                 } else if (!strcmp(g_workload_type, "compress")) {
                   #  # ]
     273                 :         10 :                         g_workload_selection = SPDK_ACCEL_OPC_COMPRESS;
     274   [ -  +  +  +  :         51 :                 } else if (!strcmp(g_workload_type, "decompress")) {
                   #  # ]
     275                 :         24 :                         g_workload_selection = SPDK_ACCEL_OPC_DECOMPRESS;
     276   [ -  +  +  +  :         27 :                 } else if (!strcmp(g_workload_type, "xor")) {
                   #  # ]
     277                 :          9 :                         g_workload_selection = SPDK_ACCEL_OPC_XOR;
     278   [ -  +  +  +  :         18 :                 } else if (!strcmp(g_workload_type, "dif_verify")) {
                   #  # ]
     279                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_DIF_VERIFY;
     280   [ -  +  -  +  :         15 :                 } else if (!strcmp(g_workload_type, "dif_verify_copy")) {
                   #  # ]
     281                 :          0 :                         g_workload_selection = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
     282   [ -  +  +  +  :         15 :                 } else if (!strcmp(g_workload_type, "dif_generate")) {
                   #  # ]
     283                 :          6 :                         g_workload_selection = SPDK_ACCEL_OPC_DIF_GENERATE;
     284   [ -  +  +  +  :          9 :                 } else if (!strcmp(g_workload_type, "dif_generate_copy")) {
                   #  # ]
     285                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
     286   [ -  +  +  +  :          6 :                 } else if (!strcmp(g_workload_type, "dix_verify")) {
                   #  # ]
     287                 :          3 :                         g_workload_selection = SPDK_ACCEL_OPC_DIX_VERIFY;
     288   [ -  +  -  +  :          3 :                 } else if (!strcmp(g_workload_type, "dix_generate")) {
                   #  # ]
     289                 :          0 :                         g_workload_selection = SPDK_ACCEL_OPC_DIX_GENERATE;
     290                 :          0 :                 } else {
     291         [ -  + ]:          3 :                         fprintf(stderr, "Unsupported workload type: %s\n", optarg);
     292                 :          3 :                         usage();
     293                 :          3 :                         return 1;
     294                 :            :                 }
     295                 :         82 :                 break;
     296                 :          0 :         case 'M':
     297                 :          0 :                 g_module_name = optarg;
     298                 :          0 :                 break;
     299                 :            : 
     300                 :          0 :         default:
     301                 :          0 :                 usage();
     302                 :          0 :                 return 1;
     303                 :            :         }
     304                 :            : 
     305                 :        299 :         return 0;
     306                 :          0 : }
     307                 :            : 
     308                 :            : static int dump_result(void);
     309                 :            : static void
     310                 :        105 : unregister_worker(void *arg1)
     311                 :            : {
     312                 :        105 :         struct worker_thread *worker = arg1;
     313                 :            : 
     314   [ +  -  #  #  :        105 :         if (worker->ch) {
                   #  # ]
     315   [ #  #  #  #  :        105 :                 spdk_accel_get_opcode_stats(worker->ch, worker->workload,
             #  #  #  # ]
     316         [ #  # ]:          0 :                                             &worker->stats, sizeof(worker->stats));
     317   [ #  #  #  # ]:        105 :                 spdk_put_io_channel(worker->ch);
     318   [ #  #  #  # ]:        105 :                 worker->ch = NULL;
     319                 :          0 :         }
     320   [ #  #  #  # ]:        105 :         free(worker->task_base);
     321                 :        105 :         spdk_thread_exit(spdk_get_thread());
     322         [ -  + ]:        105 :         pthread_mutex_lock(&g_workers_lock);
     323   [ -  +  #  # ]:        105 :         assert(g_num_workers >= 1);
     324   [ +  +  #  # ]:        105 :         if (--g_num_workers == 0) {
     325         [ -  + ]:         73 :                 pthread_mutex_unlock(&g_workers_lock);
     326                 :            :                 /* Only dump results on successful runs */
     327         [ +  - ]:         73 :                 if (g_rc == 0) {
     328                 :         73 :                         g_rc = dump_result();
     329                 :          0 :                 }
     330                 :         73 :                 spdk_app_stop(g_rc);
     331                 :          0 :         } else {
     332         [ -  + ]:         32 :                 pthread_mutex_unlock(&g_workers_lock);
     333                 :            :         }
     334                 :        105 : }
     335                 :            : 
     336                 :            : static void
     337                 :       2380 : accel_perf_construct_iovs(void *buf, uint64_t sz, struct iovec *iovs, uint32_t iovcnt)
     338                 :            : {
     339                 :            :         uint64_t ele_size;
     340                 :            :         uint8_t *data;
     341                 :            :         uint32_t i;
     342                 :            : 
     343                 :       2380 :         ele_size = spdk_divide_round_up(sz, iovcnt);
     344                 :            : 
     345                 :       2380 :         data = buf;
     346         [ +  + ]:       4760 :         for (i = 0; i < iovcnt; i++) {
     347         [ #  # ]:       2380 :                 ele_size = spdk_min(ele_size, sz);
     348   [ -  +  #  # ]:       2380 :                 assert(ele_size > 0);
     349                 :            : 
     350   [ #  #  #  #  :       2380 :                 iovs[i].iov_base = data;
                   #  # ]
     351   [ #  #  #  #  :       2380 :                 iovs[i].iov_len = ele_size;
                   #  # ]
     352                 :            : 
     353         [ #  # ]:       2380 :                 data += ele_size;
     354                 :       2380 :                 sz -= ele_size;
     355                 :          0 :         }
     356   [ -  +  #  # ]:       2380 :         assert(sz == 0);
     357                 :       2380 : }
     358                 :            : 
     359                 :            : static int
     360                 :       3456 : _get_task_data_bufs(struct ap_task *task)
     361                 :            : {
     362                 :       3456 :         uint32_t align = 0;
     363                 :       3456 :         uint32_t i = 0;
     364                 :       3456 :         int src_buff_len = g_xfer_size_bytes;
     365                 :       3456 :         int dst_buff_len = g_xfer_size_bytes;
     366                 :            :         int md_buff_len;
     367                 :        992 :         struct spdk_dif_ctx_init_ext_opts dif_opts;
     368                 :            :         uint32_t num_blocks, transfer_size_with_md;
     369                 :            :         int rc;
     370                 :            : 
     371                 :            :         /* For dualcast, the DSA HW requires 4K alignment on destination addresses but
     372                 :            :          * we do this for all modules to keep it simple.
     373                 :            :          */
     374         [ +  + ]:       3456 :         if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST) {
     375                 :         96 :                 align = ALIGN_4K;
     376                 :          0 :         }
     377                 :            : 
     378   [ +  +  #  # ]:       3456 :         if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS ||
     379         [ +  + ]:       3328 :             g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
     380   [ #  #  #  # ]:       1920 :                 task->cur_seg = STAILQ_FIRST(&g_compress_segs);
     381                 :            : 
     382         [ +  + ]:       1920 :                 if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
     383   [ #  #  #  #  :        128 :                         dst_buff_len = task->cur_seg->compressed_len_padded;
             #  #  #  # ]
     384                 :          0 :                 }
     385                 :            : 
     386   [ #  #  #  # ]:       1920 :                 task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
     387   [ -  +  #  #  :       1920 :                 if (task->dst == NULL) {
                   #  # ]
     388   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Unable to alloc dst buffer\n");
     389                 :          0 :                         return -ENOMEM;
     390                 :            :                 }
     391                 :            : 
     392   [ #  #  #  # ]:       1920 :                 task->dst_iovs = calloc(g_chained_count, sizeof(struct iovec));
     393   [ -  +  #  #  :       1920 :                 if (!task->dst_iovs) {
                   #  # ]
     394   [ #  #  #  # ]:          0 :                         fprintf(stderr, "cannot allocate task->dst_iovs for task=%p\n", task);
     395                 :          0 :                         return -ENOMEM;
     396                 :            :                 }
     397   [ #  #  #  # ]:       1920 :                 task->dst_iovcnt = g_chained_count;
     398   [ #  #  #  #  :       1920 :                 accel_perf_construct_iovs(task->dst, dst_buff_len, task->dst_iovs, task->dst_iovcnt);
          #  #  #  #  #  
                #  #  # ]
     399                 :            : 
     400                 :       1920 :                 return 0;
     401                 :            :         }
     402                 :            : 
     403         [ +  + ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY) {
     404   [ #  #  #  # ]:         96 :                 task->dst_iovcnt = g_chained_count;
     405   [ #  #  #  #  :         96 :                 task->dst_iovs = calloc(task->dst_iovcnt, sizeof(struct iovec));
             #  #  #  # ]
     406   [ -  +  #  #  :         96 :                 if (!task->dst_iovs) {
                   #  # ]
     407   [ #  #  #  # ]:          0 :                         fprintf(stderr, "cannot allocate task->dst_iovs for task=%p\n", task);
     408                 :          0 :                         return -ENOMEM;
     409                 :            :                 }
     410                 :            : 
     411         [ -  + ]:         96 :                 num_blocks = g_xfer_size_bytes / g_block_size_bytes;
     412                 :            :                 /* Add bytes for each block for metadata */
     413                 :         96 :                 transfer_size_with_md = g_xfer_size_bytes + (num_blocks * g_md_size_bytes);
     414   [ #  #  #  # ]:         96 :                 task->num_blocks = num_blocks;
     415                 :            : 
     416   [ +  +  #  #  :        192 :                 for (i = 0; i < task->dst_iovcnt; i++) {
                   #  # ]
     417   [ #  #  #  #  :         96 :                         task->dst_iovs[i].iov_base = spdk_dma_zmalloc(transfer_size_with_md, 0, NULL);
          #  #  #  #  #  
                      # ]
     418   [ -  +  #  #  :         96 :                         if (task->dst_iovs[i].iov_base == NULL) {
          #  #  #  #  #  
                #  #  # ]
     419                 :          0 :                                 return -ENOMEM;
     420                 :            :                         }
     421   [ #  #  #  #  :         96 :                         task->dst_iovs[i].iov_len = transfer_size_with_md;
          #  #  #  #  #  
                      # ]
     422                 :          0 :                 }
     423                 :            : 
     424                 :         96 :                 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     425         [ #  # ]:         96 :                 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
     426                 :            : 
     427         [ #  # ]:         96 :                 rc = spdk_dif_ctx_init(&task->dif_ctx,
     428         [ #  # ]:         96 :                                        g_block_size_bytes + g_md_size_bytes,
     429                 :          0 :                                        g_md_size_bytes, true, true,
     430                 :            :                                        SPDK_DIF_TYPE1,
     431   [ #  #  #  #  :          0 :                                        SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
                   #  # ]
     432                 :            :                                        0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
     433         [ -  + ]:         96 :                 if (rc != 0) {
     434   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
     435                 :          0 :                         return rc;
     436                 :            :                 }
     437                 :          0 :         }
     438                 :            : 
     439         [ -  + ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
     440                 :            :                 /* Allocate source buffers */
     441   [ #  #  #  # ]:          0 :                 task->src_iovcnt = g_chained_count;
     442   [ #  #  #  #  :          0 :                 task->src_iovs = calloc(task->src_iovcnt, sizeof(struct iovec));
             #  #  #  # ]
     443   [ #  #  #  #  :          0 :                 if (!task->src_iovs) {
                   #  # ]
     444   [ #  #  #  # ]:          0 :                         fprintf(stderr, "cannot allocate task->src_iovs for task=%p\n", task);
     445                 :          0 :                         return -ENOMEM;
     446                 :            :                 }
     447                 :            : 
     448         [ #  # ]:          0 :                 num_blocks = g_xfer_size_bytes / g_block_size_bytes;
     449                 :            :                 /* Add bytes for each block for metadata */
     450                 :          0 :                 transfer_size_with_md = g_xfer_size_bytes + (num_blocks * g_md_size_bytes);
     451   [ #  #  #  # ]:          0 :                 task->num_blocks = num_blocks;
     452                 :            : 
     453   [ #  #  #  #  :          0 :                 for (i = 0; i < task->src_iovcnt; i++) {
                   #  # ]
     454   [ #  #  #  #  :          0 :                         task->src_iovs[i].iov_base = spdk_dma_zmalloc(transfer_size_with_md, 0, NULL);
          #  #  #  #  #  
                      # ]
     455   [ #  #  #  #  :          0 :                         if (task->src_iovs[i].iov_base == NULL) {
          #  #  #  #  #  
                #  #  # ]
     456                 :          0 :                                 return -ENOMEM;
     457                 :            :                         }
     458   [ #  #  #  #  :          0 :                         memset(task->src_iovs[i].iov_base, DATA_PATTERN, transfer_size_with_md);
          #  #  #  #  #  
                #  #  # ]
     459   [ #  #  #  #  :          0 :                         task->src_iovs[i].iov_len = transfer_size_with_md;
          #  #  #  #  #  
                      # ]
     460                 :          0 :                 }
     461                 :            : 
     462                 :            :                 /* Allocate destination buffers */
     463   [ #  #  #  # ]:          0 :                 task->dst_iovcnt = g_chained_count;
     464   [ #  #  #  #  :          0 :                 task->dst_iovs = calloc(task->dst_iovcnt, sizeof(struct iovec));
             #  #  #  # ]
     465   [ #  #  #  #  :          0 :                 if (!task->dst_iovs) {
                   #  # ]
     466   [ #  #  #  # ]:          0 :                         fprintf(stderr, "cannot allocated task->dst_iovs fot task=%p\n", task);
     467                 :          0 :                         return -ENOMEM;
     468                 :            :                 }
     469                 :            : 
     470   [ #  #  #  #  :          0 :                 for (i = 0; i < task->dst_iovcnt; i++) {
                   #  # ]
     471   [ #  #  #  #  :          0 :                         task->dst_iovs[i].iov_base = spdk_dma_zmalloc(dst_buff_len, 0, NULL);
          #  #  #  #  #  
                      # ]
     472   [ #  #  #  #  :          0 :                         if (task->dst_iovs[i].iov_base == NULL) {
          #  #  #  #  #  
                #  #  # ]
     473                 :          0 :                                 return -ENOMEM;
     474                 :            :                         }
     475   [ #  #  #  #  :          0 :                         task->dst_iovs[i].iov_len = dst_buff_len;
          #  #  #  #  #  
                      # ]
     476                 :          0 :                 }
     477                 :            : 
     478                 :          0 :                 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     479         [ #  # ]:          0 :                 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
     480                 :            : 
     481                 :            :                 /* Init DIF ctx */
     482         [ #  # ]:          0 :                 rc = spdk_dif_ctx_init(&task->dif_ctx,
     483         [ #  # ]:          0 :                                        g_block_size_bytes + g_md_size_bytes,
     484                 :          0 :                                        g_md_size_bytes, true, true,
     485                 :            :                                        SPDK_DIF_TYPE1,
     486   [ #  #  #  #  :          0 :                                        SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
                   #  # ]
     487                 :            :                                        0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
     488         [ #  # ]:          0 :                 if (rc != 0) {
     489   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
     490                 :          0 :                         return rc;
     491                 :            :                 }
     492                 :            : 
     493   [ #  #  #  #  :          0 :                 rc = spdk_dif_generate(task->src_iovs, task->src_iovcnt, task->num_blocks, &task->dif_ctx);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     494         [ #  # ]:          0 :                 if (rc != 0) {
     495   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Generation of DIF failed, error (%d)\n", rc);
     496                 :          0 :                         return rc;
     497                 :            :                 }
     498                 :          0 :         }
     499                 :            : 
     500   [ +  +  #  # ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
     501         [ +  + ]:       1344 :             g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
     502   [ #  #  #  # ]:        384 :                 task->crc_dst = spdk_dma_zmalloc(sizeof(*task->crc_dst), 0, NULL);
     503                 :          0 :         }
     504                 :            : 
     505   [ +  +  #  # ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
     506         [ +  + ]:       1344 :             g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
     507         [ +  + ]:       1152 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
     508         [ +  + ]:       1056 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     509         [ +  + ]:        864 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY ||
     510         [ +  + ]:        768 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
     511         [ -  + ]:        672 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) {
     512   [ -  +  #  # ]:        864 :                 assert(g_chained_count > 0);
     513   [ #  #  #  # ]:        864 :                 task->src_iovcnt = g_chained_count;
     514   [ #  #  #  #  :        864 :                 task->src_iovs = calloc(task->src_iovcnt, sizeof(struct iovec));
             #  #  #  # ]
     515   [ -  +  #  #  :        864 :                 if (!task->src_iovs) {
                   #  # ]
     516   [ #  #  #  # ]:          0 :                         fprintf(stderr, "cannot allocated task->src_iovs fot task=%p\n", task);
     517                 :          0 :                         return -ENOMEM;
     518                 :            :                 }
     519                 :            : 
     520         [ +  + ]:        864 :                 if (g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C) {
     521                 :        192 :                         dst_buff_len = g_xfer_size_bytes * g_chained_count;
     522                 :          0 :                 }
     523                 :            : 
     524   [ +  +  #  # ]:        864 :                 if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     525         [ +  + ]:        672 :                     g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY) {
     526   [ -  +  #  #  :        288 :                         src_buff_len += (g_xfer_size_bytes / g_block_size_bytes) * g_md_size_bytes;
                   #  # ]
     527                 :          0 :                 }
     528                 :            : 
     529   [ +  +  #  #  :       1920 :                 for (i = 0; i < task->src_iovcnt; i++) {
                   #  # ]
     530   [ #  #  #  #  :       1056 :                         task->src_iovs[i].iov_base = spdk_dma_zmalloc(src_buff_len, 0, NULL);
          #  #  #  #  #  
                      # ]
     531   [ -  +  #  #  :       1056 :                         if (task->src_iovs[i].iov_base == NULL) {
          #  #  #  #  #  
                #  #  # ]
     532                 :          0 :                                 return -ENOMEM;
     533                 :            :                         }
     534   [ -  +  #  #  :       1056 :                         memset(task->src_iovs[i].iov_base, DATA_PATTERN, src_buff_len);
          #  #  #  #  #  
                #  #  # ]
     535   [ #  #  #  #  :       1056 :                         task->src_iovs[i].iov_len = src_buff_len;
          #  #  #  #  #  
                      # ]
     536                 :          0 :                 }
     537   [ +  -  #  # ]:        864 :                 if (g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
     538         [ +  + ]:        864 :                     g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
     539   [ -  +  #  # ]:         96 :                         md_buff_len = (g_xfer_size_bytes / g_block_size_bytes) * g_md_size_bytes *
     540                 :          0 :                                       g_chained_count;
     541   [ #  #  #  #  :         96 :                         task->md_iov.iov_base = spdk_dma_zmalloc(md_buff_len, 0, NULL);
                   #  # ]
     542   [ -  +  #  #  :         96 :                         if (task->md_iov.iov_base == NULL) {
             #  #  #  # ]
     543                 :          0 :                                 return -ENOMEM;
     544                 :            :                         }
     545   [ #  #  #  #  :         96 :                         task->md_iov.iov_len = md_buff_len;
                   #  # ]
     546                 :          0 :                 }
     547         [ +  + ]:        672 :         } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
     548   [ -  +  #  # ]:        192 :                 assert(g_xor_src_count > 1);
     549   [ #  #  #  # ]:        192 :                 task->sources = calloc(g_xor_src_count, sizeof(*task->sources));
     550   [ -  +  #  #  :        192 :                 if (!task->sources) {
                   #  # ]
     551                 :          0 :                         return -ENOMEM;
     552                 :            :                 }
     553                 :            : 
     554         [ +  + ]:        672 :                 for (i = 0; i < g_xor_src_count; i++) {
     555   [ #  #  #  #  :        480 :                         task->sources[i] = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
             #  #  #  # ]
     556   [ -  +  #  #  :        480 :                         if (!task->sources[i]) {
          #  #  #  #  #  
                      # ]
     557                 :          0 :                                 return -ENOMEM;
     558                 :            :                         }
     559   [ -  +  #  #  :        480 :                         memset(task->sources[i], DATA_PATTERN, g_xfer_size_bytes);
          #  #  #  #  #  
                      # ]
     560                 :          0 :                 }
     561                 :          0 :         } else {
     562   [ #  #  #  # ]:        480 :                 task->src = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
     563   [ -  +  #  #  :        480 :                 if (task->src == NULL) {
                   #  # ]
     564   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Unable to alloc src buffer\n");
     565                 :          0 :                         return -ENOMEM;
     566                 :            :                 }
     567                 :            : 
     568                 :            :                 /* For fill, set the entire src buffer so we can check if verify is enabled. */
     569         [ +  + ]:        480 :                 if (g_workload_selection == SPDK_ACCEL_OPC_FILL) {
     570   [ -  +  #  #  :        192 :                         memset(task->src, g_fill_pattern, g_xfer_size_bytes);
                   #  # ]
     571                 :          0 :                 } else {
     572   [ -  +  #  #  :        288 :                         memset(task->src, DATA_PATTERN, g_xfer_size_bytes);
                   #  # ]
     573                 :            :                 }
     574                 :            :         }
     575                 :            : 
     576   [ +  +  #  # ]:       1536 :         if (g_workload_selection != SPDK_ACCEL_OPC_CRC32C &&
     577         [ +  + ]:       1344 :             g_workload_selection != SPDK_ACCEL_OPC_DIF_VERIFY &&
     578         [ +  + ]:       1248 :             g_workload_selection != SPDK_ACCEL_OPC_DIF_GENERATE &&
     579         [ +  + ]:       1056 :             g_workload_selection != SPDK_ACCEL_OPC_DIF_GENERATE_COPY &&
     580         [ +  - ]:        960 :             g_workload_selection != SPDK_ACCEL_OPC_DIF_VERIFY_COPY &&
     581         [ +  + ]:        960 :             g_workload_selection != SPDK_ACCEL_OPC_DIX_VERIFY &&
     582         [ +  - ]:        864 :             g_workload_selection != SPDK_ACCEL_OPC_DIX_GENERATE) {
     583   [ #  #  #  # ]:        864 :                 task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
     584   [ -  +  #  #  :        864 :                 if (task->dst == NULL) {
                   #  # ]
     585   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Unable to alloc dst buffer\n");
     586                 :          0 :                         return -ENOMEM;
     587                 :            :                 }
     588                 :            : 
     589                 :            :                 /* For compare we want the buffers to match, otherwise not. */
     590         [ +  + ]:        864 :                 if (g_workload_selection == SPDK_ACCEL_OPC_COMPARE) {
     591   [ -  +  #  #  :         96 :                         memset(task->dst, DATA_PATTERN, dst_buff_len);
                   #  # ]
     592                 :          0 :                 } else {
     593   [ -  +  #  #  :        768 :                         memset(task->dst, ~DATA_PATTERN, dst_buff_len);
                   #  # ]
     594                 :            :                 }
     595                 :          0 :         }
     596                 :            : 
     597                 :            :         /* For dualcast 2 buffers are needed for the operation.  */
     598   [ +  +  #  # ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST ||
     599   [ +  +  -  +  :       1440 :             (g_workload_selection == SPDK_ACCEL_OPC_XOR && g_verify)) {
                   +  - ]
     600   [ #  #  #  # ]:        288 :                 task->dst2 = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
     601   [ -  +  #  #  :        288 :                 if (task->dst2 == NULL) {
                   #  # ]
     602   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Unable to alloc dst buffer\n");
     603                 :          0 :                         return -ENOMEM;
     604                 :            :                 }
     605   [ -  +  #  #  :        288 :                 memset(task->dst2, ~DATA_PATTERN, g_xfer_size_bytes);
                   #  # ]
     606                 :          0 :         }
     607                 :            : 
     608   [ +  +  #  # ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     609         [ +  + ]:       1344 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
     610         [ -  + ]:       1248 :             g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
     611                 :        288 :                 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     612         [ #  # ]:        288 :                 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
     613                 :            : 
     614   [ -  +  #  #  :        288 :                 task->num_blocks = (g_xfer_size_bytes * g_chained_count) / g_block_size_bytes;
                   #  # ]
     615                 :            : 
     616         [ #  # ]:        288 :                 rc = spdk_dif_ctx_init(&task->dif_ctx,
     617         [ #  # ]:        288 :                                        g_block_size_bytes + g_md_size_bytes,
     618                 :          0 :                                        g_md_size_bytes, true, true,
     619                 :            :                                        SPDK_DIF_TYPE1,
     620   [ #  #  #  #  :          0 :                                        SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_REFTAG_CHECK,
                   #  # ]
     621                 :            :                                        16, 0xFFFF, 10, 0, 0, &dif_opts);
     622         [ -  + ]:        288 :                 if (rc != 0) {
     623   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Initialization of DIF context failed, error (%d)\n", rc);
     624                 :          0 :                         return rc;
     625                 :            :                 }
     626                 :            : 
     627   [ +  +  #  # ]:        288 :                 if ((g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY) ||
     628         [ -  + ]:        192 :                     (g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY)) {
     629   [ #  #  #  #  :         96 :                         rc = spdk_dif_generate(task->src_iovs, task->src_iovcnt, task->num_blocks, &task->dif_ctx);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     630         [ -  + ]:         96 :                         if (rc != 0) {
     631   [ #  #  #  # ]:          0 :                                 fprintf(stderr, "Generation of DIF failed, error (%d)\n", rc);
     632                 :          0 :                                 return rc;
     633                 :            :                         }
     634                 :          0 :                 }
     635                 :          0 :         }
     636   [ +  -  #  # ]:       1536 :         if (g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE ||
     637         [ +  + ]:       1536 :             g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
     638                 :         96 :                 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     639         [ #  # ]:         96 :                 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
     640                 :            : 
     641   [ -  +  #  #  :         96 :                 task->num_blocks = (g_xfer_size_bytes * g_chained_count) / g_block_size_bytes;
                   #  # ]
     642                 :            : 
     643         [ #  # ]:         96 :                 rc = spdk_dif_ctx_init(&task->dif_ctx,
     644                 :          0 :                                        g_block_size_bytes,
     645                 :          0 :                                        g_md_size_bytes, false, true,
     646                 :            :                                        SPDK_DIF_TYPE1,
     647   [ #  #  #  # ]:          0 :                                        SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK |
     648         [ #  # ]:          0 :                                        SPDK_DIF_FLAGS_REFTAG_CHECK,
     649                 :            :                                        0x123, 0xFFFF, 0x234, 0, 0, &dif_opts);
     650         [ -  + ]:         96 :                 if (rc != 0) {
     651   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Initialization of DIX context failed, error (%d)\n", rc);
     652                 :          0 :                         return rc;
     653                 :            :                 }
     654         [ +  - ]:         96 :                 if (g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY) {
     655   [ #  #  #  #  :         96 :                         rc = spdk_dix_generate(task->src_iovs, task->src_iovcnt, &task->md_iov,
          #  #  #  #  #  
                      # ]
     656   [ #  #  #  #  :         96 :                                                task->num_blocks, &task->dif_ctx);
                   #  # ]
     657         [ -  + ]:         96 :                         if (rc != 0) {
     658   [ #  #  #  # ]:          0 :                                 fprintf(stderr, "Generation of DIX failed, error (%d)\n", rc);
     659                 :          0 :                                 return rc;
     660                 :            :                         }
     661                 :          0 :                 }
     662                 :            : 
     663                 :          0 :         }
     664                 :            : 
     665                 :       1536 :         return 0;
     666                 :          0 : }
     667                 :            : 
     668                 :            : inline static struct ap_task *
     669                 :   12320457 : _get_task(struct worker_thread *worker)
     670                 :            : {
     671                 :            :         struct ap_task *task;
     672                 :            : 
     673   [ +  -  #  #  :   12320457 :         if (!TAILQ_EMPTY(&worker->tasks_pool)) {
             #  #  #  # ]
     674   [ #  #  #  #  :   12320457 :                 task = TAILQ_FIRST(&worker->tasks_pool);
                   #  # ]
     675   [ +  +  #  #  :   12320457 :                 TAILQ_REMOVE(&worker->tasks_pool, task, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
     676                 :          0 :         } else {
     677   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Unable to get ap_task\n");
     678                 :          0 :                 return NULL;
     679                 :            :         }
     680                 :            : 
     681                 :   12320457 :         return task;
     682                 :          0 : }
     683                 :            : 
     684                 :            : /* Submit one operation using the same ap task that just completed. */
     685                 :            : static void
     686                 :   12320457 : _submit_single(struct worker_thread *worker, struct ap_task *task)
     687                 :            : {
     688                 :            :         int random_num;
     689                 :   12320457 :         int rc = 0;
     690                 :            : 
     691   [ -  +  #  # ]:   12320457 :         assert(worker);
     692                 :            : 
     693   [ +  +  +  +  :   12320457 :         switch (worker->workload) {
          +  +  +  +  +  
          +  +  +  -  -  
          +  -  #  #  #  
                      # ]
     694                 :     779392 :         case SPDK_ACCEL_OPC_COPY:
     695   [ #  #  #  #  :     779392 :                 rc = spdk_accel_submit_copy(worker->ch, task->dst, task->src,
          #  #  #  #  #  
                #  #  # ]
     696                 :          0 :                                             g_xfer_size_bytes, accel_done, task);
     697                 :     779392 :                 break;
     698                 :    1184512 :         case SPDK_ACCEL_OPC_FILL:
     699                 :            :                 /* For fill use the first byte of the task->dst buffer */
     700   [ #  #  #  #  :    1184512 :                 rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     701                 :          0 :                                             g_xfer_size_bytes, accel_done, task);
     702                 :    1184512 :                 break;
     703                 :    1961312 :         case SPDK_ACCEL_OPC_CRC32C:
     704   [ #  #  #  #  :    1961312 :                 rc = spdk_accel_submit_crc32cv(worker->ch, task->crc_dst,
             #  #  #  # ]
     705   [ #  #  #  #  :          0 :                                                task->src_iovs, task->src_iovcnt, g_crc32c_seed,
             #  #  #  # ]
     706                 :          0 :                                                accel_done, task);
     707                 :    1961312 :                 break;
     708                 :    1105568 :         case SPDK_ACCEL_OPC_COPY_CRC32C:
     709   [ #  #  #  #  :    1105568 :                 rc = spdk_accel_submit_copy_crc32cv(worker->ch, task->dst, task->src_iovs, task->src_iovcnt,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     710   [ #  #  #  # ]:          0 :                                                     task->crc_dst, g_crc32c_seed, accel_done, task);
     711                 :    1105568 :                 break;
     712                 :    1230176 :         case SPDK_ACCEL_OPC_COMPARE:
     713         [ #  # ]:    1230176 :                 random_num = rand() % 100;
     714         [ -  + ]:    1230176 :                 if (random_num < g_fail_percent_goal) {
     715   [ #  #  #  # ]:          0 :                         task->expected_status = -EILSEQ;
     716   [ #  #  #  #  :          0 :                         *(uint8_t *)task->dst = ~DATA_PATTERN;
                   #  # ]
     717                 :          0 :                 } else {
     718   [ #  #  #  # ]:    1230176 :                         task->expected_status = 0;
     719   [ #  #  #  #  :    1230176 :                         *(uint8_t *)task->dst = DATA_PATTERN;
                   #  # ]
     720                 :            :                 }
     721   [ #  #  #  #  :    1230176 :                 rc = spdk_accel_submit_compare(worker->ch, task->dst, task->src,
          #  #  #  #  #  
                #  #  # ]
     722                 :          0 :                                                g_xfer_size_bytes, accel_done, task);
     723                 :    1230176 :                 break;
     724                 :    1022240 :         case SPDK_ACCEL_OPC_DUALCAST:
     725   [ #  #  #  #  :    1022240 :                 rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
          #  #  #  #  #  
                #  #  # ]
     726   [ #  #  #  # ]:          0 :                                                 task->src, g_xfer_size_bytes, accel_done, task);
     727                 :    1022240 :                 break;
     728                 :     238792 :         case SPDK_ACCEL_OPC_COMPRESS:
     729   [ #  #  #  #  :     238792 :                 task->src_iovs = task->cur_seg->uncompressed_iovs;
          #  #  #  #  #  
                #  #  # ]
     730   [ #  #  #  #  :     238792 :                 task->src_iovcnt = task->cur_seg->uncompressed_iovcnt;
          #  #  #  #  #  
                #  #  # ]
     731   [ #  #  #  #  :     477584 :                 rc = spdk_accel_submit_compress(worker->ch, task->dst, task->cur_seg->compressed_len_padded,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     732   [ #  #  #  # ]:          0 :                                                 task->src_iovs,
     733   [ #  #  #  #  :     238792 :                                                 task->src_iovcnt, &task->compressed_sz, accel_done, task);
                   #  # ]
     734                 :     238792 :                 break;
     735                 :    1590977 :         case SPDK_ACCEL_OPC_DECOMPRESS:
     736   [ #  #  #  #  :    1590977 :                 task->src_iovs = task->cur_seg->compressed_iovs;
          #  #  #  #  #  
                #  #  # ]
     737   [ #  #  #  #  :    1590977 :                 task->src_iovcnt = task->cur_seg->compressed_iovcnt;
          #  #  #  #  #  
                #  #  # ]
     738   [ #  #  #  #  :    1590977 :                 rc = spdk_accel_submit_decompress(worker->ch, task->dst_iovs, task->dst_iovcnt, task->src_iovs,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     739   [ #  #  #  # ]:    1590977 :                                                   task->src_iovcnt, NULL, accel_done, task);
     740                 :    1590977 :                 break;
     741                 :    1839200 :         case SPDK_ACCEL_OPC_XOR:
     742   [ #  #  #  #  :    1839200 :                 rc = spdk_accel_submit_xor(worker->ch, task->dst, task->sources, g_xor_src_count,
          #  #  #  #  #  
                #  #  # ]
     743                 :          0 :                                            g_xfer_size_bytes, accel_done, task);
     744                 :    1839200 :                 break;
     745                 :     258784 :         case SPDK_ACCEL_OPC_DIF_VERIFY:
     746   [ #  #  #  #  :     517568 :                 rc = spdk_accel_submit_dif_verify(worker->ch, task->src_iovs, task->src_iovcnt, task->num_blocks,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     747   [ #  #  #  # ]:     258784 :                                                   &task->dif_ctx, &task->dif_err, accel_done, task);
     748                 :     258784 :                 break;
     749                 :     655200 :         case SPDK_ACCEL_OPC_DIF_GENERATE:
     750   [ #  #  #  #  :     655200 :                 rc = spdk_accel_submit_dif_generate(worker->ch, task->src_iovs, task->src_iovcnt, task->num_blocks,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     751         [ #  # ]:     655200 :                                                     &task->dif_ctx, accel_done, task);
     752                 :     655200 :                 break;
     753                 :     236352 :         case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
     754   [ #  #  #  #  :     472704 :                 rc = spdk_accel_submit_dif_generate_copy(worker->ch, task->dst_iovs, task->dst_iovcnt,
          #  #  #  #  #  
                #  #  # ]
     755   [ #  #  #  #  :     236352 :                                 task->src_iovs, task->src_iovcnt,
             #  #  #  # ]
     756   [ #  #  #  #  :     236352 :                                 task->num_blocks, &task->dif_ctx, accel_done, task);
                   #  # ]
     757                 :     236352 :                 break;
     758                 :          0 :         case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
     759   [ #  #  #  #  :          0 :                 rc = spdk_accel_submit_dif_verify_copy(worker->ch, task->dst_iovs, task->dst_iovcnt,
          #  #  #  #  #  
                #  #  # ]
     760   [ #  #  #  #  :          0 :                                                        task->src_iovs, task->src_iovcnt, task->num_blocks,
          #  #  #  #  #  
                #  #  # ]
     761   [ #  #  #  # ]:          0 :                                                        &task->dif_ctx, &task->dif_err, accel_done, task);
     762                 :          0 :                 break;
     763                 :          0 :         case SPDK_ACCEL_OPC_DIX_GENERATE:
     764   [ #  #  #  #  :          0 :                 rc = spdk_accel_submit_dix_generate(worker->ch, task->src_iovs, task->src_iovcnt,
          #  #  #  #  #  
                #  #  # ]
     765   [ #  #  #  #  :          0 :                                                     &task->md_iov, task->num_blocks,
                   #  # ]
     766         [ #  # ]:          0 :                                                     &task->dif_ctx, accel_done, task);
     767                 :          0 :                 break;
     768                 :     217952 :         case SPDK_ACCEL_OPC_DIX_VERIFY:
     769   [ #  #  #  #  :     435904 :                 rc = spdk_accel_submit_dix_verify(worker->ch, task->src_iovs, task->src_iovcnt,
          #  #  #  #  #  
                #  #  # ]
     770   [ #  #  #  #  :          0 :                                                   &task->md_iov, task->num_blocks,
                   #  # ]
     771   [ #  #  #  # ]:     217952 :                                                   &task->dif_ctx, &task->dif_err, accel_done, task);
     772                 :     217952 :                 break;
     773                 :          0 :         default:
     774         [ #  # ]:          0 :                 assert(false);
     775                 :            :                 break;
     776                 :            : 
     777                 :            :         }
     778                 :            : 
     779         [ #  # ]:   12320457 :         worker->current_queue_depth++;
     780         [ -  + ]:   12320457 :         if (rc) {
     781                 :          0 :                 accel_done(task, rc);
     782                 :          0 :         }
     783                 :   12320457 : }
     784                 :            : 
     785                 :            : static void
     786                 :       3456 : _free_task_buffers(struct ap_task *task)
     787                 :            : {
     788                 :            :         uint32_t i;
     789                 :            : 
     790   [ +  +  #  # ]:       3456 :         if (g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS ||
     791         [ +  + ]:       1664 :             g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
     792   [ #  #  #  # ]:       1920 :                 free(task->dst_iovs);
     793   [ +  +  #  # ]:       1536 :         } else if (g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
     794         [ +  + ]:       1344 :                    g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
     795         [ +  + ]:       1152 :                    g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
     796         [ +  + ]:       1056 :                    g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
     797         [ +  + ]:        864 :                    g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE_COPY ||
     798         [ +  - ]:        768 :                    g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY_COPY ||
     799         [ +  + ]:        768 :                    g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
     800         [ -  + ]:        672 :                    g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) {
     801   [ +  +  #  #  :        864 :                 if (task->crc_dst) {
                   #  # ]
     802   [ #  #  #  # ]:        384 :                         spdk_dma_free(task->crc_dst);
     803                 :          0 :                 }
     804   [ +  -  #  #  :        864 :                 if (task->src_iovs) {
                   #  # ]
     805   [ +  +  #  #  :       1920 :                         for (i = 0; i < task->src_iovcnt; i++) {
                   #  # ]
     806   [ +  -  #  #  :       1056 :                                 if (task->src_iovs[i].iov_base) {
          #  #  #  #  #  
                #  #  # ]
     807   [ #  #  #  #  :       1056 :                                         spdk_dma_free(task->src_iovs[i].iov_base);
          #  #  #  #  #  
                      # ]
     808                 :          0 :                                 }
     809                 :          0 :                         }
     810   [ #  #  #  # ]:        864 :                         free(task->src_iovs);
     811                 :          0 :                 }
     812   [ +  +  #  #  :        864 :                 if (task->dst_iovs) {
                   #  # ]
     813   [ +  +  #  #  :        192 :                         for (i = 0; i < task->dst_iovcnt; i++) {
                   #  # ]
     814   [ +  -  #  #  :         96 :                                 if (task->dst_iovs[i].iov_base) {
          #  #  #  #  #  
                #  #  # ]
     815   [ #  #  #  #  :         96 :                                         spdk_dma_free(task->dst_iovs[i].iov_base);
          #  #  #  #  #  
                      # ]
     816                 :          0 :                                 }
     817                 :          0 :                         }
     818   [ #  #  #  # ]:         96 :                         free(task->dst_iovs);
     819                 :          0 :                 }
     820   [ +  +  #  #  :        864 :                 if (task->md_iov.iov_base) {
             #  #  #  # ]
     821   [ #  #  #  #  :         96 :                         spdk_dma_free(task->md_iov.iov_base);
                   #  # ]
     822                 :          0 :                 }
     823         [ +  + ]:        672 :         } else if (g_workload_selection == SPDK_ACCEL_OPC_XOR) {
     824   [ +  -  #  #  :        192 :                 if (task->sources) {
                   #  # ]
     825         [ +  + ]:        672 :                         for (i = 0; i < g_xor_src_count; i++) {
     826   [ #  #  #  #  :        480 :                                 spdk_dma_free(task->sources[i]);
             #  #  #  # ]
     827                 :          0 :                         }
     828   [ #  #  #  # ]:        192 :                         free(task->sources);
     829                 :          0 :                 }
     830                 :          0 :         } else {
     831   [ #  #  #  # ]:        480 :                 spdk_dma_free(task->src);
     832                 :            :         }
     833                 :            : 
     834   [ #  #  #  # ]:       3456 :         spdk_dma_free(task->dst);
     835   [ +  +  +  + ]:       3456 :         if (g_workload_selection == SPDK_ACCEL_OPC_DUALCAST || g_workload_selection == SPDK_ACCEL_OPC_XOR) {
     836   [ #  #  #  # ]:        288 :                 spdk_dma_free(task->dst2);
     837                 :          0 :         }
     838                 :       3456 : }
     839                 :            : 
     840                 :            : static int
     841                 :    1105568 : _vector_memcmp(void *_dst, struct iovec *src_src_iovs, uint32_t iovcnt)
     842                 :            : {
     843                 :            :         uint32_t i;
     844                 :    1105568 :         uint32_t ttl_len = 0;
     845                 :    1105568 :         uint8_t *dst = (uint8_t *)_dst;
     846                 :            : 
     847         [ +  + ]:    2670624 :         for (i = 0; i < iovcnt; i++) {
     848   [ -  +  -  +  :    1565056 :                 if (memcmp(dst, src_src_iovs[i].iov_base, src_src_iovs[i].iov_len)) {
          -  +  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
     849                 :          0 :                         return -1;
     850                 :            :                 }
     851   [ #  #  #  #  :    1565056 :                 dst += src_src_iovs[i].iov_len;
             #  #  #  # ]
     852   [ #  #  #  #  :    1565056 :                 ttl_len += src_src_iovs[i].iov_len;
                   #  # ]
     853                 :          0 :         }
     854                 :            : 
     855         [ -  + ]:    1105568 :         if (ttl_len != iovcnt * g_xfer_size_bytes) {
     856                 :          0 :                 return -1;
     857                 :            :         }
     858                 :            : 
     859                 :    1105568 :         return 0;
     860                 :          0 : }
     861                 :            : 
     862                 :            : static int _worker_stop(void *arg);
     863                 :            : 
     864                 :            : static void
     865                 :   12320457 : accel_done(void *arg1, int status)
     866                 :            : {
     867                 :   12320457 :         struct ap_task *task = arg1;
     868   [ #  #  #  # ]:   12320457 :         struct worker_thread *worker = task->worker;
     869                 :            :         uint32_t sw_crc32c;
     870                 :    4316704 :         struct spdk_dif_error err_blk;
     871                 :            : 
     872   [ -  +  #  # ]:   12320457 :         assert(worker);
     873   [ -  +  #  #  :   12320457 :         assert(worker->current_queue_depth > 0);
             #  #  #  # ]
     874                 :            : 
     875   [ -  +  +  +  :   12320457 :         if (g_verify && status == 0) {
                   +  - ]
     876   [ +  +  +  +  :   10713377 :                 switch (worker->workload) {
          +  +  -  +  +  
          -  -  -  -  -  
          -  -  #  #  #  
                      # ]
     877                 :    1105568 :                 case SPDK_ACCEL_OPC_COPY_CRC32C:
     878   [ #  #  #  #  :    1105568 :                         sw_crc32c = spdk_crc32c_iov_update(task->src_iovs, task->src_iovcnt, ~g_crc32c_seed);
             #  #  #  # ]
     879   [ -  +  #  #  :    1105568 :                         if (*task->crc_dst != sw_crc32c) {
             #  #  #  # ]
     880                 :          0 :                                 SPDK_NOTICELOG("CRC-32C miscompare\n");
     881         [ #  # ]:          0 :                                 worker->xfer_failed++;
     882                 :          0 :                         }
     883   [ -  +  #  #  :    1105568 :                         if (_vector_memcmp(task->dst, task->src_iovs, task->src_iovcnt)) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     884                 :          0 :                                 SPDK_NOTICELOG("Data miscompare\n");
     885         [ #  # ]:          0 :                                 worker->xfer_failed++;
     886                 :          0 :                         }
     887                 :    1105568 :                         break;
     888                 :    1961312 :                 case SPDK_ACCEL_OPC_CRC32C:
     889   [ #  #  #  #  :    1961312 :                         sw_crc32c = spdk_crc32c_iov_update(task->src_iovs, task->src_iovcnt, ~g_crc32c_seed);
             #  #  #  # ]
     890   [ -  +  #  #  :    1961312 :                         if (*task->crc_dst != sw_crc32c) {
             #  #  #  # ]
     891                 :          0 :                                 SPDK_NOTICELOG("CRC-32C miscompare\n");
     892         [ #  # ]:          0 :                                 worker->xfer_failed++;
     893                 :          0 :                         }
     894                 :    1961312 :                         break;
     895                 :     779392 :                 case SPDK_ACCEL_OPC_COPY:
     896   [ -  +  -  +  :     779392 :                         if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
          -  +  #  #  #  
             #  #  #  #  
                      # ]
     897                 :          0 :                                 SPDK_NOTICELOG("Data miscompare\n");
     898         [ #  # ]:          0 :                                 worker->xfer_failed++;
     899                 :          0 :                         }
     900                 :     779392 :                         break;
     901                 :    1022240 :                 case SPDK_ACCEL_OPC_DUALCAST:
     902   [ -  +  -  +  :    1022240 :                         if (memcmp(task->src, task->dst, g_xfer_size_bytes)) {
          -  +  #  #  #  
             #  #  #  #  
                      # ]
     903                 :          0 :                                 SPDK_NOTICELOG("Data miscompare, first destination\n");
     904         [ #  # ]:          0 :                                 worker->xfer_failed++;
     905                 :          0 :                         }
     906   [ -  +  -  +  :    1022240 :                         if (memcmp(task->src, task->dst2, g_xfer_size_bytes)) {
          -  +  #  #  #  
             #  #  #  #  
                      # ]
     907                 :          0 :                                 SPDK_NOTICELOG("Data miscompare, second destination\n");
     908         [ #  # ]:          0 :                                 worker->xfer_failed++;
     909                 :          0 :                         }
     910                 :    1022240 :                         break;
     911                 :    1184512 :                 case SPDK_ACCEL_OPC_FILL:
     912   [ -  +  -  +  :    1184512 :                         if (memcmp(task->dst, task->src, g_xfer_size_bytes)) {
          -  +  #  #  #  
             #  #  #  #  
                      # ]
     913                 :          0 :                                 SPDK_NOTICELOG("Data miscompare\n");
     914         [ #  # ]:          0 :                                 worker->xfer_failed++;
     915                 :          0 :                         }
     916                 :    1184512 :                         break;
     917                 :    1230176 :                 case SPDK_ACCEL_OPC_COMPARE:
     918                 :    1230176 :                         break;
     919                 :          0 :                 case SPDK_ACCEL_OPC_COMPRESS:
     920                 :          0 :                         break;
     921                 :    1590977 :                 case SPDK_ACCEL_OPC_DECOMPRESS:
     922   [ -  +  -  +  :    1590977 :                         if (memcmp(task->dst, task->cur_seg->uncompressed_data, task->cur_seg->uncompressed_len)) {
          -  +  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     923                 :          0 :                                 SPDK_NOTICELOG("Data miscompare on decompression\n");
     924         [ #  # ]:          0 :                                 worker->xfer_failed++;
     925                 :          0 :                         }
     926                 :    1590977 :                         break;
     927                 :    1839200 :                 case SPDK_ACCEL_OPC_XOR:
     928   [ -  +  #  #  :    1839200 :                         if (spdk_xor_gen(task->dst2, task->sources, g_xor_src_count,
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     929                 :          0 :                                          g_xfer_size_bytes) != 0) {
     930                 :          0 :                                 SPDK_ERRLOG("Failed to generate xor for verification\n");
     931   [ -  +  -  +  :    1839200 :                         } else if (memcmp(task->dst, task->dst2, g_xfer_size_bytes)) {
          -  +  #  #  #  
             #  #  #  #  
                      # ]
     932                 :          0 :                                 SPDK_NOTICELOG("Data miscompare\n");
     933         [ #  # ]:          0 :                                 worker->xfer_failed++;
     934                 :          0 :                         }
     935                 :    1839200 :                         break;
     936                 :          0 :                 case SPDK_ACCEL_OPC_DIF_VERIFY:
     937                 :          0 :                         break;
     938                 :          0 :                 case SPDK_ACCEL_OPC_DIF_GENERATE:
     939   [ #  #  #  #  :          0 :                         if (spdk_dif_verify(task->src_iovs, task->src_iovcnt, task->num_blocks,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     940         [ #  # ]:          0 :                                             &task->dif_ctx, &err_blk) != 0) {
     941   [ #  #  #  #  :          0 :                                 SPDK_NOTICELOG("Data miscompare, "
                   #  # ]
     942                 :            :                                                "err_type %u, expected %lu, actual %lu, err_offset %u\n",
     943                 :            :                                                err_blk.err_type, err_blk.expected,
     944                 :            :                                                err_blk.actual, err_blk.err_offset);
     945         [ #  # ]:          0 :                                 worker->xfer_failed++;
     946                 :          0 :                         }
     947                 :          0 :                         break;
     948                 :          0 :                 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
     949   [ #  #  #  #  :          0 :                         if (spdk_dif_verify(task->dst_iovs, task->dst_iovcnt, task->num_blocks,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     950         [ #  # ]:          0 :                                             &task->dif_ctx, &err_blk) != 0) {
     951   [ #  #  #  #  :          0 :                                 SPDK_NOTICELOG("Data miscompare, "
                   #  # ]
     952                 :            :                                                "err_type %u, expected %lu, actual %lu, err_offset %u\n",
     953                 :            :                                                err_blk.err_type, err_blk.expected,
     954                 :            :                                                err_blk.actual, err_blk.err_offset);
     955         [ #  # ]:          0 :                                 worker->xfer_failed++;
     956                 :          0 :                         }
     957                 :          0 :                         break;
     958                 :          0 :                 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
     959                 :          0 :                         break;
     960                 :          0 :                 case SPDK_ACCEL_OPC_DIX_GENERATE:
     961   [ #  #  #  #  :          0 :                         if (spdk_dix_verify(task->src_iovs, task->src_iovcnt, &task->md_iov,
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     962   [ #  #  #  #  :          0 :                                             task->num_blocks, &task->dif_ctx, &err_blk) != 0) {
                   #  # ]
     963   [ #  #  #  #  :          0 :                                 SPDK_NOTICELOG("Data miscompare, "
                   #  # ]
     964                 :            :                                                "err_type %u, expected %lu, actual %lu, err_offset %u\n",
     965                 :            :                                                err_blk.err_type, err_blk.expected,
     966                 :            :                                                err_blk.actual, err_blk.err_offset);
     967         [ #  # ]:          0 :                                 worker->xfer_failed++;
     968                 :          0 :                         }
     969                 :          0 :                         break;
     970                 :          0 :                 case SPDK_ACCEL_OPC_DIX_VERIFY:
     971                 :          0 :                         break;
     972                 :          0 :                 default:
     973         [ #  # ]:          0 :                         assert(false);
     974                 :            :                         break;
     975                 :            :                 }
     976                 :          0 :         }
     977                 :            : 
     978   [ +  +  #  #  :   12320457 :         if (worker->workload == SPDK_ACCEL_OPC_COMPRESS ||
             #  #  #  # ]
     979         [ +  + ]:   12081665 :             g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
     980                 :            :                 /* Advance the task to the next segment */
     981   [ #  #  #  #  :    1829769 :                 task->cur_seg = STAILQ_NEXT(task->cur_seg, link);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     982   [ +  +  #  #  :    1829769 :                 if (task->cur_seg == NULL) {
                   #  # ]
     983   [ #  #  #  # ]:     271368 :                         task->cur_seg = STAILQ_FIRST(&g_compress_segs);
     984                 :          0 :                 }
     985                 :          0 :         }
     986                 :            : 
     987   [ -  +  #  #  :   12320457 :         if (task->expected_status == -EILSEQ) {
                   #  # ]
     988   [ #  #  #  # ]:          0 :                 assert(status != 0);
     989         [ #  # ]:          0 :                 worker->injected_miscompares++;
     990                 :          0 :                 status = 0;
     991         [ -  + ]:   12320457 :         } else if (status) {
     992                 :            :                 /* Expected to pass but the accel module reported an error (ex: COMPARE operation). */
     993         [ #  # ]:          0 :                 worker->xfer_failed++;
     994                 :          0 :         }
     995                 :            : 
     996         [ #  # ]:   12320457 :         worker->current_queue_depth--;
     997                 :            : 
     998   [ -  +  +  +  :   12320457 :         if (!worker->is_draining && status == 0) {
          +  -  #  #  #  
                      # ]
     999   [ #  #  #  #  :   12317001 :                 TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1000                 :   12317001 :                 task = _get_task(worker);
    1001                 :   12317001 :                 _submit_single(worker, task);
    1002                 :          0 :         } else {
    1003   [ #  #  #  #  :       3456 :                 TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1004                 :            :         }
    1005                 :   12320457 : }
    1006                 :            : 
    1007                 :            : static int
    1008                 :         73 : dump_result(void)
    1009                 :            : {
    1010                 :         73 :         uint64_t total_completed = 0;
    1011                 :         73 :         uint64_t total_failed = 0;
    1012                 :         73 :         uint64_t total_miscompared = 0;
    1013                 :         73 :         uint64_t total_xfer_per_sec, total_bw_in_MiBps = 0;
    1014                 :         73 :         struct worker_thread *worker = g_workers;
    1015                 :         22 :         char tmp[64];
    1016                 :            : 
    1017         [ -  + ]:         73 :         printf("\n%-12s %20s %16s %16s %16s\n",
    1018                 :            :                "Core,Thread", "Transfers", "Bandwidth", "Failed", "Miscompares");
    1019         [ -  + ]:         73 :         printf("------------------------------------------------------------------------------------\n");
    1020         [ +  + ]:        178 :         while (worker != NULL) {
    1021                 :            : 
    1022   [ -  +  #  #  :        105 :                 uint64_t xfer_per_sec = worker->stats.executed / g_time_in_sec;
             #  #  #  # ]
    1023   [ -  +  #  #  :        105 :                 uint64_t bw_in_MiBps = worker->stats.num_bytes /
             #  #  #  # ]
    1024   [ #  #  #  # ]:        105 :                                        (g_time_in_sec * 1024 * 1024);
    1025                 :            : 
    1026   [ #  #  #  #  :        105 :                 total_completed += worker->stats.executed;
                   #  # ]
    1027   [ #  #  #  # ]:        105 :                 total_failed += worker->xfer_failed;
    1028   [ #  #  #  # ]:        105 :                 total_miscompared += worker->injected_miscompares;
    1029                 :        105 :                 total_bw_in_MiBps += bw_in_MiBps;
    1030                 :            : 
    1031   [ -  +  #  #  :        105 :                 snprintf(tmp, sizeof(tmp), "%u,%u", worker->display.core, worker->display.thread);
          #  #  #  #  #  
                #  #  # ]
    1032         [ +  - ]:        105 :                 if (xfer_per_sec) {
    1033         [ -  + ]:        105 :                         printf("%-12s %18" PRIu64 "/s %10" PRIu64 " MiB/s %16"PRIu64 " %16" PRIu64 "\n",
    1034   [ #  #  #  # ]:          0 :                                tmp, xfer_per_sec, bw_in_MiBps, worker->xfer_failed,
    1035   [ #  #  #  # ]:          0 :                                worker->injected_miscompares);
    1036                 :          0 :                 }
    1037                 :            : 
    1038   [ #  #  #  # ]:        105 :                 worker = worker->next;
    1039                 :            :         }
    1040                 :            : 
    1041         [ -  + ]:         73 :         total_xfer_per_sec = total_completed / g_time_in_sec;
    1042                 :            : 
    1043         [ -  + ]:         73 :         printf("====================================================================================\n");
    1044         [ -  + ]:         73 :         printf("%-12s %18" PRIu64 "/s %10" PRIu64 " MiB/s %16"PRIu64 " %16" PRIu64 "\n",
    1045                 :          0 :                "Total", total_xfer_per_sec, total_bw_in_MiBps, total_failed, total_miscompared);
    1046                 :            : 
    1047                 :         73 :         return total_failed ? 1 : 0;
    1048                 :            : }
    1049                 :            : 
    1050                 :            : static inline void
    1051                 :        105 : _free_task_buffers_in_pool(struct worker_thread *worker)
    1052                 :            : {
    1053                 :            :         struct ap_task *task;
    1054                 :            : 
    1055   [ -  +  #  # ]:        105 :         assert(worker);
    1056   [ +  +  #  #  :       3561 :         while ((task = TAILQ_FIRST(&worker->tasks_pool))) {
             #  #  #  # ]
    1057   [ +  +  #  #  :       3456 :                 TAILQ_REMOVE(&worker->tasks_pool, task, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    1058                 :       3456 :                 _free_task_buffers(task);
    1059                 :            :         }
    1060                 :        105 : }
    1061                 :            : 
    1062                 :            : static int
    1063                 :       1176 : _check_draining(void *arg)
    1064                 :            : {
    1065                 :       1176 :         struct worker_thread *worker = arg;
    1066                 :            : 
    1067   [ -  +  #  # ]:       1176 :         assert(worker);
    1068                 :            : 
    1069   [ +  +  #  #  :       1176 :         if (worker->current_queue_depth == 0) {
                   #  # ]
    1070                 :        105 :                 _free_task_buffers_in_pool(worker);
    1071         [ #  # ]:        105 :                 spdk_poller_unregister(&worker->is_draining_poller);
    1072                 :        105 :                 unregister_worker(worker);
    1073                 :          0 :         }
    1074                 :            : 
    1075                 :       1176 :         return SPDK_POLLER_BUSY;
    1076                 :            : }
    1077                 :            : 
    1078                 :            : static int
    1079                 :        105 : _worker_stop(void *arg)
    1080                 :            : {
    1081                 :        105 :         struct worker_thread *worker = arg;
    1082                 :            : 
    1083   [ -  +  #  # ]:        105 :         assert(worker);
    1084                 :            : 
    1085         [ #  # ]:        105 :         spdk_poller_unregister(&worker->stop_poller);
    1086                 :            : 
    1087                 :            :         /* now let the worker drain and check it's outstanding IO with a poller */
    1088   [ #  #  #  # ]:        105 :         worker->is_draining = true;
    1089   [ #  #  #  # ]:        105 :         worker->is_draining_poller = SPDK_POLLER_REGISTER(_check_draining, worker, 0);
    1090                 :            : 
    1091                 :        105 :         return SPDK_POLLER_BUSY;
    1092                 :            : }
    1093                 :            : 
    1094                 :            : static void shutdown_cb(void);
    1095                 :            : 
    1096                 :            : static void
    1097                 :        105 : _init_thread(void *arg1)
    1098                 :            : {
    1099                 :            :         struct worker_thread *worker;
    1100                 :            :         struct ap_task *task;
    1101                 :        105 :         int i, num_tasks = g_allocate_depth;
    1102                 :        105 :         struct display_info *display = arg1;
    1103                 :            : 
    1104                 :        105 :         worker = calloc(1, sizeof(*worker));
    1105         [ -  + ]:        105 :         if (worker == NULL) {
    1106   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Unable to allocate worker\n");
    1107                 :          0 :                 free(display);
    1108                 :          0 :                 spdk_thread_exit(spdk_get_thread());
    1109                 :          0 :                 goto no_worker;
    1110                 :            :         }
    1111                 :            : 
    1112   [ #  #  #  # ]:        105 :         worker->workload = g_workload_selection;
    1113   [ #  #  #  #  :        105 :         worker->display.core = display->core;
          #  #  #  #  #  
                      # ]
    1114   [ #  #  #  #  :        105 :         worker->display.thread = display->thread;
          #  #  #  #  #  
                      # ]
    1115                 :        105 :         free(display);
    1116   [ #  #  #  # ]:        105 :         worker->core = spdk_env_get_current_core();
    1117   [ #  #  #  # ]:        105 :         worker->thread = spdk_get_thread();
    1118         [ -  + ]:        105 :         pthread_mutex_lock(&g_workers_lock);
    1119         [ #  # ]:        105 :         g_num_workers++;
    1120   [ #  #  #  # ]:        105 :         worker->next = g_workers;
    1121                 :        105 :         g_workers = worker;
    1122         [ -  + ]:        105 :         pthread_mutex_unlock(&g_workers_lock);
    1123   [ #  #  #  # ]:        105 :         worker->ch = spdk_accel_get_io_channel();
    1124   [ -  +  #  #  :        105 :         if (worker->ch == NULL) {
                   #  # ]
    1125   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Unable to get an accel channel\n");
    1126                 :          0 :                 goto error;
    1127                 :            :         }
    1128                 :            : 
    1129   [ #  #  #  #  :        105 :         TAILQ_INIT(&worker->tasks_pool);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1130                 :            : 
    1131   [ #  #  #  # ]:        105 :         worker->task_base = calloc(num_tasks, sizeof(struct ap_task));
    1132   [ -  +  #  #  :        105 :         if (worker->task_base == NULL) {
                   #  # ]
    1133   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Could not allocate task base.\n");
    1134                 :          0 :                 goto error;
    1135                 :            :         }
    1136                 :            : 
    1137   [ #  #  #  # ]:        105 :         task = worker->task_base;
    1138   [ +  +  #  # ]:       3561 :         for (i = 0; i < num_tasks; i++) {
    1139   [ #  #  #  #  :       3456 :                 TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1140   [ #  #  #  # ]:       3456 :                 task->worker = worker;
    1141         [ -  + ]:       3456 :                 if (_get_task_data_bufs(task)) {
    1142   [ #  #  #  # ]:          0 :                         fprintf(stderr, "Unable to get data bufs\n");
    1143                 :          0 :                         goto error;
    1144                 :            :                 }
    1145         [ #  # ]:       3456 :                 task++;
    1146                 :          0 :         }
    1147                 :            : 
    1148                 :            :         /* Register a poller that will stop the worker at time elapsed */
    1149   [ #  #  #  # ]:        105 :         worker->stop_poller = SPDK_POLLER_REGISTER(_worker_stop, worker,
    1150                 :            :                               g_time_in_sec * 1000000ULL);
    1151                 :            : 
    1152                 :            :         /* Load up queue depth worth of operations. */
    1153   [ +  +  #  # ]:       3561 :         for (i = 0; i < g_queue_depth; i++) {
    1154                 :       3456 :                 task = _get_task(worker);
    1155         [ -  + ]:       3456 :                 if (task == NULL) {
    1156                 :          0 :                         goto error;
    1157                 :            :                 }
    1158                 :            : 
    1159                 :       3456 :                 _submit_single(worker, task);
    1160                 :          0 :         }
    1161                 :        105 :         return;
    1162                 :          0 : error:
    1163                 :            : 
    1164                 :          0 :         _free_task_buffers_in_pool(worker);
    1165   [ #  #  #  # ]:          0 :         free(worker->task_base);
    1166   [ #  #  #  # ]:          0 :         worker->task_base = NULL;
    1167                 :          0 : no_worker:
    1168                 :          0 :         shutdown_cb();
    1169                 :          0 :         g_rc = -1;
    1170                 :          0 : }
    1171                 :            : 
    1172                 :            : static void
    1173                 :         73 : accel_perf_start(void *arg1)
    1174                 :            : {
    1175                 :         73 :         struct spdk_cpuset tmp_cpumask = {};
    1176                 :         22 :         char thread_name[32];
    1177                 :            :         uint32_t i;
    1178                 :            :         int j;
    1179                 :            :         struct spdk_thread *thread;
    1180                 :            :         struct display_info *display;
    1181                 :            : 
    1182                 :         73 :         g_tsc_rate = spdk_get_ticks_hz();
    1183                 :         73 :         g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
    1184                 :            : 
    1185                 :         73 :         dump_user_config();
    1186                 :            : 
    1187         [ -  + ]:         73 :         printf("Running for %d seconds...\n", g_time_in_sec);
    1188                 :         73 :         fflush(stdout);
    1189                 :            : 
    1190                 :            :         /* Create worker threads for each core that was specified. */
    1191         [ +  + ]:        170 :         SPDK_ENV_FOREACH_CORE(i) {
    1192   [ +  +  #  # ]:        202 :                 for (j = 0; j < g_threads_per_core; j++) {
    1193         [ -  + ]:        105 :                         snprintf(thread_name, sizeof(thread_name), "ap_worker_%u_%u", i, j);
    1194                 :        105 :                         spdk_cpuset_zero(&tmp_cpumask);
    1195                 :        105 :                         spdk_cpuset_set_cpu(&tmp_cpumask, i, true);
    1196                 :        105 :                         thread = spdk_thread_create(thread_name, &tmp_cpumask);
    1197                 :        105 :                         display = calloc(1, sizeof(*display));
    1198         [ -  + ]:        105 :                         if (display == NULL) {
    1199   [ #  #  #  # ]:          0 :                                 fprintf(stderr, "Unable to allocate memory\n");
    1200                 :          0 :                                 spdk_app_stop(-1);
    1201                 :          0 :                                 return;
    1202                 :            :                         }
    1203   [ #  #  #  # ]:        105 :                         display->core = i;
    1204   [ #  #  #  # ]:        105 :                         display->thread = j;
    1205                 :        105 :                         spdk_thread_send_msg(thread, _init_thread, display);
    1206                 :          0 :                 }
    1207                 :          0 :         }
    1208                 :          0 : }
    1209                 :            : 
    1210                 :            : static void
    1211                 :         79 : accel_perf_free_compress_segs(void)
    1212                 :            : {
    1213                 :            :         struct ap_compress_seg *seg, *tmp;
    1214                 :            : 
    1215   [ +  +  #  #  :        539 :         STAILQ_FOREACH_SAFE(seg, &g_compress_segs, link, tmp) {
          #  #  #  #  #  
                      # ]
    1216   [ #  #  #  # ]:        460 :                 free(seg->uncompressed_iovs);
    1217   [ #  #  #  # ]:        460 :                 free(seg->compressed_iovs);
    1218   [ #  #  #  # ]:        460 :                 spdk_dma_free(seg->compressed_data);
    1219   [ #  #  #  # ]:        460 :                 spdk_dma_free(seg->uncompressed_data);
    1220   [ +  +  #  #  :        460 :                 STAILQ_REMOVE_HEAD(&g_compress_segs, link);
          #  #  #  #  #  
                      # ]
    1221                 :        460 :                 free(seg);
    1222                 :          0 :         }
    1223                 :         79 : }
    1224                 :            : 
    1225                 :            : struct accel_perf_prep_ctx {
    1226                 :            :         FILE                    *file;
    1227                 :            :         long                    remaining;
    1228                 :            :         struct spdk_io_channel  *ch;
    1229                 :            :         struct ap_compress_seg  *cur_seg;
    1230                 :            : };
    1231                 :            : 
    1232                 :            : static void accel_perf_prep_process_seg(struct accel_perf_prep_ctx *ctx);
    1233                 :            : 
    1234                 :            : static void
    1235                 :        460 : accel_perf_prep_process_seg_cpl(void *ref, int status)
    1236                 :            : {
    1237                 :        460 :         struct accel_perf_prep_ctx *ctx = ref;
    1238                 :            :         struct ap_compress_seg *seg;
    1239                 :            : 
    1240         [ -  + ]:        460 :         if (status != 0) {
    1241   [ #  #  #  # ]:          0 :                 fprintf(stderr, "error (%d) on initial compress completion\n", status);
    1242   [ #  #  #  #  :          0 :                 spdk_dma_free(ctx->cur_seg->compressed_data);
             #  #  #  # ]
    1243   [ #  #  #  #  :          0 :                 spdk_dma_free(ctx->cur_seg->uncompressed_data);
             #  #  #  # ]
    1244   [ #  #  #  # ]:          0 :                 free(ctx->cur_seg);
    1245   [ #  #  #  # ]:          0 :                 spdk_put_io_channel(ctx->ch);
    1246   [ #  #  #  #  :          0 :                 fclose(ctx->file);
                   #  # ]
    1247                 :          0 :                 free(ctx);
    1248         [ #  # ]:          0 :                 spdk_app_stop(-status);
    1249                 :          0 :                 return;
    1250                 :            :         }
    1251                 :            : 
    1252   [ #  #  #  # ]:        460 :         seg = ctx->cur_seg;
    1253                 :            : 
    1254         [ +  + ]:        460 :         if (g_workload_selection == SPDK_ACCEL_OPC_DECOMPRESS) {
    1255   [ #  #  #  # ]:        348 :                 seg->compressed_iovs = calloc(g_chained_count, sizeof(struct iovec));
    1256   [ -  +  #  #  :        348 :                 if (seg->compressed_iovs == NULL) {
                   #  # ]
    1257   [ #  #  #  # ]:          0 :                         fprintf(stderr, "unable to allocate iovec\n");
    1258   [ #  #  #  # ]:          0 :                         spdk_dma_free(seg->compressed_data);
    1259   [ #  #  #  # ]:          0 :                         spdk_dma_free(seg->uncompressed_data);
    1260                 :          0 :                         free(seg);
    1261   [ #  #  #  # ]:          0 :                         spdk_put_io_channel(ctx->ch);
    1262   [ #  #  #  #  :          0 :                         fclose(ctx->file);
                   #  # ]
    1263                 :          0 :                         free(ctx);
    1264                 :          0 :                         spdk_app_stop(-ENOMEM);
    1265                 :          0 :                         return;
    1266                 :            :                 }
    1267   [ #  #  #  # ]:        348 :                 seg->compressed_iovcnt = g_chained_count;
    1268                 :            : 
    1269   [ #  #  #  #  :        348 :                 accel_perf_construct_iovs(seg->compressed_data, seg->compressed_len, seg->compressed_iovs,
          #  #  #  #  #  
                #  #  # ]
    1270   [ #  #  #  # ]:          0 :                                           seg->compressed_iovcnt);
    1271                 :          0 :         }
    1272                 :            : 
    1273   [ #  #  #  #  :        460 :         STAILQ_INSERT_TAIL(&g_compress_segs, seg, link);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1274   [ #  #  #  #  :        460 :         ctx->remaining -= seg->uncompressed_len;
          #  #  #  #  #  
                      # ]
    1275                 :            : 
    1276                 :        460 :         accel_perf_prep_process_seg(ctx);
    1277                 :          0 : }
    1278                 :            : 
    1279                 :            : static void
    1280                 :        488 : accel_perf_prep_process_seg(struct accel_perf_prep_ctx *ctx)
    1281                 :            : {
    1282                 :            :         struct ap_compress_seg *seg;
    1283                 :            :         int sz, sz_read, sz_padded;
    1284                 :            :         void *ubuf, *cbuf;
    1285                 :        122 :         struct iovec iov[1];
    1286                 :            :         int rc;
    1287                 :            : 
    1288   [ +  +  #  #  :        488 :         if (ctx->remaining == 0) {
                   #  # ]
    1289   [ #  #  #  # ]:         28 :                 spdk_put_io_channel(ctx->ch);
    1290   [ -  +  #  #  :         28 :                 fclose(ctx->file);
                   #  # ]
    1291                 :         28 :                 free(ctx);
    1292                 :         28 :                 accel_perf_start(NULL);
    1293                 :        373 :                 return;
    1294                 :            :         }
    1295                 :            : 
    1296   [ #  #  #  #  :        460 :         sz = spdk_min(ctx->remaining, g_xfer_size_bytes);
          #  #  #  #  #  
                      # ]
    1297                 :            :         /* Add 10% pad to the compress buffer for incompressible data. Note that a real app
    1298                 :            :          * would likely either deal with the failure of not having a large enough buffer
    1299                 :            :          * by submitting another operation with a larger one.  Or, like the vbdev module
    1300                 :            :          * does, just accept the error and use the data uncompressed marking it as such in
    1301                 :            :          * its own metadata so that in the future it doesn't try to decompress uncompressed
    1302                 :            :          * data, etc.
    1303                 :            :          */
    1304         [ #  # ]:        460 :         sz_padded = sz * COMP_BUF_PAD_PERCENTAGE;
    1305                 :            : 
    1306                 :        460 :         ubuf = spdk_dma_zmalloc(sz, ALIGN_4K, NULL);
    1307         [ -  + ]:        460 :         if (!ubuf) {
    1308         [ #  # ]:          0 :                 fprintf(stderr, "unable to allocate uncompress buffer\n");
    1309                 :          0 :                 rc = -ENOMEM;
    1310                 :          0 :                 goto error;
    1311                 :            :         }
    1312                 :            : 
    1313                 :        460 :         cbuf = spdk_dma_malloc(sz_padded, ALIGN_4K, NULL);
    1314         [ -  + ]:        460 :         if (!cbuf) {
    1315         [ #  # ]:          0 :                 fprintf(stderr, "unable to allocate compress buffer\n");
    1316                 :          0 :                 rc = -ENOMEM;
    1317                 :          0 :                 spdk_dma_free(ubuf);
    1318                 :          0 :                 goto error;
    1319                 :            :         }
    1320                 :            : 
    1321                 :        460 :         seg = calloc(1, sizeof(*seg));
    1322         [ -  + ]:        460 :         if (!seg) {
    1323         [ #  # ]:          0 :                 fprintf(stderr, "unable to allocate comp/decomp segment\n");
    1324                 :          0 :                 spdk_dma_free(ubuf);
    1325                 :          0 :                 spdk_dma_free(cbuf);
    1326                 :          0 :                 rc = -ENOMEM;
    1327                 :          0 :                 goto error;
    1328                 :            :         }
    1329                 :            : 
    1330   [ #  #  #  # ]:        460 :         sz_read = fread(ubuf, sizeof(uint8_t), sz, ctx->file);
    1331         [ -  + ]:        460 :         if (sz_read != sz) {
    1332         [ #  # ]:          0 :                 fprintf(stderr, "unable to read input file\n");
    1333                 :          0 :                 free(seg);
    1334                 :          0 :                 spdk_dma_free(ubuf);
    1335                 :          0 :                 spdk_dma_free(cbuf);
    1336   [ #  #  #  # ]:          0 :                 rc = -errno;
    1337                 :          0 :                 goto error;
    1338                 :            :         }
    1339                 :            : 
    1340         [ +  + ]:        460 :         if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS) {
    1341   [ #  #  #  # ]:        112 :                 seg->uncompressed_iovs = calloc(g_chained_count, sizeof(struct iovec));
    1342   [ -  +  #  #  :        112 :                 if (seg->uncompressed_iovs == NULL) {
                   #  # ]
    1343         [ #  # ]:          0 :                         fprintf(stderr, "unable to allocate iovec\n");
    1344                 :          0 :                         free(seg);
    1345                 :          0 :                         spdk_dma_free(ubuf);
    1346                 :          0 :                         spdk_dma_free(cbuf);
    1347                 :          0 :                         rc = -ENOMEM;
    1348                 :          0 :                         goto error;
    1349                 :            :                 }
    1350   [ #  #  #  # ]:        112 :                 seg->uncompressed_iovcnt = g_chained_count;
    1351   [ #  #  #  #  :        112 :                 accel_perf_construct_iovs(ubuf, sz, seg->uncompressed_iovs, seg->uncompressed_iovcnt);
             #  #  #  # ]
    1352                 :          0 :         }
    1353                 :            : 
    1354   [ #  #  #  # ]:        460 :         seg->uncompressed_data = ubuf;
    1355   [ #  #  #  # ]:        460 :         seg->uncompressed_len = sz;
    1356   [ #  #  #  # ]:        460 :         seg->compressed_data = cbuf;
    1357   [ #  #  #  # ]:        460 :         seg->compressed_len = sz;
    1358   [ #  #  #  # ]:        460 :         seg->compressed_len_padded = sz_padded;
    1359                 :            : 
    1360   [ #  #  #  # ]:        460 :         ctx->cur_seg = seg;
    1361   [ #  #  #  #  :        460 :         iov[0].iov_base = seg->uncompressed_data;
             #  #  #  # ]
    1362   [ #  #  #  #  :        460 :         iov[0].iov_len = seg->uncompressed_len;
          #  #  #  #  #  
                      # ]
    1363                 :            :         /* Note that anytime a call is made to spdk_accel_submit_compress() there's a chance
    1364                 :            :          * it will fail with -ENOMEM in the event that the destination buffer is not large enough
    1365                 :            :          * to hold the compressed data.  This example app simply adds 10% buffer for compressed data
    1366                 :            :          * but real applications may want to consider a more sophisticated method.
    1367                 :            :          */
    1368   [ #  #  #  #  :        460 :         rc = spdk_accel_submit_compress(ctx->ch, seg->compressed_data, seg->compressed_len_padded, iov, 1,
          #  #  #  #  #  
                #  #  # ]
    1369         [ #  # ]:          0 :                                         &seg->compressed_len, accel_perf_prep_process_seg_cpl, ctx);
    1370         [ -  + ]:        460 :         if (rc < 0) {
    1371         [ #  # ]:          0 :                 fprintf(stderr, "error (%d) on initial compress submission\n", rc);
    1372                 :          0 :                 goto error;
    1373                 :            :         }
    1374                 :            : 
    1375                 :        460 :         return;
    1376                 :            : 
    1377                 :          0 : error:
    1378   [ #  #  #  # ]:          0 :         spdk_put_io_channel(ctx->ch);
    1379   [ #  #  #  #  :          0 :         fclose(ctx->file);
                   #  # ]
    1380                 :          0 :         free(ctx);
    1381                 :          0 :         spdk_app_stop(rc);
    1382                 :          0 : }
    1383                 :            : 
    1384                 :            : static void
    1385                 :         79 : accel_perf_prep(void *arg1)
    1386                 :            : {
    1387                 :            :         struct accel_perf_prep_ctx *ctx;
    1388                 :         79 :         const char *module_name = NULL;
    1389                 :         79 :         int rc = 0;
    1390                 :            : 
    1391         [ -  + ]:         79 :         if (g_module_name) {
    1392                 :          0 :                 rc = spdk_accel_get_opc_module_name(g_workload_selection, &module_name);
    1393   [ #  #  #  #  :          0 :                 if (rc != 0 || strcmp(g_module_name, module_name) != 0) {
             #  #  #  # ]
    1394         [ #  # ]:          0 :                         fprintf(stderr, "Module '%s' was assigned via JSON config or RPC, instead of '%s'\n",
    1395                 :          0 :                                 module_name, g_module_name);
    1396         [ #  # ]:          0 :                         fprintf(stderr, "-M option is not compatible with accel_assign_opc RPC\n");
    1397                 :          0 :                         rc = -EINVAL;
    1398                 :          0 :                         goto error_end;
    1399                 :            :                 }
    1400                 :          0 :         }
    1401                 :            : 
    1402   [ +  +  #  # ]:         79 :         if (g_workload_selection != SPDK_ACCEL_OPC_COMPRESS &&
    1403         [ +  + ]:         69 :             g_workload_selection != SPDK_ACCEL_OPC_DECOMPRESS) {
    1404                 :         45 :                 accel_perf_start(arg1);
    1405                 :         66 :                 return;
    1406                 :            :         }
    1407                 :            : 
    1408         [ +  + ]:         34 :         if (g_cd_file_in_name == NULL) {
    1409         [ -  + ]:          3 :                 fprintf(stdout, "A filename is required.\n");
    1410                 :          3 :                 rc = -EINVAL;
    1411                 :          3 :                 goto error_end;
    1412                 :            :         }
    1413                 :            : 
    1414   [ +  +  -  +  :         31 :         if (g_workload_selection == SPDK_ACCEL_OPC_COMPRESS && g_verify) {
                   +  + ]
    1415         [ -  + ]:          3 :                 fprintf(stdout, "\nCompression does not support the verify option, aborting.\n");
    1416                 :          3 :                 rc = -ENOTSUP;
    1417                 :          3 :                 goto error_end;
    1418                 :            :         }
    1419                 :            : 
    1420                 :         28 :         printf("Preparing input file...\n");
    1421                 :            : 
    1422                 :         28 :         ctx = calloc(1, sizeof(*ctx));
    1423         [ -  + ]:         28 :         if (ctx == NULL) {
    1424                 :          0 :                 rc = -ENOMEM;
    1425                 :          0 :                 goto error_end;
    1426                 :            :         }
    1427                 :            : 
    1428   [ #  #  #  # ]:         28 :         ctx->file = fopen(g_cd_file_in_name, "r");
    1429   [ -  +  #  #  :         28 :         if (ctx->file == NULL) {
                   #  # ]
    1430         [ #  # ]:          0 :                 fprintf(stderr, "Could not open file %s.\n", g_cd_file_in_name);
    1431   [ #  #  #  # ]:          0 :                 rc = -errno;
    1432                 :          0 :                 goto error_ctx;
    1433                 :            :         }
    1434                 :            : 
    1435   [ #  #  #  # ]:         28 :         fseek(ctx->file, 0L, SEEK_END);
    1436   [ #  #  #  #  :         28 :         ctx->remaining = ftell(ctx->file);
             #  #  #  # ]
    1437   [ #  #  #  # ]:         28 :         fseek(ctx->file, 0L, SEEK_SET);
    1438                 :            : 
    1439   [ #  #  #  # ]:         28 :         ctx->ch = spdk_accel_get_io_channel();
    1440   [ -  +  #  #  :         28 :         if (ctx->ch == NULL) {
                   #  # ]
    1441                 :          0 :                 rc = -EAGAIN;
    1442                 :          0 :                 goto error_file;
    1443                 :            :         }
    1444                 :            : 
    1445         [ +  + ]:         28 :         if (g_xfer_size_bytes == 0) {
    1446                 :            :                 /* size of 0 means "file at a time" */
    1447   [ #  #  #  # ]:         12 :                 g_xfer_size_bytes = ctx->remaining;
    1448                 :          0 :         }
    1449                 :            : 
    1450                 :         28 :         accel_perf_prep_process_seg(ctx);
    1451                 :         28 :         return;
    1452                 :            : 
    1453                 :          0 : error_file:
    1454   [ #  #  #  #  :          0 :         fclose(ctx->file);
                   #  # ]
    1455                 :          0 : error_ctx:
    1456                 :          0 :         free(ctx);
    1457                 :          6 : error_end:
    1458                 :          6 :         spdk_app_stop(rc);
    1459                 :          0 : }
    1460                 :            : 
    1461                 :            : static void
    1462                 :          0 : worker_shutdown(void *ctx)
    1463                 :            : {
    1464                 :          0 :         _worker_stop(ctx);
    1465                 :          0 : }
    1466                 :            : 
    1467                 :            : static void
    1468                 :          0 : shutdown_cb(void)
    1469                 :            : {
    1470                 :            :         struct worker_thread *worker;
    1471                 :            : 
    1472         [ #  # ]:          0 :         pthread_mutex_lock(&g_workers_lock);
    1473         [ #  # ]:          0 :         if (!g_workers) {
    1474                 :          0 :                 spdk_app_stop(1);
    1475                 :          0 :                 goto unlock;
    1476                 :            :         }
    1477                 :            : 
    1478                 :          0 :         worker = g_workers;
    1479         [ #  # ]:          0 :         while (worker) {
    1480   [ #  #  #  # ]:          0 :                 spdk_thread_send_msg(worker->thread, worker_shutdown, worker);
    1481   [ #  #  #  # ]:          0 :                 worker = worker->next;
    1482                 :            :         }
    1483                 :          0 : unlock:
    1484         [ #  # ]:          0 :         pthread_mutex_unlock(&g_workers_lock);
    1485                 :          0 : }
    1486                 :            : 
    1487                 :            : int
    1488                 :         88 : main(int argc, char **argv)
    1489                 :            : {
    1490                 :            :         struct worker_thread *worker, *tmp;
    1491                 :            :         int rc;
    1492                 :            : 
    1493         [ -  + ]:         88 :         pthread_mutex_init(&g_workers_lock, NULL);
    1494                 :         88 :         spdk_app_opts_init(&g_opts, sizeof(g_opts));
    1495                 :         88 :         g_opts.name = "accel_perf";
    1496                 :         88 :         g_opts.reactor_mask = "0x1";
    1497                 :         88 :         g_opts.shutdown_cb = shutdown_cb;
    1498                 :         88 :         g_opts.rpc_addr = NULL;
    1499                 :            : 
    1500                 :         88 :         rc = spdk_app_parse_args(argc, argv, &g_opts, "a:C:o:q:t:yw:M:P:f:T:l:S:x:", NULL,
    1501                 :            :                                  parse_args, usage);
    1502         [ +  + ]:         88 :         if (rc != SPDK_APP_PARSE_ARGS_SUCCESS) {
    1503                 :          9 :                 return rc == SPDK_APP_PARSE_ARGS_HELP ? 0 : 1;
    1504                 :            :         }
    1505                 :            : 
    1506         [ -  + ]:         79 :         if (g_workload_selection == SPDK_ACCEL_OPC_LAST) {
    1507   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Must provide a workload type\n");
    1508                 :          0 :                 usage();
    1509                 :          0 :                 return -1;
    1510                 :            :         }
    1511                 :            : 
    1512   [ +  +  -  + ]:         79 :         if (g_allocate_depth > 0 && g_queue_depth > g_allocate_depth) {
    1513   [ #  #  #  # ]:          0 :                 fprintf(stdout, "allocate depth must be at least as big as queue depth\n");
    1514                 :          0 :                 usage();
    1515                 :          0 :                 return -1;
    1516                 :            :         }
    1517                 :            : 
    1518         [ +  + ]:         79 :         if (g_allocate_depth == 0) {
    1519                 :         76 :                 g_allocate_depth = g_queue_depth;
    1520                 :          0 :         }
    1521                 :            : 
    1522   [ +  +  #  # ]:         79 :         if ((g_workload_selection == SPDK_ACCEL_OPC_CRC32C ||
    1523         [ +  + ]:         73 :              g_workload_selection == SPDK_ACCEL_OPC_COPY_CRC32C ||
    1524         [ +  + ]:         67 :              g_workload_selection == SPDK_ACCEL_OPC_DIF_VERIFY ||
    1525         [ +  + ]:         64 :              g_workload_selection == SPDK_ACCEL_OPC_DIF_GENERATE ||
    1526         [ +  + ]:         58 :              g_workload_selection == SPDK_ACCEL_OPC_DIX_VERIFY ||
    1527         [ -  + ]:         55 :              g_workload_selection == SPDK_ACCEL_OPC_DIX_GENERATE) &&
    1528         [ -  + ]:         24 :             g_chained_count == 0) {
    1529                 :          0 :                 usage();
    1530                 :          0 :                 return -1;
    1531                 :            :         }
    1532                 :            : 
    1533   [ +  +  -  + ]:         79 :         if (g_workload_selection == SPDK_ACCEL_OPC_XOR && g_xor_src_count < 2) {
    1534                 :          0 :                 usage();
    1535                 :          0 :                 return -1;
    1536                 :            :         }
    1537                 :            : 
    1538   [ -  +  -  - ]:         79 :         if (g_module_name && spdk_accel_assign_opc(g_workload_selection, g_module_name)) {
    1539   [ #  #  #  # ]:          0 :                 fprintf(stderr, "Was not able to assign '%s' module to the workload\n", g_module_name);
    1540                 :          0 :                 usage();
    1541                 :          0 :                 return -1;
    1542                 :            :         }
    1543                 :            : 
    1544                 :         79 :         g_rc = spdk_app_start(&g_opts, accel_perf_prep, NULL);
    1545         [ +  + ]:         79 :         if (g_rc) {
    1546                 :          6 :                 SPDK_ERRLOG("ERROR starting application\n");
    1547                 :          0 :         }
    1548                 :            : 
    1549         [ -  + ]:         79 :         pthread_mutex_destroy(&g_workers_lock);
    1550                 :            : 
    1551                 :         79 :         worker = g_workers;
    1552         [ +  + ]:        184 :         while (worker) {
    1553   [ #  #  #  # ]:        105 :                 tmp = worker->next;
    1554                 :        105 :                 free(worker);
    1555                 :        105 :                 worker = tmp;
    1556                 :            :         }
    1557                 :         79 :         accel_perf_free_compress_segs();
    1558                 :         79 :         spdk_app_fini();
    1559                 :         79 :         return g_rc;
    1560                 :          0 : }

Generated by: LCOV version 1.15