LCOV - code coverage report
Current view: top level - spdk/module/accel/dpdk_cryptodev - accel_dpdk_cryptodev.c (source / functions) Hit Total Coverage
Test: Combined Lines: 488 686 71.1 %
Date: 2024-07-15 14:29:31 Functions: 33 35 94.3 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 253 446 56.7 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2018 Intel Corporation.
       3                 :            :  *   Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
       4                 :            :  *   All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #include "accel_dpdk_cryptodev.h"
       8                 :            : 
       9                 :            : #include "spdk/accel.h"
      10                 :            : #include "spdk/accel_module.h"
      11                 :            : #include "spdk/env.h"
      12                 :            : #include "spdk/likely.h"
      13                 :            : #include "spdk/thread.h"
      14                 :            : #include "spdk/util.h"
      15                 :            : #include "spdk/log.h"
      16                 :            : #include "spdk/json.h"
      17                 :            : #include "spdk_internal/sgl.h"
      18                 :            : 
      19                 :            : #include <rte_bus_vdev.h>
      20                 :            : #include <rte_crypto.h>
      21                 :            : #include <rte_cryptodev.h>
      22                 :            : #include <rte_mbuf_dyn.h>
      23                 :            : #include <rte_version.h>
      24                 :            : 
      25                 :            : /* The VF spread is the number of queue pairs between virtual functions, we use this to
      26                 :            :  * load balance the QAT device.
      27                 :            :  */
      28                 :            : #define ACCEL_DPDK_CRYPTODEV_QAT_VF_SPREAD              32
      29                 :            : 
      30                 :            : /* This controls how many ops will be dequeued from the crypto driver in one run
      31                 :            :  * of the poller. It is mainly a performance knob as it effectively determines how
      32                 :            :  * much work the poller has to do.  However even that can vary between crypto drivers
      33                 :            :  * as the ACCEL_DPDK_CRYPTODEV_AESNI_MB driver for example does all the crypto work on dequeue whereas the
      34                 :            :  * QAT driver just dequeues what has been completed already.
      35                 :            :  */
      36                 :            : #define ACCEL_DPDK_CRYPTODEV_MAX_DEQUEUE_BURST_SIZE     64
      37                 :            : 
      38                 :            : #define ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE (128)
      39                 :            : 
      40                 :            : /* The number of MBUFS we need must be a power of two and to support other small IOs
      41                 :            :  * in addition to the limits mentioned above, we go to the next power of two. It is
      42                 :            :  * big number because it is one mempool for source and destination mbufs. It may
      43                 :            :  * need to be bigger to support multiple crypto drivers at once.
      44                 :            :  */
      45                 :            : #define ACCEL_DPDK_CRYPTODEV_NUM_MBUFS                  32768
      46                 :            : #define ACCEL_DPDK_CRYPTODEV_POOL_CACHE_SIZE            256
      47                 :            : #define ACCEL_DPDK_CRYPTODEV_MAX_CRYPTO_VOLUMES         128
      48                 :            : #define ACCEL_DPDK_CRYPTODEV_NUM_SESSIONS               (2 * ACCEL_DPDK_CRYPTODEV_MAX_CRYPTO_VOLUMES)
      49                 :            : #define ACCEL_DPDK_CRYPTODEV_SESS_MEMPOOL_CACHE_SIZE    0
      50                 :            : 
      51                 :            : /* This is the max number of IOs we can supply to any crypto device QP at one time.
      52                 :            :  * It can vary between drivers.
      53                 :            :  */
      54                 :            : #define ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS             2048
      55                 :            : 
      56                 :            : /* At this moment DPDK descriptors allocation for mlx5 has some issues. We use 512
      57                 :            :  * as a compromise value between performance and the time spent for initialization. */
      58                 :            : #define ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS_MLX5        512
      59                 :            : 
      60                 :            : #define ACCEL_DPDK_CRYPTODEV_AESNI_MB_NUM_QP            64
      61                 :            : 
      62                 :            : /* Common for suported devices. */
      63                 :            : #define ACCEL_DPDK_CRYPTODEV_DEFAULT_NUM_XFORMS         2
      64                 :            : #define ACCEL_DPDK_CRYPTODEV_IV_OFFSET (sizeof(struct rte_crypto_op) + \
      65                 :            :                 sizeof(struct rte_crypto_sym_op) + \
      66                 :            :                 (ACCEL_DPDK_CRYPTODEV_DEFAULT_NUM_XFORMS * \
      67                 :            :                  sizeof(struct rte_crypto_sym_xform)))
      68                 :            : #define ACCEL_DPDK_CRYPTODEV_IV_LENGTH                  16
      69                 :            : 
      70                 :            : /* Driver names */
      71                 :            : #define ACCEL_DPDK_CRYPTODEV_AESNI_MB   "crypto_aesni_mb"
      72                 :            : #define ACCEL_DPDK_CRYPTODEV_QAT        "crypto_qat"
      73                 :            : #define ACCEL_DPDK_CRYPTODEV_QAT_ASYM   "crypto_qat_asym"
      74                 :            : #define ACCEL_DPDK_CRYPTODEV_MLX5       "mlx5_pci"
      75                 :            : #define ACCEL_DPDK_CRYPTODEV_UADK       "crypto_uadk"
      76                 :            : 
      77                 :            : /* Supported ciphers */
      78                 :            : #define ACCEL_DPDK_CRYPTODEV_AES_CBC    "AES_CBC" /* QAT and ACCEL_DPDK_CRYPTODEV_AESNI_MB */
      79                 :            : #define ACCEL_DPDK_CRYPTODEV_AES_XTS    "AES_XTS" /* QAT and MLX5 */
      80                 :            : 
      81                 :            : /* Specific to AES_CBC. */
      82                 :            : #define ACCEL_DPDK_CRYPTODEV_AES_CBC_KEY_LENGTH                 16
      83                 :            : 
      84                 :            : /* Limit of the max memory len attached to mbuf - rte_pktmbuf_attach_extbuf has uint16_t `buf_len`
      85                 :            :  * parameter, we use closes aligned value 32768 for better performance */
      86                 :            : #define ACCEL_DPDK_CRYPTODEV_MAX_MBUF_LEN                       32768
      87                 :            : 
      88                 :            : /* Used to store IO context in mbuf */
      89                 :            : static const struct rte_mbuf_dynfield rte_mbuf_dynfield_io_context = {
      90                 :            :         .name = "context_accel_dpdk_cryptodev",
      91                 :            :         .size = sizeof(uint64_t),
      92                 :            :         .align = __alignof__(uint64_t),
      93                 :            :         .flags = 0,
      94                 :            : };
      95                 :            : 
      96                 :            : struct accel_dpdk_cryptodev_device;
      97                 :            : 
      98                 :            : enum accel_dpdk_cryptodev_driver_type {
      99                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB = 0,
     100                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_QAT,
     101                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI,
     102                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_UADK,
     103                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_LAST
     104                 :            : };
     105                 :            : 
     106                 :            : struct accel_dpdk_cryptodev_qp {
     107                 :            :         struct accel_dpdk_cryptodev_device *device;     /* ptr to crypto device */
     108                 :            :         uint32_t num_enqueued_ops;      /* Used to decide whether to poll the qp or not */
     109                 :            :         uint8_t qp; /* queue identifier */
     110                 :            :         bool in_use; /* whether this node is in use or not */
     111                 :            :         uint8_t index; /* used by QAT to load balance placement of qpairs */
     112                 :            :         TAILQ_ENTRY(accel_dpdk_cryptodev_qp) link;
     113                 :            : };
     114                 :            : 
     115                 :            : struct accel_dpdk_cryptodev_device {
     116                 :            :         enum accel_dpdk_cryptodev_driver_type type;
     117                 :            :         struct rte_cryptodev_info cdev_info; /* includes DPDK device friendly name */
     118                 :            :         uint32_t qp_desc_nr; /* max number of qp descriptors to be enqueued in burst */
     119                 :            :         uint8_t cdev_id; /* identifier for the device */
     120                 :            :         TAILQ_HEAD(, accel_dpdk_cryptodev_qp) qpairs;
     121                 :            :         TAILQ_ENTRY(accel_dpdk_cryptodev_device) link;
     122                 :            : };
     123                 :            : 
     124                 :            : struct accel_dpdk_cryptodev_key_handle {
     125                 :            :         struct accel_dpdk_cryptodev_device *device;
     126                 :            :         TAILQ_ENTRY(accel_dpdk_cryptodev_key_handle) link;
     127                 :            :         void *session_encrypt;  /* encryption session for this key */
     128                 :            :         void *session_decrypt;  /* decryption session for this key */
     129                 :            :         struct rte_crypto_sym_xform cipher_xform;               /* crypto control struct for this key */
     130                 :            : };
     131                 :            : 
     132                 :            : struct accel_dpdk_cryptodev_key_priv {
     133                 :            :         enum accel_dpdk_cryptodev_driver_type driver;
     134                 :            :         enum spdk_accel_cipher cipher;
     135                 :            :         char *xts_key;
     136                 :            :         TAILQ_HEAD(, accel_dpdk_cryptodev_key_handle) dev_keys;
     137                 :            : };
     138                 :            : 
     139                 :            : /* The crypto channel struct. It is allocated and freed on my behalf by the io channel code.
     140                 :            :  * We store things in here that are needed on per thread basis like the base_channel for this thread,
     141                 :            :  * and the poller for this thread.
     142                 :            :  */
     143                 :            : struct accel_dpdk_cryptodev_io_channel {
     144                 :            :         /* completion poller */
     145                 :            :         struct spdk_poller *poller;
     146                 :            :         /* Array of qpairs for each available device. The specific device will be selected depending on the crypto key */
     147                 :            :         struct accel_dpdk_cryptodev_qp *device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_LAST];
     148                 :            :         /* Used to queue tasks when qpair is full or only part of crypto ops was submitted to the PMD */
     149                 :            :         TAILQ_HEAD(, accel_dpdk_cryptodev_task) queued_tasks;
     150                 :            :         /* Used to queue tasks that were completed in submission path - to avoid calling cpl_cb and possibly overflow
     151                 :            :          * call stack */
     152                 :            :         TAILQ_HEAD(, accel_dpdk_cryptodev_task) completed_tasks;
     153                 :            : };
     154                 :            : 
     155                 :            : struct accel_dpdk_cryptodev_task {
     156                 :            :         struct spdk_accel_task base;
     157                 :            :         uint32_t cryop_completed;       /* The number of crypto operations completed by HW */
     158                 :            :         uint32_t cryop_submitted;       /* The number of crypto operations submitted to HW */
     159                 :            :         uint32_t cryop_total;           /* Total number of crypto operations in this task */
     160                 :            :         bool is_failed;
     161                 :            :         bool inplace;
     162                 :            :         TAILQ_ENTRY(accel_dpdk_cryptodev_task) link;
     163                 :            : };
     164                 :            : 
     165                 :            : /* Shared mempools between all devices on this system */
     166                 :            : static struct rte_mempool *g_session_mp = NULL;
     167                 :            : static struct rte_mempool *g_session_mp_priv = NULL;
     168                 :            : static struct rte_mempool *g_mbuf_mp = NULL;            /* mbuf mempool */
     169                 :            : static int g_mbuf_offset;
     170                 :            : static struct rte_mempool *g_crypto_op_mp = NULL;       /* crypto operations, must be rte* mempool */
     171                 :            : 
     172                 :            : static struct rte_mbuf_ext_shared_info g_shinfo = {};   /* used by DPDK mbuf macro */
     173                 :            : 
     174                 :            : static uint8_t g_qat_total_qp = 0;
     175                 :            : static uint8_t g_next_qat_index;
     176                 :            : 
     177                 :            : static const char *g_driver_names[] = {
     178                 :            :         [ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB]  = ACCEL_DPDK_CRYPTODEV_AESNI_MB,
     179                 :            :         [ACCEL_DPDK_CRYPTODEV_DRIVER_QAT]       = ACCEL_DPDK_CRYPTODEV_QAT,
     180                 :            :         [ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI]  = ACCEL_DPDK_CRYPTODEV_MLX5,
     181                 :            :         [ACCEL_DPDK_CRYPTODEV_DRIVER_UADK]      = ACCEL_DPDK_CRYPTODEV_UADK
     182                 :            : };
     183                 :            : static const char *g_cipher_names[] = {
     184                 :            :         [SPDK_ACCEL_CIPHER_AES_CBC]     = ACCEL_DPDK_CRYPTODEV_AES_CBC,
     185                 :            :         [SPDK_ACCEL_CIPHER_AES_XTS]     = ACCEL_DPDK_CRYPTODEV_AES_XTS,
     186                 :            : };
     187                 :            : 
     188                 :            : static enum accel_dpdk_cryptodev_driver_type g_dpdk_cryptodev_driver =
     189                 :            :         ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB;
     190                 :            : 
     191                 :            : /* Global list of all crypto devices */
     192                 :            : static TAILQ_HEAD(, accel_dpdk_cryptodev_device) g_crypto_devices = TAILQ_HEAD_INITIALIZER(
     193                 :            :                         g_crypto_devices);
     194                 :            : static pthread_mutex_t g_device_lock = PTHREAD_MUTEX_INITIALIZER;
     195                 :            : 
     196                 :            : static struct spdk_accel_module_if g_accel_dpdk_cryptodev_module;
     197                 :            : 
     198                 :            : static int accel_dpdk_cryptodev_process_task(struct accel_dpdk_cryptodev_io_channel *crypto_ch,
     199                 :            :                 struct accel_dpdk_cryptodev_task *task);
     200                 :            : 
     201                 :            : void
     202                 :         23 : accel_dpdk_cryptodev_enable(void)
     203                 :            : {
     204                 :         23 :         spdk_accel_module_list_add(&g_accel_dpdk_cryptodev_module);
     205                 :         23 : }
     206                 :            : 
     207                 :            : int
     208                 :         22 : accel_dpdk_cryptodev_set_driver(const char *driver_name)
     209                 :            : {
     210   [ -  +  +  + ]:         22 :         if (strcmp(driver_name, ACCEL_DPDK_CRYPTODEV_QAT) == 0) {
     211                 :          9 :                 g_dpdk_cryptodev_driver = ACCEL_DPDK_CRYPTODEV_DRIVER_QAT;
     212   [ -  +  +  - ]:         13 :         } else if (strcmp(driver_name, ACCEL_DPDK_CRYPTODEV_AESNI_MB) == 0) {
     213                 :         13 :                 g_dpdk_cryptodev_driver = ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB;
     214   [ #  #  #  # ]:          0 :         } else if (strcmp(driver_name, ACCEL_DPDK_CRYPTODEV_MLX5) == 0) {
     215                 :          0 :                 g_dpdk_cryptodev_driver = ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI;
     216   [ #  #  #  # ]:          0 :         } else if (strcmp(driver_name, ACCEL_DPDK_CRYPTODEV_UADK) == 0) {
     217                 :          0 :                 g_dpdk_cryptodev_driver = ACCEL_DPDK_CRYPTODEV_DRIVER_UADK;
     218                 :            :         } else {
     219                 :          0 :                 SPDK_ERRLOG("Unsupported driver %s\n", driver_name);
     220                 :          0 :                 return -EINVAL;
     221                 :            :         }
     222                 :            : 
     223                 :         22 :         SPDK_NOTICELOG("Using driver %s\n", driver_name);
     224                 :            : 
     225                 :         22 :         return 0;
     226                 :            : }
     227                 :            : 
     228                 :            : const char *
     229                 :          0 : accel_dpdk_cryptodev_get_driver(void)
     230                 :            : {
     231                 :          0 :         return g_driver_names[g_dpdk_cryptodev_driver];
     232                 :            : }
     233                 :            : 
     234                 :            : static inline uint16_t
     235                 :     472838 : accel_dpdk_cryptodev_poll_qp(struct accel_dpdk_cryptodev_qp *qp,
     236                 :            :                              struct accel_dpdk_cryptodev_io_channel *crypto_ch)
     237                 :            : {
     238                 :         81 :         struct rte_crypto_op *dequeued_ops[ACCEL_DPDK_CRYPTODEV_MAX_DEQUEUE_BURST_SIZE];
     239                 :         81 :         struct rte_mbuf *mbufs_to_free[2 * ACCEL_DPDK_CRYPTODEV_MAX_DEQUEUE_BURST_SIZE];
     240                 :            :         struct accel_dpdk_cryptodev_task *task;
     241                 :     472838 :         uint32_t num_mbufs = 0;
     242                 :            :         int i;
     243                 :            :         uint16_t num_dequeued_ops;
     244                 :            : 
     245                 :            :         /* Each run of the poller will get just what the device has available
     246                 :            :          * at the moment we call it, we don't check again after draining the
     247                 :            :          * first batch.
     248                 :            :          */
     249                 :     472838 :         num_dequeued_ops = rte_cryptodev_dequeue_burst(qp->device->cdev_id, qp->qp,
     250                 :            :                            dequeued_ops, ACCEL_DPDK_CRYPTODEV_MAX_DEQUEUE_BURST_SIZE);
     251                 :            :         /* Check if operation was processed successfully */
     252         [ +  + ]:   10847705 :         for (i = 0; i < num_dequeued_ops; i++) {
     253                 :            : 
     254                 :            :                 /* We don't know the order or association of the crypto ops wrt any
     255                 :            :                  * particular task so need to look at each and determine if it's
     256                 :            :                  * the last one for it's task or not.
     257                 :            :                  */
     258                 :   10374867 :                 task = (struct accel_dpdk_cryptodev_task *)*RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src,
     259                 :            :                                 g_mbuf_offset, uint64_t *);
     260         [ -  + ]:   10374867 :                 assert(task != NULL);
     261                 :            : 
     262         [ -  + ]:   10374867 :                 if (dequeued_ops[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
     263                 :          0 :                         SPDK_ERRLOG("error with op %d status %u\n", i, dequeued_ops[i]->status);
     264                 :            :                         /* Update the task status to error, we'll still process the
     265                 :            :                          * rest of the crypto ops for this task though so they
     266                 :            :                          * aren't left hanging.
     267                 :            :                          */
     268                 :          0 :                         task->is_failed = true;
     269                 :            :                 }
     270                 :            : 
     271                 :            :                 /* Return the associated src and dst mbufs by collecting them into
     272                 :            :                  * an array that we can use the bulk API to free after the loop.
     273                 :            :                  */
     274                 :   10374867 :                 *RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src, g_mbuf_offset, uint64_t *) = 0;
     275                 :   10374867 :                 mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_src;
     276         [ +  + ]:   10374867 :                 if (dequeued_ops[i]->sym->m_dst) {
     277                 :   10374575 :                         mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_dst;
     278                 :            :                 }
     279                 :            : 
     280                 :   10374867 :                 task->cryop_completed++;
     281         [ +  + ]:   10374867 :                 if (task->cryop_completed == task->cryop_total) {
     282                 :            :                         /* Complete the IO */
     283   [ -  +  -  + ]:    2938498 :                         spdk_accel_task_complete(&task->base, task->is_failed ? -EINVAL : 0);
     284         [ +  + ]:    7436369 :                 } else if (task->cryop_completed == task->cryop_submitted) {
     285                 :            :                         /* submit remaining crypto ops */
     286                 :      13654 :                         int rc = accel_dpdk_cryptodev_process_task(crypto_ch, task);
     287                 :            : 
     288         [ +  + ]:      13654 :                         if (spdk_unlikely(rc)) {
     289         [ +  - ]:       7099 :                                 if (rc == -ENOMEM) {
     290                 :       7099 :                                         TAILQ_INSERT_TAIL(&crypto_ch->queued_tasks, task, link);
     291                 :       7099 :                                         continue;
     292         [ #  # ]:          0 :                                 } else if (rc == -EALREADY) {
     293                 :            :                                         /* -EALREADY means that a task is completed, but it might be unsafe to complete
     294                 :            :                                          * it if we are in the submission path. Since we are in the poller context, we can
     295                 :            :                                          * complete th task immediately */
     296                 :          0 :                                         rc = 0;
     297                 :            :                                 }
     298                 :          0 :                                 spdk_accel_task_complete(&task->base, rc);
     299                 :            :                         }
     300                 :            :                 }
     301                 :            :         }
     302                 :            : 
     303                 :            :         /* Now bulk free both mbufs and crypto operations. */
     304         [ +  + ]:     472838 :         if (num_dequeued_ops > 0) {
     305                 :     472209 :                 rte_mempool_put_bulk(g_crypto_op_mp, (void **)dequeued_ops, num_dequeued_ops);
     306         [ -  + ]:     472209 :                 assert(num_mbufs > 0);
     307                 :            :                 /* This also releases chained mbufs if any. */
     308                 :     472209 :                 rte_pktmbuf_free_bulk(mbufs_to_free, num_mbufs);
     309                 :            :         }
     310                 :            : 
     311         [ -  + ]:     472838 :         assert(qp->num_enqueued_ops >= num_dequeued_ops);
     312                 :     472838 :         qp->num_enqueued_ops -= num_dequeued_ops;
     313                 :            : 
     314                 :     472838 :         return num_dequeued_ops;
     315                 :            : }
     316                 :            : 
     317                 :            : /* This is the poller for the crypto module. It uses a single API to dequeue whatever is ready at
     318                 :            :  * the device. Then we need to decide if what we've got so far (including previous poller
     319                 :            :  * runs) totals up to one or more complete task */
     320                 :            : static int
     321                 :   10684470 : accel_dpdk_cryptodev_poller(void *args)
     322                 :            : {
     323                 :   10684470 :         struct accel_dpdk_cryptodev_io_channel *crypto_ch = args;
     324                 :            :         struct accel_dpdk_cryptodev_qp *qp;
     325                 :            :         struct accel_dpdk_cryptodev_task *task, *task_tmp;
     326                 :    7111128 :         TAILQ_HEAD(, accel_dpdk_cryptodev_task) queued_tasks_tmp;
     327                 :   10684470 :         uint32_t num_dequeued_ops = 0, num_enqueued_ops = 0, num_completed_tasks = 0;
     328                 :            :         int i, rc;
     329                 :            : 
     330         [ +  + ]:   53422350 :         for (i = 0; i < ACCEL_DPDK_CRYPTODEV_DRIVER_LAST; i++) {
     331                 :   42737880 :                 qp = crypto_ch->device_qp[i];
     332                 :            :                 /* Avoid polling "idle" qps since it may affect performance */
     333   [ +  +  +  + ]:   42737880 :                 if (qp && qp->num_enqueued_ops) {
     334                 :     472838 :                         num_dequeued_ops += accel_dpdk_cryptodev_poll_qp(qp, crypto_ch);
     335                 :            :                 }
     336                 :            :         }
     337                 :            : 
     338         [ +  + ]:   10684470 :         if (!TAILQ_EMPTY(&crypto_ch->queued_tasks)) {
     339                 :      21430 :                 TAILQ_INIT(&queued_tasks_tmp);
     340                 :            : 
     341         [ +  + ]:     976824 :                 TAILQ_FOREACH_SAFE(task, &crypto_ch->queued_tasks, link, task_tmp) {
     342         [ +  + ]:     955394 :                         TAILQ_REMOVE(&crypto_ch->queued_tasks, task, link);
     343                 :     955394 :                         rc = accel_dpdk_cryptodev_process_task(crypto_ch, task);
     344         [ +  + ]:     955394 :                         if (spdk_unlikely(rc)) {
     345         [ +  - ]:     912875 :                                 if (rc == -ENOMEM) {
     346                 :     912875 :                                         TAILQ_INSERT_TAIL(&queued_tasks_tmp, task, link);
     347                 :            :                                         /* Other queued tasks may belong to other qpairs,
     348                 :            :                                          * so process the whole list */
     349                 :     912875 :                                         continue;
     350         [ #  # ]:          0 :                                 } else if (rc == -EALREADY) {
     351                 :            :                                         /* -EALREADY means that a task is completed, but it might be unsafe to complete
     352                 :            :                                          * it if we are in the submission path. Since we are in the poller context, we can
     353                 :            :                                          * complete th task immediately */
     354                 :          0 :                                         rc = 0;
     355                 :            :                                 }
     356                 :          0 :                                 spdk_accel_task_complete(&task->base, rc);
     357                 :          0 :                                 num_completed_tasks++;
     358                 :            :                         } else {
     359                 :      42519 :                                 num_enqueued_ops++;
     360                 :            :                         }
     361                 :            :                 }
     362                 :            : 
     363   [ +  +  -  + ]:      21430 :                 TAILQ_SWAP(&crypto_ch->queued_tasks, &queued_tasks_tmp, accel_dpdk_cryptodev_task, link);
     364                 :            :         }
     365                 :            : 
     366         [ -  + ]:   10684470 :         TAILQ_FOREACH_SAFE(task, &crypto_ch->completed_tasks, link, task_tmp) {
     367         [ #  # ]:          0 :                 TAILQ_REMOVE(&crypto_ch->completed_tasks, task, link);
     368                 :          0 :                 spdk_accel_task_complete(&task->base, 0);
     369                 :          0 :                 num_completed_tasks++;
     370                 :            :         }
     371                 :            : 
     372                 :   10684470 :         return !!(num_dequeued_ops + num_enqueued_ops + num_completed_tasks);
     373                 :            : }
     374                 :            : 
     375                 :            : /* Allocate the new mbuf of @remainder size with data pointed by @addr and attach
     376                 :            :  * it to the @orig_mbuf. */
     377                 :            : static inline int
     378                 :          0 : accel_dpdk_cryptodev_mbuf_chain_remainder(struct accel_dpdk_cryptodev_task *task,
     379                 :            :                 struct rte_mbuf *orig_mbuf, uint8_t *addr, uint64_t *_remainder)
     380                 :            : {
     381                 :          0 :         uint64_t phys_addr, phys_len, remainder = *_remainder;
     382                 :          0 :         struct rte_mbuf *chain_mbuf;
     383                 :            :         int rc;
     384                 :            : 
     385                 :          0 :         phys_len = remainder;
     386                 :          0 :         phys_addr = spdk_vtophys((void *)addr, &phys_len);
     387         [ #  # ]:          0 :         if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR)) {
     388                 :          0 :                 return -EFAULT;
     389                 :            :         }
     390                 :          0 :         remainder = spdk_min(remainder, phys_len);
     391                 :          0 :         remainder = spdk_min(remainder, ACCEL_DPDK_CRYPTODEV_MAX_MBUF_LEN);
     392                 :          0 :         rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, (struct rte_mbuf **)&chain_mbuf, 1);
     393         [ #  # ]:          0 :         if (spdk_unlikely(rc)) {
     394                 :          0 :                 return -ENOMEM;
     395                 :            :         }
     396                 :            :         /* Store context in every mbuf as we don't know anything about completion order */
     397                 :          0 :         *RTE_MBUF_DYNFIELD(chain_mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)task;
     398                 :          0 :         rte_pktmbuf_attach_extbuf(chain_mbuf, addr, phys_addr, remainder, &g_shinfo);
     399                 :          0 :         rte_pktmbuf_append(chain_mbuf, remainder);
     400                 :            : 
     401                 :            :         /* Chained buffer is released by rte_pktbuf_free_bulk() automagicaly. */
     402                 :          0 :         rte_pktmbuf_chain(orig_mbuf, chain_mbuf);
     403                 :          0 :         *_remainder = remainder;
     404                 :            : 
     405                 :          0 :         return 0;
     406                 :            : }
     407                 :            : 
     408                 :            : /* Attach data buffer pointed by @addr to @mbuf. Return utilized len of the
     409                 :            :  * contiguous space that was physically available. */
     410                 :            : static inline uint64_t
     411                 :   21508786 : accel_dpdk_cryptodev_mbuf_attach_buf(struct accel_dpdk_cryptodev_task *task, struct rte_mbuf *mbuf,
     412                 :            :                                      uint8_t *addr, uint32_t len)
     413                 :            : {
     414                 :        482 :         uint64_t phys_addr, phys_len;
     415                 :            : 
     416                 :            :         /* Store context in every mbuf as we don't know anything about completion order */
     417                 :   21508786 :         *RTE_MBUF_DYNFIELD(mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)task;
     418                 :            : 
     419                 :   21508786 :         phys_len = len;
     420                 :   21508786 :         phys_addr = spdk_vtophys((void *)addr, &phys_len);
     421   [ +  -  -  + ]:   21508786 :         if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR || phys_len == 0)) {
     422                 :          0 :                 return 0;
     423                 :            :         }
     424         [ -  + ]:   21508786 :         assert(phys_len <= len);
     425                 :   21508786 :         phys_len = spdk_min(phys_len, ACCEL_DPDK_CRYPTODEV_MAX_MBUF_LEN);
     426                 :            : 
     427                 :            :         /* Set the mbuf elements address and length. */
     428                 :   21508786 :         rte_pktmbuf_attach_extbuf(mbuf, addr, phys_addr, phys_len, &g_shinfo);
     429                 :   21508786 :         rte_pktmbuf_append(mbuf, phys_len);
     430                 :            : 
     431                 :   21508786 :         return phys_len;
     432                 :            : }
     433                 :            : 
     434                 :            : static inline struct accel_dpdk_cryptodev_key_handle *
     435                 :    3348881 : accel_dpdk_find_key_handle_in_channel(struct accel_dpdk_cryptodev_io_channel *crypto_ch,
     436                 :            :                                       struct accel_dpdk_cryptodev_key_priv *key)
     437                 :            : {
     438                 :            :         struct accel_dpdk_cryptodev_key_handle *key_handle;
     439                 :            : 
     440         [ -  + ]:    3348881 :         if (key->driver == ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI) {
     441                 :            :                 /* Crypto key is registered on all available devices while io_channel opens CQ/QP on a single device.
     442                 :            :                  * We need to iterate a list of key entries to find a suitable device */
     443         [ #  # ]:          0 :                 TAILQ_FOREACH(key_handle, &key->dev_keys, link) {
     444                 :          0 :                         if (key_handle->device->cdev_id ==
     445         [ #  # ]:          0 :                             crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI]->device->cdev_id) {
     446                 :          0 :                                 return key_handle;
     447                 :            :                         }
     448                 :            :                 }
     449                 :          0 :                 return NULL;
     450                 :            :         } else {
     451                 :    3348881 :                 return TAILQ_FIRST(&key->dev_keys);
     452                 :            :         }
     453                 :            : }
     454                 :            : 
     455                 :            : static inline int
     456                 :    3348881 : accel_dpdk_cryptodev_task_alloc_resources(struct rte_mbuf **src_mbufs, struct rte_mbuf **dst_mbufs,
     457                 :            :                 struct rte_crypto_op **crypto_ops, int count)
     458                 :            : {
     459                 :            :         int rc;
     460                 :            : 
     461                 :            :         /* Get the number of source mbufs that we need. These will always be 1:1 because we
     462                 :            :          * don't support chaining. The reason we don't is because of our decision to use
     463                 :            :          * LBA as IV, there can be no case where we'd need >1 mbuf per crypto op or the
     464                 :            :          * op would be > 1 LBA.
     465                 :            :          */
     466                 :    3348881 :         rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, count);
     467         [ +  + ]:    3348881 :         if (rc) {
     468                 :      21026 :                 SPDK_ERRLOG("Failed to get src_mbufs!\n");
     469                 :      21026 :                 return -ENOMEM;
     470                 :            :         }
     471                 :            : 
     472                 :            :         /* Get the same amount to describe destination. If crypto operation is inline then we don't just skip it */
     473         [ +  + ]:    3327855 :         if (dst_mbufs) {
     474                 :    3327796 :                 rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, dst_mbufs, count);
     475         [ +  + ]:    3327796 :                 if (rc) {
     476                 :       1503 :                         SPDK_ERRLOG("Failed to get dst_mbufs!\n");
     477                 :       1503 :                         goto err_free_src;
     478                 :            :                 }
     479                 :            :         }
     480                 :            : 
     481                 :            : #ifdef __clang_analyzer__
     482                 :            :         /* silence scan-build false positive */
     483                 :            :         SPDK_CLANG_ANALYZER_PREINIT_PTR_ARRAY(crypto_ops, ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE,
     484                 :            :                                               0x1000);
     485                 :            : #endif
     486                 :            :         /* Allocate crypto operations. */
     487                 :    3326352 :         rc = rte_crypto_op_bulk_alloc(g_crypto_op_mp,
     488                 :            :                                       RTE_CRYPTO_OP_TYPE_SYMMETRIC,
     489                 :            :                                       crypto_ops, count);
     490         [ -  + ]:    3326352 :         if (rc < count) {
     491                 :          0 :                 SPDK_ERRLOG("Failed to allocate crypto ops! rc %d\n", rc);
     492                 :          0 :                 goto err_free_ops;
     493                 :            :         }
     494                 :            : 
     495                 :    3326352 :         return 0;
     496                 :            : 
     497                 :          0 : err_free_ops:
     498         [ #  # ]:          0 :         if (rc > 0) {
     499                 :          0 :                 rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, rc);
     500                 :            :         }
     501         [ #  # ]:          0 :         if (dst_mbufs) {
     502                 :            :                 /* This also releases chained mbufs if any. */
     503                 :          0 :                 rte_pktmbuf_free_bulk(dst_mbufs, count);
     504                 :            :         }
     505                 :          0 : err_free_src:
     506                 :            :         /* This also releases chained mbufs if any. */
     507                 :       1503 :         rte_pktmbuf_free_bulk(src_mbufs, count);
     508                 :            : 
     509                 :       1503 :         return -ENOMEM;
     510                 :            : }
     511                 :            : 
     512                 :            : static inline int
     513                 :   21508786 : accel_dpdk_cryptodev_mbuf_add_single_block(struct spdk_iov_sgl *sgl, struct rte_mbuf *mbuf,
     514                 :            :                 struct accel_dpdk_cryptodev_task *task)
     515                 :            : {
     516                 :            :         int rc;
     517                 :            :         uint8_t *buf_addr;
     518                 :            :         uint64_t phys_len;
     519                 :            :         uint64_t remainder;
     520                 :        482 :         uint64_t buf_len;
     521                 :            : 
     522         [ -  + ]:   21508786 :         assert(sgl->iov->iov_len > sgl->iov_offset);
     523                 :   21508786 :         buf_len = spdk_min(task->base.block_size, sgl->iov->iov_len - sgl->iov_offset);
     524                 :   21508786 :         buf_addr = sgl->iov->iov_base + sgl->iov_offset;
     525                 :   21508786 :         phys_len = accel_dpdk_cryptodev_mbuf_attach_buf(task, mbuf, buf_addr, buf_len);
     526         [ -  + ]:   21508786 :         if (spdk_unlikely(phys_len == 0)) {
     527                 :          0 :                 return -EFAULT;
     528                 :            :         }
     529                 :   21508786 :         buf_len = spdk_min(buf_len, phys_len);
     530                 :   21508786 :         spdk_iov_sgl_advance(sgl, buf_len);
     531                 :            : 
     532                 :            :         /* Handle the case of page boundary. */
     533         [ -  + ]:   21508786 :         assert(task->base.block_size >= buf_len);
     534                 :   21508786 :         remainder = task->base.block_size - buf_len;
     535         [ -  + ]:   21508786 :         while (remainder) {
     536                 :          0 :                 buf_len = spdk_min(remainder, sgl->iov->iov_len - sgl->iov_offset);
     537                 :          0 :                 buf_addr = sgl->iov->iov_base + sgl->iov_offset;
     538                 :          0 :                 rc = accel_dpdk_cryptodev_mbuf_chain_remainder(task, mbuf, buf_addr, &buf_len);
     539         [ #  # ]:          0 :                 if (spdk_unlikely(rc)) {
     540                 :          0 :                         return rc;
     541                 :            :                 }
     542                 :          0 :                 spdk_iov_sgl_advance(sgl, buf_len);
     543                 :          0 :                 remainder -= buf_len;
     544                 :            :         }
     545                 :            : 
     546                 :   21508786 :         return 0;
     547                 :            : }
     548                 :            : 
     549                 :            : static inline void
     550                 :   10754539 : accel_dpdk_cryptodev_op_set_iv(struct rte_crypto_op *crypto_op, uint64_t iv)
     551                 :            : {
     552                 :   10754539 :         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(crypto_op, uint8_t *, ACCEL_DPDK_CRYPTODEV_IV_OFFSET);
     553                 :            : 
     554                 :            :         /* Set the IV - we use the LBA of the crypto_op */
     555   [ -  +  -  + ]:   10754539 :         memset(iv_ptr, 0, ACCEL_DPDK_CRYPTODEV_IV_LENGTH);
     556                 :            :         rte_memcpy(iv_ptr, &iv, sizeof(uint64_t));
     557                 :   10754539 : }
     558                 :            : 
     559                 :            : static inline void
     560                 :       5472 : accel_dpdk_cryptodev_update_resources_from_pools(struct rte_crypto_op **crypto_ops,
     561                 :            :                 struct rte_mbuf **src_mbufs, struct rte_mbuf **dst_mbufs,
     562                 :            :                 uint32_t num_enqueued_ops, uint32_t cryop_cnt)
     563                 :            : {
     564   [ -  +  -  + ]:       5472 :         memmove(crypto_ops, &crypto_ops[num_enqueued_ops], sizeof(crypto_ops[0]) * cryop_cnt);
     565   [ -  +  -  + ]:       5472 :         memmove(src_mbufs, &src_mbufs[num_enqueued_ops], sizeof(src_mbufs[0]) * cryop_cnt);
     566         [ +  - ]:       5472 :         if (dst_mbufs) {
     567   [ -  +  -  + ]:       5472 :                 memmove(dst_mbufs, &dst_mbufs[num_enqueued_ops], sizeof(dst_mbufs[0]) * cryop_cnt);
     568                 :            :         }
     569                 :       5472 : }
     570                 :            : 
     571                 :            : static int
     572                 :    3907546 : accel_dpdk_cryptodev_process_task(struct accel_dpdk_cryptodev_io_channel *crypto_ch,
     573                 :            :                                   struct accel_dpdk_cryptodev_task *task)
     574                 :            : {
     575                 :            :         uint16_t num_enqueued_ops;
     576                 :            :         uint32_t cryop_cnt;
     577                 :    3907546 :         uint32_t crypto_len = task->base.block_size;
     578                 :            :         uint64_t dst_length, total_length;
     579                 :            :         uint32_t sgl_offset;
     580                 :            :         uint32_t qp_capacity;
     581                 :            :         uint64_t iv_start;
     582                 :            :         uint32_t i, crypto_index;
     583                 :         97 :         struct rte_crypto_op *crypto_ops[ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE];
     584                 :         97 :         struct rte_mbuf *src_mbufs[ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE];
     585                 :         97 :         struct rte_mbuf *dst_mbufs[ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE];
     586                 :            :         void *session;
     587                 :            :         struct accel_dpdk_cryptodev_key_priv *priv;
     588                 :            :         struct accel_dpdk_cryptodev_key_handle *key_handle;
     589                 :            :         struct accel_dpdk_cryptodev_qp *qp;
     590                 :            :         struct accel_dpdk_cryptodev_device *dev;
     591                 :    3907546 :         struct spdk_iov_sgl src, dst = {};
     592                 :            :         int rc;
     593         [ -  + ]:    3907546 :         bool inplace = task->inplace;
     594                 :            : 
     595   [ +  -  -  + ]:    3907546 :         if (spdk_unlikely(!task->base.crypto_key ||
     596                 :            :                           task->base.crypto_key->module_if != &g_accel_dpdk_cryptodev_module)) {
     597                 :          0 :                 return -EINVAL;
     598                 :            :         }
     599                 :            : 
     600                 :    3907546 :         priv = task->base.crypto_key->priv;
     601         [ -  + ]:    3907546 :         assert(priv->driver < ACCEL_DPDK_CRYPTODEV_DRIVER_LAST);
     602                 :            : 
     603         [ +  + ]:    3907546 :         if (task->cryop_completed) {
     604                 :            :                 /* We continue to process remaining blocks */
     605         [ -  + ]:     529579 :                 assert(task->cryop_submitted == task->cryop_completed);
     606         [ -  + ]:     529579 :                 assert(task->cryop_total > task->cryop_completed);
     607                 :     529579 :                 cryop_cnt = task->cryop_total - task->cryop_completed;
     608                 :     529579 :                 sgl_offset = task->cryop_completed * crypto_len;
     609                 :     529579 :                 iv_start = task->base.iv + task->cryop_completed;
     610                 :            :         } else {
     611                 :            :                 /* That is a new task */
     612                 :    3377967 :                 total_length = 0;
     613         [ +  + ]:    6756383 :                 for (i = 0; i < task->base.s.iovcnt; i++) {
     614                 :    3378416 :                         total_length += task->base.s.iovs[i].iov_len;
     615                 :            :                 }
     616                 :    3377967 :                 dst_length = 0;
     617         [ +  + ]:    6756471 :                 for (i = 0; i < task->base.d.iovcnt; i++) {
     618                 :    3378504 :                         dst_length += task->base.d.iovs[i].iov_len;
     619                 :            :                 }
     620                 :            : 
     621   [ +  -  -  + ]:    3377967 :                 if (spdk_unlikely(total_length != dst_length || !total_length)) {
     622                 :          0 :                         return -ERANGE;
     623                 :            :                 }
     624   [ -  +  -  + ]:    3377967 :                 if (spdk_unlikely(total_length % task->base.block_size != 0)) {
     625                 :          0 :                         return -EINVAL;
     626                 :            :                 }
     627                 :            : 
     628         [ -  + ]:    3377967 :                 cryop_cnt = total_length / task->base.block_size;
     629                 :    3377967 :                 task->cryop_total = cryop_cnt;
     630                 :    3377967 :                 sgl_offset = 0;
     631                 :    3377967 :                 iv_start = task->base.iv;
     632                 :            :         }
     633                 :            : 
     634                 :            :         /* Limit the number of crypto ops that we can process once */
     635                 :    3907546 :         cryop_cnt = spdk_min(cryop_cnt, ACCEL_DPDK_CRYPTODEV_MAX_ENQUEUE_ARRAY_SIZE);
     636                 :            : 
     637                 :    3907546 :         qp = crypto_ch->device_qp[priv->driver];
     638         [ -  + ]:    3907546 :         assert(qp);
     639                 :    3907546 :         dev = qp->device;
     640         [ -  + ]:    3907546 :         assert(dev);
     641         [ -  + ]:    3907546 :         assert(dev->qp_desc_nr >= qp->num_enqueued_ops);
     642                 :            : 
     643                 :    3907546 :         qp_capacity = dev->qp_desc_nr - qp->num_enqueued_ops;
     644                 :    3907546 :         cryop_cnt = spdk_min(cryop_cnt, qp_capacity);
     645         [ +  + ]:    3907546 :         if (spdk_unlikely(cryop_cnt == 0)) {
     646                 :            :                 /* QP is full */
     647                 :     558665 :                 return -ENOMEM;
     648                 :            :         }
     649                 :            : 
     650                 :    3348881 :         key_handle = accel_dpdk_find_key_handle_in_channel(crypto_ch, priv);
     651         [ -  + ]:    3348881 :         if (spdk_unlikely(!key_handle)) {
     652                 :          0 :                 SPDK_ERRLOG("Failed to find a key handle, driver %s, cipher %s\n", g_driver_names[priv->driver],
     653                 :            :                             g_cipher_names[priv->cipher]);
     654                 :          0 :                 return -EINVAL;
     655                 :            :         }
     656                 :            :         /* mlx5_pci binds keys to a specific device, we can't use a key with any device */
     657   [ +  +  -  + ]:    3348881 :         assert(dev == key_handle->device || priv->driver != ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI);
     658                 :            : 
     659         [ +  + ]:    3348881 :         if (task->base.op_code == SPDK_ACCEL_OPC_ENCRYPT) {
     660                 :    2257691 :                 session = key_handle->session_encrypt;
     661         [ +  - ]:    1091190 :         } else if (task->base.op_code == SPDK_ACCEL_OPC_DECRYPT) {
     662                 :    1091190 :                 session = key_handle->session_decrypt;
     663                 :            :         } else {
     664                 :          0 :                 return -EINVAL;
     665                 :            :         }
     666                 :            : 
     667         [ +  + ]:    3348881 :         rc = accel_dpdk_cryptodev_task_alloc_resources(src_mbufs, inplace ? NULL : dst_mbufs,
     668                 :            :                         crypto_ops, cryop_cnt);
     669         [ +  + ]:    3348881 :         if (rc) {
     670                 :      22529 :                 return rc;
     671                 :            :         }
     672                 :            : 
     673                 :            :         /* As we don't support chaining because of a decision to use LBA as IV, construction
     674                 :            :          * of crypto operations is straightforward. We build both the op, the mbuf and the
     675                 :            :          * dst_mbuf in our local arrays by looping through the length of the accel task and
     676                 :            :          * picking off LBA sized blocks of memory from the IOVs as we walk through them. Each
     677                 :            :          * LBA sized chunk of memory will correspond 1:1 to a crypto operation and a single
     678                 :            :          * mbuf per crypto operation.
     679                 :            :          */
     680                 :    3326352 :         spdk_iov_sgl_init(&src, task->base.s.iovs, task->base.s.iovcnt, 0);
     681                 :    3326352 :         spdk_iov_sgl_advance(&src, sgl_offset);
     682         [ +  + ]:    3326352 :         if (!inplace) {
     683                 :    3326293 :                 spdk_iov_sgl_init(&dst, task->base.d.iovs, task->base.d.iovcnt, 0);
     684                 :    3326293 :                 spdk_iov_sgl_advance(&dst, sgl_offset);
     685                 :            :         }
     686                 :            : 
     687         [ +  + ]:   14080891 :         for (crypto_index = 0; crypto_index < cryop_cnt; crypto_index++) {
     688                 :   10754539 :                 rc = accel_dpdk_cryptodev_mbuf_add_single_block(&src, src_mbufs[crypto_index], task);
     689         [ -  + ]:   10754539 :                 if (spdk_unlikely(rc)) {
     690                 :          0 :                         goto free_ops;
     691                 :            :                 }
     692                 :   10754539 :                 accel_dpdk_cryptodev_op_set_iv(crypto_ops[crypto_index], iv_start);
     693                 :   10754539 :                 iv_start++;
     694                 :            : 
     695                 :            :                 /* Set the data to encrypt/decrypt length */
     696                 :   10754539 :                 crypto_ops[crypto_index]->sym->cipher.data.length = crypto_len;
     697                 :   10754539 :                 crypto_ops[crypto_index]->sym->cipher.data.offset = 0;
     698                 :   10754539 :                 rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], session);
     699                 :            : 
     700                 :            :                 /* link the mbuf to the crypto op. */
     701                 :   10754539 :                 crypto_ops[crypto_index]->sym->m_src = src_mbufs[crypto_index];
     702                 :            : 
     703         [ +  + ]:   10754539 :                 if (inplace) {
     704                 :        292 :                         crypto_ops[crypto_index]->sym->m_dst = NULL;
     705                 :            :                 } else {
     706                 :            : #ifndef __clang_analyzer__
     707                 :            :                         /* scan-build thinks that dst_mbufs is not initialized */
     708                 :   10754247 :                         rc = accel_dpdk_cryptodev_mbuf_add_single_block(&dst, dst_mbufs[crypto_index], task);
     709         [ -  + ]:   10754247 :                         if (spdk_unlikely(rc)) {
     710                 :          0 :                                 goto free_ops;
     711                 :            :                         }
     712                 :   10754247 :                         crypto_ops[crypto_index]->sym->m_dst = dst_mbufs[crypto_index];
     713                 :            : #endif
     714                 :            :                 }
     715                 :            :         }
     716                 :            : 
     717                 :            :         /* Enqueue everything we've got but limit by the max number of descriptors we
     718                 :            :          * configured the crypto device for.
     719                 :            :          */
     720                 :    3326352 :         num_enqueued_ops = rte_cryptodev_enqueue_burst(dev->cdev_id, qp->qp, crypto_ops, cryop_cnt);
     721                 :            :         /* This value is used in the completion callback to determine when the accel task is complete. */
     722                 :    3326352 :         task->cryop_submitted += num_enqueued_ops;
     723                 :    3326352 :         qp->num_enqueued_ops += num_enqueued_ops;
     724                 :            :         /* We were unable to enqueue everything but did get some, so need to decide what
     725                 :            :          * to do based on the status of the last op.
     726                 :            :          */
     727         [ +  + ]:    3326352 :         if (num_enqueued_ops < cryop_cnt) {
     728      [ -  +  - ]:     379672 :                 switch (crypto_ops[num_enqueued_ops]->status) {
     729                 :          0 :                 case RTE_CRYPTO_OP_STATUS_SUCCESS:
     730                 :            :                         /* Crypto operation might be completed successfully but enqueuing to a completion ring might fail.
     731                 :            :                          * That might happen with SW PMDs like openssl
     732                 :            :                          * We can't retry such operation on next turn since if crypto operation was inplace, we can encrypt/
     733                 :            :                          * decrypt already processed buffer. See github issue #2907 for more details.
     734                 :            :                          * Handle this case as the crypto op was completed successfully - increment cryop_submitted and
     735                 :            :                          * cryop_completed.
     736                 :            :                          * We won't receive a completion for such operation, so we need to cleanup mbufs and crypto_ops */
     737         [ #  # ]:          0 :                         assert(task->cryop_total > task->cryop_completed);
     738                 :          0 :                         task->cryop_completed++;
     739                 :          0 :                         task->cryop_submitted++;
     740         [ #  # ]:          0 :                         if (task->cryop_completed == task->cryop_total) {
     741         [ #  # ]:          0 :                                 assert(num_enqueued_ops == 0);
     742                 :            :                                 /* All crypto ops are completed. We can't complete the task immediately since this function might be
     743                 :            :                                  * called in scope of spdk_accel_submit_* function and user's logic in the completion callback
     744                 :            :                                  * might lead to stack overflow */
     745                 :          0 :                                 cryop_cnt -= num_enqueued_ops;
     746         [ #  # ]:          0 :                                 accel_dpdk_cryptodev_update_resources_from_pools(crypto_ops, src_mbufs, inplace ? NULL : dst_mbufs,
     747                 :            :                                                 num_enqueued_ops, cryop_cnt);
     748                 :          0 :                                 rc = -EALREADY;
     749                 :          0 :                                 goto free_ops;
     750                 :            :                         }
     751                 :            :                 /* fallthrough */
     752                 :            :                 case RTE_CRYPTO_OP_STATUS_NOT_PROCESSED:
     753         [ +  + ]:     379672 :                         if (num_enqueued_ops == 0) {
     754                 :            :                                 /* Nothing was submitted. Free crypto ops and mbufs, treat this case as NOMEM */
     755                 :     374200 :                                 rc = -ENOMEM;
     756                 :     374200 :                                 goto free_ops;
     757                 :            :                         }
     758                 :            :                         /* Part of the crypto operations were not submitted, release mbufs and crypto ops.
     759                 :            :                          * The rest crypto ops will be submitted again once current batch is completed */
     760                 :       5472 :                         cryop_cnt -= num_enqueued_ops;
     761         [ -  + ]:       5472 :                         accel_dpdk_cryptodev_update_resources_from_pools(crypto_ops, src_mbufs, inplace ? NULL : dst_mbufs,
     762                 :            :                                         num_enqueued_ops, cryop_cnt);
     763                 :       5472 :                         rc = 0;
     764                 :       5472 :                         goto free_ops;
     765                 :          0 :                 default:
     766                 :            :                         /* For all other statuses, mark task as failed so that the poller will pick
     767                 :            :                          * the failure up for the overall task status.
     768                 :            :                          */
     769                 :          0 :                         task->is_failed = true;
     770         [ #  # ]:          0 :                         if (num_enqueued_ops == 0) {
     771                 :            :                                 /* If nothing was enqueued, but the last one wasn't because of
     772                 :            :                                  * busy, fail it now as the poller won't know anything about it.
     773                 :            :                                  */
     774                 :          0 :                                 rc = -EINVAL;
     775                 :          0 :                                 goto free_ops;
     776                 :            :                         }
     777                 :          0 :                         break;
     778                 :            :                 }
     779                 :            :         }
     780                 :            : 
     781                 :    2946680 :         return 0;
     782                 :            : 
     783                 :            :         /* Error cleanup paths. */
     784                 :     379672 : free_ops:
     785         [ +  - ]:     379672 :         if (!inplace) {
     786                 :            :                 /* This also releases chained mbufs if any. */
     787                 :     379672 :                 rte_pktmbuf_free_bulk(dst_mbufs, cryop_cnt);
     788                 :            :         }
     789                 :     379672 :         rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, cryop_cnt);
     790                 :            :         /* This also releases chained mbufs if any. */
     791                 :     379672 :         rte_pktmbuf_free_bulk(src_mbufs, cryop_cnt);
     792                 :     379672 :         return rc;
     793                 :            : }
     794                 :            : 
     795                 :            : static inline struct accel_dpdk_cryptodev_qp *
     796                 :        549 : accel_dpdk_cryptodev_get_next_device_qpair(enum accel_dpdk_cryptodev_driver_type type)
     797                 :            : {
     798                 :            :         struct accel_dpdk_cryptodev_device *device, *device_tmp;
     799                 :            :         struct accel_dpdk_cryptodev_qp *qpair;
     800                 :            : 
     801         [ +  + ]:      23519 :         TAILQ_FOREACH_SAFE(device, &g_crypto_devices, link, device_tmp) {
     802         [ +  + ]:      23079 :                 if (device->type != type) {
     803                 :      22970 :                         continue;
     804                 :            :                 }
     805         [ +  - ]:        193 :                 TAILQ_FOREACH(qpair, &device->qpairs, link) {
     806   [ -  +  +  + ]:        193 :                         if (!qpair->in_use) {
     807                 :        109 :                                 qpair->in_use = true;
     808                 :        109 :                                 return qpair;
     809                 :            :                         }
     810                 :            :                 }
     811                 :            :         }
     812                 :            : 
     813                 :        440 :         return NULL;
     814                 :            : }
     815                 :            : 
     816                 :            : /* Helper function for the channel creation callback.
     817                 :            :  * Returns the number of drivers assigned to the channel */
     818                 :            : static uint32_t
     819                 :        183 : accel_dpdk_cryptodev_assign_device_qps(struct accel_dpdk_cryptodev_io_channel *crypto_ch)
     820                 :            : {
     821                 :            :         struct accel_dpdk_cryptodev_device *device;
     822                 :            :         struct accel_dpdk_cryptodev_qp *device_qp;
     823                 :        183 :         uint32_t num_drivers = 0;
     824                 :        183 :         bool qat_found = false;
     825                 :            : 
     826         [ -  + ]:        183 :         pthread_mutex_lock(&g_device_lock);
     827                 :            : 
     828         [ +  + ]:       7876 :         TAILQ_FOREACH(device, &g_crypto_devices, link) {
     829   [ +  +  +  + ]:       7693 :                 if (device->type == ACCEL_DPDK_CRYPTODEV_DRIVER_QAT && !qat_found) {
     830                 :            :                         /* For some QAT devices, the optimal qp to use is every 32nd as this spreads the
     831                 :            :                          * workload out over the multiple virtual functions in the device. For the devices
     832                 :            :                          * where this isn't the case, it doesn't hurt.
     833                 :            :                          */
     834         [ +  + ]:       7064 :                         TAILQ_FOREACH(device_qp, &device->qpairs, link) {
     835         [ +  + ]:       4768 :                                 if (device_qp->index != g_next_qat_index) {
     836                 :       4596 :                                         continue;
     837                 :            :                                 }
     838   [ -  +  +  + ]:        172 :                                 if (device_qp->in_use == false) {
     839         [ -  + ]:        158 :                                         assert(crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_QAT] == NULL);
     840                 :        158 :                                         crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_QAT] = device_qp;
     841                 :        158 :                                         device_qp->in_use = true;
     842         [ -  + ]:        158 :                                         g_next_qat_index = (g_next_qat_index + ACCEL_DPDK_CRYPTODEV_QAT_VF_SPREAD) % g_qat_total_qp;
     843                 :        158 :                                         qat_found = true;
     844                 :        158 :                                         num_drivers++;
     845                 :        158 :                                         break;
     846                 :            :                                 } else {
     847                 :            :                                         /* if the preferred index is used, skip to the next one in this set. */
     848         [ -  + ]:         14 :                                         g_next_qat_index = (g_next_qat_index + 1) % g_qat_total_qp;
     849                 :            :                                 }
     850                 :            :                         }
     851                 :            :                 }
     852                 :            :         }
     853                 :            : 
     854                 :            :         /* For ACCEL_DPDK_CRYPTODEV_AESNI_MB and MLX5_PCI select devices in round-robin manner */
     855                 :        183 :         device_qp = accel_dpdk_cryptodev_get_next_device_qpair(ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB);
     856         [ +  + ]:        183 :         if (device_qp) {
     857         [ -  + ]:        109 :                 assert(crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB] == NULL);
     858                 :        109 :                 crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB] = device_qp;
     859                 :        109 :                 num_drivers++;
     860                 :            :         }
     861                 :            : 
     862                 :        183 :         device_qp = accel_dpdk_cryptodev_get_next_device_qpair(ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI);
     863         [ -  + ]:        183 :         if (device_qp) {
     864         [ #  # ]:          0 :                 assert(crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI] == NULL);
     865                 :          0 :                 crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI] = device_qp;
     866                 :          0 :                 num_drivers++;
     867                 :            :         }
     868                 :            : 
     869                 :        183 :         device_qp = accel_dpdk_cryptodev_get_next_device_qpair(ACCEL_DPDK_CRYPTODEV_DRIVER_UADK);
     870         [ -  + ]:        183 :         if (device_qp) {
     871         [ #  # ]:          0 :                 assert(crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_UADK] == NULL);
     872                 :          0 :                 crypto_ch->device_qp[ACCEL_DPDK_CRYPTODEV_DRIVER_UADK] = device_qp;
     873                 :          0 :                 num_drivers++;
     874                 :            :         }
     875         [ -  + ]:        183 :         pthread_mutex_unlock(&g_device_lock);
     876                 :            : 
     877                 :        183 :         return num_drivers;
     878                 :            : }
     879                 :            : 
     880                 :            : static void
     881                 :        183 : _accel_dpdk_cryptodev_destroy_cb(void *io_device, void *ctx_buf)
     882                 :            : {
     883                 :        183 :         struct accel_dpdk_cryptodev_io_channel *crypto_ch = (struct accel_dpdk_cryptodev_io_channel *)
     884                 :            :                         ctx_buf;
     885                 :            :         int i;
     886                 :            : 
     887         [ -  + ]:        183 :         pthread_mutex_lock(&g_device_lock);
     888         [ +  + ]:        915 :         for (i = 0; i < ACCEL_DPDK_CRYPTODEV_DRIVER_LAST; i++) {
     889         [ +  + ]:        732 :                 if (crypto_ch->device_qp[i]) {
     890                 :        267 :                         crypto_ch->device_qp[i]->in_use = false;
     891                 :            :                 }
     892                 :            :         }
     893         [ -  + ]:        183 :         pthread_mutex_unlock(&g_device_lock);
     894                 :            : 
     895                 :        183 :         spdk_poller_unregister(&crypto_ch->poller);
     896                 :        183 : }
     897                 :            : 
     898                 :            : static int
     899                 :        183 : _accel_dpdk_cryptodev_create_cb(void *io_device, void *ctx_buf)
     900                 :            : {
     901                 :        183 :         struct accel_dpdk_cryptodev_io_channel *crypto_ch = (struct accel_dpdk_cryptodev_io_channel *)
     902                 :            :                         ctx_buf;
     903                 :            : 
     904                 :        183 :         crypto_ch->poller = SPDK_POLLER_REGISTER(accel_dpdk_cryptodev_poller, crypto_ch, 0);
     905         [ -  + ]:        183 :         if (!accel_dpdk_cryptodev_assign_device_qps(crypto_ch)) {
     906                 :          0 :                 SPDK_ERRLOG("No crypto drivers assigned\n");
     907                 :          0 :                 spdk_poller_unregister(&crypto_ch->poller);
     908                 :          0 :                 return -EINVAL;
     909                 :            :         }
     910                 :            : 
     911                 :            :         /* We use this to queue tasks when qpair is full or no resources in pools */
     912                 :        183 :         TAILQ_INIT(&crypto_ch->queued_tasks);
     913                 :        183 :         TAILQ_INIT(&crypto_ch->completed_tasks);
     914                 :            : 
     915                 :        183 :         return 0;
     916                 :            : }
     917                 :            : 
     918                 :            : static struct spdk_io_channel *
     919                 :        366 : accel_dpdk_cryptodev_get_io_channel(void)
     920                 :            : {
     921                 :        366 :         return spdk_get_io_channel(&g_accel_dpdk_cryptodev_module);
     922                 :            : }
     923                 :            : 
     924                 :            : static size_t
     925                 :         46 : accel_dpdk_cryptodev_ctx_size(void)
     926                 :            : {
     927                 :         46 :         return sizeof(struct accel_dpdk_cryptodev_task);
     928                 :            : }
     929                 :            : 
     930                 :            : static bool
     931                 :        859 : accel_dpdk_cryptodev_supports_opcode(enum spdk_accel_opcode opc)
     932                 :            : {
     933         [ +  + ]:        859 :         switch (opc) {
     934                 :        560 :         case SPDK_ACCEL_OPC_ENCRYPT:
     935                 :            :         case SPDK_ACCEL_OPC_DECRYPT:
     936                 :        560 :                 return true;
     937                 :        299 :         default:
     938                 :        299 :                 return false;
     939                 :            :         }
     940                 :            : }
     941                 :            : 
     942                 :            : static int
     943                 :    2938498 : accel_dpdk_cryptodev_submit_tasks(struct spdk_io_channel *_ch, struct spdk_accel_task *_task)
     944                 :            : {
     945                 :    2938498 :         struct accel_dpdk_cryptodev_task *task = SPDK_CONTAINEROF(_task, struct accel_dpdk_cryptodev_task,
     946                 :            :                         base);
     947                 :    2938498 :         struct accel_dpdk_cryptodev_io_channel *ch = spdk_io_channel_get_ctx(_ch);
     948                 :            :         int rc;
     949                 :            : 
     950                 :    2938498 :         task->cryop_completed = 0;
     951                 :    2938498 :         task->cryop_submitted = 0;
     952                 :    2938498 :         task->cryop_total = 0;
     953                 :    2938498 :         task->inplace = true;
     954                 :    2938498 :         task->is_failed = false;
     955                 :            : 
     956                 :            :         /* Check if crypto operation is inplace: no destination or source == destination */
     957         [ +  + ]:    2938498 :         if (task->base.s.iovcnt == task->base.d.iovcnt) {
     958   [ -  +  -  +  :    2938424 :                 if (memcmp(task->base.s.iovs, task->base.d.iovs, sizeof(struct iovec) * task->base.s.iovcnt) != 0) {
                   +  + ]
     959                 :    2938365 :                         task->inplace = false;
     960                 :            :                 }
     961         [ +  - ]:         74 :         } else if (task->base.d.iovcnt != 0) {
     962                 :         74 :                 task->inplace = false;
     963                 :            :         }
     964                 :            : 
     965                 :    2938498 :         rc = accel_dpdk_cryptodev_process_task(ch, task);
     966         [ +  + ]:    2938498 :         if (spdk_unlikely(rc)) {
     967         [ +  - ]:      35420 :                 if (rc == -ENOMEM) {
     968                 :      35420 :                         TAILQ_INSERT_TAIL(&ch->queued_tasks, task, link);
     969                 :      35420 :                         rc = 0;
     970         [ #  # ]:          0 :                 } else if (rc == -EALREADY) {
     971                 :            :                         /* -EALREADY means that a task is completed, but it might be unsafe to complete
     972                 :            :                          * it if we are in the submission path. Hence put it into a dedicated queue to and
     973                 :            :                          * process it during polling */
     974                 :          0 :                         TAILQ_INSERT_TAIL(&ch->completed_tasks, task, link);
     975                 :          0 :                         rc = 0;
     976                 :            :                 }
     977                 :            :         }
     978                 :            : 
     979                 :    2938498 :         return rc;
     980                 :            : }
     981                 :            : 
     982                 :            : /* Dummy function used by DPDK to free ext attached buffers to mbufs, we free them ourselves but
     983                 :            :  * this callback has to be here. */
     984                 :            : static void
     985                 :        326 : shinfo_free_cb(void *arg1, void *arg2)
     986                 :            : {
     987                 :        326 : }
     988                 :            : 
     989                 :            : static int
     990                 :       1934 : accel_dpdk_cryptodev_create(uint8_t index, uint16_t num_lcores)
     991                 :            : {
     992                 :       1934 :         struct rte_cryptodev_qp_conf qp_conf = {
     993                 :            :                 .mp_session = g_session_mp,
     994                 :            : #if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0)
     995                 :            :                 .mp_session_private = g_session_mp_priv
     996                 :            : #endif
     997                 :            :         };
     998                 :            :         /* Setup queue pairs. */
     999                 :       1934 :         struct rte_cryptodev_config conf = { .socket_id = SPDK_ENV_SOCKET_ID_ANY };
    1000                 :            :         struct accel_dpdk_cryptodev_device *device;
    1001                 :            :         uint8_t j, cdev_id, cdrv_id;
    1002                 :            :         struct accel_dpdk_cryptodev_qp *dev_qp;
    1003                 :            :         int rc;
    1004                 :            : 
    1005                 :       1934 :         device = calloc(1, sizeof(*device));
    1006         [ -  + ]:       1934 :         if (!device) {
    1007                 :          0 :                 return -ENOMEM;
    1008                 :            :         }
    1009                 :            : 
    1010                 :            :         /* Get details about this device. */
    1011                 :       1934 :         rte_cryptodev_info_get(index, &device->cdev_info);
    1012                 :       1934 :         cdrv_id = device->cdev_info.driver_id;
    1013                 :       1934 :         cdev_id = device->cdev_id = index;
    1014                 :            : 
    1015   [ -  +  +  + ]:       1934 :         if (strcmp(device->cdev_info.driver_name, ACCEL_DPDK_CRYPTODEV_QAT) == 0) {
    1016                 :        960 :                 device->qp_desc_nr = ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS;
    1017                 :        960 :                 device->type = ACCEL_DPDK_CRYPTODEV_DRIVER_QAT;
    1018   [ -  +  +  + ]:        974 :         } else if (strcmp(device->cdev_info.driver_name, ACCEL_DPDK_CRYPTODEV_AESNI_MB) == 0) {
    1019                 :         14 :                 device->qp_desc_nr = ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS;
    1020                 :         14 :                 device->type = ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB;
    1021   [ -  +  -  + ]:        960 :         } else if (strcmp(device->cdev_info.driver_name, ACCEL_DPDK_CRYPTODEV_MLX5) == 0) {
    1022                 :          0 :                 device->qp_desc_nr = ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS_MLX5;
    1023                 :          0 :                 device->type = ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI;
    1024   [ -  +  +  - ]:        960 :         } else if (strcmp(device->cdev_info.driver_name, ACCEL_DPDK_CRYPTODEV_QAT_ASYM) == 0) {
    1025                 :            :                 /* ACCEL_DPDK_CRYPTODEV_QAT_ASYM devices are not supported at this time. */
    1026                 :        960 :                 rc = 0;
    1027                 :        960 :                 goto err;
    1028   [ #  #  #  # ]:          0 :         } else if (strcmp(device->cdev_info.driver_name, ACCEL_DPDK_CRYPTODEV_UADK) == 0) {
    1029                 :          0 :                 device->qp_desc_nr = ACCEL_DPDK_CRYPTODEV_QP_DESCRIPTORS;
    1030                 :          0 :                 device->type = ACCEL_DPDK_CRYPTODEV_DRIVER_UADK;
    1031                 :            :         } else {
    1032                 :          0 :                 SPDK_ERRLOG("Failed to start device %u. Invalid driver name \"%s\"\n",
    1033                 :            :                             cdev_id, device->cdev_info.driver_name);
    1034                 :          0 :                 rc = -EINVAL;
    1035                 :          0 :                 goto err;
    1036                 :            :         }
    1037                 :            : 
    1038                 :            :         /* Before going any further, make sure we have enough resources for this
    1039                 :            :          * device type to function.  We need a unique queue pair per core accross each
    1040                 :            :          * device type to remain lockless....
    1041                 :            :          */
    1042                 :        974 :         if ((rte_cryptodev_device_count_by_driver(cdrv_id) *
    1043         [ -  + ]:        974 :              device->cdev_info.max_nb_queue_pairs) < num_lcores) {
    1044                 :          0 :                 SPDK_ERRLOG("Insufficient unique queue pairs available for %s\n",
    1045                 :            :                             device->cdev_info.driver_name);
    1046                 :          0 :                 SPDK_ERRLOG("Either add more crypto devices or decrease core count\n");
    1047                 :          0 :                 rc = -EINVAL;
    1048                 :          0 :                 goto err;
    1049                 :            :         }
    1050                 :            : 
    1051                 :        974 :         conf.nb_queue_pairs = device->cdev_info.max_nb_queue_pairs;
    1052                 :        974 :         rc = rte_cryptodev_configure(cdev_id, &conf);
    1053         [ -  + ]:        974 :         if (rc < 0) {
    1054                 :          0 :                 SPDK_ERRLOG("Failed to configure cryptodev %u: error %d\n",
    1055                 :            :                             cdev_id, rc);
    1056                 :          0 :                 rc = -EINVAL;
    1057                 :          0 :                 goto err;
    1058                 :            :         }
    1059                 :            : 
    1060                 :            :         /* Pre-setup all potential qpairs now and assign them in the channel
    1061                 :            :          * callback. If we were to create them there, we'd have to stop the
    1062                 :            :          * entire device affecting all other threads that might be using it
    1063                 :            :          * even on other queue pairs.
    1064                 :            :          */
    1065                 :        974 :         qp_conf.nb_descriptors = device->qp_desc_nr;
    1066         [ +  + ]:       3790 :         for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) {
    1067                 :       2816 :                 rc = rte_cryptodev_queue_pair_setup(cdev_id, j, &qp_conf, SOCKET_ID_ANY);
    1068         [ -  + ]:       2816 :                 if (rc < 0) {
    1069                 :          0 :                         SPDK_ERRLOG("Failed to setup queue pair %u on "
    1070                 :            :                                     "cryptodev %u: error %d\n", j, cdev_id, rc);
    1071                 :          0 :                         rc = -EINVAL;
    1072                 :          0 :                         goto err_qp_setup;
    1073                 :            :                 }
    1074                 :            :         }
    1075                 :            : 
    1076                 :        974 :         rc = rte_cryptodev_start(cdev_id);
    1077         [ -  + ]:        974 :         if (rc < 0) {
    1078                 :          0 :                 SPDK_ERRLOG("Failed to start device %u: error %d\n", cdev_id, rc);
    1079                 :          0 :                 rc = -EINVAL;
    1080                 :          0 :                 goto err_dev_start;
    1081                 :            :         }
    1082                 :            : 
    1083                 :        974 :         TAILQ_INIT(&device->qpairs);
    1084                 :            :         /* Build up lists of device/qp combinations per PMD */
    1085         [ +  + ]:       3790 :         for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) {
    1086                 :       2816 :                 dev_qp = calloc(1, sizeof(*dev_qp));
    1087         [ -  + ]:       2816 :                 if (!dev_qp) {
    1088                 :          0 :                         rc = -ENOMEM;
    1089                 :          0 :                         goto err_qp_alloc;
    1090                 :            :                 }
    1091                 :       2816 :                 dev_qp->device = device;
    1092                 :       2816 :                 dev_qp->qp = j;
    1093                 :       2816 :                 dev_qp->in_use = false;
    1094                 :       2816 :                 TAILQ_INSERT_TAIL(&device->qpairs, dev_qp, link);
    1095         [ +  + ]:       2816 :                 if (device->type == ACCEL_DPDK_CRYPTODEV_DRIVER_QAT) {
    1096                 :       1920 :                         dev_qp->index = g_qat_total_qp++;
    1097                 :            :                 }
    1098                 :            :         }
    1099                 :            :         /* Add to our list of available crypto devices. */
    1100                 :        974 :         TAILQ_INSERT_TAIL(&g_crypto_devices, device, link);
    1101                 :            : 
    1102                 :        974 :         return 0;
    1103                 :            : 
    1104                 :          0 : err_qp_alloc:
    1105         [ #  # ]:          0 :         TAILQ_FOREACH(dev_qp, &device->qpairs, link) {
    1106         [ #  # ]:          0 :                 if (dev_qp->device->cdev_id != device->cdev_id) {
    1107                 :          0 :                         continue;
    1108                 :            :                 }
    1109                 :          0 :                 free(dev_qp);
    1110         [ #  # ]:          0 :                 if (device->type == ACCEL_DPDK_CRYPTODEV_DRIVER_QAT) {
    1111         [ #  # ]:          0 :                         assert(g_qat_total_qp);
    1112                 :          0 :                         g_qat_total_qp--;
    1113                 :            :                 }
    1114                 :            :         }
    1115                 :          0 :         rte_cryptodev_stop(cdev_id);
    1116                 :          0 : err_dev_start:
    1117                 :          0 : err_qp_setup:
    1118                 :          0 :         rte_cryptodev_close(cdev_id);
    1119                 :        960 : err:
    1120                 :        960 :         free(device);
    1121                 :            : 
    1122                 :        960 :         return rc;
    1123                 :            : }
    1124                 :            : 
    1125                 :            : static void
    1126                 :        974 : accel_dpdk_cryptodev_release(struct accel_dpdk_cryptodev_device *device)
    1127                 :            : {
    1128                 :            :         struct accel_dpdk_cryptodev_qp *dev_qp, *tmp;
    1129                 :            : 
    1130         [ -  + ]:        974 :         assert(device);
    1131                 :            : 
    1132         [ +  + ]:       3790 :         TAILQ_FOREACH_SAFE(dev_qp, &device->qpairs, link, tmp) {
    1133                 :       2816 :                 free(dev_qp);
    1134                 :            :         }
    1135         [ +  + ]:        974 :         if (device->type == ACCEL_DPDK_CRYPTODEV_DRIVER_QAT) {
    1136         [ -  + ]:        960 :                 assert(g_qat_total_qp >= device->cdev_info.max_nb_queue_pairs);
    1137                 :        960 :                 g_qat_total_qp -= device->cdev_info.max_nb_queue_pairs;
    1138                 :            :         }
    1139                 :        974 :         rte_cryptodev_stop(device->cdev_id);
    1140                 :        974 :         rte_cryptodev_close(device->cdev_id);
    1141                 :        974 :         free(device);
    1142                 :        974 : }
    1143                 :            : 
    1144                 :            : static int
    1145                 :         23 : accel_dpdk_cryptodev_init(void)
    1146                 :            : {
    1147                 :            :         uint8_t cdev_count;
    1148                 :            :         uint8_t cdev_id;
    1149                 :            :         int i, rc;
    1150                 :         23 :         const char *driver_name = g_driver_names[g_dpdk_cryptodev_driver];
    1151                 :            :         struct accel_dpdk_cryptodev_device *device, *tmp_dev;
    1152                 :         23 :         unsigned int max_sess_size = 0, sess_size;
    1153                 :         23 :         uint16_t num_lcores = rte_lcore_count();
    1154                 :          3 :         char init_args[32];
    1155                 :            : 
    1156                 :            :         /* Only the first call via module init should init the crypto drivers. */
    1157         [ -  + ]:         23 :         if (g_session_mp != NULL) {
    1158                 :          0 :                 return 0;
    1159                 :            :         }
    1160                 :            : 
    1161         [ +  + ]:         23 :         if (g_dpdk_cryptodev_driver == ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB ||
    1162         [ -  + ]:          9 :             g_dpdk_cryptodev_driver == ACCEL_DPDK_CRYPTODEV_DRIVER_UADK) {
    1163         [ -  + ]:         14 :                 snprintf(init_args, sizeof(init_args), "max_nb_queue_pairs=%d",
    1164                 :            :                          ACCEL_DPDK_CRYPTODEV_AESNI_MB_NUM_QP);
    1165                 :         14 :                 rc = rte_vdev_init(driver_name, init_args);
    1166         [ -  + ]:         14 :                 if (rc) {
    1167                 :          0 :                         SPDK_NOTICELOG("Failed to create virtual PMD %s: error %d. "
    1168                 :            :                                        "Possibly %s is not supported by DPDK library. "
    1169                 :            :                                        "Keep going...\n", driver_name, rc, driver_name);
    1170                 :            :                 }
    1171                 :            :         }
    1172                 :            : 
    1173                 :            :         /* If we have no crypto devices, report error to fallback on other modules. */
    1174                 :         23 :         cdev_count = rte_cryptodev_count();
    1175         [ -  + ]:         23 :         if (cdev_count == 0) {
    1176                 :          0 :                 return -ENODEV;
    1177                 :            :         }
    1178                 :         23 :         SPDK_NOTICELOG("Found crypto devices: %d\n", (int)cdev_count);
    1179                 :            : 
    1180                 :         23 :         g_mbuf_offset = rte_mbuf_dynfield_register(&rte_mbuf_dynfield_io_context);
    1181         [ -  + ]:         23 :         if (g_mbuf_offset < 0) {
    1182                 :          0 :                 SPDK_ERRLOG("error registering dynamic field with DPDK\n");
    1183                 :          0 :                 return -EINVAL;
    1184                 :            :         }
    1185                 :            : 
    1186                 :            :         /* Create global mempools, shared by all devices regardless of type */
    1187                 :            :         /* First determine max session size, most pools are shared by all the devices,
    1188                 :            :          * so we need to find the global max sessions size. */
    1189         [ +  + ]:       1957 :         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
    1190                 :       1934 :                 sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id);
    1191         [ +  + ]:       1934 :                 if (sess_size > max_sess_size) {
    1192                 :         34 :                         max_sess_size = sess_size;
    1193                 :            :                 }
    1194                 :            :         }
    1195                 :            : 
    1196                 :            : #if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0)
    1197                 :            :         g_session_mp_priv = rte_mempool_create("dpdk_crypto_ses_mp_priv",
    1198                 :            :                                                ACCEL_DPDK_CRYPTODEV_NUM_SESSIONS, max_sess_size, ACCEL_DPDK_CRYPTODEV_SESS_MEMPOOL_CACHE_SIZE, 0,
    1199                 :            :                                                NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
    1200                 :            :         if (g_session_mp_priv == NULL) {
    1201                 :            :                 SPDK_ERRLOG("Cannot create private session pool max size 0x%x\n", max_sess_size);
    1202                 :            :                 return -ENOMEM;
    1203                 :            :         }
    1204                 :            : 
    1205                 :            :         /* When session private data mempool allocated, the element size for the session mempool
    1206                 :            :          * should be 0. */
    1207                 :            :         max_sess_size = 0;
    1208                 :            : #endif
    1209                 :            : 
    1210                 :         23 :         g_session_mp = rte_cryptodev_sym_session_pool_create("dpdk_crypto_ses_mp",
    1211                 :            :                         ACCEL_DPDK_CRYPTODEV_NUM_SESSIONS, max_sess_size, ACCEL_DPDK_CRYPTODEV_SESS_MEMPOOL_CACHE_SIZE, 0,
    1212                 :            :                         SOCKET_ID_ANY);
    1213         [ -  + ]:         23 :         if (g_session_mp == NULL) {
    1214                 :          0 :                 SPDK_ERRLOG("Cannot create session pool max size 0x%x\n", max_sess_size);
    1215                 :          0 :                 rc = -ENOMEM;
    1216                 :          0 :                 goto error_create_session_mp;
    1217                 :            :         }
    1218                 :            : 
    1219                 :         23 :         g_mbuf_mp = rte_pktmbuf_pool_create("dpdk_crypto_mbuf_mp", ACCEL_DPDK_CRYPTODEV_NUM_MBUFS,
    1220                 :            :                                             ACCEL_DPDK_CRYPTODEV_POOL_CACHE_SIZE,
    1221                 :            :                                             0, 0, SPDK_ENV_SOCKET_ID_ANY);
    1222         [ -  + ]:         23 :         if (g_mbuf_mp == NULL) {
    1223                 :          0 :                 SPDK_ERRLOG("Cannot create mbuf pool\n");
    1224                 :          0 :                 rc = -ENOMEM;
    1225                 :          0 :                 goto error_create_mbuf;
    1226                 :            :         }
    1227                 :            : 
    1228                 :            :         /* We use per op private data as suggested by DPDK and to store the IV and
    1229                 :            :          * our own struct for queueing ops. */
    1230                 :         23 :         g_crypto_op_mp = rte_crypto_op_pool_create("dpdk_crypto_op_mp",
    1231                 :            :                          RTE_CRYPTO_OP_TYPE_SYMMETRIC, ACCEL_DPDK_CRYPTODEV_NUM_MBUFS, ACCEL_DPDK_CRYPTODEV_POOL_CACHE_SIZE,
    1232                 :            :                          (ACCEL_DPDK_CRYPTODEV_DEFAULT_NUM_XFORMS * sizeof(struct rte_crypto_sym_xform)) +
    1233                 :         23 :                          ACCEL_DPDK_CRYPTODEV_IV_LENGTH, rte_socket_id());
    1234         [ -  + ]:         23 :         if (g_crypto_op_mp == NULL) {
    1235                 :          0 :                 SPDK_ERRLOG("Cannot create op pool\n");
    1236                 :          0 :                 rc = -ENOMEM;
    1237                 :          0 :                 goto error_create_op;
    1238                 :            :         }
    1239                 :            : 
    1240                 :            :         /* Init all devices */
    1241         [ +  + ]:       1957 :         for (i = 0; i < cdev_count; i++) {
    1242                 :       1934 :                 rc = accel_dpdk_cryptodev_create(i, num_lcores);
    1243         [ -  + ]:       1934 :                 if (rc) {
    1244                 :          0 :                         goto err;
    1245                 :            :                 }
    1246                 :            :         }
    1247                 :            : 
    1248                 :         23 :         g_shinfo.free_cb = shinfo_free_cb;
    1249                 :            : 
    1250                 :         23 :         spdk_io_device_register(&g_accel_dpdk_cryptodev_module, _accel_dpdk_cryptodev_create_cb,
    1251                 :            :                                 _accel_dpdk_cryptodev_destroy_cb, sizeof(struct accel_dpdk_cryptodev_io_channel),
    1252                 :            :                                 "accel_dpdk_cryptodev");
    1253                 :            : 
    1254                 :         23 :         return 0;
    1255                 :            : 
    1256                 :            :         /* Error cleanup paths. */
    1257                 :          0 : err:
    1258         [ #  # ]:          0 :         TAILQ_FOREACH_SAFE(device, &g_crypto_devices, link, tmp_dev) {
    1259         [ #  # ]:          0 :                 TAILQ_REMOVE(&g_crypto_devices, device, link);
    1260                 :          0 :                 accel_dpdk_cryptodev_release(device);
    1261                 :            :         }
    1262                 :          0 :         rte_mempool_free(g_crypto_op_mp);
    1263                 :          0 :         g_crypto_op_mp = NULL;
    1264                 :          0 : error_create_op:
    1265                 :          0 :         rte_mempool_free(g_mbuf_mp);
    1266                 :          0 :         g_mbuf_mp = NULL;
    1267                 :          0 : error_create_mbuf:
    1268                 :          0 :         rte_mempool_free(g_session_mp);
    1269                 :          0 :         g_session_mp = NULL;
    1270                 :          0 : error_create_session_mp:
    1271         [ #  # ]:          0 :         if (g_session_mp_priv != NULL) {
    1272                 :          0 :                 rte_mempool_free(g_session_mp_priv);
    1273                 :          0 :                 g_session_mp_priv = NULL;
    1274                 :            :         }
    1275                 :          0 :         return rc;
    1276                 :            : }
    1277                 :            : 
    1278                 :            : static void
    1279                 :         23 : accel_dpdk_cryptodev_fini_cb(void *io_device)
    1280                 :            : {
    1281                 :            :         struct accel_dpdk_cryptodev_device *device, *tmp;
    1282                 :            : 
    1283         [ +  + ]:        997 :         TAILQ_FOREACH_SAFE(device, &g_crypto_devices, link, tmp) {
    1284         [ +  + ]:        974 :                 TAILQ_REMOVE(&g_crypto_devices, device, link);
    1285                 :        974 :                 accel_dpdk_cryptodev_release(device);
    1286                 :            :         }
    1287                 :            : 
    1288         [ +  + ]:         23 :         if (g_dpdk_cryptodev_driver == ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB ||
    1289         [ -  + ]:          9 :             g_dpdk_cryptodev_driver == ACCEL_DPDK_CRYPTODEV_DRIVER_UADK) {
    1290                 :         14 :                 rte_vdev_uninit(g_driver_names[g_dpdk_cryptodev_driver]);
    1291                 :            :         }
    1292                 :            : 
    1293                 :         23 :         rte_mempool_free(g_crypto_op_mp);
    1294                 :         23 :         rte_mempool_free(g_mbuf_mp);
    1295                 :         23 :         rte_mempool_free(g_session_mp);
    1296         [ -  + ]:         23 :         if (g_session_mp_priv != NULL) {
    1297                 :          0 :                 rte_mempool_free(g_session_mp_priv);
    1298                 :            :         }
    1299                 :            : 
    1300                 :         23 :         spdk_accel_module_finish();
    1301                 :         23 : }
    1302                 :            : 
    1303                 :            : /* Called when the entire module is being torn down. */
    1304                 :            : static void
    1305                 :         23 : accel_dpdk_cryptodev_fini(void *ctx)
    1306                 :            : {
    1307         [ +  - ]:         23 :         if (g_crypto_op_mp) {
    1308                 :         23 :                 spdk_io_device_unregister(&g_accel_dpdk_cryptodev_module, accel_dpdk_cryptodev_fini_cb);
    1309                 :            :         }
    1310                 :         23 : }
    1311                 :            : 
    1312                 :            : static void
    1313                 :        156 : accel_dpdk_cryptodev_key_handle_session_free(struct accel_dpdk_cryptodev_device *device,
    1314                 :            :                 void *session)
    1315                 :            : {
    1316                 :            : #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)
    1317         [ -  + ]:        156 :         assert(device != NULL);
    1318                 :            : 
    1319                 :        156 :         rte_cryptodev_sym_session_free(device->cdev_id, session);
    1320                 :            : #else
    1321                 :            :         rte_cryptodev_sym_session_free(session);
    1322                 :            : #endif
    1323                 :        156 : }
    1324                 :            : 
    1325                 :            : static void *
    1326                 :        156 : accel_dpdk_cryptodev_key_handle_session_create(struct accel_dpdk_cryptodev_device *device,
    1327                 :            :                 struct rte_crypto_sym_xform *cipher_xform)
    1328                 :            : {
    1329                 :            :         void *session;
    1330                 :            : 
    1331                 :            : #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)
    1332                 :        156 :         session = rte_cryptodev_sym_session_create(device->cdev_id, cipher_xform, g_session_mp);
    1333                 :            : #else
    1334                 :            :         session = rte_cryptodev_sym_session_create(g_session_mp);
    1335                 :            :         if (!session) {
    1336                 :            :                 return NULL;
    1337                 :            :         }
    1338                 :            : 
    1339                 :            :         if (rte_cryptodev_sym_session_init(device->cdev_id, session, cipher_xform, g_session_mp_priv) < 0) {
    1340                 :            :                 accel_dpdk_cryptodev_key_handle_session_free(device, session);
    1341                 :            :                 return NULL;
    1342                 :            :         }
    1343                 :            : #endif
    1344                 :            : 
    1345                 :        156 :         return session;
    1346                 :            : }
    1347                 :            : 
    1348                 :            : static int
    1349                 :         78 : accel_dpdk_cryptodev_key_handle_configure(struct spdk_accel_crypto_key *key,
    1350                 :            :                 struct accel_dpdk_cryptodev_key_handle *key_handle)
    1351                 :            : {
    1352                 :         78 :         struct accel_dpdk_cryptodev_key_priv *priv = key->priv;
    1353                 :            : 
    1354                 :         78 :         key_handle->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
    1355                 :         78 :         key_handle->cipher_xform.cipher.iv.offset = ACCEL_DPDK_CRYPTODEV_IV_OFFSET;
    1356                 :         78 :         key_handle->cipher_xform.cipher.iv.length = ACCEL_DPDK_CRYPTODEV_IV_LENGTH;
    1357                 :            : 
    1358      [ +  +  - ]:         78 :         switch (priv->cipher) {
    1359                 :         60 :         case SPDK_ACCEL_CIPHER_AES_CBC:
    1360                 :         60 :                 key_handle->cipher_xform.cipher.key.data = key->key;
    1361                 :         60 :                 key_handle->cipher_xform.cipher.key.length = key->key_size;
    1362                 :         60 :                 key_handle->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
    1363                 :         60 :                 break;
    1364                 :         18 :         case SPDK_ACCEL_CIPHER_AES_XTS:
    1365                 :         18 :                 key_handle->cipher_xform.cipher.key.data = priv->xts_key;
    1366                 :         18 :                 key_handle->cipher_xform.cipher.key.length = key->key_size + key->key2_size;
    1367                 :         18 :                 key_handle->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_XTS;
    1368                 :         18 :                 break;
    1369                 :          0 :         default:
    1370                 :          0 :                 SPDK_ERRLOG("Invalid cipher name %s.\n", key->param.cipher);
    1371                 :          0 :                 return -EINVAL;
    1372                 :            :         }
    1373                 :            : 
    1374                 :         78 :         key_handle->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
    1375                 :         78 :         key_handle->session_encrypt = accel_dpdk_cryptodev_key_handle_session_create(key_handle->device,
    1376                 :            :                                       &key_handle->cipher_xform);
    1377         [ -  + ]:         78 :         if (!key_handle->session_encrypt) {
    1378                 :          0 :                 SPDK_ERRLOG("Failed to init encrypt session\n");
    1379                 :          0 :                 return -EINVAL;
    1380                 :            :         }
    1381                 :            : 
    1382                 :         78 :         key_handle->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
    1383                 :         78 :         key_handle->session_decrypt = accel_dpdk_cryptodev_key_handle_session_create(key_handle->device,
    1384                 :            :                                       &key_handle->cipher_xform);
    1385         [ -  + ]:         78 :         if (!key_handle->session_decrypt) {
    1386                 :          0 :                 SPDK_ERRLOG("Failed to init decrypt session:");
    1387                 :          0 :                 accel_dpdk_cryptodev_key_handle_session_free(key_handle->device, key_handle->session_encrypt);
    1388                 :          0 :                 return -EINVAL;
    1389                 :            :         }
    1390                 :            : 
    1391                 :         78 :         return 0;
    1392                 :            : }
    1393                 :            : 
    1394                 :            : static void
    1395                 :         78 : accel_dpdk_cryptodev_key_deinit(struct spdk_accel_crypto_key *key)
    1396                 :            : {
    1397                 :            :         struct accel_dpdk_cryptodev_key_handle *key_handle, *key_handle_tmp;
    1398                 :         78 :         struct accel_dpdk_cryptodev_key_priv *priv = key->priv;
    1399                 :            : 
    1400         [ +  + ]:        156 :         TAILQ_FOREACH_SAFE(key_handle, &priv->dev_keys, link, key_handle_tmp) {
    1401                 :         78 :                 accel_dpdk_cryptodev_key_handle_session_free(key_handle->device, key_handle->session_encrypt);
    1402                 :         78 :                 accel_dpdk_cryptodev_key_handle_session_free(key_handle->device, key_handle->session_decrypt);
    1403         [ -  + ]:         78 :                 TAILQ_REMOVE(&priv->dev_keys, key_handle, link);
    1404                 :         78 :                 spdk_memset_s(key_handle, sizeof(*key_handle), 0, sizeof(*key_handle));
    1405                 :         78 :                 free(key_handle);
    1406                 :            :         }
    1407                 :            : 
    1408         [ +  + ]:         78 :         if (priv->xts_key) {
    1409                 :         18 :                 spdk_memset_s(priv->xts_key, key->key_size + key->key2_size, 0, key->key_size + key->key2_size);
    1410                 :            :         }
    1411                 :         78 :         free(priv->xts_key);
    1412                 :         78 :         free(priv);
    1413                 :         78 : }
    1414                 :            : 
    1415                 :            : static bool
    1416                 :         78 : accel_dpdk_cryptodev_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size)
    1417                 :            : {
    1418      [ +  -  - ]:         78 :         switch (g_dpdk_cryptodev_driver) {
    1419                 :         78 :         case ACCEL_DPDK_CRYPTODEV_DRIVER_QAT:
    1420                 :            :         case ACCEL_DPDK_CRYPTODEV_DRIVER_UADK:
    1421                 :            :         case ACCEL_DPDK_CRYPTODEV_DRIVER_AESNI_MB:
    1422      [ +  +  - ]:         78 :                 switch (cipher) {
    1423                 :         18 :                 case SPDK_ACCEL_CIPHER_AES_XTS:
    1424                 :         18 :                         return key_size == SPDK_ACCEL_AES_XTS_128_KEY_SIZE;
    1425                 :         60 :                 case SPDK_ACCEL_CIPHER_AES_CBC:
    1426                 :         60 :                         return key_size == ACCEL_DPDK_CRYPTODEV_AES_CBC_KEY_LENGTH;
    1427                 :          0 :                 default:
    1428                 :          0 :                         return false;
    1429                 :            :                 }
    1430                 :          0 :         case ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI:
    1431         [ #  # ]:          0 :                 switch (cipher) {
    1432                 :          0 :                 case SPDK_ACCEL_CIPHER_AES_XTS:
    1433   [ #  #  #  # ]:          0 :                         return key_size == SPDK_ACCEL_AES_XTS_128_KEY_SIZE || key_size == SPDK_ACCEL_AES_XTS_256_KEY_SIZE;
    1434                 :          0 :                 default:
    1435                 :          0 :                         return false;
    1436                 :            :                 }
    1437                 :          0 :         default:
    1438                 :          0 :                 return false;
    1439                 :            :         }
    1440                 :            : }
    1441                 :            : 
    1442                 :            : static int
    1443                 :         78 : accel_dpdk_cryptodev_key_init(struct spdk_accel_crypto_key *key)
    1444                 :            : {
    1445                 :            :         struct accel_dpdk_cryptodev_device *device;
    1446                 :            :         struct accel_dpdk_cryptodev_key_priv *priv;
    1447                 :            :         struct accel_dpdk_cryptodev_key_handle *key_handle;
    1448                 :            :         enum accel_dpdk_cryptodev_driver_type driver;
    1449                 :            :         int rc;
    1450                 :            : 
    1451                 :         78 :         driver = g_dpdk_cryptodev_driver;
    1452                 :            : 
    1453                 :         78 :         priv = calloc(1, sizeof(*priv));
    1454         [ -  + ]:         78 :         if (!priv) {
    1455                 :          0 :                 SPDK_ERRLOG("Memory allocation failed\n");
    1456                 :          0 :                 return -ENOMEM;
    1457                 :            :         }
    1458                 :         78 :         key->priv = priv;
    1459                 :         78 :         priv->driver = driver;
    1460                 :         78 :         priv->cipher = key->cipher;
    1461                 :         78 :         TAILQ_INIT(&priv->dev_keys);
    1462                 :            : 
    1463         [ +  + ]:         78 :         if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
    1464                 :            :                 /* DPDK expects the keys to be concatenated together. */
    1465                 :         18 :                 priv->xts_key = calloc(key->key_size + key->key2_size + 1, sizeof(char));
    1466         [ -  + ]:         18 :                 if (!priv->xts_key) {
    1467                 :          0 :                         SPDK_ERRLOG("Memory allocation failed\n");
    1468                 :          0 :                         accel_dpdk_cryptodev_key_deinit(key);
    1469                 :          0 :                         return -ENOMEM;
    1470                 :            :                 }
    1471   [ -  +  -  + ]:         18 :                 memcpy(priv->xts_key, key->key, key->key_size);
    1472   [ -  +  -  + ]:         18 :                 memcpy(priv->xts_key + key->key_size, key->key2, key->key2_size);
    1473                 :            :         }
    1474                 :            : 
    1475         [ -  + ]:         78 :         pthread_mutex_lock(&g_device_lock);
    1476         [ +  - ]:       1902 :         TAILQ_FOREACH(device, &g_crypto_devices, link) {
    1477         [ +  + ]:       1902 :                 if (device->type != driver) {
    1478                 :       1824 :                         continue;
    1479                 :            :                 }
    1480                 :         78 :                 key_handle = calloc(1, sizeof(*key_handle));
    1481         [ -  + ]:         78 :                 if (!key_handle) {
    1482         [ #  # ]:          0 :                         pthread_mutex_unlock(&g_device_lock);
    1483                 :          0 :                         accel_dpdk_cryptodev_key_deinit(key);
    1484                 :          0 :                         return -ENOMEM;
    1485                 :            :                 }
    1486                 :         78 :                 key_handle->device = device;
    1487                 :         78 :                 TAILQ_INSERT_TAIL(&priv->dev_keys, key_handle, link);
    1488                 :         78 :                 rc = accel_dpdk_cryptodev_key_handle_configure(key, key_handle);
    1489         [ -  + ]:         78 :                 if (rc) {
    1490         [ #  # ]:          0 :                         pthread_mutex_unlock(&g_device_lock);
    1491                 :          0 :                         accel_dpdk_cryptodev_key_deinit(key);
    1492                 :          0 :                         return rc;
    1493                 :            :                 }
    1494         [ +  - ]:         78 :                 if (driver != ACCEL_DPDK_CRYPTODEV_DRIVER_MLX5_PCI) {
    1495                 :            :                         /* For MLX5_PCI we need to register a key on each device since
    1496                 :            :                          * the key is bound to a specific Protection Domain,
    1497                 :            :                          * so don't break the loop */
    1498                 :         78 :                         break;
    1499                 :            :                 }
    1500                 :            :         }
    1501         [ -  + ]:         78 :         pthread_mutex_unlock(&g_device_lock);
    1502                 :            : 
    1503         [ -  + ]:         78 :         if (TAILQ_EMPTY(&priv->dev_keys)) {
    1504                 :          0 :                 free(priv);
    1505                 :          0 :                 return -ENODEV;
    1506                 :            :         }
    1507                 :            : 
    1508                 :         78 :         return 0;
    1509                 :            : }
    1510                 :            : 
    1511                 :            : static void
    1512                 :          7 : accel_dpdk_cryptodev_write_config_json(struct spdk_json_write_ctx *w)
    1513                 :            : {
    1514                 :          7 :         spdk_json_write_object_begin(w);
    1515                 :          7 :         spdk_json_write_named_string(w, "method", "dpdk_cryptodev_scan_accel_module");
    1516                 :          7 :         spdk_json_write_object_end(w);
    1517                 :            : 
    1518                 :          7 :         spdk_json_write_object_begin(w);
    1519                 :          7 :         spdk_json_write_named_string(w, "method", "dpdk_cryptodev_set_driver");
    1520                 :          7 :         spdk_json_write_named_object_begin(w, "params");
    1521                 :          7 :         spdk_json_write_named_string(w, "driver_name", g_driver_names[g_dpdk_cryptodev_driver]);
    1522                 :          7 :         spdk_json_write_object_end(w);
    1523                 :          7 :         spdk_json_write_object_end(w);
    1524                 :          7 : }
    1525                 :            : 
    1526                 :            : static int
    1527                 :        468 : accel_dpdk_cryptodev_get_operation_info(enum spdk_accel_opcode opcode,
    1528                 :            :                                         const struct spdk_accel_operation_exec_ctx *ctx,
    1529                 :            :                                         struct spdk_accel_opcode_info *info)
    1530                 :            : {
    1531         [ -  + ]:        468 :         if (!accel_dpdk_cryptodev_supports_opcode(opcode)) {
    1532                 :          0 :                 SPDK_ERRLOG("Received unexpected opcode: %d", opcode);
    1533                 :          0 :                 assert(false);
    1534                 :            :                 return -EINVAL;
    1535                 :            :         }
    1536                 :            : 
    1537         [ +  + ]:        468 :         switch (g_dpdk_cryptodev_driver) {
    1538                 :        216 :         case ACCEL_DPDK_CRYPTODEV_DRIVER_QAT:
    1539                 :        216 :                 info->required_alignment = spdk_u32log2(ctx->block_size);
    1540                 :        216 :                 break;
    1541                 :        252 :         default:
    1542                 :        252 :                 info->required_alignment = 0;
    1543                 :        252 :                 break;
    1544                 :            :         }
    1545                 :            : 
    1546                 :        468 :         return 0;
    1547                 :            : }
    1548                 :            : 
    1549                 :            : static struct spdk_accel_module_if g_accel_dpdk_cryptodev_module = {
    1550                 :            :         .module_init            = accel_dpdk_cryptodev_init,
    1551                 :            :         .module_fini            = accel_dpdk_cryptodev_fini,
    1552                 :            :         .write_config_json      = accel_dpdk_cryptodev_write_config_json,
    1553                 :            :         .get_ctx_size           = accel_dpdk_cryptodev_ctx_size,
    1554                 :            :         .name                   = "dpdk_cryptodev",
    1555                 :            :         .supports_opcode        = accel_dpdk_cryptodev_supports_opcode,
    1556                 :            :         .get_io_channel         = accel_dpdk_cryptodev_get_io_channel,
    1557                 :            :         .submit_tasks           = accel_dpdk_cryptodev_submit_tasks,
    1558                 :            :         .crypto_key_init        = accel_dpdk_cryptodev_key_init,
    1559                 :            :         .crypto_key_deinit      = accel_dpdk_cryptodev_key_deinit,
    1560                 :            :         .crypto_supports_cipher = accel_dpdk_cryptodev_supports_cipher,
    1561                 :            :         .get_operation_info     = accel_dpdk_cryptodev_get_operation_info,
    1562                 :            : };

Generated by: LCOV version 1.14