LCOV - code coverage report
Current view: top level - include/spdk_internal - nvme_tcp.h (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 240 390 61.5 %
Date: 2024-07-12 11:42:48 Functions: 15 19 78.9 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2018 Intel Corporation. All rights reserved.
       3             :  *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
       4             :  *   Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5             :  */
       6             : 
       7             : #ifndef SPDK_INTERNAL_NVME_TCP_H
       8             : #define SPDK_INTERNAL_NVME_TCP_H
       9             : 
      10             : #include "spdk/likely.h"
      11             : #include "spdk/sock.h"
      12             : #include "spdk/dif.h"
      13             : #include "spdk/hexlify.h"
      14             : #include "spdk/nvmf_spec.h"
      15             : #include "spdk/util.h"
      16             : #include "spdk/base64.h"
      17             : 
      18             : #include "sgl.h"
      19             : 
      20             : #include "openssl/evp.h"
      21             : #include "openssl/kdf.h"
      22             : #include "openssl/sha.h"
      23             : 
      24             : #define SPDK_CRC32C_XOR                         0xffffffffUL
      25             : #define SPDK_NVME_TCP_DIGEST_LEN                4
      26             : #define SPDK_NVME_TCP_DIGEST_ALIGNMENT          4
      27             : #define SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT        30
      28             : #define SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR      8
      29             : #define SPDK_NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE  8192u
      30             : /*
      31             :  * Maximum number of SGL elements.
      32             :  */
      33             : #define NVME_TCP_MAX_SGL_DESCRIPTORS    (16)
      34             : 
      35             : #define MAKE_DIGEST_WORD(BUF, CRC32C) \
      36             :         (   ((*((uint8_t *)(BUF)+0)) = (uint8_t)((uint32_t)(CRC32C) >> 0)), \
      37             :             ((*((uint8_t *)(BUF)+1)) = (uint8_t)((uint32_t)(CRC32C) >> 8)), \
      38             :             ((*((uint8_t *)(BUF)+2)) = (uint8_t)((uint32_t)(CRC32C) >> 16)), \
      39             :             ((*((uint8_t *)(BUF)+3)) = (uint8_t)((uint32_t)(CRC32C) >> 24)))
      40             : 
      41             : #define MATCH_DIGEST_WORD(BUF, CRC32C) \
      42             :         (    ((((uint32_t) *((uint8_t *)(BUF)+0)) << 0)         \
      43             :             | (((uint32_t) *((uint8_t *)(BUF)+1)) << 8)         \
      44             :             | (((uint32_t) *((uint8_t *)(BUF)+2)) << 16)        \
      45             :             | (((uint32_t) *((uint8_t *)(BUF)+3)) << 24))       \
      46             :             == (CRC32C))
      47             : 
      48             : #define DGET32(B)                                                               \
      49             :         (((  (uint32_t) *((uint8_t *)(B)+0)) << 0)                              \
      50             :          | (((uint32_t) *((uint8_t *)(B)+1)) << 8)                              \
      51             :          | (((uint32_t) *((uint8_t *)(B)+2)) << 16)                             \
      52             :          | (((uint32_t) *((uint8_t *)(B)+3)) << 24))
      53             : 
      54             : #define DSET32(B,D)                                                             \
      55             :         (((*((uint8_t *)(B)+0)) = (uint8_t)((uint32_t)(D) >> 0)),               \
      56             :          ((*((uint8_t *)(B)+1)) = (uint8_t)((uint32_t)(D) >> 8)),               \
      57             :          ((*((uint8_t *)(B)+2)) = (uint8_t)((uint32_t)(D) >> 16)),              \
      58             :          ((*((uint8_t *)(B)+3)) = (uint8_t)((uint32_t)(D) >> 24)))
      59             : 
      60             : /* The PSK identity comprises of following components:
      61             :  * 4-character format specifier "NVMe" +
      62             :  * 1-character TLS protocol version indicator +
      63             :  * 1-character PSK type indicator, specifying the used PSK +
      64             :  * 2-characters hash specifier +
      65             :  * NQN of the host (SPDK_NVMF_NQN_MAX_LEN -> 223) +
      66             :  * NQN of the subsystem (SPDK_NVMF_NQN_MAX_LEN -> 223) +
      67             :  * 2 space character separators +
      68             :  * 1 null terminator =
      69             :  * 457 characters. */
      70             : #define NVMF_PSK_IDENTITY_LEN (SPDK_NVMF_NQN_MAX_LEN + SPDK_NVMF_NQN_MAX_LEN + 11)
      71             : 
      72             : /* The maximum size of hkdf_info is defined by RFC 8446, 514B (2 + 256 + 256). */
      73             : #define NVME_TCP_HKDF_INFO_MAX_LEN 514
      74             : 
      75             : #define PSK_ID_PREFIX "NVMe0R"
      76             : 
      77             : enum nvme_tcp_cipher_suite {
      78             :         NVME_TCP_CIPHER_AES_128_GCM_SHA256,
      79             :         NVME_TCP_CIPHER_AES_256_GCM_SHA384,
      80             : };
      81             : 
      82             : typedef void (*nvme_tcp_qpair_xfer_complete_cb)(void *cb_arg);
      83             : 
      84             : struct nvme_tcp_pdu {
      85             :         union {
      86             :                 /* to hold error pdu data */
      87             :                 uint8_t                                 raw[SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE];
      88             :                 struct spdk_nvme_tcp_common_pdu_hdr     common;
      89             :                 struct spdk_nvme_tcp_ic_req             ic_req;
      90             :                 struct spdk_nvme_tcp_term_req_hdr       term_req;
      91             :                 struct spdk_nvme_tcp_cmd                capsule_cmd;
      92             :                 struct spdk_nvme_tcp_h2c_data_hdr       h2c_data;
      93             :                 struct spdk_nvme_tcp_ic_resp            ic_resp;
      94             :                 struct spdk_nvme_tcp_rsp                capsule_resp;
      95             :                 struct spdk_nvme_tcp_c2h_data_hdr       c2h_data;
      96             :                 struct spdk_nvme_tcp_r2t_hdr            r2t;
      97             : 
      98             :         } hdr;
      99             : 
     100             :         bool                                            has_hdgst;
     101             :         bool                                            ddgst_enable;
     102             :         uint32_t                                        data_digest_crc32;
     103             :         uint8_t                                         data_digest[SPDK_NVME_TCP_DIGEST_LEN];
     104             : 
     105             :         uint8_t                                         ch_valid_bytes;
     106             :         uint8_t                                         psh_valid_bytes;
     107             :         uint8_t                                         psh_len;
     108             : 
     109             :         nvme_tcp_qpair_xfer_complete_cb                 cb_fn;
     110             :         void                                            *cb_arg;
     111             : 
     112             :         /* The sock request ends with a 0 length iovec. Place the actual iovec immediately
     113             :          * after it. There is a static assert below to check if the compiler inserted
     114             :          * any unwanted padding */
     115             :         struct spdk_sock_request                        sock_req;
     116             :         struct iovec                                    iov[NVME_TCP_MAX_SGL_DESCRIPTORS * 2];
     117             : 
     118             :         struct iovec                                    data_iov[NVME_TCP_MAX_SGL_DESCRIPTORS];
     119             :         uint32_t                                        data_iovcnt;
     120             :         uint32_t                                        data_len;
     121             : 
     122             :         uint32_t                                        rw_offset;
     123             :         TAILQ_ENTRY(nvme_tcp_pdu)                       tailq;
     124             :         uint32_t                                        remaining;
     125             :         uint32_t                                        padding_len;
     126             : 
     127             :         struct spdk_dif_ctx                             *dif_ctx;
     128             : 
     129             :         void                                            *req; /* data tied to a tcp request */
     130             :         void                                            *qpair;
     131             :         SLIST_ENTRY(nvme_tcp_pdu)                       slist;
     132             : };
     133             : SPDK_STATIC_ASSERT(offsetof(struct nvme_tcp_pdu,
     134             :                             sock_req) + sizeof(struct spdk_sock_request) == offsetof(struct nvme_tcp_pdu, iov),
     135             :                    "Compiler inserted padding between iov and sock_req");
     136             : 
     137             : enum nvme_tcp_pdu_recv_state {
     138             :         /* Ready to wait for PDU */
     139             :         NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY,
     140             : 
     141             :         /* Active tqpair waiting for any PDU common header */
     142             :         NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH,
     143             : 
     144             :         /* Active tqpair waiting for any PDU specific header */
     145             :         NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH,
     146             : 
     147             :         /* Active tqpair waiting for a tcp request, only use in target side */
     148             :         NVME_TCP_PDU_RECV_STATE_AWAIT_REQ,
     149             : 
     150             :         /* Active tqpair waiting for payload */
     151             :         NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD,
     152             : 
     153             :         /* Active tqpair waiting for all outstanding PDUs to complete */
     154             :         NVME_TCP_PDU_RECV_STATE_QUIESCING,
     155             : 
     156             :         /* Active tqpair does not wait for payload */
     157             :         NVME_TCP_PDU_RECV_STATE_ERROR,
     158             : };
     159             : 
     160             : enum nvme_tcp_error_codes {
     161             :         NVME_TCP_PDU_IN_PROGRESS        = 0,
     162             :         NVME_TCP_CONNECTION_FATAL       = -1,
     163             :         NVME_TCP_PDU_FATAL              = -2,
     164             : };
     165             : 
     166             : enum nvme_tcp_qpair_state {
     167             :         NVME_TCP_QPAIR_STATE_INVALID = 0,
     168             :         NVME_TCP_QPAIR_STATE_INITIALIZING = 1,
     169             :         NVME_TCP_QPAIR_STATE_FABRIC_CONNECT_SEND = 2,
     170             :         NVME_TCP_QPAIR_STATE_FABRIC_CONNECT_POLL = 3,
     171             :         NVME_TCP_QPAIR_STATE_RUNNING = 4,
     172             :         NVME_TCP_QPAIR_STATE_EXITING = 5,
     173             :         NVME_TCP_QPAIR_STATE_EXITED = 6,
     174             : };
     175             : 
     176             : static const bool g_nvme_tcp_hdgst[] = {
     177             :         [SPDK_NVME_TCP_PDU_TYPE_IC_REQ]         = false,
     178             :         [SPDK_NVME_TCP_PDU_TYPE_IC_RESP]        = false,
     179             :         [SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ]   = false,
     180             :         [SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ]   = false,
     181             :         [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD]    = true,
     182             :         [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP]   = true,
     183             :         [SPDK_NVME_TCP_PDU_TYPE_H2C_DATA]       = true,
     184             :         [SPDK_NVME_TCP_PDU_TYPE_C2H_DATA]       = true,
     185             :         [SPDK_NVME_TCP_PDU_TYPE_R2T]            = true
     186             : };
     187             : 
     188             : static const bool g_nvme_tcp_ddgst[] = {
     189             :         [SPDK_NVME_TCP_PDU_TYPE_IC_REQ]         = false,
     190             :         [SPDK_NVME_TCP_PDU_TYPE_IC_RESP]        = false,
     191             :         [SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ]   = false,
     192             :         [SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ]   = false,
     193             :         [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD]    = true,
     194             :         [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP]   = false,
     195             :         [SPDK_NVME_TCP_PDU_TYPE_H2C_DATA]       = true,
     196             :         [SPDK_NVME_TCP_PDU_TYPE_C2H_DATA]       = true,
     197             :         [SPDK_NVME_TCP_PDU_TYPE_R2T]            = false
     198             : };
     199             : 
     200             : static uint32_t
     201           4 : nvme_tcp_pdu_calc_header_digest(struct nvme_tcp_pdu *pdu)
     202             : {
     203             :         uint32_t crc32c;
     204           4 :         uint32_t hlen = pdu->hdr.common.hlen;
     205             : 
     206           4 :         crc32c = spdk_crc32c_update(&pdu->hdr.raw, hlen, ~0);
     207           4 :         crc32c = crc32c ^ SPDK_CRC32C_XOR;
     208           4 :         return crc32c;
     209             : }
     210             : 
     211             : static uint32_t
     212           3 : nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu)
     213             : {
     214           3 :         uint32_t crc32c = SPDK_CRC32C_XOR;
     215             :         uint32_t mod;
     216             : 
     217           3 :         assert(pdu->data_len != 0);
     218             : 
     219           3 :         if (spdk_likely(!pdu->dif_ctx)) {
     220           3 :                 crc32c = spdk_crc32c_iov_update(pdu->data_iov, pdu->data_iovcnt, crc32c);
     221             :         } else {
     222           0 :                 spdk_dif_update_crc32c_stream(pdu->data_iov, pdu->data_iovcnt,
     223           0 :                                               0, pdu->data_len, &crc32c, pdu->dif_ctx);
     224             :         }
     225             : 
     226           3 :         mod = pdu->data_len % SPDK_NVME_TCP_DIGEST_ALIGNMENT;
     227           3 :         if (mod != 0) {
     228           0 :                 uint32_t pad_length = SPDK_NVME_TCP_DIGEST_ALIGNMENT - mod;
     229           0 :                 uint8_t pad[3] = {0, 0, 0};
     230             : 
     231           0 :                 assert(pad_length > 0);
     232           0 :                 assert(pad_length <= sizeof(pad));
     233           0 :                 crc32c = spdk_crc32c_update(pad, pad_length, crc32c);
     234             :         }
     235           3 :         return crc32c;
     236             : }
     237             : 
     238             : static inline void
     239          52 : _nvme_tcp_sgl_get_buf(struct spdk_iov_sgl *s, void **_buf, uint32_t *_buf_len)
     240             : {
     241          52 :         if (_buf != NULL) {
     242          52 :                 *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset;
     243             :         }
     244          52 :         if (_buf_len != NULL) {
     245          52 :                 *_buf_len = s->iov->iov_len - s->iov_offset;
     246             :         }
     247          52 : }
     248             : 
     249             : static inline bool
     250          29 : _nvme_tcp_sgl_append_multi(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt)
     251             : {
     252             :         int i;
     253             : 
     254          75 :         for (i = 0; i < iovcnt; i++) {
     255          46 :                 if (!spdk_iov_sgl_append(s, iov[i].iov_base, iov[i].iov_len)) {
     256           0 :                         return false;
     257             :                 }
     258             :         }
     259             : 
     260          29 :         return true;
     261             : }
     262             : 
     263             : static inline uint32_t
     264           0 : _get_iov_array_size(struct iovec *iov, int iovcnt)
     265             : {
     266             :         int i;
     267           0 :         uint32_t size = 0;
     268             : 
     269           0 :         for (i = 0; i < iovcnt; i++) {
     270           0 :                 size += iov[i].iov_len;
     271             :         }
     272             : 
     273           0 :         return size;
     274             : }
     275             : 
     276             : static inline bool
     277           1 : _nvme_tcp_sgl_append_multi_with_md(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt,
     278             :                                    uint32_t data_len, const struct spdk_dif_ctx *dif_ctx)
     279             : {
     280             :         int rc;
     281           1 :         uint32_t mapped_len = 0;
     282             : 
     283           1 :         if (s->iov_offset >= data_len) {
     284           0 :                 s->iov_offset -= _get_iov_array_size(iov, iovcnt);
     285             :         } else {
     286           1 :                 rc = spdk_dif_set_md_interleave_iovs(s->iov, s->iovcnt, iov, iovcnt,
     287           1 :                                                      s->iov_offset, data_len - s->iov_offset,
     288             :                                                      &mapped_len, dif_ctx);
     289           1 :                 if (rc < 0) {
     290           0 :                         SPDK_ERRLOG("Failed to setup iovs for DIF insert/strip.\n");
     291           0 :                         return false;
     292             :                 }
     293             : 
     294           1 :                 s->total_size += mapped_len;
     295           1 :                 s->iov_offset = 0;
     296           1 :                 assert(s->iovcnt >= rc);
     297           1 :                 s->iovcnt -= rc;
     298           1 :                 s->iov += rc;
     299             : 
     300           1 :                 if (s->iovcnt == 0) {
     301           0 :                         return false;
     302             :                 }
     303             :         }
     304             : 
     305           1 :         return true;
     306             : }
     307             : 
     308             : static int
     309          49 : nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
     310             :                     bool hdgst_enable, bool ddgst_enable, uint32_t *_mapped_length)
     311             : {
     312             :         uint32_t hlen;
     313             :         uint32_t plen __attribute__((unused));
     314          49 :         struct spdk_iov_sgl sgl;
     315             : 
     316          49 :         if (iovcnt == 0) {
     317           0 :                 return 0;
     318             :         }
     319             : 
     320          49 :         spdk_iov_sgl_init(&sgl, iov, iovcnt, 0);
     321          49 :         hlen = pdu->hdr.common.hlen;
     322             : 
     323             :         /* Header Digest */
     324          49 :         if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && hdgst_enable) {
     325           7 :                 hlen += SPDK_NVME_TCP_DIGEST_LEN;
     326             :         }
     327             : 
     328          49 :         plen = hlen;
     329          49 :         if (!pdu->data_len) {
     330             :                 /* PDU header + possible header digest */
     331          19 :                 spdk_iov_sgl_append(&sgl, (uint8_t *)&pdu->hdr.raw, hlen);
     332          19 :                 goto end;
     333             :         }
     334             : 
     335             :         /* Padding */
     336          30 :         if (pdu->padding_len > 0) {
     337           1 :                 hlen += pdu->padding_len;
     338           1 :                 plen = hlen;
     339             :         }
     340             : 
     341          30 :         if (!spdk_iov_sgl_append(&sgl, (uint8_t *)&pdu->hdr.raw, hlen)) {
     342           0 :                 goto end;
     343             :         }
     344             : 
     345             :         /* Data Segment */
     346          30 :         plen += pdu->data_len;
     347          30 :         if (spdk_likely(!pdu->dif_ctx)) {
     348          29 :                 if (!_nvme_tcp_sgl_append_multi(&sgl, pdu->data_iov, pdu->data_iovcnt)) {
     349           0 :                         goto end;
     350             :                 }
     351             :         } else {
     352           1 :                 if (!_nvme_tcp_sgl_append_multi_with_md(&sgl, pdu->data_iov, pdu->data_iovcnt,
     353           1 :                                                         pdu->data_len, pdu->dif_ctx)) {
     354           0 :                         goto end;
     355             :                 }
     356             :         }
     357             : 
     358             :         /* Data Digest */
     359          30 :         if (g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && ddgst_enable) {
     360           6 :                 plen += SPDK_NVME_TCP_DIGEST_LEN;
     361           6 :                 spdk_iov_sgl_append(&sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
     362             :         }
     363             : 
     364          30 :         assert(plen == pdu->hdr.common.plen);
     365             : 
     366          49 : end:
     367          49 :         if (_mapped_length != NULL) {
     368          49 :                 *_mapped_length = sgl.total_size;
     369             :         }
     370             : 
     371          49 :         return iovcnt - sgl.iovcnt;
     372             : }
     373             : 
     374             : static int
     375           0 : nvme_tcp_build_payload_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
     376             :                             bool ddgst_enable, uint32_t *_mapped_length)
     377             : {
     378           0 :         struct spdk_iov_sgl sgl;
     379             : 
     380           0 :         if (iovcnt == 0) {
     381           0 :                 return 0;
     382             :         }
     383             : 
     384           0 :         spdk_iov_sgl_init(&sgl, iov, iovcnt, pdu->rw_offset);
     385             : 
     386           0 :         if (spdk_likely(!pdu->dif_ctx)) {
     387           0 :                 if (!_nvme_tcp_sgl_append_multi(&sgl, pdu->data_iov, pdu->data_iovcnt)) {
     388           0 :                         goto end;
     389             :                 }
     390             :         } else {
     391           0 :                 if (!_nvme_tcp_sgl_append_multi_with_md(&sgl, pdu->data_iov, pdu->data_iovcnt,
     392           0 :                                                         pdu->data_len, pdu->dif_ctx)) {
     393           0 :                         goto end;
     394             :                 }
     395             :         }
     396             : 
     397             :         /* Data Digest */
     398           0 :         if (ddgst_enable) {
     399           0 :                 spdk_iov_sgl_append(&sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
     400             :         }
     401             : 
     402           0 : end:
     403           0 :         if (_mapped_length != NULL) {
     404           0 :                 *_mapped_length = sgl.total_size;
     405             :         }
     406           0 :         return iovcnt - sgl.iovcnt;
     407             : }
     408             : 
     409             : static int
     410           4 : nvme_tcp_read_data(struct spdk_sock *sock, int bytes,
     411             :                    void *buf)
     412             : {
     413             :         int ret;
     414             : 
     415           4 :         ret = spdk_sock_recv(sock, buf, bytes);
     416             : 
     417           4 :         if (ret > 0) {
     418           4 :                 return ret;
     419             :         }
     420             : 
     421           0 :         if (ret < 0) {
     422           0 :                 if (errno == EAGAIN || errno == EWOULDBLOCK) {
     423           0 :                         return 0;
     424             :                 }
     425             : 
     426             :                 /* For connect reset issue, do not output error log */
     427           0 :                 if (errno != ECONNRESET) {
     428           0 :                         SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
     429             :                                     errno, spdk_strerror(errno));
     430             :                 }
     431             :         }
     432             : 
     433             :         /* connection closed */
     434           0 :         return NVME_TCP_CONNECTION_FATAL;
     435             : }
     436             : 
     437             : static int
     438           0 : nvme_tcp_readv_data(struct spdk_sock *sock, struct iovec *iov, int iovcnt)
     439             : {
     440             :         int ret;
     441             : 
     442           0 :         assert(sock != NULL);
     443           0 :         if (iov == NULL || iovcnt == 0) {
     444           0 :                 return 0;
     445             :         }
     446             : 
     447           0 :         if (iovcnt == 1) {
     448           0 :                 return nvme_tcp_read_data(sock, iov->iov_len, iov->iov_base);
     449             :         }
     450             : 
     451           0 :         ret = spdk_sock_readv(sock, iov, iovcnt);
     452             : 
     453           0 :         if (ret > 0) {
     454           0 :                 return ret;
     455             :         }
     456             : 
     457           0 :         if (ret < 0) {
     458           0 :                 if (errno == EAGAIN || errno == EWOULDBLOCK) {
     459           0 :                         return 0;
     460             :                 }
     461             : 
     462             :                 /* For connect reset issue, do not output error log */
     463           0 :                 if (errno != ECONNRESET) {
     464           0 :                         SPDK_ERRLOG("spdk_sock_readv() failed, errno %d: %s\n",
     465             :                                     errno, spdk_strerror(errno));
     466             :                 }
     467             :         }
     468             : 
     469             :         /* connection closed */
     470           0 :         return NVME_TCP_CONNECTION_FATAL;
     471             : }
     472             : 
     473             : 
     474             : static int
     475           0 : nvme_tcp_read_payload_data(struct spdk_sock *sock, struct nvme_tcp_pdu *pdu)
     476             : {
     477           0 :         struct iovec iov[NVME_TCP_MAX_SGL_DESCRIPTORS + 1];
     478             :         int iovcnt;
     479             : 
     480           0 :         iovcnt = nvme_tcp_build_payload_iovs(iov, NVME_TCP_MAX_SGL_DESCRIPTORS + 1, pdu,
     481           0 :                                              pdu->ddgst_enable, NULL);
     482           0 :         assert(iovcnt >= 0);
     483             : 
     484           0 :         return nvme_tcp_readv_data(sock, iov, iovcnt);
     485             : }
     486             : 
     487             : static void
     488          35 : _nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
     489             : {
     490          35 :         pdu->data_iov[0].iov_base = data;
     491          35 :         pdu->data_iov[0].iov_len = data_len;
     492          35 :         pdu->data_iovcnt = 1;
     493          35 : }
     494             : 
     495             : static void
     496          30 : nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
     497             : {
     498          30 :         _nvme_tcp_pdu_set_data(pdu, data, data_len);
     499          30 :         pdu->data_len = data_len;
     500          30 : }
     501             : 
     502             : static void
     503          21 : nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
     504             :                           struct iovec *iov, int iovcnt,
     505             :                           uint32_t data_offset, uint32_t data_len)
     506             : {
     507          21 :         uint32_t buf_offset, buf_len, remain_len, len;
     508          21 :         uint8_t *buf;
     509          21 :         struct spdk_iov_sgl pdu_sgl, buf_sgl;
     510             : 
     511          21 :         pdu->data_len = data_len;
     512             : 
     513          21 :         if (spdk_likely(!pdu->dif_ctx)) {
     514          15 :                 buf_offset = data_offset;
     515          15 :                 buf_len = data_len;
     516             :         } else {
     517           6 :                 spdk_dif_ctx_set_data_offset(pdu->dif_ctx, data_offset);
     518           6 :                 spdk_dif_get_range_with_md(data_offset, data_len,
     519           6 :                                            &buf_offset, &buf_len, pdu->dif_ctx);
     520             :         }
     521             : 
     522          21 :         if (iovcnt == 1) {
     523           5 :                 _nvme_tcp_pdu_set_data(pdu, (void *)((uint64_t)iov[0].iov_base + buf_offset), buf_len);
     524             :         } else {
     525          16 :                 spdk_iov_sgl_init(&pdu_sgl, pdu->data_iov, NVME_TCP_MAX_SGL_DESCRIPTORS, 0);
     526          16 :                 spdk_iov_sgl_init(&buf_sgl, iov, iovcnt, 0);
     527             : 
     528          16 :                 spdk_iov_sgl_advance(&buf_sgl, buf_offset);
     529          16 :                 remain_len = buf_len;
     530             : 
     531          67 :                 while (remain_len > 0) {
     532          52 :                         _nvme_tcp_sgl_get_buf(&buf_sgl, (void *)&buf, &len);
     533          52 :                         len = spdk_min(len, remain_len);
     534             : 
     535          52 :                         spdk_iov_sgl_advance(&buf_sgl, len);
     536          52 :                         remain_len -= len;
     537             : 
     538          52 :                         if (!spdk_iov_sgl_append(&pdu_sgl, buf, len)) {
     539           1 :                                 break;
     540             :                         }
     541             :                 }
     542             : 
     543          16 :                 assert(remain_len == 0);
     544          16 :                 assert(pdu_sgl.total_size == buf_len);
     545             : 
     546          16 :                 pdu->data_iovcnt = NVME_TCP_MAX_SGL_DESCRIPTORS - pdu_sgl.iovcnt;
     547             :         }
     548          21 : }
     549             : 
     550             : static void
     551           3 : nvme_tcp_pdu_calc_psh_len(struct nvme_tcp_pdu *pdu, bool hdgst_enable)
     552             : {
     553             :         uint8_t psh_len, pdo, padding_len;
     554             : 
     555           3 :         psh_len = pdu->hdr.common.hlen;
     556             : 
     557           3 :         if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && hdgst_enable) {
     558           0 :                 pdu->has_hdgst = true;
     559           0 :                 psh_len += SPDK_NVME_TCP_DIGEST_LEN;
     560             :         }
     561           3 :         if (pdu->hdr.common.plen > psh_len) {
     562           0 :                 switch (pdu->hdr.common.pdu_type) {
     563           0 :                 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD:
     564             :                 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA:
     565             :                 case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
     566           0 :                         pdo = pdu->hdr.common.pdo;
     567           0 :                         padding_len = pdo - psh_len;
     568           0 :                         if (padding_len > 0) {
     569           0 :                                 psh_len = pdo;
     570             :                         }
     571           0 :                         break;
     572           0 :                 default:
     573             :                         /* There is no padding for other PDU types */
     574           0 :                         break;
     575             :                 }
     576           3 :         }
     577             : 
     578           3 :         psh_len -= sizeof(struct spdk_nvme_tcp_common_pdu_hdr);
     579           3 :         pdu->psh_len = psh_len;
     580           3 : }
     581             : 
     582             : static inline int
     583           4 : nvme_tcp_generate_psk_identity(char *out_id, size_t out_id_len, const char *hostnqn,
     584             :                                const char *subnqn, enum nvme_tcp_cipher_suite tls_cipher_suite)
     585             : {
     586             :         int rc;
     587             : 
     588           4 :         assert(out_id != NULL);
     589             : 
     590           4 :         if (out_id_len < strlen(PSK_ID_PREFIX) + strlen(hostnqn) + strlen(subnqn) + 5) {
     591           1 :                 SPDK_ERRLOG("Out buffer too small!\n");
     592           1 :                 return -1;
     593             :         }
     594             : 
     595           3 :         if (tls_cipher_suite == NVME_TCP_CIPHER_AES_128_GCM_SHA256) {
     596           2 :                 rc = snprintf(out_id, out_id_len, "%s%s %s %s", PSK_ID_PREFIX, "01",
     597             :                               hostnqn, subnqn);
     598           1 :         } else if (tls_cipher_suite == NVME_TCP_CIPHER_AES_256_GCM_SHA384) {
     599           0 :                 rc = snprintf(out_id, out_id_len, "%s%s %s %s", PSK_ID_PREFIX, "02",
     600             :                               hostnqn, subnqn);
     601             :         } else {
     602           1 :                 SPDK_ERRLOG("Unknown cipher suite requested!\n");
     603           1 :                 return -EOPNOTSUPP;
     604             :         }
     605             : 
     606           2 :         if (rc < 0) {
     607           0 :                 SPDK_ERRLOG("Could not generate PSK identity\n");
     608           0 :                 return -1;
     609             :         }
     610             : 
     611           2 :         return 0;
     612             : }
     613             : 
     614             : enum nvme_tcp_hash_algorithm {
     615             :         NVME_TCP_HASH_ALGORITHM_NONE,
     616             :         NVME_TCP_HASH_ALGORITHM_SHA256,
     617             :         NVME_TCP_HASH_ALGORITHM_SHA384,
     618             : };
     619             : 
     620             : static inline int
     621           8 : nvme_tcp_derive_retained_psk(const uint8_t *psk_in, uint64_t psk_in_size, const char *hostnqn,
     622             :                              uint8_t *psk_out, uint64_t psk_out_len, enum nvme_tcp_hash_algorithm psk_retained_hash)
     623             : {
     624             :         EVP_PKEY_CTX *ctx;
     625           8 :         uint64_t digest_len;
     626           8 :         uint8_t hkdf_info[NVME_TCP_HKDF_INFO_MAX_LEN] = {};
     627           8 :         const char *label = "tls13 HostNQN";
     628             :         size_t pos, labellen, nqnlen;
     629             :         const EVP_MD *hash;
     630             :         int rc, hkdf_info_size;
     631             : 
     632           8 :         labellen = strlen(label);
     633           8 :         nqnlen = strlen(hostnqn);
     634           8 :         assert(nqnlen <= SPDK_NVMF_NQN_MAX_LEN);
     635             : 
     636           8 :         *(uint16_t *)&hkdf_info[0] = htons(psk_in_size);
     637           8 :         pos = sizeof(uint16_t);
     638           8 :         hkdf_info[pos] = (uint8_t)labellen;
     639           8 :         pos += sizeof(uint8_t);
     640           8 :         memcpy(&hkdf_info[pos], label, labellen);
     641           8 :         pos += labellen;
     642           8 :         hkdf_info[pos] = (uint8_t)nqnlen;
     643           8 :         pos += sizeof(uint8_t);
     644           8 :         memcpy(&hkdf_info[pos], hostnqn, nqnlen);
     645           8 :         pos += nqnlen;
     646           8 :         hkdf_info_size = pos;
     647             : 
     648           8 :         switch (psk_retained_hash) {
     649           6 :         case NVME_TCP_HASH_ALGORITHM_SHA256:
     650           6 :                 digest_len = SHA256_DIGEST_LENGTH;
     651           6 :                 hash = EVP_sha256();
     652           6 :                 break;
     653           1 :         case NVME_TCP_HASH_ALGORITHM_SHA384:
     654           1 :                 digest_len = SHA384_DIGEST_LENGTH;
     655           1 :                 hash = EVP_sha384();
     656           1 :                 break;
     657           1 :         default:
     658           1 :                 SPDK_ERRLOG("Unknown PSK hash requested!\n");
     659           1 :                 return -EOPNOTSUPP;
     660             :         }
     661             : 
     662           7 :         if (digest_len > psk_out_len) {
     663           1 :                 SPDK_ERRLOG("Insufficient buffer size for out key!\n");
     664           1 :                 return -EINVAL;
     665             :         }
     666             : 
     667           6 :         ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
     668           6 :         if (!ctx) {
     669           0 :                 SPDK_ERRLOG("Unable to initialize EVP_PKEY_CTX!\n");
     670           0 :                 return -ENOMEM;
     671             :         }
     672             : 
     673             :         /* EVP_PKEY_* functions returns 1 as a success code and 0 or negative on failure. */
     674           6 :         if (EVP_PKEY_derive_init(ctx) != 1) {
     675           0 :                 SPDK_ERRLOG("Unable to initialize key derivation ctx for HKDF!\n");
     676           0 :                 rc = -ENOMEM;
     677           0 :                 goto end;
     678             :         }
     679           6 :         if (EVP_PKEY_CTX_set_hkdf_md(ctx, hash) != 1) {
     680           0 :                 SPDK_ERRLOG("Unable to set hash for HKDF!\n");
     681           0 :                 rc = -EOPNOTSUPP;
     682           0 :                 goto end;
     683             :         }
     684           6 :         if (EVP_PKEY_CTX_set1_hkdf_key(ctx, psk_in, psk_in_size) != 1) {
     685           0 :                 SPDK_ERRLOG("Unable to set PSK key for HKDF!\n");
     686           0 :                 rc = -ENOBUFS;
     687           0 :                 goto end;
     688             :         }
     689             : 
     690           6 :         if (EVP_PKEY_CTX_add1_hkdf_info(ctx, hkdf_info, hkdf_info_size) != 1) {
     691           0 :                 SPDK_ERRLOG("Unable to set info label for HKDF!\n");
     692           0 :                 rc = -ENOBUFS;
     693           0 :                 goto end;
     694             :         }
     695           6 :         if (EVP_PKEY_CTX_set1_hkdf_salt(ctx, NULL, 0) != 1) {
     696           0 :                 SPDK_ERRLOG("Unable to set salt for HKDF!\n");
     697           0 :                 rc = -EINVAL;
     698           0 :                 goto end;
     699             :         }
     700           6 :         if (EVP_PKEY_derive(ctx, psk_out, &digest_len) != 1) {
     701           0 :                 SPDK_ERRLOG("Unable to derive the PSK key!\n");
     702           0 :                 rc = -EINVAL;
     703           0 :                 goto end;
     704             :         }
     705             : 
     706           6 :         rc = digest_len;
     707             : 
     708           6 : end:
     709           6 :         EVP_PKEY_CTX_free(ctx);
     710           6 :         return rc;
     711             : }
     712             : 
     713             : static inline int
     714           4 : nvme_tcp_derive_tls_psk(const uint8_t *psk_in, uint64_t psk_in_size, const char *psk_identity,
     715             :                         uint8_t *psk_out, uint64_t psk_out_size, enum nvme_tcp_cipher_suite tls_cipher_suite)
     716             : {
     717             :         EVP_PKEY_CTX *ctx;
     718           4 :         uint64_t digest_len = 0;
     719           4 :         char hkdf_info[NVME_TCP_HKDF_INFO_MAX_LEN] = {};
     720           4 :         const char *label = "tls13 nvme-tls-psk";
     721             :         size_t pos, labellen, idlen;
     722             :         const EVP_MD *hash;
     723             :         int rc, hkdf_info_size;
     724             : 
     725           4 :         if (tls_cipher_suite == NVME_TCP_CIPHER_AES_128_GCM_SHA256) {
     726           2 :                 digest_len = SHA256_DIGEST_LENGTH;
     727           2 :                 hash = EVP_sha256();
     728           2 :         } else if (tls_cipher_suite == NVME_TCP_CIPHER_AES_256_GCM_SHA384) {
     729           1 :                 digest_len = SHA384_DIGEST_LENGTH;
     730           1 :                 hash = EVP_sha384();
     731             :         } else {
     732           1 :                 SPDK_ERRLOG("Unknown cipher suite requested!\n");
     733           1 :                 return -EOPNOTSUPP;
     734             :         }
     735             : 
     736           3 :         labellen = strlen(label);
     737           3 :         idlen = strlen(psk_identity);
     738           3 :         if (idlen > UINT8_MAX) {
     739           0 :                 SPDK_ERRLOG("Invalid PSK ID: too long\n");
     740           0 :                 return -1;
     741             :         }
     742             : 
     743           3 :         *(uint16_t *)&hkdf_info[0] = htons(psk_in_size);
     744           3 :         pos = sizeof(uint16_t);
     745           3 :         hkdf_info[pos] = (uint8_t)labellen;
     746           3 :         pos += sizeof(uint8_t);
     747           3 :         memcpy(&hkdf_info[pos], label, labellen);
     748           3 :         pos += labellen;
     749           3 :         hkdf_info[pos] = (uint8_t)idlen;
     750           3 :         pos += sizeof(uint8_t);
     751           3 :         memcpy(&hkdf_info[pos], psk_identity, idlen);
     752           3 :         pos += idlen;
     753           3 :         hkdf_info_size = pos;
     754             : 
     755           3 :         if (digest_len > psk_out_size) {
     756           1 :                 SPDK_ERRLOG("Insufficient buffer size for out key!\n");
     757           1 :                 return -1;
     758             :         }
     759             : 
     760           2 :         ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
     761           2 :         if (!ctx) {
     762           0 :                 SPDK_ERRLOG("Unable to initialize EVP_PKEY_CTX!\n");
     763           0 :                 return -1;
     764             :         }
     765             : 
     766           2 :         if (EVP_PKEY_derive_init(ctx) != 1) {
     767           0 :                 SPDK_ERRLOG("Unable to initialize key derivation ctx for HKDF!\n");
     768           0 :                 rc = -ENOMEM;
     769           0 :                 goto end;
     770             :         }
     771           2 :         if (EVP_PKEY_CTX_set_hkdf_md(ctx, hash) != 1) {
     772           0 :                 SPDK_ERRLOG("Unable to set hash method for HKDF!\n");
     773           0 :                 rc = -EOPNOTSUPP;
     774           0 :                 goto end;
     775             :         }
     776           2 :         if (EVP_PKEY_CTX_set1_hkdf_key(ctx, psk_in, psk_in_size) != 1) {
     777           0 :                 SPDK_ERRLOG("Unable to set PSK key for HKDF!\n");
     778           0 :                 rc = -ENOBUFS;
     779           0 :                 goto end;
     780             :         }
     781           2 :         if (EVP_PKEY_CTX_add1_hkdf_info(ctx, hkdf_info, hkdf_info_size) != 1) {
     782           0 :                 SPDK_ERRLOG("Unable to set info label for HKDF!\n");
     783           0 :                 rc = -ENOBUFS;
     784           0 :                 goto end;
     785             :         }
     786           2 :         if (EVP_PKEY_CTX_set1_hkdf_salt(ctx, NULL, 0) != 1) {
     787           0 :                 SPDK_ERRLOG("Unable to set salt for HKDF!\n");
     788           0 :                 rc = -EINVAL;
     789           0 :                 goto end;
     790             :         }
     791           2 :         if (EVP_PKEY_derive(ctx, psk_out, &digest_len) != 1) {
     792           0 :                 SPDK_ERRLOG("Unable to derive the PSK key!\n");
     793           0 :                 rc = -EINVAL;
     794           0 :                 goto end;
     795             :         }
     796             : 
     797           2 :         rc = digest_len;
     798             : 
     799           2 : end:
     800           2 :         EVP_PKEY_CTX_free(ctx);
     801           2 :         return rc;
     802             : }
     803             : 
     804             : static inline int
     805           1 : nvme_tcp_parse_interchange_psk(const char *psk_in, uint8_t *psk_out, size_t psk_out_size,
     806             :                                uint64_t *psk_out_decoded_size, uint8_t *hash)
     807             : {
     808           1 :         const char *delim = ":";
     809           1 :         char psk_cpy[SPDK_TLS_PSK_MAX_LEN] = {};
     810           1 :         uint8_t psk_base64_decoded[SPDK_TLS_PSK_MAX_LEN] = {};
     811           1 :         uint64_t psk_configured_size = 0;
     812             :         uint32_t crc32_calc, crc32;
     813             :         char *psk_base64;
     814           1 :         uint64_t psk_base64_decoded_size = 0;
     815             :         int rc;
     816             : 
     817             :         /* Verify PSK format. */
     818           1 :         if (sscanf(psk_in, "NVMeTLSkey-1:%02hhx:", hash) != 1 || psk_in[strlen(psk_in) - 1] != delim[0]) {
     819           0 :                 SPDK_ERRLOG("Invalid format of PSK interchange!\n");
     820           0 :                 return -EINVAL;
     821             :         }
     822             : 
     823           1 :         if (strlen(psk_in) >= SPDK_TLS_PSK_MAX_LEN) {
     824           0 :                 SPDK_ERRLOG("PSK interchange exceeds maximum %d characters!\n", SPDK_TLS_PSK_MAX_LEN);
     825           0 :                 return -EINVAL;
     826             :         }
     827           1 :         if (*hash != NVME_TCP_HASH_ALGORITHM_NONE && *hash != NVME_TCP_HASH_ALGORITHM_SHA256 &&
     828           0 :             *hash != NVME_TCP_HASH_ALGORITHM_SHA384) {
     829           0 :                 SPDK_ERRLOG("Invalid PSK length!\n");
     830           0 :                 return -EINVAL;
     831             :         }
     832             : 
     833             :         /* Check provided hash function string. */
     834           1 :         memcpy(psk_cpy, psk_in, strlen(psk_in));
     835           1 :         strtok(psk_cpy, delim);
     836           1 :         strtok(NULL, delim);
     837             : 
     838           1 :         psk_base64 = strtok(NULL, delim);
     839           1 :         if (psk_base64 == NULL) {
     840           0 :                 SPDK_ERRLOG("Could not get base64 string from PSK interchange!\n");
     841           0 :                 return -EINVAL;
     842             :         }
     843             : 
     844           1 :         rc = spdk_base64_decode(psk_base64_decoded, &psk_base64_decoded_size, psk_base64);
     845           1 :         if (rc) {
     846           0 :                 SPDK_ERRLOG("Could not decode base64 PSK!\n");
     847           0 :                 return -EINVAL;
     848             :         }
     849             : 
     850           1 :         switch (*hash) {
     851           1 :         case NVME_TCP_HASH_ALGORITHM_SHA256:
     852           1 :                 psk_configured_size = SHA256_DIGEST_LENGTH;
     853           1 :                 break;
     854           0 :         case NVME_TCP_HASH_ALGORITHM_SHA384:
     855           0 :                 psk_configured_size = SHA384_DIGEST_LENGTH;
     856           0 :                 break;
     857           0 :         case NVME_TCP_HASH_ALGORITHM_NONE:
     858           0 :                 if (psk_base64_decoded_size == SHA256_DIGEST_LENGTH + SPDK_CRC32_SIZE_BYTES) {
     859           0 :                         psk_configured_size = SHA256_DIGEST_LENGTH;
     860           0 :                 } else if (psk_base64_decoded_size == SHA384_DIGEST_LENGTH + SPDK_CRC32_SIZE_BYTES) {
     861           0 :                         psk_configured_size = SHA384_DIGEST_LENGTH;
     862             :                 }
     863           0 :                 break;
     864           0 :         default:
     865           0 :                 SPDK_ERRLOG("Invalid key: unsupported key hash\n");
     866           0 :                 assert(0);
     867             :                 return -EINVAL;
     868             :         }
     869           1 :         if (psk_base64_decoded_size != psk_configured_size + SPDK_CRC32_SIZE_BYTES) {
     870           0 :                 SPDK_ERRLOG("Invalid key: unsupported key length\n");
     871           0 :                 return -EINVAL;
     872             :         }
     873             : 
     874           1 :         crc32 = from_le32(&psk_base64_decoded[psk_configured_size]);
     875             : 
     876           1 :         crc32_calc = spdk_crc32_ieee_update(psk_base64_decoded, psk_configured_size, ~0);
     877           1 :         crc32_calc = ~crc32_calc;
     878             : 
     879           1 :         if (crc32 != crc32_calc) {
     880           0 :                 SPDK_ERRLOG("CRC-32 checksums do not match!\n");
     881           0 :                 return -EINVAL;
     882             :         }
     883             : 
     884           1 :         if (psk_configured_size > psk_out_size) {
     885           0 :                 SPDK_ERRLOG("Insufficient buffer size: %lu for configured PSK of size: %lu!\n",
     886             :                             psk_out_size, psk_configured_size);
     887           0 :                 return -ENOBUFS;
     888             :         }
     889           1 :         memcpy(psk_out, psk_base64_decoded, psk_configured_size);
     890           1 :         *psk_out_decoded_size = psk_configured_size;
     891             : 
     892           1 :         return rc;
     893             : }
     894             : 
     895             : #endif /* SPDK_INTERNAL_NVME_TCP_H */

Generated by: LCOV version 1.15