LCOV - code coverage report
Current view: top level - lib/nvme - nvme_transport.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 57 388 14.7 %
Date: 2024-07-15 18:22:12 Functions: 6 57 10.5 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2016 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  *   Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
       5             :  *   Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       6             :  */
       7             : 
       8             : /*
       9             :  * NVMe transport abstraction
      10             :  */
      11             : 
      12             : #include "nvme_internal.h"
      13             : #include "spdk/queue.h"
      14             : 
      15             : #define SPDK_MAX_NUM_OF_TRANSPORTS 16
      16             : 
      17             : struct spdk_nvme_transport {
      18             :         struct spdk_nvme_transport_ops  ops;
      19             :         TAILQ_ENTRY(spdk_nvme_transport)        link;
      20             : };
      21             : 
      22             : TAILQ_HEAD(nvme_transport_list, spdk_nvme_transport) g_spdk_nvme_transports =
      23             :         TAILQ_HEAD_INITIALIZER(g_spdk_nvme_transports);
      24             : 
      25             : struct spdk_nvme_transport g_spdk_transports[SPDK_MAX_NUM_OF_TRANSPORTS] = {};
      26             : int g_current_transport_index = 0;
      27             : 
      28             : struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts = {
      29             :         .rdma_srq_size = 0,
      30             :         .rdma_max_cq_size = 0,
      31             :         .rdma_cm_event_timeout_ms = 1000
      32             : };
      33             : 
      34             : const struct spdk_nvme_transport *
      35           0 : nvme_get_first_transport(void)
      36             : {
      37           0 :         return TAILQ_FIRST(&g_spdk_nvme_transports);
      38             : }
      39             : 
      40             : const struct spdk_nvme_transport *
      41           0 : nvme_get_next_transport(const struct spdk_nvme_transport *transport)
      42             : {
      43           0 :         return TAILQ_NEXT(transport, link);
      44             : }
      45             : 
      46             : /*
      47             :  * Unfortunately, due to NVMe PCIe multiprocess support, we cannot store the
      48             :  * transport object in either the controller struct or the admin qpair. This means
      49             :  * that a lot of admin related transport calls will have to call nvme_get_transport
      50             :  * in order to know which functions to call.
      51             :  * In the I/O path, we have the ability to store the transport struct in the I/O
      52             :  * qpairs to avoid taking a performance hit.
      53             :  */
      54             : const struct spdk_nvme_transport *
      55           4 : nvme_get_transport(const char *transport_name)
      56             : {
      57             :         struct spdk_nvme_transport *registered_transport;
      58             : 
      59           4 :         TAILQ_FOREACH(registered_transport, &g_spdk_nvme_transports, link) {
      60           3 :                 if (strcasecmp(transport_name, registered_transport->ops.name) == 0) {
      61           3 :                         return registered_transport;
      62             :                 }
      63             :         }
      64             : 
      65           1 :         return NULL;
      66             : }
      67             : 
      68             : bool
      69           0 : spdk_nvme_transport_available(enum spdk_nvme_transport_type trtype)
      70             : {
      71           0 :         return nvme_get_transport(spdk_nvme_transport_id_trtype_str(trtype)) == NULL ? false : true;
      72             : }
      73             : 
      74             : bool
      75           0 : spdk_nvme_transport_available_by_name(const char *transport_name)
      76             : {
      77           0 :         return nvme_get_transport(transport_name) == NULL ? false : true;
      78             : }
      79             : 
      80             : void
      81           0 : spdk_nvme_transport_register(const struct spdk_nvme_transport_ops *ops)
      82             : {
      83             :         struct spdk_nvme_transport *new_transport;
      84             : 
      85           0 :         if (nvme_get_transport(ops->name)) {
      86           0 :                 SPDK_ERRLOG("Double registering NVMe transport %s is prohibited.\n", ops->name);
      87           0 :                 assert(false);
      88             :         }
      89             : 
      90           0 :         if (g_current_transport_index == SPDK_MAX_NUM_OF_TRANSPORTS) {
      91           0 :                 SPDK_ERRLOG("Unable to register new NVMe transport.\n");
      92           0 :                 assert(false);
      93             :                 return;
      94             :         }
      95           0 :         new_transport = &g_spdk_transports[g_current_transport_index++];
      96             : 
      97           0 :         new_transport->ops = *ops;
      98           0 :         TAILQ_INSERT_TAIL(&g_spdk_nvme_transports, new_transport, link);
      99             : }
     100             : 
     101           0 : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
     102             :                 const struct spdk_nvme_ctrlr_opts *opts,
     103             :                 void *devhandle)
     104             : {
     105           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(trid->trstring);
     106             :         struct spdk_nvme_ctrlr *ctrlr;
     107             : 
     108           0 :         if (transport == NULL) {
     109           0 :                 SPDK_ERRLOG("Transport %s doesn't exist.", trid->trstring);
     110           0 :                 return NULL;
     111             :         }
     112             : 
     113           0 :         ctrlr = transport->ops.ctrlr_construct(trid, opts, devhandle);
     114             : 
     115           0 :         return ctrlr;
     116             : }
     117             : 
     118             : int
     119           0 : nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
     120             :                           bool direct_connect)
     121             : {
     122           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
     123             : 
     124           0 :         if (transport == NULL) {
     125           0 :                 SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
     126           0 :                 return -ENOENT;
     127             :         }
     128             : 
     129           0 :         return transport->ops.ctrlr_scan(probe_ctx, direct_connect);
     130             : }
     131             : 
     132             : int
     133           0 : nvme_transport_ctrlr_scan_attached(struct spdk_nvme_probe_ctx *probe_ctx)
     134             : {
     135           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
     136             : 
     137           0 :         if (transport == NULL) {
     138           0 :                 SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
     139           0 :                 return -ENOENT;
     140             :         }
     141             : 
     142           0 :         if (transport->ops.ctrlr_scan_attached != NULL) {
     143           0 :                 return transport->ops.ctrlr_scan_attached(probe_ctx);
     144             :         }
     145           0 :         SPDK_ERRLOG("Transport %s does not support ctrlr_scan_attached callback\n",
     146             :                     probe_ctx->trid.trstring);
     147           0 :         return -ENOTSUP;
     148             : }
     149             : 
     150             : int
     151           0 : nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
     152             : {
     153           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     154             : 
     155           0 :         assert(transport != NULL);
     156           0 :         return transport->ops.ctrlr_destruct(ctrlr);
     157             : }
     158             : 
     159             : int
     160           0 : nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
     161             : {
     162           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     163             : 
     164           0 :         assert(transport != NULL);
     165           0 :         return transport->ops.ctrlr_enable(ctrlr);
     166             : }
     167             : 
     168             : int
     169           0 : nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr)
     170             : {
     171           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     172             : 
     173           0 :         assert(transport != NULL);
     174           0 :         if (transport->ops.ctrlr_ready) {
     175           0 :                 return transport->ops.ctrlr_ready(ctrlr);
     176             :         }
     177             : 
     178           0 :         return 0;
     179             : }
     180             : 
     181             : int
     182           0 : nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
     183             : {
     184           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     185             : 
     186           0 :         assert(transport != NULL);
     187           0 :         return transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
     188             : }
     189             : 
     190             : int
     191           0 : nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
     192             : {
     193           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     194             : 
     195           0 :         assert(transport != NULL);
     196           0 :         return transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
     197             : }
     198             : 
     199             : int
     200           0 : nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
     201             : {
     202           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     203             : 
     204           0 :         assert(transport != NULL);
     205           0 :         return transport->ops.ctrlr_get_reg_4(ctrlr, offset, value);
     206             : }
     207             : 
     208             : int
     209           0 : nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
     210             : {
     211           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     212             : 
     213           0 :         assert(transport != NULL);
     214           0 :         return transport->ops.ctrlr_get_reg_8(ctrlr, offset, value);
     215             : }
     216             : 
     217             : static int
     218           0 : nvme_queue_register_operation_completion(struct spdk_nvme_ctrlr *ctrlr, uint64_t value,
     219             :                 spdk_nvme_reg_cb cb_fn, void *cb_ctx)
     220             : {
     221             :         struct nvme_register_completion *ctx;
     222             : 
     223           0 :         ctx = spdk_zmalloc(sizeof(*ctx), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
     224           0 :         if (ctx == NULL) {
     225           0 :                 return -ENOMEM;
     226             :         }
     227             : 
     228           0 :         ctx->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
     229           0 :         ctx->cpl.status.sc = SPDK_NVME_SC_SUCCESS;
     230           0 :         ctx->cb_fn = cb_fn;
     231           0 :         ctx->cb_ctx = cb_ctx;
     232           0 :         ctx->value = value;
     233           0 :         ctx->pid = getpid();
     234             : 
     235           0 :         nvme_ctrlr_lock(ctrlr);
     236           0 :         STAILQ_INSERT_TAIL(&ctrlr->register_operations, ctx, stailq);
     237           0 :         nvme_ctrlr_unlock(ctrlr);
     238             : 
     239           0 :         return 0;
     240             : }
     241             : 
     242             : int
     243           0 : nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value,
     244             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     245             : {
     246           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     247             :         int rc;
     248             : 
     249           0 :         assert(transport != NULL);
     250           0 :         if (transport->ops.ctrlr_set_reg_4_async == NULL) {
     251           0 :                 rc = transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
     252           0 :                 if (rc != 0) {
     253           0 :                         return rc;
     254             :                 }
     255             : 
     256           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     257             :         }
     258             : 
     259           0 :         return transport->ops.ctrlr_set_reg_4_async(ctrlr, offset, value, cb_fn, cb_arg);
     260             : }
     261             : 
     262             : int
     263           0 : nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value,
     264             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     265             : 
     266             : {
     267           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     268             :         int rc;
     269             : 
     270           0 :         assert(transport != NULL);
     271           0 :         if (transport->ops.ctrlr_set_reg_8_async == NULL) {
     272           0 :                 rc = transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
     273           0 :                 if (rc != 0) {
     274           0 :                         return rc;
     275             :                 }
     276             : 
     277           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     278             :         }
     279             : 
     280           0 :         return transport->ops.ctrlr_set_reg_8_async(ctrlr, offset, value, cb_fn, cb_arg);
     281             : }
     282             : 
     283             : int
     284           0 : nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
     285             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     286             : {
     287           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     288           0 :         uint32_t value;
     289             :         int rc;
     290             : 
     291           0 :         assert(transport != NULL);
     292           0 :         if (transport->ops.ctrlr_get_reg_4_async == NULL) {
     293           0 :                 rc = transport->ops.ctrlr_get_reg_4(ctrlr, offset, &value);
     294           0 :                 if (rc != 0) {
     295           0 :                         return rc;
     296             :                 }
     297             : 
     298           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     299             :         }
     300             : 
     301           0 :         return transport->ops.ctrlr_get_reg_4_async(ctrlr, offset, cb_fn, cb_arg);
     302             : }
     303             : 
     304             : int
     305           0 : nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
     306             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     307             : {
     308           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     309           0 :         uint64_t value;
     310             :         int rc;
     311             : 
     312           0 :         assert(transport != NULL);
     313           0 :         if (transport->ops.ctrlr_get_reg_8_async == NULL) {
     314           0 :                 rc = transport->ops.ctrlr_get_reg_8(ctrlr, offset, &value);
     315           0 :                 if (rc != 0) {
     316           0 :                         return rc;
     317             :                 }
     318             : 
     319           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     320             :         }
     321             : 
     322           0 :         return transport->ops.ctrlr_get_reg_8_async(ctrlr, offset, cb_fn, cb_arg);
     323             : }
     324             : 
     325             : uint32_t
     326           0 : nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
     327             : {
     328           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     329             : 
     330           0 :         assert(transport != NULL);
     331           0 :         return transport->ops.ctrlr_get_max_xfer_size(ctrlr);
     332             : }
     333             : 
     334             : uint16_t
     335           0 : nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
     336             : {
     337           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     338             : 
     339           0 :         assert(transport != NULL);
     340           0 :         return transport->ops.ctrlr_get_max_sges(ctrlr);
     341             : }
     342             : 
     343             : int
     344           0 : nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
     345             : {
     346           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     347             : 
     348           0 :         assert(transport != NULL);
     349           0 :         if (transport->ops.ctrlr_reserve_cmb != NULL) {
     350           0 :                 return transport->ops.ctrlr_reserve_cmb(ctrlr);
     351             :         }
     352             : 
     353           0 :         return -ENOTSUP;
     354             : }
     355             : 
     356             : void *
     357           0 : nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
     358             : {
     359           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     360             : 
     361           0 :         assert(transport != NULL);
     362           0 :         if (transport->ops.ctrlr_map_cmb != NULL) {
     363           0 :                 return transport->ops.ctrlr_map_cmb(ctrlr, size);
     364             :         }
     365             : 
     366           0 :         return NULL;
     367             : }
     368             : 
     369             : int
     370           0 : nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
     371             : {
     372           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     373             : 
     374           0 :         assert(transport != NULL);
     375           0 :         if (transport->ops.ctrlr_unmap_cmb != NULL) {
     376           0 :                 return transport->ops.ctrlr_unmap_cmb(ctrlr);
     377             :         }
     378             : 
     379           0 :         return 0;
     380             : }
     381             : 
     382             : int
     383           0 : nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr)
     384             : {
     385           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     386             : 
     387           0 :         assert(transport != NULL);
     388           0 :         if (transport->ops.ctrlr_enable_pmr != NULL) {
     389           0 :                 return transport->ops.ctrlr_enable_pmr(ctrlr);
     390             :         }
     391             : 
     392           0 :         return -ENOSYS;
     393             : }
     394             : 
     395             : int
     396           0 : nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr)
     397             : {
     398           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     399             : 
     400           0 :         assert(transport != NULL);
     401           0 :         if (transport->ops.ctrlr_disable_pmr != NULL) {
     402           0 :                 return transport->ops.ctrlr_disable_pmr(ctrlr);
     403             :         }
     404             : 
     405           0 :         return -ENOSYS;
     406             : }
     407             : 
     408             : void *
     409           0 : nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
     410             : {
     411           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     412             : 
     413           0 :         assert(transport != NULL);
     414           0 :         if (transport->ops.ctrlr_map_pmr != NULL) {
     415           0 :                 return transport->ops.ctrlr_map_pmr(ctrlr, size);
     416             :         }
     417             : 
     418           0 :         return NULL;
     419             : }
     420             : 
     421             : int
     422           0 : nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr)
     423             : {
     424           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     425             : 
     426           0 :         assert(transport != NULL);
     427           0 :         if (transport->ops.ctrlr_unmap_pmr != NULL) {
     428           0 :                 return transport->ops.ctrlr_unmap_pmr(ctrlr);
     429             :         }
     430             : 
     431           0 :         return -ENOSYS;
     432             : }
     433             : 
     434             : struct spdk_nvme_qpair *
     435           0 : nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
     436             :                                      const struct spdk_nvme_io_qpair_opts *opts)
     437             : {
     438             :         struct spdk_nvme_qpair *qpair;
     439           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     440             : 
     441           0 :         assert(transport != NULL);
     442           0 :         qpair = transport->ops.ctrlr_create_io_qpair(ctrlr, qid, opts);
     443           0 :         if (qpair != NULL && !nvme_qpair_is_admin_queue(qpair)) {
     444           0 :                 qpair->transport = transport;
     445             :         }
     446             : 
     447           0 :         return qpair;
     448             : }
     449             : 
     450             : void
     451           0 : nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     452             : {
     453           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     454             :         int rc;
     455             : 
     456           0 :         assert(transport != NULL);
     457             : 
     458             :         /* Do not rely on qpair->transport.  For multi-process cases, a foreign process may delete
     459             :          * the IO qpair, in which case the transport object would be invalid (each process has their
     460             :          * own unique transport objects since they contain function pointers).  So we look up the
     461             :          * transport object in the delete_io_qpair case.
     462             :          */
     463           0 :         rc = transport->ops.ctrlr_delete_io_qpair(ctrlr, qpair);
     464           0 :         if (rc != 0) {
     465           0 :                 SPDK_ERRLOG("transport %s returned non-zero for ctrlr_delete_io_qpair op\n",
     466             :                             transport->ops.name);
     467           0 :                 assert(false);
     468             :         }
     469           0 : }
     470             : 
     471             : static void
     472           0 : nvme_transport_connect_qpair_fail(struct spdk_nvme_qpair *qpair, void *unused)
     473             : {
     474           0 :         struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
     475             : 
     476             :         /* If the qpair was unable to reconnect, restore the original failure reason */
     477           0 :         qpair->transport_failure_reason = qpair->last_transport_failure_reason;
     478           0 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
     479           0 : }
     480             : 
     481             : int
     482           0 : nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     483             : {
     484           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     485             :         int rc;
     486             : 
     487           0 :         assert(transport != NULL);
     488           0 :         if (!nvme_qpair_is_admin_queue(qpair) && qpair->transport == NULL) {
     489           0 :                 qpair->transport = transport;
     490             :         }
     491             : 
     492           0 :         qpair->last_transport_failure_reason = qpair->transport_failure_reason;
     493           0 :         qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_NONE;
     494             : 
     495           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTING);
     496           0 :         rc = transport->ops.ctrlr_connect_qpair(ctrlr, qpair);
     497           0 :         if (rc != 0) {
     498           0 :                 goto err;
     499             :         }
     500             : 
     501           0 :         if (qpair->poll_group) {
     502           0 :                 rc = nvme_poll_group_connect_qpair(qpair);
     503           0 :                 if (rc) {
     504           0 :                         goto err;
     505             :                 }
     506             :         }
     507             : 
     508           0 :         if (!qpair->async) {
     509             :                 /* Busy wait until the qpair exits the connecting state */
     510           0 :                 while (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
     511           0 :                         if (qpair->poll_group && spdk_nvme_ctrlr_is_fabrics(ctrlr)) {
     512           0 :                                 rc = spdk_nvme_poll_group_process_completions(
     513           0 :                                              qpair->poll_group->group, 0,
     514             :                                              nvme_transport_connect_qpair_fail);
     515             :                         } else {
     516           0 :                                 rc = spdk_nvme_qpair_process_completions(qpair, 0);
     517             :                         }
     518             : 
     519           0 :                         if (rc < 0) {
     520           0 :                                 goto err;
     521             :                         }
     522             :                 }
     523             :         }
     524             : 
     525           0 :         return 0;
     526           0 : err:
     527           0 :         nvme_transport_connect_qpair_fail(qpair, NULL);
     528           0 :         if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {
     529           0 :                 assert(qpair->async == true);
     530             :                 /* Let the caller to poll the qpair until it is actually disconnected. */
     531           0 :                 return 0;
     532             :         }
     533             : 
     534           0 :         return rc;
     535             : }
     536             : 
     537             : void
     538           0 : nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     539             : {
     540           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     541             : 
     542           0 :         if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING ||
     543           0 :             nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTED) {
     544           0 :                 return;
     545             :         }
     546             : 
     547           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTING);
     548           0 :         assert(transport != NULL);
     549             : 
     550           0 :         if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) {
     551           0 :                 nvme_poll_group_disconnect_qpair(qpair);
     552             :         }
     553             : 
     554           0 :         transport->ops.ctrlr_disconnect_qpair(ctrlr, qpair);
     555             : }
     556             : 
     557             : void
     558           0 : nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair)
     559             : {
     560           0 :         if (qpair->active_proc == nvme_ctrlr_get_current_process(qpair->ctrlr) ||
     561           0 :             nvme_qpair_is_admin_queue(qpair)) {
     562           0 :                 nvme_qpair_abort_all_queued_reqs(qpair);
     563             :         }
     564           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
     565           0 : }
     566             : 
     567             : int
     568           2 : nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
     569             :                                         struct spdk_memory_domain **domains, int array_size)
     570             : {
     571           2 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     572             : 
     573           2 :         assert(transport != NULL);
     574           2 :         if (transport->ops.ctrlr_get_memory_domains) {
     575           1 :                 return transport->ops.ctrlr_get_memory_domains(ctrlr, domains, array_size);
     576             :         }
     577             : 
     578           1 :         return 0;
     579             : }
     580             : 
     581             : void
     582           0 : nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair)
     583             : {
     584             :         const struct spdk_nvme_transport *transport;
     585             : 
     586           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     587           0 :                 qpair->transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
     588             :         } else {
     589           0 :                 transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     590           0 :                 assert(transport != NULL);
     591           0 :                 transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
     592             :         }
     593           0 : }
     594             : 
     595             : int
     596           0 : nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair)
     597             : {
     598             :         const struct spdk_nvme_transport *transport;
     599             : 
     600           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     601           0 :                 return qpair->transport->ops.qpair_reset(qpair);
     602             :         }
     603             : 
     604           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     605           0 :         assert(transport != NULL);
     606           0 :         return transport->ops.qpair_reset(qpair);
     607             : }
     608             : 
     609             : int
     610           0 : nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
     611             : {
     612             :         const struct spdk_nvme_transport *transport;
     613             : 
     614           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     615           0 :                 return qpair->transport->ops.qpair_submit_request(qpair, req);
     616             :         }
     617             : 
     618           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     619           0 :         assert(transport != NULL);
     620           0 :         return transport->ops.qpair_submit_request(qpair, req);
     621             : }
     622             : 
     623             : int32_t
     624           0 : nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
     625             : {
     626             :         const struct spdk_nvme_transport *transport;
     627             : 
     628           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     629           0 :                 return qpair->transport->ops.qpair_process_completions(qpair, max_completions);
     630             :         }
     631             : 
     632           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     633           0 :         assert(transport != NULL);
     634           0 :         return transport->ops.qpair_process_completions(qpair, max_completions);
     635             : }
     636             : 
     637             : int
     638           0 : nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
     639             :                                       int (*iter_fn)(struct nvme_request *req, void *arg),
     640             :                                       void *arg)
     641             : {
     642             :         const struct spdk_nvme_transport *transport;
     643             : 
     644           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     645           0 :                 return qpair->transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
     646             :         }
     647             : 
     648           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     649           0 :         assert(transport != NULL);
     650           0 :         return transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
     651             : }
     652             : 
     653             : void
     654           0 : nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
     655             : {
     656           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     657             : 
     658           0 :         assert(transport != NULL);
     659           0 :         transport->ops.admin_qpair_abort_aers(qpair);
     660           0 : }
     661             : 
     662             : struct spdk_nvme_transport_poll_group *
     663           0 : nvme_transport_poll_group_create(const struct spdk_nvme_transport *transport)
     664             : {
     665           0 :         struct spdk_nvme_transport_poll_group *group = NULL;
     666             : 
     667           0 :         group = transport->ops.poll_group_create();
     668           0 :         if (group) {
     669           0 :                 group->transport = transport;
     670           0 :                 STAILQ_INIT(&group->connected_qpairs);
     671           0 :                 STAILQ_INIT(&group->disconnected_qpairs);
     672           0 :                 group->num_connected_qpairs = 0;
     673             :         }
     674             : 
     675           0 :         return group;
     676             : }
     677             : 
     678             : struct spdk_nvme_transport_poll_group *
     679           0 : nvme_transport_qpair_get_optimal_poll_group(const struct spdk_nvme_transport *transport,
     680             :                 struct spdk_nvme_qpair *qpair)
     681             : {
     682           0 :         if (transport->ops.qpair_get_optimal_poll_group) {
     683           0 :                 return transport->ops.qpair_get_optimal_poll_group(qpair);
     684             :         } else {
     685           0 :                 return NULL;
     686             :         }
     687             : }
     688             : 
     689             : int
     690           1 : nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
     691             :                               struct spdk_nvme_qpair *qpair)
     692             : {
     693             :         int rc;
     694             : 
     695           1 :         rc = tgroup->transport->ops.poll_group_add(tgroup, qpair);
     696           1 :         if (rc == 0) {
     697           1 :                 qpair->poll_group = tgroup;
     698           1 :                 assert(nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTED);
     699           1 :                 qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
     700           1 :                 STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
     701             :         }
     702             : 
     703           1 :         return rc;
     704             : }
     705             : 
     706             : int
     707           3 : nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
     708             :                                  struct spdk_nvme_qpair *qpair)
     709             : {
     710             :         int rc __attribute__((unused));
     711             : 
     712           3 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     713           1 :                 return -EINVAL;
     714           2 :         } else if (qpair->poll_group_tailq_head != &tgroup->disconnected_qpairs) {
     715           1 :                 return -ENOENT;
     716             :         }
     717             : 
     718           1 :         rc = tgroup->transport->ops.poll_group_remove(tgroup, qpair);
     719           1 :         assert(rc == 0);
     720             : 
     721           1 :         STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     722             : 
     723           1 :         qpair->poll_group = NULL;
     724           1 :         qpair->poll_group_tailq_head = NULL;
     725             : 
     726           1 :         return 0;
     727             : }
     728             : 
     729             : int64_t
     730           0 : nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
     731             :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
     732             : {
     733           0 :         return tgroup->transport->ops.poll_group_process_completions(tgroup, completions_per_qpair,
     734             :                         disconnected_qpair_cb);
     735             : }
     736             : 
     737             : int
     738           0 : nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
     739             : {
     740           0 :         return tgroup->transport->ops.poll_group_destroy(tgroup);
     741             : }
     742             : 
     743             : int
     744           3 : nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
     745             : {
     746             :         struct spdk_nvme_transport_poll_group *tgroup;
     747             :         int rc __attribute__((unused));
     748             : 
     749           3 :         tgroup = qpair->poll_group;
     750             : 
     751           3 :         if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
     752           1 :                 return 0;
     753             :         }
     754             : 
     755           2 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     756           1 :                 rc = tgroup->transport->ops.poll_group_disconnect_qpair(qpair);
     757           1 :                 assert(rc == 0);
     758             : 
     759           1 :                 qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
     760           1 :                 STAILQ_REMOVE(&tgroup->connected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     761           1 :                 assert(tgroup->num_connected_qpairs > 0);
     762           1 :                 tgroup->num_connected_qpairs--;
     763           1 :                 STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
     764             : 
     765           1 :                 return 0;
     766             :         }
     767             : 
     768           1 :         return -EINVAL;
     769             : }
     770             : 
     771             : int
     772           3 : nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
     773             : {
     774             :         struct spdk_nvme_transport_poll_group *tgroup;
     775             :         int rc;
     776             : 
     777           3 :         tgroup = qpair->poll_group;
     778             : 
     779           3 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     780           1 :                 return 0;
     781             :         }
     782             : 
     783           2 :         if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
     784           1 :                 rc = tgroup->transport->ops.poll_group_connect_qpair(qpair);
     785           1 :                 if (rc == 0) {
     786           1 :                         qpair->poll_group_tailq_head = &tgroup->connected_qpairs;
     787           1 :                         STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     788           1 :                         STAILQ_INSERT_TAIL(&tgroup->connected_qpairs, qpair, poll_group_stailq);
     789           1 :                         tgroup->num_connected_qpairs++;
     790             :                 }
     791             : 
     792           1 :                 return rc == -EINPROGRESS ? 0 : rc;
     793             :         }
     794             : 
     795             : 
     796           1 :         return -EINVAL;
     797             : }
     798             : 
     799             : int
     800           0 : nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
     801             :                                     struct spdk_nvme_transport_poll_group_stat **stats)
     802             : {
     803           0 :         if (tgroup->transport->ops.poll_group_get_stats) {
     804           0 :                 return tgroup->transport->ops.poll_group_get_stats(tgroup, stats);
     805             :         }
     806           0 :         return -ENOTSUP;
     807             : }
     808             : 
     809             : void
     810           0 : nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
     811             :                                      struct spdk_nvme_transport_poll_group_stat *stats)
     812             : {
     813           0 :         if (tgroup->transport->ops.poll_group_free_stats) {
     814           0 :                 tgroup->transport->ops.poll_group_free_stats(tgroup, stats);
     815             :         }
     816           0 : }
     817             : 
     818             : spdk_nvme_transport_type_t
     819           0 : nvme_transport_get_trtype(const struct spdk_nvme_transport *transport)
     820             : {
     821           0 :         return transport->ops.type;
     822             : }
     823             : 
     824             : void
     825           0 : spdk_nvme_transport_get_opts(struct spdk_nvme_transport_opts *opts, size_t opts_size)
     826             : {
     827           0 :         if (opts == NULL) {
     828           0 :                 SPDK_ERRLOG("opts should not be NULL.\n");
     829           0 :                 return;
     830             :         }
     831             : 
     832           0 :         if (opts_size == 0) {
     833           0 :                 SPDK_ERRLOG("opts_size should not be zero.\n");
     834           0 :                 return;
     835             :         }
     836             : 
     837           0 :         opts->opts_size = opts_size;
     838             : 
     839             : #define SET_FIELD(field) \
     840             :         if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts_size) { \
     841             :                 opts->field = g_spdk_nvme_transport_opts.field; \
     842             :         } \
     843             : 
     844           0 :         SET_FIELD(rdma_srq_size);
     845           0 :         SET_FIELD(rdma_max_cq_size);
     846           0 :         SET_FIELD(rdma_cm_event_timeout_ms);
     847             : 
     848             :         /* Do not remove this statement, you should always update this statement when you adding a new field,
     849             :          * and do not forget to add the SET_FIELD statement for your added field. */
     850             :         SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 24, "Incorrect size");
     851             : 
     852             : #undef SET_FIELD
     853             : }
     854             : 
     855             : int
     856           0 : spdk_nvme_transport_set_opts(const struct spdk_nvme_transport_opts *opts, size_t opts_size)
     857             : {
     858           0 :         if (opts == NULL) {
     859           0 :                 SPDK_ERRLOG("opts should not be NULL.\n");
     860           0 :                 return -EINVAL;
     861             :         }
     862             : 
     863           0 :         if (opts_size == 0) {
     864           0 :                 SPDK_ERRLOG("opts_size should not be zero.\n");
     865           0 :                 return -EINVAL;
     866             :         }
     867             : 
     868             : #define SET_FIELD(field) \
     869             :         if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
     870             :                 g_spdk_nvme_transport_opts.field = opts->field; \
     871             :         } \
     872             : 
     873           0 :         SET_FIELD(rdma_srq_size);
     874           0 :         SET_FIELD(rdma_max_cq_size);
     875           0 :         SET_FIELD(rdma_cm_event_timeout_ms);
     876             : 
     877           0 :         g_spdk_nvme_transport_opts.opts_size = opts->opts_size;
     878             : 
     879             : #undef SET_FIELD
     880             : 
     881           0 :         return 0;
     882             : }
     883             : 
     884             : volatile struct spdk_nvme_registers *
     885           0 : spdk_nvme_ctrlr_get_registers(struct spdk_nvme_ctrlr *ctrlr)
     886             : {
     887           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     888             : 
     889           0 :         if (transport == NULL) {
     890             :                 /* Transport does not exist. */
     891           0 :                 return NULL;
     892             :         }
     893             : 
     894           0 :         if (transport->ops.ctrlr_get_registers) {
     895           0 :                 return transport->ops.ctrlr_get_registers(ctrlr);
     896             :         }
     897             : 
     898           0 :         return NULL;
     899             : }

Generated by: LCOV version 1.15