LCOV - code coverage report
Current view: top level - spdk/lib/nvme - nvme_internal.h (source / functions) Hit Total Coverage
Test: Combined Lines: 132 141 93.6 %
Date: 2024-11-15 10:35:21 Functions: 22 22 100.0 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 157 600 26.2 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2015 Intel Corporation. All rights reserved.
       3                 :            :  *   Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
       4                 :            :  *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #ifndef __NVME_INTERNAL_H__
       8                 :            : #define __NVME_INTERNAL_H__
       9                 :            : 
      10                 :            : #include "spdk/config.h"
      11                 :            : #include "spdk/likely.h"
      12                 :            : #include "spdk/stdinc.h"
      13                 :            : 
      14                 :            : #include "spdk/nvme.h"
      15                 :            : 
      16                 :            : #if defined(__i386__) || defined(__x86_64__)
      17                 :            : #include <x86intrin.h>
      18                 :            : #endif
      19                 :            : 
      20                 :            : #include "spdk/queue.h"
      21                 :            : #include "spdk/barrier.h"
      22                 :            : #include "spdk/bit_array.h"
      23                 :            : #include "spdk/mmio.h"
      24                 :            : #include "spdk/pci_ids.h"
      25                 :            : #include "spdk/util.h"
      26                 :            : #include "spdk/memory.h"
      27                 :            : #include "spdk/nvme_intel.h"
      28                 :            : #include "spdk/nvmf_spec.h"
      29                 :            : #include "spdk/tree.h"
      30                 :            : #include "spdk/uuid.h"
      31                 :            : #include "spdk/fd_group.h"
      32                 :            : 
      33                 :            : #include "spdk_internal/assert.h"
      34                 :            : #include "spdk/log.h"
      35                 :            : 
      36                 :            : extern pid_t g_spdk_nvme_pid;
      37                 :            : 
      38                 :            : extern struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts;
      39                 :            : 
      40                 :            : /*
      41                 :            :  * Some Intel devices support vendor-unique read latency log page even
      42                 :            :  * though the log page directory says otherwise.
      43                 :            :  */
      44                 :            : #define NVME_INTEL_QUIRK_READ_LATENCY 0x1
      45                 :            : 
      46                 :            : /*
      47                 :            :  * Some Intel devices support vendor-unique write latency log page even
      48                 :            :  * though the log page directory says otherwise.
      49                 :            :  */
      50                 :            : #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
      51                 :            : 
      52                 :            : /*
      53                 :            :  * The controller needs a delay before starts checking the device
      54                 :            :  * readiness, which is done by reading the NVME_CSTS_RDY bit.
      55                 :            :  */
      56                 :            : #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
      57                 :            : 
      58                 :            : /*
      59                 :            :  * The controller performs best when I/O is split on particular
      60                 :            :  * LBA boundaries.
      61                 :            :  */
      62                 :            : #define NVME_INTEL_QUIRK_STRIPING 0x8
      63                 :            : 
      64                 :            : /*
      65                 :            :  * The controller needs a delay after allocating an I/O queue pair
      66                 :            :  * before it is ready to accept I/O commands.
      67                 :            :  */
      68                 :            : #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
      69                 :            : 
      70                 :            : /*
      71                 :            :  * Earlier NVMe devices do not indicate whether unmapped blocks
      72                 :            :  * will read all zeroes or not. This define indicates that the
      73                 :            :  * device does in fact read all zeroes after an unmap event
      74                 :            :  */
      75                 :            : #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
      76                 :            : 
      77                 :            : /*
      78                 :            :  * The controller doesn't handle Identify value others than 0 or 1 correctly.
      79                 :            :  */
      80                 :            : #define NVME_QUIRK_IDENTIFY_CNS 0x40
      81                 :            : 
      82                 :            : /*
      83                 :            :  * The controller supports Open Channel command set if matching additional
      84                 :            :  * condition, like the first byte (value 0x1) in the vendor specific
      85                 :            :  * bits of the namespace identify structure is set.
      86                 :            :  */
      87                 :            : #define NVME_QUIRK_OCSSD 0x80
      88                 :            : 
      89                 :            : /*
      90                 :            :  * The controller has an Intel vendor ID but does not support Intel vendor-specific
      91                 :            :  * log pages.  This is primarily for QEMU emulated SSDs which report an Intel vendor
      92                 :            :  * ID but do not support these log pages.
      93                 :            :  */
      94                 :            : #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
      95                 :            : 
      96                 :            : /*
      97                 :            :  * The controller does not set SHST_COMPLETE in a reasonable amount of time.  This
      98                 :            :  * is primarily seen in virtual VMWare NVMe SSDs.  This quirk merely adds an additional
      99                 :            :  * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
     100                 :            :  */
     101                 :            : #define NVME_QUIRK_SHST_COMPLETE 0x200
     102                 :            : 
     103                 :            : /*
     104                 :            :  * The controller requires an extra delay before starting the initialization process
     105                 :            :  * during attach.
     106                 :            :  */
     107                 :            : #define NVME_QUIRK_DELAY_BEFORE_INIT 0x400
     108                 :            : 
     109                 :            : /*
     110                 :            :  * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size.
     111                 :            :  * This quirk will increase the default to 1024 which matches other operating
     112                 :            :  * systems, at the cost of some extra memory usage.  Users can still override
     113                 :            :  * the increased default by changing the spdk_nvme_io_qpair_opts when allocating
     114                 :            :  * a new queue pair.
     115                 :            :  */
     116                 :            : #define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800
     117                 :            : 
     118                 :            : /**
     119                 :            :  * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or
     120                 :            :  * SSE instructions to optimize the memory access(memcpy or memset) larger than
     121                 :            :  * 8 Bytes.
     122                 :            :  */
     123                 :            : #define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000
     124                 :            : 
     125                 :            : /**
     126                 :            :  * The SSD does not support OPAL even through it sets the security bit in OACS.
     127                 :            :  */
     128                 :            : #define NVME_QUIRK_OACS_SECURITY 0x2000
     129                 :            : 
     130                 :            : /**
     131                 :            :  * Intel P55XX SSDs can't support Dataset Management command with SGL format,
     132                 :            :  * so use PRP with DSM command.
     133                 :            :  */
     134                 :            : #define NVME_QUIRK_NO_SGL_FOR_DSM 0x4000
     135                 :            : 
     136                 :            : /**
     137                 :            :  * Maximum Data Transfer Size(MDTS) excludes interleaved metadata.
     138                 :            :  */
     139                 :            : #define NVME_QUIRK_MDTS_EXCLUDE_MD 0x8000
     140                 :            : 
     141                 :            : /**
     142                 :            :  * Force not to use SGL even the controller report that it can
     143                 :            :  * support it.
     144                 :            :  */
     145                 :            : #define NVME_QUIRK_NOT_USE_SGL 0x10000
     146                 :            : 
     147                 :            : /*
     148                 :            :  * Some SSDs require the admin submission queue size to equate to an even
     149                 :            :  * 4KiB multiple.
     150                 :            :  */
     151                 :            : #define NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE 0x20000
     152                 :            : 
     153                 :            : #define NVME_MAX_ASYNC_EVENTS   (8)
     154                 :            : 
     155                 :            : #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS  (30)
     156                 :            : 
     157                 :            : /* Maximum log page size to fetch for AERs. */
     158                 :            : #define NVME_MAX_AER_LOG_SIZE           (4096)
     159                 :            : 
     160                 :            : /*
     161                 :            :  * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
     162                 :            :  *  define specifies the maximum number of queues this driver will actually
     163                 :            :  *  try to configure, if available.
     164                 :            :  */
     165                 :            : #define DEFAULT_MAX_IO_QUEUES           (1024)
     166                 :            : #define MAX_IO_QUEUES_WITH_INTERRUPTS   (256)
     167                 :            : #define DEFAULT_ADMIN_QUEUE_SIZE        (32)
     168                 :            : #define DEFAULT_IO_QUEUE_SIZE           (256)
     169                 :            : #define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */
     170                 :            : 
     171                 :            : #define DEFAULT_IO_QUEUE_REQUESTS       (512)
     172                 :            : 
     173                 :            : #define SPDK_NVME_DEFAULT_RETRY_COUNT   (4)
     174                 :            : 
     175                 :            : #define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED        (0)
     176                 :            : #define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED
     177                 :            : 
     178                 :            : #define SPDK_NVME_TRANSPORT_TOS_DISABLED        (0)
     179                 :            : 
     180                 :            : #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS    (10000)
     181                 :            : 
     182                 :            : /* We want to fit submission and completion rings each in a single 2MB
     183                 :            :  * hugepage to ensure physical address contiguity.
     184                 :            :  */
     185                 :            : #define MAX_IO_QUEUE_ENTRIES            (VALUE_2MB / spdk_max( \
     186                 :            :                                                 sizeof(struct spdk_nvme_cmd), \
     187                 :            :                                                 sizeof(struct spdk_nvme_cpl)))
     188                 :            : 
     189                 :            : /* Default timeout for fabrics connect commands. */
     190                 :            : #ifdef DEBUG
     191                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 0
     192                 :            : #else
     193                 :            : /* 500 millisecond timeout. */
     194                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 500000
     195                 :            : #endif
     196                 :            : 
     197                 :            : /* This value indicates that a read from a PCIe register is invalid. This can happen when a device is no longer present */
     198                 :            : #define SPDK_NVME_INVALID_REGISTER_VALUE 0xFFFFFFFFu
     199                 :            : 
     200                 :            : enum nvme_payload_type {
     201                 :            :         NVME_PAYLOAD_TYPE_INVALID = 0,
     202                 :            : 
     203                 :            :         /** nvme_request::u.payload.contig_buffer is valid for this request */
     204                 :            :         NVME_PAYLOAD_TYPE_CONTIG,
     205                 :            : 
     206                 :            :         /** nvme_request::u.sgl is valid for this request */
     207                 :            :         NVME_PAYLOAD_TYPE_SGL,
     208                 :            : };
     209                 :            : 
     210                 :            : /** Boot partition write states */
     211                 :            : enum nvme_bp_write_state {
     212                 :            :         SPDK_NVME_BP_WS_DOWNLOADING     = 0x0,
     213                 :            :         SPDK_NVME_BP_WS_DOWNLOADED      = 0x1,
     214                 :            :         SPDK_NVME_BP_WS_REPLACE         = 0x2,
     215                 :            :         SPDK_NVME_BP_WS_ACTIVATE        = 0x3,
     216                 :            : };
     217                 :            : 
     218                 :            : /**
     219                 :            :  * Descriptor for a request data payload.
     220                 :            :  */
     221                 :            : struct nvme_payload {
     222                 :            :         /**
     223                 :            :          * Functions for retrieving physical addresses for scattered payloads.
     224                 :            :          */
     225                 :            :         spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
     226                 :            :         spdk_nvme_req_next_sge_cb next_sge_fn;
     227                 :            : 
     228                 :            :         /**
     229                 :            :          * Extended IO options passed by the user
     230                 :            :          */
     231                 :            :         struct spdk_nvme_ns_cmd_ext_io_opts *opts;
     232                 :            :         /**
     233                 :            :          * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
     234                 :            :          * virtual memory address of a single virtually contiguous buffer.
     235                 :            :          *
     236                 :            :          * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
     237                 :            :          * cb_arg that will be passed to the SGL callback functions.
     238                 :            :          */
     239                 :            :         void *contig_or_cb_arg;
     240                 :            : 
     241                 :            :         /** Virtual memory address of a single virtually contiguous metadata buffer */
     242                 :            :         void *md;
     243                 :            : };
     244                 :            : 
     245                 :            : #define NVME_PAYLOAD_CONTIG(contig_, md_) \
     246                 :            :         (struct nvme_payload) { \
     247                 :            :                 .reset_sgl_fn = NULL, \
     248                 :            :                 .next_sge_fn = NULL, \
     249                 :            :                 .contig_or_cb_arg = (contig_), \
     250                 :            :                 .md = (md_), \
     251                 :            :         }
     252                 :            : 
     253                 :            : #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
     254                 :            :         (struct nvme_payload) { \
     255                 :            :                 .reset_sgl_fn = (reset_sgl_fn_), \
     256                 :            :                 .next_sge_fn = (next_sge_fn_), \
     257                 :            :                 .contig_or_cb_arg = (cb_arg_), \
     258                 :            :                 .md = (md_), \
     259                 :            :         }
     260                 :            : 
     261                 :            : static inline enum nvme_payload_type
     262                 :  202348110 : nvme_payload_type(const struct nvme_payload *payload) {
     263   [ +  +  +  - ]:  202348110 :         return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG;
     264                 :            : }
     265                 :            : 
     266                 :            : struct nvme_error_cmd {
     267                 :            :         bool                            do_not_submit;
     268                 :            :         uint64_t                        timeout_tsc;
     269                 :            :         uint32_t                        err_count;
     270                 :            :         uint8_t                         opc;
     271                 :            :         struct spdk_nvme_status         status;
     272                 :            :         TAILQ_ENTRY(nvme_error_cmd)     link;
     273                 :            : };
     274                 :            : 
     275                 :            : struct nvme_request {
     276                 :            :         struct spdk_nvme_cmd            cmd;
     277                 :            : 
     278                 :            :         uint8_t                         retries;
     279                 :            : 
     280                 :            :         uint8_t                         timed_out : 1;
     281                 :            : 
     282                 :            :         /**
     283                 :            :          * True if the request is in the queued_req list.
     284                 :            :          */
     285                 :            :         uint8_t                         queued : 1;
     286                 :            :         uint8_t                         reserved : 6;
     287                 :            : 
     288                 :            :         /**
     289                 :            :          * Number of children requests still outstanding for this
     290                 :            :          *  request which was split into multiple child requests.
     291                 :            :          */
     292                 :            :         uint16_t                        num_children;
     293                 :            : 
     294                 :            :         /**
     295                 :            :          * Offset in bytes from the beginning of payload for this request.
     296                 :            :          * This is used for I/O commands that are split into multiple requests.
     297                 :            :          */
     298                 :            :         uint32_t                        payload_offset;
     299                 :            :         uint32_t                        md_offset;
     300                 :            : 
     301                 :            :         uint32_t                        payload_size;
     302                 :            : 
     303                 :            :         /**
     304                 :            :          * Timeout ticks for error injection requests, can be extended in future
     305                 :            :          * to support per-request timeout feature.
     306                 :            :          */
     307                 :            :         uint64_t                        timeout_tsc;
     308                 :            : 
     309                 :            :         /**
     310                 :            :          * Data payload for this request's command.
     311                 :            :          */
     312                 :            :         struct nvme_payload             payload;
     313                 :            : 
     314                 :            :         spdk_nvme_cmd_cb                cb_fn;
     315                 :            :         void                            *cb_arg;
     316                 :            :         STAILQ_ENTRY(nvme_request)      stailq;
     317                 :            : 
     318                 :            :         struct spdk_nvme_qpair          *qpair;
     319                 :            : 
     320                 :            :         /*
     321                 :            :          * The value of spdk_get_ticks() when the request was submitted to the hardware.
     322                 :            :          * Only set if ctrlr->timeout_enabled is true.
     323                 :            :          */
     324                 :            :         uint64_t                        submit_tick;
     325                 :            : 
     326                 :            :         /**
     327                 :            :          * The active admin request can be moved to a per process pending
     328                 :            :          *  list based on the saved pid to tell which process it belongs
     329                 :            :          *  to. The cpl saves the original completion information which
     330                 :            :          *  is used in the completion callback.
     331                 :            :          * NOTE: these below two fields are only used for admin request.
     332                 :            :          */
     333                 :            :         pid_t                           pid;
     334                 :            :         struct spdk_nvme_cpl            cpl;
     335                 :            : 
     336                 :            :         uint32_t                        md_size;
     337                 :            : 
     338                 :            :         /**
     339                 :            :          * The following members should not be reordered with members
     340                 :            :          *  above.  These members are only needed when splitting
     341                 :            :          *  requests which is done rarely, and the driver is careful
     342                 :            :          *  to not touch the following fields until a split operation is
     343                 :            :          *  needed, to avoid touching an extra cacheline.
     344                 :            :          */
     345                 :            : 
     346                 :            :         /**
     347                 :            :          * Points to the outstanding child requests for a parent request.
     348                 :            :          *  Only valid if a request was split into multiple children
     349                 :            :          *  requests, and is not initialized for non-split requests.
     350                 :            :          */
     351                 :            :         TAILQ_HEAD(, nvme_request)      children;
     352                 :            : 
     353                 :            :         /**
     354                 :            :          * Linked-list pointers for a child request in its parent's list.
     355                 :            :          */
     356                 :            :         TAILQ_ENTRY(nvme_request)       child_tailq;
     357                 :            : 
     358                 :            :         /**
     359                 :            :          * Points to a parent request if part of a split request,
     360                 :            :          *   NULL otherwise.
     361                 :            :          */
     362                 :            :         struct nvme_request             *parent;
     363                 :            : 
     364                 :            :         /**
     365                 :            :          * Completion status for a parent request.  Initialized to all 0's
     366                 :            :          *  (SUCCESS) before child requests are submitted.  If a child
     367                 :            :          *  request completes with error, the error status is copied here,
     368                 :            :          *  to ensure that the parent request is also completed with error
     369                 :            :          *  status once all child requests are completed.
     370                 :            :          */
     371                 :            :         struct spdk_nvme_cpl            parent_status;
     372                 :            : 
     373                 :            :         /**
     374                 :            :          * The user_cb_fn and user_cb_arg fields are used for holding the original
     375                 :            :          * callback data when using nvme_allocate_request_user_copy.
     376                 :            :          */
     377                 :            :         spdk_nvme_cmd_cb                user_cb_fn;
     378                 :            :         void                            *user_cb_arg;
     379                 :            :         void                            *user_buffer;
     380                 :            : 
     381                 :            :         /** Sequence of accel operations associated with this request */
     382                 :            :         void                            *accel_sequence;
     383                 :            : };
     384                 :            : 
     385                 :            : struct nvme_completion_poll_status {
     386                 :            :         struct spdk_nvme_cpl    cpl;
     387                 :            :         uint64_t                timeout_tsc;
     388                 :            :         /**
     389                 :            :          * DMA buffer retained throughout the duration of the command.  It'll be released
     390                 :            :          * automatically if the command times out, otherwise the user is responsible for freeing it.
     391                 :            :          */
     392                 :            :         void                    *dma_data;
     393                 :            :         bool                    done;
     394                 :            :         /* This flag indicates that the request has been timed out and the memory
     395                 :            :            must be freed in a completion callback */
     396                 :            :         bool                    timed_out;
     397                 :            : };
     398                 :            : 
     399                 :            : struct nvme_async_event_request {
     400                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     401                 :            :         struct nvme_request             *req;
     402                 :            :         struct spdk_nvme_cpl            cpl;
     403                 :            : };
     404                 :            : 
     405                 :            : enum nvme_qpair_state {
     406                 :            :         NVME_QPAIR_DISCONNECTED,
     407                 :            :         NVME_QPAIR_DISCONNECTING,
     408                 :            :         NVME_QPAIR_CONNECTING,
     409                 :            :         NVME_QPAIR_CONNECTED,
     410                 :            :         NVME_QPAIR_ENABLING,
     411                 :            :         NVME_QPAIR_ENABLED,
     412                 :            :         NVME_QPAIR_DESTROYING,
     413                 :            : };
     414                 :            : 
     415                 :            : enum nvme_qpair_auth_state {
     416                 :            :         NVME_QPAIR_AUTH_STATE_NEGOTIATE,
     417                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_NEGOTIATE,
     418                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_CHALLENGE,
     419                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_REPLY,
     420                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS1,
     421                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS2,
     422                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_FAILURE2,
     423                 :            :         NVME_QPAIR_AUTH_STATE_DONE,
     424                 :            : };
     425                 :            : 
     426                 :            : /* Authentication transaction required (authreq.atr) */
     427                 :            : #define NVME_QPAIR_AUTH_FLAG_ATR        (1 << 0)
     428                 :            : /* Authentication and secure channel required (authreq.ascr) */
     429                 :            : #define NVME_QPAIR_AUTH_FLAG_ASCR       (1 << 1)
     430                 :            : 
     431                 :            : /* Maximum size of a digest */
     432                 :            : #define NVME_AUTH_DIGEST_MAX_SIZE       64
     433                 :            : 
     434                 :            : struct nvme_auth {
     435                 :            :         /* State of the authentication */
     436                 :            :         enum nvme_qpair_auth_state      state;
     437                 :            :         /* Status of the authentication */
     438                 :            :         int                             status;
     439                 :            :         /* Transaction ID */
     440                 :            :         uint16_t                        tid;
     441                 :            :         /* Flags */
     442                 :            :         uint32_t                        flags;
     443                 :            :         /* Selected hash function */
     444                 :            :         uint8_t                         hash;
     445                 :            :         /* Buffer used for controller challenge */
     446                 :            :         uint8_t                         challenge[NVME_AUTH_DIGEST_MAX_SIZE];
     447                 :            :         /* User's auth cb fn/ctx */
     448                 :            :         spdk_nvme_authenticate_cb       cb_fn;
     449                 :            :         void                            *cb_ctx;
     450                 :            : };
     451                 :            : 
     452                 :            : struct spdk_nvme_qpair {
     453                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
     454                 :            : 
     455                 :            :         uint16_t                                id;
     456                 :            : 
     457                 :            :         uint8_t                                 qprio: 2;
     458                 :            : 
     459                 :            :         uint8_t                                 state: 3;
     460                 :            : 
     461                 :            :         uint8_t                                 async: 1;
     462                 :            : 
     463                 :            :         uint8_t                                 is_new_qpair: 1;
     464                 :            : 
     465                 :            :         uint8_t                                 abort_dnr: 1;
     466                 :            :         /*
     467                 :            :          * Members for handling IO qpair deletion inside of a completion context.
     468                 :            :          * These are specifically defined as single bits, so that they do not
     469                 :            :          *  push this data structure out to another cacheline.
     470                 :            :          */
     471                 :            :         uint8_t                                 in_completion_context: 1;
     472                 :            :         uint8_t                                 delete_after_completion_context: 1;
     473                 :            : 
     474                 :            :         /*
     475                 :            :          * Set when no deletion notification is needed. For example, the process
     476                 :            :          * which allocated this qpair exited unexpectedly.
     477                 :            :          */
     478                 :            :         uint8_t                                 no_deletion_notification_needed: 1;
     479                 :            : 
     480                 :            :         uint8_t                                 last_fuse: 2;
     481                 :            : 
     482                 :            :         uint8_t                                 transport_failure_reason: 3;
     483                 :            :         uint8_t                                 last_transport_failure_reason: 3;
     484                 :            : 
     485                 :            :         /* The user is destroying qpair */
     486                 :            :         uint8_t                                 destroy_in_progress: 1;
     487                 :            : 
     488                 :            :         /* Number of IO outstanding at transport level */
     489                 :            :         uint16_t                                queue_depth;
     490                 :            : 
     491                 :            :         enum spdk_nvme_transport_type           trtype;
     492                 :            : 
     493                 :            :         uint32_t                                num_outstanding_reqs;
     494                 :            : 
     495                 :            :         /* request object used only for this qpair's FABRICS/CONNECT command (if needed) */
     496                 :            :         struct nvme_request                     *reserved_req;
     497                 :            : 
     498                 :            :         STAILQ_HEAD(, nvme_request)             free_req;
     499                 :            :         STAILQ_HEAD(, nvme_request)             queued_req;
     500                 :            : 
     501                 :            :         /* List entry for spdk_nvme_transport_poll_group::qpairs */
     502                 :            :         STAILQ_ENTRY(spdk_nvme_qpair)           poll_group_stailq;
     503                 :            : 
     504                 :            :         /** Commands opcode in this list will return error */
     505                 :            :         TAILQ_HEAD(, nvme_error_cmd)            err_cmd_head;
     506                 :            :         /** Requests in this list will return error */
     507                 :            :         STAILQ_HEAD(, nvme_request)             err_req_head;
     508                 :            : 
     509                 :            :         struct spdk_nvme_ctrlr_process          *active_proc;
     510                 :            : 
     511                 :            :         struct spdk_nvme_transport_poll_group   *poll_group;
     512                 :            : 
     513                 :            :         void                                    *poll_group_tailq_head;
     514                 :            : 
     515                 :            :         const struct spdk_nvme_transport        *transport;
     516                 :            : 
     517                 :            :         /* Entries below here are not touched in the main I/O path. */
     518                 :            : 
     519                 :            :         struct nvme_completion_poll_status      *poll_status;
     520                 :            : 
     521                 :            :         /* List entry for spdk_nvme_ctrlr::active_io_qpairs */
     522                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            tailq;
     523                 :            : 
     524                 :            :         /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
     525                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            per_process_tailq;
     526                 :            : 
     527                 :            :         STAILQ_HEAD(, nvme_request)             aborting_queued_req;
     528                 :            : 
     529                 :            :         void                                    *req_buf;
     530                 :            : 
     531                 :            :         /* In-band authentication state */
     532                 :            :         struct nvme_auth                        auth;
     533                 :            : };
     534                 :            : 
     535                 :            : struct spdk_nvme_poll_group {
     536                 :            :         void                                            *ctx;
     537                 :            :         struct spdk_nvme_accel_fn_table                 accel_fn_table;
     538                 :            :         STAILQ_HEAD(, spdk_nvme_transport_poll_group)   tgroups;
     539                 :            :         bool                                            in_process_completions;
     540                 :            :         bool                                            enable_interrupts;
     541                 :            :         bool                                            enable_interrupts_is_valid;
     542                 :            :         int                                             disconnect_qpair_fd;
     543                 :            :         struct spdk_fd_group                            *fgrp;
     544                 :            : };
     545                 :            : 
     546                 :            : struct spdk_nvme_transport_poll_group {
     547                 :            :         struct spdk_nvme_poll_group                     *group;
     548                 :            :         const struct spdk_nvme_transport                *transport;
     549                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  connected_qpairs;
     550                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  disconnected_qpairs;
     551                 :            :         STAILQ_ENTRY(spdk_nvme_transport_poll_group)    link;
     552                 :            :         uint32_t                                        num_connected_qpairs;
     553                 :            : };
     554                 :            : 
     555                 :            : struct spdk_nvme_ns {
     556                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     557                 :            :         uint32_t                        sector_size;
     558                 :            : 
     559                 :            :         /*
     560                 :            :          * Size of data transferred as part of each block,
     561                 :            :          * including metadata if FLBAS indicates the metadata is transferred
     562                 :            :          * as part of the data buffer at the end of each LBA.
     563                 :            :          */
     564                 :            :         uint32_t                        extended_lba_size;
     565                 :            : 
     566                 :            :         uint32_t                        md_size;
     567                 :            :         uint32_t                        pi_type;
     568                 :            :         uint32_t                        pi_format;
     569                 :            :         uint32_t                        sectors_per_max_io;
     570                 :            :         uint32_t                        sectors_per_max_io_no_md;
     571                 :            :         uint32_t                        sectors_per_stripe;
     572                 :            :         uint32_t                        id;
     573                 :            :         uint16_t                        flags;
     574                 :            :         bool                            active;
     575                 :            : 
     576                 :            :         /* Command Set Identifier */
     577                 :            :         enum spdk_nvme_csi              csi;
     578                 :            : 
     579                 :            :         /* Namespace Identification Descriptor List (CNS = 03h) */
     580                 :            :         uint8_t                         id_desc_list[4096];
     581                 :            : 
     582                 :            :         uint32_t                        ana_group_id;
     583                 :            :         enum spdk_nvme_ana_state        ana_state;
     584                 :            : 
     585                 :            :         /* Identify Namespace data. */
     586                 :            :         struct spdk_nvme_ns_data        nsdata;
     587                 :            : 
     588                 :            :         /* Zoned Namespace Command Set Specific Identify Namespace data. */
     589                 :            :         struct spdk_nvme_zns_ns_data    *nsdata_zns;
     590                 :            : 
     591                 :            :         struct spdk_nvme_nvm_ns_data    *nsdata_nvm;
     592                 :            : 
     593                 :            :         RB_ENTRY(spdk_nvme_ns)          node;
     594                 :            : };
     595                 :            : 
     596                 :            : #define CTRLR_STRING(ctrlr) \
     597                 :            :         (spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype) ? \
     598                 :            :         ctrlr->trid.subnqn : ctrlr->trid.traddr)
     599                 :            : 
     600                 :            : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
     601                 :            :         SPDK_ERRLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
     602                 :            : 
     603                 :            : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
     604                 :            :         SPDK_WARNLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
     605                 :            : 
     606                 :            : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
     607                 :            :         SPDK_NOTICELOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
     608                 :            : 
     609                 :            : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
     610                 :            :         SPDK_INFOLOG(nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
     611                 :            : 
     612                 :            : #ifdef DEBUG
     613                 :            : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
     614                 :            :         SPDK_DEBUGLOG(nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
     615                 :            : #else
     616                 :            : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
     617                 :            : #endif
     618                 :            : 
     619                 :            : /**
     620                 :            :  * State of struct spdk_nvme_ctrlr (in particular, during initialization).
     621                 :            :  */
     622                 :            : enum nvme_ctrlr_state {
     623                 :            :         /**
     624                 :            :          * Wait before initializing the controller.
     625                 :            :          */
     626                 :            :         NVME_CTRLR_STATE_INIT_DELAY,
     627                 :            : 
     628                 :            :         /**
     629                 :            :          * Connect the admin queue.
     630                 :            :          */
     631                 :            :         NVME_CTRLR_STATE_CONNECT_ADMINQ,
     632                 :            : 
     633                 :            :         /**
     634                 :            :          * Controller has not started initialized yet.
     635                 :            :          */
     636                 :            :         NVME_CTRLR_STATE_INIT = NVME_CTRLR_STATE_CONNECT_ADMINQ,
     637                 :            : 
     638                 :            :         /**
     639                 :            :          * Waiting for admin queue to connect.
     640                 :            :          */
     641                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,
     642                 :            : 
     643                 :            :         /**
     644                 :            :          * Read Version (VS) register.
     645                 :            :          */
     646                 :            :         NVME_CTRLR_STATE_READ_VS,
     647                 :            : 
     648                 :            :         /**
     649                 :            :          * Waiting for Version (VS) register to be read.
     650                 :            :          */
     651                 :            :         NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS,
     652                 :            : 
     653                 :            :         /**
     654                 :            :          * Read Capabilities (CAP) register.
     655                 :            :          */
     656                 :            :         NVME_CTRLR_STATE_READ_CAP,
     657                 :            : 
     658                 :            :         /**
     659                 :            :          * Waiting for Capabilities (CAP) register to be read.
     660                 :            :          */
     661                 :            :         NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP,
     662                 :            : 
     663                 :            :         /**
     664                 :            :          * Check EN to prepare for controller initialization.
     665                 :            :          */
     666                 :            :         NVME_CTRLR_STATE_CHECK_EN,
     667                 :            : 
     668                 :            :         /**
     669                 :            :          * Waiting for CC to be read as part of EN check.
     670                 :            :          */
     671                 :            :         NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC,
     672                 :            : 
     673                 :            :         /**
     674                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
     675                 :            :          */
     676                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
     677                 :            : 
     678                 :            :         /**
     679                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     680                 :            :          */
     681                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     682                 :            : 
     683                 :            :         /**
     684                 :            :          * Disabling the controller by setting CC.EN to 0.
     685                 :            :          */
     686                 :            :         NVME_CTRLR_STATE_SET_EN_0,
     687                 :            : 
     688                 :            :         /**
     689                 :            :          * Waiting for the CC register to be read as part of disabling the controller.
     690                 :            :          */
     691                 :            :         NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC,
     692                 :            : 
     693                 :            :         /**
     694                 :            :          * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
     695                 :            :          */
     696                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
     697                 :            : 
     698                 :            :         /**
     699                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 0.
     700                 :            :          */
     701                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS,
     702                 :            : 
     703                 :            :         /**
     704                 :            :          * The controller is disabled. (CC.EN and CSTS.RDY are 0.)
     705                 :            :          */
     706                 :            :         NVME_CTRLR_STATE_DISABLED,
     707                 :            : 
     708                 :            :         /**
     709                 :            :          * Enable the controller by writing CC.EN to 1
     710                 :            :          */
     711                 :            :         NVME_CTRLR_STATE_ENABLE,
     712                 :            : 
     713                 :            :         /**
     714                 :            :          * Waiting for CC register to be written as part of enabling the controller.
     715                 :            :          */
     716                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC,
     717                 :            : 
     718                 :            :         /**
     719                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
     720                 :            :          */
     721                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
     722                 :            : 
     723                 :            :         /**
     724                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     725                 :            :          */
     726                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     727                 :            : 
     728                 :            :         /**
     729                 :            :          * Reset the Admin queue of the controller.
     730                 :            :          */
     731                 :            :         NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
     732                 :            : 
     733                 :            :         /**
     734                 :            :          * Identify Controller command will be sent to then controller.
     735                 :            :          */
     736                 :            :         NVME_CTRLR_STATE_IDENTIFY,
     737                 :            : 
     738                 :            :         /**
     739                 :            :          * Waiting for Identify Controller command be completed.
     740                 :            :          */
     741                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
     742                 :            : 
     743                 :            :         /**
     744                 :            :          * Configure AER of the controller.
     745                 :            :          */
     746                 :            :         NVME_CTRLR_STATE_CONFIGURE_AER,
     747                 :            : 
     748                 :            :         /**
     749                 :            :          * Waiting for the Configure AER to be completed.
     750                 :            :          */
     751                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
     752                 :            : 
     753                 :            :         /**
     754                 :            :          * Set Keep Alive Timeout of the controller.
     755                 :            :          */
     756                 :            :         NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
     757                 :            : 
     758                 :            :         /**
     759                 :            :          * Waiting for Set Keep Alive Timeout to be completed.
     760                 :            :          */
     761                 :            :         NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
     762                 :            : 
     763                 :            :         /**
     764                 :            :          * Get Identify I/O Command Set Specific Controller data structure.
     765                 :            :          */
     766                 :            :         NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
     767                 :            : 
     768                 :            :         /**
     769                 :            :          * Waiting for Identify I/O Command Set Specific Controller command to be completed.
     770                 :            :          */
     771                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC,
     772                 :            : 
     773                 :            :         /**
     774                 :            :          * Get Commands Supported and Effects log page for the Zoned Namespace Command Set.
     775                 :            :          */
     776                 :            :         NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG,
     777                 :            : 
     778                 :            :         /**
     779                 :            :          * Waiting for the Get Log Page command to be completed.
     780                 :            :          */
     781                 :            :         NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG,
     782                 :            : 
     783                 :            :         /**
     784                 :            :          * Set Number of Queues of the controller.
     785                 :            :          */
     786                 :            :         NVME_CTRLR_STATE_SET_NUM_QUEUES,
     787                 :            : 
     788                 :            :         /**
     789                 :            :          * Waiting for Set Num of Queues command to be completed.
     790                 :            :          */
     791                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
     792                 :            : 
     793                 :            :         /**
     794                 :            :          * Get active Namespace list of the controller.
     795                 :            :          */
     796                 :            :         NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
     797                 :            : 
     798                 :            :         /**
     799                 :            :          * Waiting for the Identify Active Namespace commands to be completed.
     800                 :            :          */
     801                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
     802                 :            : 
     803                 :            :         /**
     804                 :            :          * Get Identify Namespace Data structure for each NS.
     805                 :            :          */
     806                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS,
     807                 :            : 
     808                 :            :         /**
     809                 :            :          * Waiting for the Identify Namespace commands to be completed.
     810                 :            :          */
     811                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
     812                 :            : 
     813                 :            :         /**
     814                 :            :          * Get Identify Namespace Identification Descriptors.
     815                 :            :          */
     816                 :            :         NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
     817                 :            : 
     818                 :            :         /**
     819                 :            :          * Get Identify I/O Command Set Specific Namespace data structure for each NS.
     820                 :            :          */
     821                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
     822                 :            : 
     823                 :            :         /**
     824                 :            :          * Waiting for the Identify I/O Command Set Specific Namespace commands to be completed.
     825                 :            :          */
     826                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
     827                 :            : 
     828                 :            :         /**
     829                 :            :          * Waiting for the Identify Namespace Identification
     830                 :            :          * Descriptors to be completed.
     831                 :            :          */
     832                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
     833                 :            : 
     834                 :            :         /**
     835                 :            :          * Set supported log pages of the controller.
     836                 :            :          */
     837                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
     838                 :            : 
     839                 :            :         /**
     840                 :            :          * Set supported log pages of INTEL controller.
     841                 :            :          */
     842                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES,
     843                 :            : 
     844                 :            :         /**
     845                 :            :          * Waiting for supported log pages of INTEL controller.
     846                 :            :          */
     847                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES,
     848                 :            : 
     849                 :            :         /**
     850                 :            :          * Set supported features of the controller.
     851                 :            :          */
     852                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     853                 :            : 
     854                 :            :         /**
     855                 :            :          * Set the Host Behavior Support feature of the controller.
     856                 :            :          */
     857                 :            :         NVME_CTRLR_STATE_SET_HOST_FEATURE,
     858                 :            : 
     859                 :            :         /**
     860                 :            :          * Waiting for the Host Behavior Support feature of the controller.
     861                 :            :          */
     862                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE,
     863                 :            : 
     864                 :            :         /**
     865                 :            :          * Set Doorbell Buffer Config of the controller.
     866                 :            :          */
     867                 :            :         NVME_CTRLR_STATE_SET_DB_BUF_CFG,
     868                 :            : 
     869                 :            :         /**
     870                 :            :          * Waiting for Doorbell Buffer Config to be completed.
     871                 :            :          */
     872                 :            :         NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
     873                 :            : 
     874                 :            :         /**
     875                 :            :          * Set Host ID of the controller.
     876                 :            :          */
     877                 :            :         NVME_CTRLR_STATE_SET_HOST_ID,
     878                 :            : 
     879                 :            :         /**
     880                 :            :          * Waiting for Set Host ID to be completed.
     881                 :            :          */
     882                 :            :         NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
     883                 :            : 
     884                 :            :         /**
     885                 :            :          * Let transport layer do its part of initialization.
     886                 :            :          */
     887                 :            :         NVME_CTRLR_STATE_TRANSPORT_READY,
     888                 :            : 
     889                 :            :         /**
     890                 :            :          * Controller initialization has completed and the controller is ready.
     891                 :            :          */
     892                 :            :         NVME_CTRLR_STATE_READY,
     893                 :            : 
     894                 :            :         /**
     895                 :            :          * Controller initialization has an error.
     896                 :            :          */
     897                 :            :         NVME_CTRLR_STATE_ERROR,
     898                 :            : 
     899                 :            :         /**
     900                 :            :          * Admin qpair was disconnected, controller needs to be re-initialized
     901                 :            :          */
     902                 :            :         NVME_CTRLR_STATE_DISCONNECTED,
     903                 :            : };
     904                 :            : 
     905                 :            : #define NVME_TIMEOUT_INFINITE           0
     906                 :            : #define NVME_TIMEOUT_KEEP_EXISTING      UINT64_MAX
     907                 :            : 
     908                 :            : struct spdk_nvme_ctrlr_aer_completion {
     909                 :            :         struct spdk_nvme_cpl    cpl;
     910                 :            :         STAILQ_ENTRY(spdk_nvme_ctrlr_aer_completion) link;
     911                 :            : };
     912                 :            : 
     913                 :            : /*
     914                 :            :  * Used to track properties for all processes accessing the controller.
     915                 :            :  */
     916                 :            : struct spdk_nvme_ctrlr_process {
     917                 :            :         /** Whether it is the primary process  */
     918                 :            :         bool                                            is_primary;
     919                 :            : 
     920                 :            :         /** Process ID */
     921                 :            :         pid_t                                           pid;
     922                 :            : 
     923                 :            :         /** Active admin requests to be completed */
     924                 :            :         STAILQ_HEAD(, nvme_request)                     active_reqs;
     925                 :            : 
     926                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr_process)            tailq;
     927                 :            : 
     928                 :            :         /** Per process PCI device handle */
     929                 :            :         struct spdk_pci_device                          *devhandle;
     930                 :            : 
     931                 :            :         /** Reference to track the number of attachment to this controller. */
     932                 :            :         int                                             ref;
     933                 :            : 
     934                 :            :         /** Allocated IO qpairs */
     935                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)                   allocated_io_qpairs;
     936                 :            : 
     937                 :            :         spdk_nvme_aer_cb                                aer_cb_fn;
     938                 :            :         void                                            *aer_cb_arg;
     939                 :            : 
     940                 :            :         /**
     941                 :            :          * A function pointer to timeout callback function
     942                 :            :          */
     943                 :            :         spdk_nvme_timeout_cb            timeout_cb_fn;
     944                 :            :         void                            *timeout_cb_arg;
     945                 :            :         /** separate timeout values for io vs. admin reqs */
     946                 :            :         uint64_t                        timeout_io_ticks;
     947                 :            :         uint64_t                        timeout_admin_ticks;
     948                 :            : 
     949                 :            :         /** List to publish AENs to all procs in multiprocess setup */
     950                 :            :         STAILQ_HEAD(, spdk_nvme_ctrlr_aer_completion)      async_events;
     951                 :            : };
     952                 :            : 
     953                 :            : struct nvme_register_completion {
     954                 :            :         struct spdk_nvme_cpl                    cpl;
     955                 :            :         uint64_t                                value;
     956                 :            :         spdk_nvme_reg_cb                        cb_fn;
     957                 :            :         void                                    *cb_ctx;
     958                 :            :         STAILQ_ENTRY(nvme_register_completion)  stailq;
     959                 :            :         pid_t                                   pid;
     960                 :            : };
     961                 :            : 
     962                 :            : struct spdk_nvme_ctrlr {
     963                 :            :         /* Hot data (accessed in I/O path) starts here. */
     964                 :            : 
     965                 :            :         /* Tree of namespaces */
     966                 :            :         RB_HEAD(nvme_ns_tree, spdk_nvme_ns)     ns;
     967                 :            : 
     968                 :            :         /* The number of active namespaces */
     969                 :            :         uint32_t                        active_ns_count;
     970                 :            : 
     971                 :            :         bool                            is_removed;
     972                 :            : 
     973                 :            :         bool                            is_resetting;
     974                 :            : 
     975                 :            :         bool                            is_failed;
     976                 :            : 
     977                 :            :         bool                            is_destructed;
     978                 :            : 
     979                 :            :         bool                            timeout_enabled;
     980                 :            : 
     981                 :            :         /* The application is preparing to reset the controller.  Transports
     982                 :            :          * can use this to skip unnecessary parts of the qpair deletion process
     983                 :            :          * for example, like the DELETE_SQ/CQ commands.
     984                 :            :          */
     985                 :            :         bool                            prepare_for_reset;
     986                 :            : 
     987                 :            :         bool                            is_disconnecting;
     988                 :            : 
     989                 :            :         bool                            needs_io_msg_update;
     990                 :            : 
     991                 :            :         uint16_t                        max_sges;
     992                 :            : 
     993                 :            :         uint16_t                        cntlid;
     994                 :            : 
     995                 :            :         /** Controller support flags */
     996                 :            :         uint64_t                        flags;
     997                 :            : 
     998                 :            :         /** NVMEoF in-capsule data size in bytes */
     999                 :            :         uint32_t                        ioccsz_bytes;
    1000                 :            : 
    1001                 :            :         /** NVMEoF in-capsule data offset in 16 byte units */
    1002                 :            :         uint16_t                        icdoff;
    1003                 :            : 
    1004                 :            :         /* Cold data (not accessed in normal I/O path) is after this point. */
    1005                 :            : 
    1006                 :            :         struct spdk_nvme_transport_id   trid;
    1007                 :            : 
    1008                 :            :         struct {
    1009                 :            :                 /** Is numa.id valid? Ensures numa.id == 0 is interpreted correctly. */
    1010                 :            :                 uint32_t                id_valid : 1;
    1011                 :            :                 int32_t                 id : 31;
    1012                 :            :         } numa;
    1013                 :            : 
    1014                 :            :         union spdk_nvme_cap_register    cap;
    1015                 :            :         union spdk_nvme_vs_register     vs;
    1016                 :            : 
    1017                 :            :         int                             state;
    1018                 :            :         uint64_t                        state_timeout_tsc;
    1019                 :            : 
    1020                 :            :         uint64_t                        next_keep_alive_tick;
    1021                 :            :         uint64_t                        keep_alive_interval_ticks;
    1022                 :            : 
    1023                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr)    tailq;
    1024                 :            : 
    1025                 :            :         /** All the log pages supported */
    1026                 :            :         bool                            log_page_supported[256];
    1027                 :            : 
    1028                 :            :         /** All the features supported */
    1029                 :            :         bool                            feature_supported[256];
    1030                 :            : 
    1031                 :            :         /** maximum i/o size in bytes */
    1032                 :            :         uint32_t                        max_xfer_size;
    1033                 :            : 
    1034                 :            :         /** minimum page size supported by this controller in bytes */
    1035                 :            :         uint32_t                        min_page_size;
    1036                 :            : 
    1037                 :            :         /** selected memory page size for this controller in bytes */
    1038                 :            :         uint32_t                        page_size;
    1039                 :            : 
    1040                 :            :         uint32_t                        num_aers;
    1041                 :            :         struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
    1042                 :            : 
    1043                 :            :         /** guards access to the controller itself, including admin queues */
    1044                 :            :         pthread_mutex_t                 ctrlr_lock;
    1045                 :            : 
    1046                 :            :         struct spdk_nvme_qpair          *adminq;
    1047                 :            : 
    1048                 :            :         /** shadow doorbell buffer */
    1049                 :            :         uint32_t                        *shadow_doorbell;
    1050                 :            :         /** eventidx buffer */
    1051                 :            :         uint32_t                        *eventidx;
    1052                 :            : 
    1053                 :            :         /**
    1054                 :            :          * Identify Controller data.
    1055                 :            :          */
    1056                 :            :         struct spdk_nvme_ctrlr_data     cdata;
    1057                 :            : 
    1058                 :            :         /**
    1059                 :            :          * Zoned Namespace Command Set Specific Identify Controller data.
    1060                 :            :          */
    1061                 :            :         struct spdk_nvme_zns_ctrlr_data *cdata_zns;
    1062                 :            : 
    1063                 :            :         struct spdk_bit_array           *free_io_qids;
    1064                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)   active_io_qpairs;
    1065                 :            : 
    1066                 :            :         struct spdk_nvme_ctrlr_opts     opts;
    1067                 :            : 
    1068                 :            :         uint64_t                        quirks;
    1069                 :            : 
    1070                 :            :         /* Extra sleep time during controller initialization */
    1071                 :            :         uint64_t                        sleep_timeout_tsc;
    1072                 :            : 
    1073                 :            :         /** Track all the processes manage this controller */
    1074                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr_process)   active_procs;
    1075                 :            : 
    1076                 :            : 
    1077                 :            :         STAILQ_HEAD(, nvme_request)     queued_aborts;
    1078                 :            :         uint32_t                        outstanding_aborts;
    1079                 :            : 
    1080                 :            :         uint32_t                        lock_depth;
    1081                 :            : 
    1082                 :            :         /* CB to notify the user when the ctrlr is removed/failed. */
    1083                 :            :         spdk_nvme_remove_cb                     remove_cb;
    1084                 :            :         void                                    *cb_ctx;
    1085                 :            : 
    1086                 :            :         struct spdk_nvme_qpair          *external_io_msgs_qpair;
    1087                 :            :         pthread_mutex_t                 external_io_msgs_lock;
    1088                 :            :         struct spdk_ring                *external_io_msgs;
    1089                 :            : 
    1090                 :            :         STAILQ_HEAD(, nvme_io_msg_producer) io_producers;
    1091                 :            : 
    1092                 :            :         struct spdk_nvme_ana_page               *ana_log_page;
    1093                 :            :         struct spdk_nvme_ana_group_descriptor   *copied_ana_desc;
    1094                 :            :         uint32_t                                ana_log_page_size;
    1095                 :            : 
    1096                 :            :         /* scratchpad pointer that can be used to send data between two NVME_CTRLR_STATEs */
    1097                 :            :         void                            *tmp_ptr;
    1098                 :            : 
    1099                 :            :         /* maximum zone append size in bytes */
    1100                 :            :         uint32_t                        max_zone_append_size;
    1101                 :            : 
    1102                 :            :         /* PMR size in bytes */
    1103                 :            :         uint64_t                        pmr_size;
    1104                 :            : 
    1105                 :            :         /* Boot Partition Info */
    1106                 :            :         enum nvme_bp_write_state        bp_ws;
    1107                 :            :         uint32_t                        bpid;
    1108                 :            :         spdk_nvme_cmd_cb                bp_write_cb_fn;
    1109                 :            :         void                            *bp_write_cb_arg;
    1110                 :            : 
    1111                 :            :         /* Firmware Download */
    1112                 :            :         void                            *fw_payload;
    1113                 :            :         unsigned int                    fw_size_remaining;
    1114                 :            :         unsigned int                    fw_offset;
    1115                 :            :         unsigned int                    fw_transfer_size;
    1116                 :            : 
    1117                 :            :         /* Completed register operations */
    1118                 :            :         STAILQ_HEAD(, nvme_register_completion) register_operations;
    1119                 :            : 
    1120                 :            :         union spdk_nvme_cc_register             process_init_cc;
    1121                 :            : 
    1122                 :            :         /* Authentication transaction ID */
    1123                 :            :         uint16_t                                auth_tid;
    1124                 :            :         /* Authentication sequence number */
    1125                 :            :         uint32_t                                auth_seqnum;
    1126                 :            : };
    1127                 :            : 
    1128                 :            : struct spdk_nvme_probe_ctx {
    1129                 :            :         struct spdk_nvme_transport_id           trid;
    1130                 :            :         const struct spdk_nvme_ctrlr_opts       *opts;
    1131                 :            :         void                                    *cb_ctx;
    1132                 :            :         spdk_nvme_probe_cb                      probe_cb;
    1133                 :            :         spdk_nvme_attach_cb                     attach_cb;
    1134                 :            :         spdk_nvme_attach_fail_cb                attach_fail_cb;
    1135                 :            :         spdk_nvme_remove_cb                     remove_cb;
    1136                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)           init_ctrlrs;
    1137                 :            : };
    1138                 :            : 
    1139                 :            : typedef void (*nvme_ctrlr_detach_cb)(struct spdk_nvme_ctrlr *ctrlr);
    1140                 :            : 
    1141                 :            : enum nvme_ctrlr_detach_state {
    1142                 :            :         NVME_CTRLR_DETACH_SET_CC,
    1143                 :            :         NVME_CTRLR_DETACH_CHECK_CSTS,
    1144                 :            :         NVME_CTRLR_DETACH_GET_CSTS,
    1145                 :            :         NVME_CTRLR_DETACH_GET_CSTS_DONE,
    1146                 :            : };
    1147                 :            : 
    1148                 :            : struct nvme_ctrlr_detach_ctx {
    1149                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
    1150                 :            :         nvme_ctrlr_detach_cb                    cb_fn;
    1151                 :            :         uint64_t                                shutdown_start_tsc;
    1152                 :            :         uint32_t                                shutdown_timeout_ms;
    1153                 :            :         bool                                    shutdown_complete;
    1154                 :            :         enum nvme_ctrlr_detach_state            state;
    1155                 :            :         union spdk_nvme_csts_register           csts;
    1156                 :            :         TAILQ_ENTRY(nvme_ctrlr_detach_ctx)      link;
    1157                 :            : };
    1158                 :            : 
    1159                 :            : struct spdk_nvme_detach_ctx {
    1160                 :            :         TAILQ_HEAD(, nvme_ctrlr_detach_ctx)     head;
    1161                 :            : };
    1162                 :            : 
    1163                 :            : struct nvme_driver {
    1164                 :            :         pthread_mutex_t                 lock;
    1165                 :            : 
    1166                 :            :         /** Multi-process shared attached controller list */
    1167                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)   shared_attached_ctrlrs;
    1168                 :            : 
    1169                 :            :         bool                            initialized;
    1170                 :            :         struct spdk_uuid                default_extended_host_id;
    1171                 :            : 
    1172                 :            :         /** netlink socket fd for hotplug messages */
    1173                 :            :         int                             hotplug_fd;
    1174                 :            : };
    1175                 :            : 
    1176                 :            : #define nvme_ns_cmd_get_ext_io_opt(opts, field, defval) \
    1177                 :            :        ((opts) != NULL && offsetof(struct spdk_nvme_ns_cmd_ext_io_opts, field) + \
    1178                 :            :         sizeof((opts)->field) <= (opts)->size ? (opts)->field : (defval))
    1179                 :            : 
    1180                 :            : extern struct nvme_driver *g_spdk_nvme_driver;
    1181                 :            : 
    1182                 :            : int nvme_driver_init(void);
    1183                 :            : 
    1184                 :            : #define nvme_delay              usleep
    1185                 :            : 
    1186                 :            : static inline bool
    1187                 : 2546601522 : nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
    1188                 :            : {
    1189   [ +  -  +  - ]: 2546601522 :         return qpair->id == 0;
    1190                 :            : }
    1191                 :            : 
    1192                 :            : static inline bool
    1193                 :            : nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
    1194                 :            : {
    1195                 :            :         return qpair->id != 0;
    1196                 :            : }
    1197                 :            : 
    1198                 :            : static inline int
    1199                 :  164429844 : nvme_robust_mutex_lock(pthread_mutex_t *mtx)
    1200                 :            : {
    1201         [ +  + ]:  164429844 :         int rc = pthread_mutex_lock(mtx);
    1202                 :            : 
    1203                 :            : #ifndef __FreeBSD__
    1204         [ +  + ]:  164359672 :         if (rc == EOWNERDEAD) {
    1205         [ #  # ]:          0 :                 rc = pthread_mutex_consistent(mtx);
    1206                 :          0 :         }
    1207                 :            : #endif
    1208                 :            : 
    1209                 :  164429844 :         return rc;
    1210                 :            : }
    1211                 :            : 
    1212                 :            : static inline int
    1213                 :  162932337 : nvme_ctrlr_lock(struct spdk_nvme_ctrlr *ctrlr)
    1214                 :            : {
    1215                 :            :         int rc;
    1216                 :            : 
    1217         [ +  - ]:  162932337 :         rc = nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
    1218         [ +  - ]:  162932337 :         ctrlr->lock_depth++;
    1219                 :  162932337 :         return rc;
    1220                 :            : }
    1221                 :            : 
    1222                 :            : static inline int
    1223                 :  164429836 : nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
    1224                 :            : {
    1225         [ +  + ]:  164429836 :         return pthread_mutex_unlock(mtx);
    1226                 :            : }
    1227                 :            : 
    1228                 :            : static inline int
    1229                 :  162932329 : nvme_ctrlr_unlock(struct spdk_nvme_ctrlr *ctrlr)
    1230                 :            : {
    1231         [ +  - ]:  162932329 :         ctrlr->lock_depth--;
    1232         [ +  - ]:  162932329 :         return nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
    1233                 :            : }
    1234                 :            : 
    1235                 :            : /* Poll group management functions. */
    1236                 :            : int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1237                 :            : int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1238                 :            : void nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group);
    1239                 :            : 
    1240                 :            : /* Admin functions */
    1241                 :            : int     nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
    1242                 :            :                                 uint8_t cns, uint16_t cntid, uint32_t nsid,
    1243                 :            :                                 uint8_t csi, void *payload, size_t payload_size,
    1244                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1245                 :            : int     nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1246                 :            :                                       uint32_t num_queues, spdk_nvme_cmd_cb cb_fn,
    1247                 :            :                                       void *cb_arg);
    1248                 :            : int     nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1249                 :            :                                       spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1250                 :            : int     nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
    1251                 :            :                 union spdk_nvme_feat_async_event_configuration config,
    1252                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1253                 :            : int     nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
    1254                 :            :                                    spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1255                 :            : int     nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1256                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1257                 :            : int     nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1258                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1259                 :            : int     nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
    1260                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1261                 :            : int     nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr,
    1262                 :            :                 uint64_t prp1, uint64_t prp2,
    1263                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1264                 :            : int     nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
    1265                 :            :                                  void *cb_arg);
    1266                 :            : int     nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1267                 :            :                               struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1268                 :            : int     nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
    1269                 :            :                                  const struct spdk_nvme_fw_commit *fw_commit,
    1270                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1271                 :            : int     nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
    1272                 :            :                 uint32_t size, uint32_t offset, void *payload,
    1273                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1274                 :            : int     nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1275                 :            :                                 struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
    1276                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1277                 :            : void    nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl);
    1278                 :            : int     nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
    1279                 :            :                                  struct nvme_completion_poll_status *status);
    1280                 :            : int     nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair,
    1281                 :            :                 struct nvme_completion_poll_status *status,
    1282                 :            :                 pthread_mutex_t *robust_mutex);
    1283                 :            : int     nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
    1284                 :            :                 struct nvme_completion_poll_status *status,
    1285                 :            :                 uint64_t timeout_in_usecs);
    1286                 :            : int     nvme_wait_for_completion_robust_lock_timeout(struct spdk_nvme_qpair *qpair,
    1287                 :            :                 struct nvme_completion_poll_status *status,
    1288                 :            :                 pthread_mutex_t *robust_mutex,
    1289                 :            :                 uint64_t timeout_in_usecs);
    1290                 :            : int     nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair,
    1291                 :            :                 struct nvme_completion_poll_status *status,
    1292                 :            :                 pthread_mutex_t *robust_mutex);
    1293                 :            : 
    1294                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr,
    1295                 :            :                 pid_t pid);
    1296                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr);
    1297                 :            : int     nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle);
    1298                 :            : void    nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr);
    1299                 :            : struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr);
    1300                 :            : 
    1301                 :            : int     nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
    1302                 :            :                          struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle);
    1303                 :            : 
    1304                 :            : int     nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
    1305                 :            : void    nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
    1306                 :            : void    nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1307                 :            : void    nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
    1308                 :            :                                   struct nvme_ctrlr_detach_ctx *ctx);
    1309                 :            : int     nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
    1310                 :            :                                        struct nvme_ctrlr_detach_ctx *ctx);
    1311                 :            : void    nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
    1312                 :            : int     nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
    1313                 :            : void    nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr);
    1314                 :            : int     nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr);
    1315                 :            : void    nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
    1316                 :            :                              struct spdk_nvme_ctrlr *ctrlr);
    1317                 :            : 
    1318                 :            : int     nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
    1319                 :            :                                         struct nvme_request *req);
    1320                 :            : int     nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap);
    1321                 :            : int     nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs);
    1322                 :            : int     nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz);
    1323                 :            : int     nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap);
    1324                 :            : int     nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo);
    1325                 :            : int     nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel);
    1326                 :            : int     nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value);
    1327                 :            : bool    nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr);
    1328                 :            : void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1329                 :            : void nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr);
    1330                 :            : int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
    1331                 :            :                     struct spdk_nvme_ctrlr *ctrlr,
    1332                 :            :                     enum spdk_nvme_qprio qprio,
    1333                 :            :                     uint32_t num_requests, bool async);
    1334                 :            : void    nvme_qpair_deinit(struct spdk_nvme_qpair *qpair);
    1335                 :            : void    nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair);
    1336                 :            : int     nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
    1337                 :            :                                   struct nvme_request *req);
    1338                 :            : void    nvme_qpair_abort_all_queued_reqs(struct spdk_nvme_qpair *qpair);
    1339                 :            : uint32_t nvme_qpair_abort_queued_reqs_with_cbarg(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg);
    1340                 :            : void    nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair);
    1341                 :            : void    nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests);
    1342                 :            : int     nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr);
    1343                 :            : void    nvme_ns_set_identify_data(struct spdk_nvme_ns *ns);
    1344                 :            : void    nvme_ns_set_id_desc_list_data(struct spdk_nvme_ns *ns);
    1345                 :            : void    nvme_ns_free_zns_specific_data(struct spdk_nvme_ns *ns);
    1346                 :            : void    nvme_ns_free_nvm_specific_data(struct spdk_nvme_ns *ns);
    1347                 :            : void    nvme_ns_free_iocs_specific_data(struct spdk_nvme_ns *ns);
    1348                 :            : bool    nvme_ns_has_supported_iocs_specific_data(struct spdk_nvme_ns *ns);
    1349                 :            : int     nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
    1350                 :            :                           struct spdk_nvme_ctrlr *ctrlr);
    1351                 :            : void    nvme_ns_destruct(struct spdk_nvme_ns *ns);
    1352                 :            : int     nvme_ns_cmd_zone_append_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1353                 :            :                                         void *buffer, void *metadata, uint64_t zslba,
    1354                 :            :                                         uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
    1355                 :            :                                         uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag);
    1356                 :            : int nvme_ns_cmd_zone_appendv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1357                 :            :                                      uint64_t zslba, uint32_t lba_count,
    1358                 :            :                                      spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
    1359                 :            :                                      spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
    1360                 :            :                                      spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
    1361                 :            :                                      uint16_t apptag_mask, uint16_t apptag);
    1362                 :            : 
    1363                 :            : int     nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1364                 :            : int     nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1365                 :            : int     nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1366                 :            : int     nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1367                 :            : int     nvme_fabric_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1368                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1369                 :            : int     nvme_fabric_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1370                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1371                 :            : int     nvme_fabric_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1372                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1373                 :            : int     nvme_fabric_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1374                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1375                 :            : int     nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1376                 :            : int     nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
    1377                 :            :                                    struct spdk_nvme_probe_ctx *probe_ctx);
    1378                 :            : int     nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1379                 :            : int     nvme_fabric_qpair_connect_async(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1380                 :            : int     nvme_fabric_qpair_connect_poll(struct spdk_nvme_qpair *qpair);
    1381                 :            : bool    nvme_fabric_qpair_auth_required(struct spdk_nvme_qpair *qpair);
    1382                 :            : int     nvme_fabric_qpair_authenticate_async(struct spdk_nvme_qpair *qpair);
    1383                 :            : int     nvme_fabric_qpair_authenticate_poll(struct spdk_nvme_qpair *qpair);
    1384                 :            : 
    1385                 :            : typedef int (*spdk_nvme_parse_ana_log_page_cb)(
    1386                 :            :         const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
    1387                 :            : int     nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr,
    1388                 :            :                                       spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg);
    1389                 :            : 
    1390                 :            : static inline void
    1391                 :  106765529 : nvme_request_clear(struct nvme_request *req)
    1392                 :            : {
    1393                 :            :         /*
    1394                 :            :          * Only memset/zero fields that need it.  All other fields
    1395                 :            :          *  will be initialized appropriately either later in this
    1396                 :            :          *  function, or before they are needed later in the
    1397                 :            :          *  submission patch.  For example, the children
    1398                 :            :          *  TAILQ_ENTRY and following members are
    1399                 :            :          *  only used as part of I/O splitting so we avoid
    1400                 :            :          *  memsetting them until it is actually needed.
    1401                 :            :          *  They will be initialized in nvme_request_add_child()
    1402                 :            :          *  if the request is split.
    1403                 :            :          */
    1404         [ +  + ]:  106765529 :         memset(req, 0, offsetof(struct nvme_request, payload_size));
    1405                 :  106765529 : }
    1406                 :            : 
    1407                 :            : #define NVME_INIT_REQUEST(req, _cb_fn, _cb_arg, _payload, _payload_size, _md_size)      \
    1408                 :            :         do {                                            \
    1409                 :            :                 nvme_request_clear(req);                \
    1410                 :            :                 req->cb_fn = _cb_fn;                 \
    1411                 :            :                 req->cb_arg = _cb_arg;                       \
    1412                 :            :                 req->payload = _payload;             \
    1413                 :            :                 req->payload_size = _payload_size;   \
    1414                 :            :                 req->md_size = _md_size;             \
    1415                 :            :                 req->pid = g_spdk_nvme_pid;          \
    1416                 :            :                 req->submit_tick = 0;                        \
    1417                 :            :                 req->accel_sequence = NULL;          \
    1418                 :            :         } while (0);
    1419                 :            : 
    1420                 :            : static inline struct nvme_request *
    1421                 :  107625220 : nvme_allocate_request(struct spdk_nvme_qpair *qpair,
    1422                 :            :                       const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size,
    1423                 :            :                       spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1424                 :            : {
    1425                 :            :         struct nvme_request *req;
    1426                 :            : 
    1427   [ +  -  +  -  :  107625220 :         req = STAILQ_FIRST(&qpair->free_req);
                   +  - ]
    1428         [ +  + ]:  107625220 :         if (req == NULL) {
    1429                 :     871331 :                 return req;
    1430                 :            :         }
    1431                 :            : 
    1432   [ +  +  +  -  :  106753889 :         STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  -  
          +  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1433         [ +  - ]:  106753889 :         qpair->num_outstanding_reqs++;
    1434                 :            : 
    1435   [ +  -  +  -  :  106753889 :         NVME_INIT_REQUEST(req, cb_fn, cb_arg, *payload, payload_size, md_size);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                      - ]
    1436                 :            : 
    1437                 :  106753889 :         return req;
    1438                 :    1320197 : }
    1439                 :            : 
    1440                 :            : static inline struct nvme_request *
    1441                 :    5065087 : nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair,
    1442                 :            :                              void *buffer, uint32_t payload_size,
    1443                 :            :                              spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1444                 :            : {
    1445                 :     383733 :         struct nvme_payload payload;
    1446                 :            : 
    1447                 :    5065087 :         payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
    1448                 :            : 
    1449                 :    5065087 :         return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg);
    1450                 :            : }
    1451                 :            : 
    1452                 :            : static inline struct nvme_request *
    1453                 :    2398704 : nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1454                 :            : {
    1455                 :    2398704 :         return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg);
    1456                 :            : }
    1457                 :            : 
    1458                 :            : struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
    1459                 :            :                 void *buffer, uint32_t payload_size,
    1460                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);
    1461                 :            : 
    1462                 :            : static inline void
    1463                 :  106764668 : _nvme_free_request(struct nvme_request *req, struct spdk_nvme_qpair *qpair)
    1464                 :            : {
    1465   [ +  +  #  # ]:  106764668 :         assert(req != NULL);
    1466   [ +  +  +  -  :  106764668 :         assert(req->num_children == 0);
             +  -  #  # ]
    1467   [ +  +  #  # ]:  106764668 :         assert(qpair != NULL);
    1468                 :            : 
    1469                 :            :         /* The reserved_req does not go in the free_req STAILQ - it is
    1470                 :            :          * saved only for use with a FABRICS/CONNECT command.
    1471                 :            :          */
    1472   [ +  +  +  -  :  106764668 :         if (spdk_likely(qpair->reserved_req != req)) {
                   +  + ]
    1473   [ +  +  +  -  :  106752932 :                 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
          +  -  +  -  +  
          -  +  -  +  -  
          #  #  #  #  #  
          #  #  #  #  #  
          +  -  +  -  +  
                      - ]
    1474                 :            : 
    1475   [ +  +  +  -  :  106752932 :                 assert(qpair->num_outstanding_reqs > 0);
             +  -  #  # ]
    1476         [ +  - ]:  106752932 :                 qpair->num_outstanding_reqs--;
    1477                 :    1319425 :         }
    1478                 :  106764704 : }
    1479                 :            : 
    1480                 :            : static inline void
    1481                 :   52914016 : nvme_free_request(struct nvme_request *req)
    1482                 :            : {
    1483   [ #  #  #  # ]:   52914016 :         _nvme_free_request(req, req->qpair);
    1484                 :   52914016 : }
    1485                 :            : 
    1486                 :            : static inline void
    1487                 :   53850829 : nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,
    1488                 :            :                       struct nvme_request *req, struct spdk_nvme_cpl *cpl)
    1489                 :            : {
    1490                 :   10938661 :         struct spdk_nvme_cpl            err_cpl;
    1491                 :            :         struct nvme_error_cmd           *cmd;
    1492                 :            : 
    1493   [ +  +  +  -  :   53850829 :         if (spdk_unlikely(req->accel_sequence != NULL)) {
                   +  - ]
    1494   [ #  #  #  #  :          0 :                 struct spdk_nvme_poll_group *pg = qpair->poll_group->group;
             #  #  #  # ]
    1495                 :            : 
    1496                 :            :                 /* Transports are required to execute the sequence and clear req->accel_sequence.
    1497                 :            :                  * If it's left non-NULL it must mean the request is failed. */
    1498   [ #  #  #  #  :          0 :                 assert(spdk_nvme_cpl_is_error(cpl));
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    1499   [ #  #  #  #  :          0 :                 pg->accel_fn_table.abort_sequence(req->accel_sequence);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1500   [ #  #  #  # ]:          0 :                 req->accel_sequence = NULL;
    1501                 :          0 :         }
    1502                 :            : 
    1503                 :            :         /* error injection at completion path,
    1504                 :            :          * only inject for successful completed commands
    1505                 :            :          */
    1506   [ +  +  +  +  :   53850829 :         if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) &&
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  +  
                      - ]
    1507                 :            :                           !spdk_nvme_cpl_is_error(cpl))) {
    1508   [ +  +  #  #  :    1320891 :                 TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1509                 :            : 
    1510   [ +  +  +  +  :     660451 :                         if (cmd->do_not_submit) {
             #  #  #  # ]
    1511                 :     660425 :                                 continue;
    1512                 :            :                         }
    1513                 :            : 
    1514   [ +  -  +  +  :         26 :                         if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1515                 :            : 
    1516                 :         11 :                                 err_cpl = *cpl;
    1517   [ #  #  #  #  :         11 :                                 err_cpl.status.sct = cmd->status.sct;
             #  #  #  # ]
    1518   [ #  #  #  #  :         11 :                                 err_cpl.status.sc = cmd->status.sc;
             #  #  #  # ]
    1519                 :            : 
    1520                 :         11 :                                 cpl = &err_cpl;
    1521         [ #  # ]:         11 :                                 cmd->err_count--;
    1522                 :         11 :                                 break;
    1523                 :            :                         }
    1524                 :          2 :                 }
    1525                 :         17 :         }
    1526                 :            : 
    1527                 :            :         /* For PCIe completions, we want to avoid touching the req itself to avoid
    1528                 :            :          * dependencies on loading those cachelines. So call the internal helper
    1529                 :            :          * function instead using the qpair that was passed by the caller, instead
    1530                 :            :          * of getting it from the req.
    1531                 :            :          */
    1532                 :   53850829 :         _nvme_free_request(req, qpair);
    1533                 :            : 
    1534         [ +  + ]:   53850829 :         if (spdk_likely(cb_fn)) {
    1535   [ -  +  +  - ]:   53850545 :                 cb_fn(cb_arg, cpl);
    1536                 :    1320666 :         }
    1537                 :   53850829 : }
    1538                 :            : 
    1539                 :            : static inline void
    1540                 :     102034 : nvme_cleanup_user_req(struct nvme_request *req)
    1541                 :            : {
    1542   [ +  +  +  -  :     102034 :         if (req->user_buffer && req->payload_size) {
          +  +  +  -  +  
                -  +  - ]
    1543   [ +  -  +  -  :      91050 :                 spdk_free(req->payload.contig_or_cb_arg);
                   +  - ]
    1544   [ +  -  +  - ]:      91050 :                 req->user_buffer = NULL;
    1545                 :       3321 :         }
    1546                 :            : 
    1547   [ +  -  +  - ]:     102034 :         req->user_cb_arg = NULL;
    1548   [ +  -  +  - ]:     102034 :         req->user_cb_fn = NULL;
    1549                 :     102034 : }
    1550                 :            : 
    1551                 :            : static inline bool
    1552                 :   33245648 : nvme_request_abort_match(struct nvme_request *req, void *cmd_cb_arg)
    1553                 :            : {
    1554   [ +  +  #  #  :   33245650 :         return req->cb_arg == cmd_cb_arg ||
                   #  # ]
    1555   [ +  +  +  -  :   65933590 :                req->user_cb_arg == cmd_cb_arg ||
                   #  # ]
    1556   [ +  +  -  -  :   32687944 :                (req->parent != NULL && req->parent->cb_arg == cmd_cb_arg);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1557                 :            : }
    1558                 :            : 
    1559                 :            : static inline void
    1560                 :      80711 : nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state)
    1561                 :            : {
    1562         [ +  - ]:      80711 :         qpair->state = state;
    1563         [ +  + ]:      80711 :         if (state == NVME_QPAIR_ENABLED) {
    1564         [ +  - ]:      10017 :                 qpair->is_new_qpair = false;
    1565                 :       2404 :         }
    1566                 :      80711 : }
    1567                 :            : 
    1568                 :            : static inline enum nvme_qpair_state
    1569                 : 2309115990 : nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) {
    1570         [ +  - ]: 2309115990 :         return qpair->state;
    1571                 :            : }
    1572                 :            : 
    1573                 :            : static inline void
    1574                 :   54224475 : nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child)
    1575                 :            : {
    1576   [ +  +  #  # ]:   54224475 :         assert(parent != NULL);
    1577   [ +  +  #  # ]:   54224475 :         assert(child != NULL);
    1578   [ +  +  #  #  :   54224475 :         assert(child->parent == parent);
             #  #  #  # ]
    1579   [ -  +  #  #  :   54224475 :         assert(parent->num_children != 0);
             #  #  #  # ]
    1580                 :            : 
    1581         [ #  # ]:   54224475 :         parent->num_children--;
    1582   [ #  #  #  # ]:   54224475 :         child->parent = NULL;
    1583   [ +  +  #  #  :   54224475 :         TAILQ_REMOVE(&parent->children, child, child_tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    1584                 :   54224475 : }
    1585                 :            : 
    1586                 :            : static inline void
    1587                 :     987879 : nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
    1588                 :            : {
    1589                 :     987879 :         struct nvme_request *child = child_arg;
    1590   [ #  #  #  # ]:     987879 :         struct nvme_request *parent = child->parent;
    1591                 :            : 
    1592                 :     987879 :         nvme_request_remove_child(parent, child);
    1593                 :            : 
    1594   [ +  -  -  +  :     987879 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1595   [ #  #  #  #  :          0 :                 memcpy(&parent->parent_status, cpl, sizeof(*cpl));
                   #  # ]
    1596                 :          0 :         }
    1597                 :            : 
    1598   [ +  +  #  #  :     987879 :         if (parent->num_children == 0) {
                   #  # ]
    1599   [ #  #  #  #  :     297550 :                 nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
          #  #  #  #  #  
                #  #  # ]
    1600         [ #  # ]:          3 :                                       parent, &parent->parent_status);
    1601                 :          3 :         }
    1602                 :     987879 : }
    1603                 :            : 
    1604                 :            : static inline void
    1605                 :   53794889 : nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
    1606                 :            : {
    1607   [ +  +  #  #  :   53794889 :         assert(parent->num_children != UINT16_MAX);
             #  #  #  # ]
    1608                 :            : 
    1609   [ +  +  #  #  :   53794889 :         if (parent->num_children == 0) {
                   #  # ]
    1610                 :            :                 /*
    1611                 :            :                  * Defer initialization of the children TAILQ since it falls
    1612                 :            :                  *  on a separate cacheline.  This ensures we do not touch this
    1613                 :            :                  *  cacheline except on request splitting cases, which are
    1614                 :            :                  *  relatively rare.
    1615                 :            :                  */
    1616   [ #  #  #  #  :     401977 :                 TAILQ_INIT(&parent->children);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1617   [ #  #  #  # ]:     401977 :                 parent->parent = NULL;
    1618   [ -  +  #  # ]:     401977 :                 memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
    1619                 :         18 :         }
    1620                 :            : 
    1621         [ #  # ]:   53794889 :         parent->num_children++;
    1622   [ #  #  #  #  :   53794889 :         TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1623   [ #  #  #  # ]:   53794889 :         child->parent = parent;
    1624   [ #  #  #  # ]:   53794889 :         child->cb_fn = nvme_cb_complete_child;
    1625   [ #  #  #  # ]:   53794889 :         child->cb_arg = child;
    1626                 :   53794889 : }
    1627                 :            : 
    1628                 :            : static inline void
    1629                 :   52911488 : nvme_request_free_children(struct nvme_request *req)
    1630                 :            : {
    1631                 :            :         struct nvme_request *child, *tmp;
    1632                 :            : 
    1633   [ +  +  #  #  :   52911488 :         if (req->num_children == 0) {
                   #  # ]
    1634                 :   52807070 :                 return;
    1635                 :            :         }
    1636                 :            : 
    1637                 :            :         /* free all child nvme_request */
    1638   [ +  +  +  +  :   52911400 :         TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1639                 :   52806982 :                 nvme_request_remove_child(req, child);
    1640                 :   52806982 :                 nvme_request_free_children(child);
    1641                 :   52806982 :                 nvme_free_request(child);
    1642                 :         50 :         }
    1643                 :         69 : }
    1644                 :            : 
    1645                 :            : int     nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
    1646                 :            :                                    struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick);
    1647                 :            : uint64_t nvme_get_quirks(const struct spdk_pci_id *id);
    1648                 :            : 
    1649                 :            : int     nvme_robust_mutex_init_shared(pthread_mutex_t *mtx);
    1650                 :            : int     nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx);
    1651                 :            : 
    1652                 :            : bool    nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
    1653                 :            : 
    1654                 :            : struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe(
    1655                 :            :         const struct spdk_nvme_transport_id *trid, const char *hostnqn);
    1656                 :            : 
    1657                 :            : const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name);
    1658                 :            : const struct spdk_nvme_transport *nvme_get_first_transport(void);
    1659                 :            : const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport
    1660                 :            :                 *transport);
    1661                 :            : void  nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr);
    1662                 :            : 
    1663                 :            : /* Transport specific functions */
    1664                 :            : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
    1665                 :            :                 const struct spdk_nvme_ctrlr_opts *opts,
    1666                 :            :                 void *devhandle);
    1667                 :            : int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1668                 :            : int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1669                 :            : int nvme_transport_ctrlr_scan_attached(struct spdk_nvme_probe_ctx *probe_ctx);
    1670                 :            : int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr);
    1671                 :            : int nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr);
    1672                 :            : int nvme_transport_ctrlr_enable_interrupts(struct spdk_nvme_ctrlr *ctrlr);
    1673                 :            : int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1674                 :            : int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1675                 :            : int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1676                 :            : int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1677                 :            : int nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1678                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1679                 :            : int nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1680                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1681                 :            : int nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1682                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1683                 :            : int nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1684                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1685                 :            : uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr);
    1686                 :            : uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr);
    1687                 :            : struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1688                 :            :                 uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts);
    1689                 :            : int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1690                 :            : void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1691                 :            : int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1692                 :            : int nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1693                 :            : int nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1694                 :            : void *nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1695                 :            : int nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1696                 :            : void nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1697                 :            :                 struct spdk_nvme_qpair *qpair);
    1698                 :            : int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1699                 :            :                                        struct spdk_nvme_qpair *qpair);
    1700                 :            : void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1701                 :            :                 struct spdk_nvme_qpair *qpair);
    1702                 :            : void nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair);
    1703                 :            : int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
    1704                 :            :                 struct spdk_memory_domain **domains, int array_size);
    1705                 :            : void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair);
    1706                 :            : int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
    1707                 :            : int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
    1708                 :            : int nvme_transport_qpair_get_fd(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
    1709                 :            :                                 struct spdk_event_handler_opts *opts);
    1710                 :            : int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair,
    1711                 :            :                 uint32_t max_completions);
    1712                 :            : void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair);
    1713                 :            : int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
    1714                 :            :                 int (*iter_fn)(struct nvme_request *req, void *arg),
    1715                 :            :                 void *arg);
    1716                 :            : int nvme_transport_qpair_authenticate(struct spdk_nvme_qpair *qpair);
    1717                 :            : 
    1718                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create(
    1719                 :            :         const struct spdk_nvme_transport *transport);
    1720                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_qpair_get_optimal_poll_group(
    1721                 :            :         const struct spdk_nvme_transport *transport,
    1722                 :            :         struct spdk_nvme_qpair *qpair);
    1723                 :            : int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
    1724                 :            :                                   struct spdk_nvme_qpair *qpair);
    1725                 :            : int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
    1726                 :            :                                      struct spdk_nvme_qpair *qpair);
    1727                 :            : int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1728                 :            : int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1729                 :            : int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
    1730                 :            :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
    1731                 :            : void nvme_transport_poll_group_check_disconnected_qpairs(
    1732                 :            :         struct spdk_nvme_transport_poll_group *tgroup,
    1733                 :            :         spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
    1734                 :            : int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);
    1735                 :            : int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1736                 :            :                                         struct spdk_nvme_transport_poll_group_stat **stats);
    1737                 :            : void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1738                 :            :                 struct spdk_nvme_transport_poll_group_stat *stats);
    1739                 :            : enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport
    1740                 :            :                 *transport);
    1741                 :            : /*
    1742                 :            :  * Below ref related functions must be called with the global
    1743                 :            :  *  driver lock held for the multi-process condition.
    1744                 :            :  *  Within these functions, the per ctrlr ctrlr_lock is also
    1745                 :            :  *  acquired for the multi-thread condition.
    1746                 :            :  */
    1747                 :            : void    nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr);
    1748                 :            : void    nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr);
    1749                 :            : int     nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr);
    1750                 :            : 
    1751                 :            : int     nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
    1752                 :            : int     nvme_parse_addr(struct sockaddr_storage *sa, int family,
    1753                 :            :                         const char *addr, const char *service, long int *port);
    1754                 :            : int     nvme_get_default_hostnqn(char *buf, int len);
    1755                 :            : 
    1756                 :            : static inline bool
    1757                 :    2916268 : _is_page_aligned(uint64_t address, uint64_t page_size)
    1758                 :            : {
    1759                 :    2916268 :         return (address & (page_size - 1)) == 0;
    1760                 :            : }
    1761                 :            : 
    1762                 :            : #endif /* __NVME_INTERNAL_H__ */

Generated by: LCOV version 1.14