Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
6 : : */
7 : :
8 : : #include "spdk/stdinc.h"
9 : :
10 : : #include "bdev_nvme.h"
11 : :
12 : : #include "spdk/accel.h"
13 : : #include "spdk/config.h"
14 : : #include "spdk/endian.h"
15 : : #include "spdk/bdev.h"
16 : : #include "spdk/json.h"
17 : : #include "spdk/keyring.h"
18 : : #include "spdk/likely.h"
19 : : #include "spdk/nvme.h"
20 : : #include "spdk/nvme_ocssd.h"
21 : : #include "spdk/nvme_zns.h"
22 : : #include "spdk/opal.h"
23 : : #include "spdk/thread.h"
24 : : #include "spdk/trace.h"
25 : : #include "spdk/string.h"
26 : : #include "spdk/util.h"
27 : : #include "spdk/uuid.h"
28 : :
29 : : #include "spdk/bdev_module.h"
30 : : #include "spdk/log.h"
31 : :
32 : : #include "spdk_internal/usdt.h"
33 : : #include "spdk_internal/trace_defs.h"
34 : :
35 : : #define CTRLR_STRING(nvme_ctrlr) \
36 : : (spdk_nvme_trtype_is_fabrics(nvme_ctrlr->active_path_id->trid.trtype) ? \
37 : : nvme_ctrlr->active_path_id->trid.subnqn : nvme_ctrlr->active_path_id->trid.traddr)
38 : :
39 : : #define CTRLR_ID(nvme_ctrlr) (spdk_nvme_ctrlr_get_id(nvme_ctrlr->ctrlr))
40 : :
41 : : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
42 : : SPDK_ERRLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
43 : :
44 : : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
45 : : SPDK_WARNLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
46 : :
47 : : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
48 : : SPDK_NOTICELOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
49 : :
50 : : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
51 : : SPDK_INFOLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
52 : :
53 : : #ifdef DEBUG
54 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
55 : : SPDK_DEBUGLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
56 : : #else
57 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
58 : : #endif
59 : :
60 : : #define BDEV_STRING(nbdev) (nbdev->disk.name)
61 : :
62 : : #define NVME_BDEV_ERRLOG(nbdev, format, ...) \
63 : : SPDK_ERRLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
64 : :
65 : : #define NVME_BDEV_WARNLOG(nbdev, format, ...) \
66 : : SPDK_WARNLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
67 : :
68 : : #define NVME_BDEV_NOTICELOG(nbdev, format, ...) \
69 : : SPDK_NOTICELOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
70 : :
71 : : #define NVME_BDEV_INFOLOG(nbdev, format, ...) \
72 : : SPDK_INFOLOG(bdev_nvme, "[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
73 : :
74 : : #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true
75 : : #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000)
76 : :
77 : : #define NSID_STR_LEN 10
78 : :
79 : : #define SPDK_CONTROLLER_NAME_MAX 512
80 : :
81 : : static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
82 : :
83 : : struct nvme_bdev_io {
84 : : /** array of iovecs to transfer. */
85 : : struct iovec *iovs;
86 : :
87 : : /** Number of iovecs in iovs array. */
88 : : int iovcnt;
89 : :
90 : : /** Current iovec position. */
91 : : int iovpos;
92 : :
93 : : /** Offset in current iovec. */
94 : : uint32_t iov_offset;
95 : :
96 : : /** Offset in current iovec. */
97 : : uint32_t fused_iov_offset;
98 : :
99 : : /** array of iovecs to transfer. */
100 : : struct iovec *fused_iovs;
101 : :
102 : : /** Number of iovecs in iovs array. */
103 : : int fused_iovcnt;
104 : :
105 : : /** Current iovec position. */
106 : : int fused_iovpos;
107 : :
108 : : /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
109 : : * being reset in a reset I/O.
110 : : */
111 : : struct nvme_io_path *io_path;
112 : :
113 : : /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
114 : : struct spdk_nvme_cpl cpl;
115 : :
116 : : /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
117 : : struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
118 : :
119 : : /** Keeps track if first of fused commands was submitted */
120 : : bool first_fused_submitted;
121 : :
122 : : /** Keeps track if first of fused commands was completed */
123 : : bool first_fused_completed;
124 : :
125 : : /* How many times the current I/O was retried. */
126 : : int32_t retry_count;
127 : :
128 : : /** Expiration value in ticks to retry the current I/O. */
129 : : uint64_t retry_ticks;
130 : :
131 : : /** Temporary pointer to zone report buffer */
132 : : struct spdk_nvme_zns_zone_report *zone_report_buf;
133 : :
134 : : /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
135 : : uint64_t handled_zones;
136 : :
137 : : /* Current tsc at submit time. */
138 : : uint64_t submit_tsc;
139 : :
140 : : /* Used to put nvme_bdev_io into the list */
141 : : TAILQ_ENTRY(nvme_bdev_io) retry_link;
142 : : };
143 : :
144 : : struct nvme_probe_skip_entry {
145 : : struct spdk_nvme_transport_id trid;
146 : : TAILQ_ENTRY(nvme_probe_skip_entry) tailq;
147 : : };
148 : : /* All the controllers deleted by users via RPC are skipped by hotplug monitor */
149 : : static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(
150 : : g_skipped_nvme_ctrlrs);
151 : :
152 : : #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \
153 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \
154 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512))
155 : :
156 : : #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \
157 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \
158 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \
159 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \
160 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \
161 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192))
162 : :
163 : : static struct spdk_bdev_nvme_opts g_opts = {
164 : : .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
165 : : .timeout_us = 0,
166 : : .timeout_admin_us = 0,
167 : : .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS,
168 : : .transport_retry_count = 4,
169 : : .arbitration_burst = 0,
170 : : .low_priority_weight = 0,
171 : : .medium_priority_weight = 0,
172 : : .high_priority_weight = 0,
173 : : .nvme_adminq_poll_period_us = 10000ULL,
174 : : .nvme_ioq_poll_period_us = 0,
175 : : .io_queue_requests = 0,
176 : : .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT,
177 : : .bdev_retry_count = 3,
178 : : .transport_ack_timeout = 0,
179 : : .ctrlr_loss_timeout_sec = 0,
180 : : .reconnect_delay_sec = 0,
181 : : .fast_io_fail_timeout_sec = 0,
182 : : .disable_auto_failback = false,
183 : : .generate_uuids = false,
184 : : .transport_tos = 0,
185 : : .nvme_error_stat = false,
186 : : .io_path_stat = false,
187 : : .allow_accel_sequence = false,
188 : : .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS,
189 : : .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS,
190 : : };
191 : :
192 : : #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL
193 : : #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL
194 : :
195 : : static int g_hot_insert_nvme_controller_index = 0;
196 : : static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
197 : : static bool g_nvme_hotplug_enabled = false;
198 : : struct spdk_thread *g_bdev_nvme_init_thread;
199 : : static struct spdk_poller *g_hotplug_poller;
200 : : static struct spdk_poller *g_hotplug_probe_poller;
201 : : static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
202 : :
203 : : static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
204 : : struct nvme_async_probe_ctx *ctx);
205 : : static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
206 : : struct nvme_async_probe_ctx *ctx);
207 : : static int bdev_nvme_library_init(void);
208 : : static void bdev_nvme_library_fini(void);
209 : : static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch,
210 : : struct spdk_bdev_io *bdev_io);
211 : : static void bdev_nvme_submit_request(struct spdk_io_channel *ch,
212 : : struct spdk_bdev_io *bdev_io);
213 : : static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
214 : : void *md, uint64_t lba_count, uint64_t lba,
215 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
216 : : struct spdk_accel_sequence *seq);
217 : : static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
218 : : void *md, uint64_t lba_count, uint64_t lba);
219 : : static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
220 : : void *md, uint64_t lba_count, uint64_t lba,
221 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
222 : : struct spdk_accel_sequence *seq,
223 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13);
224 : : static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
225 : : void *md, uint64_t lba_count,
226 : : uint64_t zslba, uint32_t flags);
227 : : static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
228 : : void *md, uint64_t lba_count, uint64_t lba,
229 : : uint32_t flags);
230 : : static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio,
231 : : struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov,
232 : : int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba,
233 : : uint32_t flags);
234 : : static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id,
235 : : uint32_t num_zones, struct spdk_bdev_zone_info *info);
236 : : static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
237 : : enum spdk_bdev_zone_action action);
238 : : static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch,
239 : : struct nvme_bdev_io *bio,
240 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes);
241 : : static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
242 : : void *buf, size_t nbytes);
243 : : static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
244 : : void *buf, size_t nbytes, void *md_buf, size_t md_len);
245 : : static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
246 : : struct iovec *iov, int iovcnt, size_t nbytes,
247 : : void *md_buf, size_t md_len);
248 : : static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch,
249 : : struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort);
250 : : static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio);
251 : : static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
252 : : static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
253 : : static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
254 : : static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
255 : :
256 : : static struct nvme_ns *nvme_ns_alloc(void);
257 : : static void nvme_ns_free(struct nvme_ns *ns);
258 : :
259 : : static int
260 : 1875 : nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
261 : : {
262 [ + + + - : 1875 : return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
+ - + - -
+ + - + -
+ - + - ]
263 : : }
264 : :
265 [ + + + + : 21850 : RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
+ + + + +
+ + + + +
+ + + + -
+ + - + -
+ + + + +
+ + - + -
+ - - - -
- - + + +
+ + + + -
+ - - - -
- - - - -
- - + # #
# # # # #
# + - + -
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
+ - # # #
# # # # #
# # # # -
+ # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
# # # # #
# # # - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
- + + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
266 : :
267 : : struct spdk_nvme_qpair *
268 : 3 : bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
269 : : {
270 : : struct nvme_ctrlr_channel *ctrlr_ch;
271 : :
272 [ - + # # ]: 3 : assert(ctrlr_io_ch != NULL);
273 : :
274 : 3 : ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch);
275 : :
276 [ # # # # : 3 : return ctrlr_ch->qpair->qpair;
# # # # ]
277 : : }
278 : :
279 : : static int
280 : 1940 : bdev_nvme_get_ctx_size(void)
281 : : {
282 : 1940 : return sizeof(struct nvme_bdev_io);
283 : : }
284 : :
285 : : static struct spdk_bdev_module nvme_if = {
286 : : .name = "nvme",
287 : : .async_fini = true,
288 : : .module_init = bdev_nvme_library_init,
289 : : .module_fini = bdev_nvme_library_fini,
290 : : .config_json = bdev_nvme_config_json,
291 : : .get_ctx_size = bdev_nvme_get_ctx_size,
292 : :
293 : : };
294 : 2127 : SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if)
295 : :
296 : : struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs);
297 : : pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
298 : : bool g_bdev_nvme_module_finish;
299 : :
300 : : struct nvme_bdev_ctrlr *
301 : 47911 : nvme_bdev_ctrlr_get_by_name(const char *name)
302 : : {
303 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
304 : :
305 [ + + # # : 49839 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
306 [ + + + + : 42605 : if (strcmp(name, nbdev_ctrlr->name) == 0) {
+ + + - -
+ ]
307 : 40677 : break;
308 : : }
309 : 0 : }
310 : :
311 : 47911 : return nbdev_ctrlr;
312 : : }
313 : :
314 : : static struct nvme_ctrlr *
315 : 672 : nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr,
316 : : const struct spdk_nvme_transport_id *trid, const char *hostnqn)
317 : : {
318 : : const struct spdk_nvme_ctrlr_opts *opts;
319 : : struct nvme_ctrlr *nvme_ctrlr;
320 : :
321 [ + + # # : 1293 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
322 [ # # # # ]: 720 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
323 [ + + # # : 720 : if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 &&
# # # # #
# ]
324 [ + + - + : 102 : strcmp(hostnqn, opts->hostnqn) == 0) {
+ + ]
325 : 99 : break;
326 : : }
327 : 0 : }
328 : :
329 : 672 : return nvme_ctrlr;
330 : : }
331 : :
332 : : struct nvme_ctrlr *
333 : 0 : nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr,
334 : : uint16_t cntlid)
335 : : {
336 : : struct nvme_ctrlr *nvme_ctrlr;
337 : : const struct spdk_nvme_ctrlr_data *cdata;
338 : :
339 [ # # # # : 0 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
340 [ # # # # ]: 0 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
341 [ # # # # : 0 : if (cdata->cntlid == cntlid) {
# # ]
342 : 0 : break;
343 : : }
344 : 0 : }
345 : :
346 : 0 : return nvme_ctrlr;
347 : : }
348 : :
349 : : static struct nvme_bdev *
350 : 1576 : nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid)
351 : : {
352 : : struct nvme_bdev *bdev;
353 : :
354 [ + + ]: 1576 : pthread_mutex_lock(&g_bdev_nvme_mutex);
355 [ + + + - : 1842 : TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) {
+ - + - #
# # # #
# ]
356 [ + + # # : 386 : if (bdev->nsid == nsid) {
# # ]
357 : 120 : break;
358 : : }
359 : 0 : }
360 [ + + ]: 1576 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
361 : :
362 : 1576 : return bdev;
363 : : }
364 : :
365 : : struct nvme_ns *
366 : 3241 : nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid)
367 : : {
368 : 1180 : struct nvme_ns ns;
369 : :
370 [ + + # # ]: 3241 : assert(nsid > 0);
371 : :
372 : 3241 : ns.id = nsid;
373 [ + - ]: 3241 : return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns);
374 : : }
375 : :
376 : : struct nvme_ns *
377 : 3582 : nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr)
378 : : {
379 [ + - ]: 3582 : return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces);
380 : : }
381 : :
382 : : struct nvme_ns *
383 : 1580 : nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
384 : : {
385 [ - + ]: 1580 : if (ns == NULL) {
386 : 0 : return NULL;
387 : : }
388 : :
389 : 1580 : return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
390 : 1 : }
391 : :
392 : : static struct nvme_ctrlr *
393 : 1714 : nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn)
394 : : {
395 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
396 : 1714 : struct nvme_ctrlr *nvme_ctrlr = NULL;
397 : :
398 [ + + ]: 1714 : pthread_mutex_lock(&g_bdev_nvme_mutex);
399 [ + + # # : 2269 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
400 : 555 : nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn);
401 [ - + ]: 555 : if (nvme_ctrlr != NULL) {
402 : 0 : break;
403 : : }
404 : 0 : }
405 [ + + ]: 1714 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
406 : :
407 : 1714 : return nvme_ctrlr;
408 : : }
409 : :
410 : : struct nvme_ctrlr *
411 : 3500 : nvme_ctrlr_get_by_name(const char *name)
412 : : {
413 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
414 : 3500 : struct nvme_ctrlr *nvme_ctrlr = NULL;
415 : :
416 [ + + ]: 3500 : if (name == NULL) {
417 : 0 : return NULL;
418 : : }
419 : :
420 [ + + ]: 3500 : pthread_mutex_lock(&g_bdev_nvme_mutex);
421 : 3500 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
422 [ + + ]: 3500 : if (nbdev_ctrlr != NULL) {
423 [ + - + - : 285 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
+ - ]
424 : 1 : }
425 [ + + ]: 3500 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
426 : :
427 : 3500 : return nvme_ctrlr;
428 : 3 : }
429 : :
430 : : void
431 : 763 : nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx)
432 : : {
433 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
434 : :
435 [ - + ]: 763 : pthread_mutex_lock(&g_bdev_nvme_mutex);
436 [ + + # # : 1481 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
437 [ # # # # ]: 718 : fn(nbdev_ctrlr, ctx);
438 : 0 : }
439 [ - + ]: 763 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
440 : 763 : }
441 : :
442 : : struct nvme_ctrlr_channel_iter {
443 : : nvme_ctrlr_for_each_channel_msg fn;
444 : : nvme_ctrlr_for_each_channel_done cpl;
445 : : struct spdk_io_channel_iter *i;
446 : : void *ctx;
447 : : };
448 : :
449 : : void
450 : 3118 : nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status)
451 : : {
452 [ # # # # ]: 3118 : spdk_for_each_channel_continue(iter->i, status);
453 : 3118 : }
454 : :
455 : : static void
456 : 3118 : nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i)
457 : : {
458 : 3118 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
459 : 3118 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
460 : 3118 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
461 : 3118 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
462 : :
463 [ # # # # ]: 3118 : iter->i = i;
464 [ # # # # : 3118 : iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx);
# # # # #
# # # ]
465 : 3118 : }
466 : :
467 : : static void
468 : 3004 : nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
469 : : {
470 : 3004 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
471 : 3004 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
472 : :
473 [ # # # # ]: 3004 : iter->i = i;
474 [ # # # # : 3004 : iter->cpl(nvme_ctrlr, iter->ctx, status);
# # # # #
# # # ]
475 : :
476 : 3004 : free(iter);
477 : 3004 : }
478 : :
479 : : void
480 : 3004 : nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr,
481 : : nvme_ctrlr_for_each_channel_msg fn, void *ctx,
482 : : nvme_ctrlr_for_each_channel_done cpl)
483 : : {
484 : : struct nvme_ctrlr_channel_iter *iter;
485 : :
486 [ + - + - ]: 3004 : assert(nvme_ctrlr != NULL && fn != NULL);
487 : :
488 : 3004 : iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter));
489 [ - + ]: 3004 : if (iter == NULL) {
490 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
491 [ # # ]: 0 : assert(false);
492 : : return;
493 : : }
494 : :
495 [ # # # # ]: 3004 : iter->fn = fn;
496 [ # # # # ]: 3004 : iter->cpl = cpl;
497 [ # # # # ]: 3004 : iter->ctx = ctx;
498 : :
499 : 3004 : spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg,
500 : 0 : iter, nvme_ctrlr_each_channel_cpl);
501 : 0 : }
502 : :
503 : : struct nvme_bdev_channel_iter {
504 : : nvme_bdev_for_each_channel_msg fn;
505 : : nvme_bdev_for_each_channel_done cpl;
506 : : struct spdk_io_channel_iter *i;
507 : : void *ctx;
508 : : };
509 : :
510 : : void
511 : 295 : nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status)
512 : : {
513 [ # # # # ]: 295 : spdk_for_each_channel_continue(iter->i, status);
514 : 295 : }
515 : :
516 : : static void
517 : 295 : nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i)
518 : : {
519 : 295 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
520 : 295 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
521 : 295 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
522 : 295 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
523 : :
524 [ # # # # ]: 295 : iter->i = i;
525 [ # # # # : 295 : iter->fn(iter, nbdev, nbdev_ch, iter->ctx);
# # # # #
# # # ]
526 : 295 : }
527 : :
528 : : static void
529 : 293 : nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
530 : : {
531 : 293 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
532 : 293 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
533 : :
534 [ # # # # ]: 293 : iter->i = i;
535 [ # # # # : 293 : iter->cpl(nbdev, iter->ctx, status);
# # # # #
# # # ]
536 : :
537 : 293 : free(iter);
538 : 293 : }
539 : :
540 : : void
541 : 293 : nvme_bdev_for_each_channel(struct nvme_bdev *nbdev,
542 : : nvme_bdev_for_each_channel_msg fn, void *ctx,
543 : : nvme_bdev_for_each_channel_done cpl)
544 : : {
545 : : struct nvme_bdev_channel_iter *iter;
546 : :
547 [ + - + - ]: 293 : assert(nbdev != NULL && fn != NULL);
548 : :
549 : 293 : iter = calloc(1, sizeof(struct nvme_bdev_channel_iter));
550 [ - + ]: 293 : if (iter == NULL) {
551 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
552 [ # # ]: 0 : assert(false);
553 : : return;
554 : : }
555 : :
556 [ # # # # ]: 293 : iter->fn = fn;
557 [ # # # # ]: 293 : iter->cpl = cpl;
558 [ # # # # ]: 293 : iter->ctx = ctx;
559 : :
560 : 293 : spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter,
561 : : nvme_bdev_each_channel_cpl);
562 : 0 : }
563 : :
564 : : void
565 : 2181 : nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w)
566 : : {
567 : : const char *trtype_str;
568 : : const char *adrfam_str;
569 : :
570 [ # # # # ]: 2181 : trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype);
571 [ + - ]: 2181 : if (trtype_str) {
572 : 2181 : spdk_json_write_named_string(w, "trtype", trtype_str);
573 : 0 : }
574 : :
575 [ # # # # ]: 2181 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
576 [ + + ]: 2181 : if (adrfam_str) {
577 : 1154 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
578 : 0 : }
579 : :
580 [ + - # # : 2181 : if (trid->traddr[0] != '\0') {
# # # # #
# ]
581 [ # # ]: 2181 : spdk_json_write_named_string(w, "traddr", trid->traddr);
582 : 0 : }
583 : :
584 [ + + # # : 2181 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
585 [ # # ]: 1154 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
586 : 0 : }
587 : :
588 [ + + # # : 2181 : if (trid->subnqn[0] != '\0') {
# # # # #
# ]
589 [ # # ]: 1154 : spdk_json_write_named_string(w, "subnqn", trid->subnqn);
590 : 0 : }
591 : 2181 : }
592 : :
593 : : static void
594 : 1714 : nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr,
595 : : struct nvme_ctrlr *nvme_ctrlr)
596 : : {
597 : 512 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name);
598 [ + + ]: 1714 : pthread_mutex_lock(&g_bdev_nvme_mutex);
599 : :
600 [ + + + - : 1714 : TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
601 [ + + + - : 1714 : if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) {
+ - - + ]
602 [ - + ]: 58 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
603 : :
604 : 58 : return;
605 : : }
606 [ + + + - : 1656 : TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
607 : :
608 [ + + ]: 1656 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
609 : :
610 [ + + + - : 1656 : assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs));
+ - + - #
# ]
611 : :
612 [ + - + - ]: 1656 : free(nbdev_ctrlr->name);
613 : 1656 : free(nbdev_ctrlr);
614 : 1 : }
615 : :
616 : : static void
617 : 1717 : _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
618 : : {
619 : : struct nvme_path_id *path_id, *tmp_path;
620 : : struct nvme_ns *ns, *tmp_ns;
621 : :
622 [ + - + - ]: 1717 : free(nvme_ctrlr->copied_ana_desc);
623 [ + - + - ]: 1717 : spdk_free(nvme_ctrlr->ana_log_page);
624 : :
625 [ + + + - : 1717 : if (nvme_ctrlr->opal_dev) {
+ - ]
626 [ # # # # ]: 32 : spdk_opal_dev_destruct(nvme_ctrlr->opal_dev);
627 [ # # # # ]: 32 : nvme_ctrlr->opal_dev = NULL;
628 : 0 : }
629 : :
630 [ + + + - : 1717 : if (nvme_ctrlr->nbdev_ctrlr) {
- + ]
631 [ + - + - ]: 1714 : nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr);
632 : 1 : }
633 : :
634 [ + + + - : 1717 : RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) {
- + ]
635 [ # # ]: 0 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
636 : 0 : nvme_ns_free(ns);
637 : 0 : }
638 : :
639 [ + + + - : 3442 : TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) {
+ - + + +
- + - + -
+ + ]
640 [ + + + - : 1725 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
641 : 1725 : free(path_id);
642 : 1 : }
643 : :
644 [ + + + - ]: 1717 : pthread_mutex_destroy(&nvme_ctrlr->mutex);
645 [ + - + - ]: 1717 : spdk_keyring_put_key(nvme_ctrlr->psk);
646 [ + - + - ]: 1717 : spdk_keyring_put_key(nvme_ctrlr->dhchap_key);
647 [ + - + - ]: 1717 : spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key);
648 : 1717 : free(nvme_ctrlr);
649 : :
650 [ + + ]: 1717 : pthread_mutex_lock(&g_bdev_nvme_mutex);
651 [ + + + + : 1717 : if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
+ + ]
652 [ + + ]: 541 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
653 : 541 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
654 : 541 : spdk_bdev_module_fini_done();
655 : 541 : return;
656 : : }
657 [ - + ]: 1176 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
658 : 1 : }
659 : :
660 : : static int
661 : 211916 : nvme_detach_poller(void *arg)
662 : : {
663 : 211916 : struct nvme_ctrlr *nvme_ctrlr = arg;
664 : : int rc;
665 : :
666 [ + - + - ]: 211916 : rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx);
667 [ + + ]: 211916 : if (rc != -EAGAIN) {
668 [ + - ]: 1717 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
669 : 1717 : _nvme_ctrlr_delete(nvme_ctrlr);
670 : 1 : }
671 : :
672 : 211916 : return SPDK_POLLER_BUSY;
673 : : }
674 : :
675 : : static void
676 : 1717 : nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
677 : : {
678 : : int rc;
679 : :
680 [ + - ]: 1717 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
681 : :
682 : : /* First, unregister the adminq poller, as the driver will poll adminq if necessary */
683 [ + - ]: 1717 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
684 : :
685 : : /* If we got here, the reset/detach poller cannot be active */
686 [ + + + - : 1717 : assert(nvme_ctrlr->reset_detach_poller == NULL);
+ - # # ]
687 [ + - + - ]: 1717 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller,
688 : : nvme_ctrlr, 1000);
689 [ + + + - : 1717 : if (nvme_ctrlr->reset_detach_poller == NULL) {
+ - ]
690 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to register detach poller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
691 : 0 : goto error;
692 : : }
693 : :
694 [ + - + - : 1717 : rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx);
+ - ]
695 [ - + ]: 1717 : if (rc != 0) {
696 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to detach the NVMe controller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
697 : 0 : goto error;
698 : : }
699 : :
700 : 1717 : return;
701 : 0 : error:
702 : : /* We don't have a good way to handle errors here, so just do what we can and delete the
703 : : * controller without detaching the underlying NVMe device.
704 : : */
705 [ # # ]: 0 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
706 : 0 : _nvme_ctrlr_delete(nvme_ctrlr);
707 : 1 : }
708 : :
709 : : static void
710 : 1714 : nvme_ctrlr_unregister_cb(void *io_device)
711 : : {
712 : 1714 : struct nvme_ctrlr *nvme_ctrlr = io_device;
713 : :
714 : 1714 : nvme_ctrlr_delete(nvme_ctrlr);
715 : 1714 : }
716 : :
717 : : static void
718 : 1714 : nvme_ctrlr_unregister(void *ctx)
719 : : {
720 : 1714 : struct nvme_ctrlr *nvme_ctrlr = ctx;
721 : :
722 : 1714 : spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);
723 : 1714 : }
724 : :
725 : : static bool
726 : 8034 : nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr)
727 : : {
728 [ + + + + ]: 8034 : if (!nvme_ctrlr->destruct) {
729 : 4741 : return false;
730 : : }
731 : :
732 [ + + + - : 3293 : if (nvme_ctrlr->ref > 0) {
+ + ]
733 : 1564 : return false;
734 : : }
735 : :
736 [ + + - + ]: 1729 : if (nvme_ctrlr->resetting) {
737 : 15 : return false;
738 : : }
739 : :
740 [ + + - + ]: 1714 : if (nvme_ctrlr->ana_log_page_updating) {
741 : 0 : return false;
742 : : }
743 : :
744 [ - + - + ]: 1714 : if (nvme_ctrlr->io_path_cache_clearing) {
745 : 0 : return false;
746 : : }
747 : :
748 : 1714 : return true;
749 : 3 : }
750 : :
751 : : static void
752 : 5683 : nvme_ctrlr_release(struct nvme_ctrlr *nvme_ctrlr)
753 : : {
754 [ + + + - ]: 5683 : pthread_mutex_lock(&nvme_ctrlr->mutex);
755 : 1535 : SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref);
756 : :
757 [ + + + - : 5683 : assert(nvme_ctrlr->ref > 0);
+ - # # ]
758 [ + - + - ]: 5683 : nvme_ctrlr->ref--;
759 : :
760 [ + + ]: 5683 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
761 [ + + + - ]: 3984 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
762 : 3984 : return;
763 : : }
764 : :
765 [ + + + - ]: 1699 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
766 : :
767 [ + - + - ]: 1699 : spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr);
768 : 3 : }
769 : :
770 : : static void
771 : 422782 : bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch)
772 : : {
773 [ + - + - ]: 422782 : nbdev_ch->current_io_path = NULL;
774 [ + - + - ]: 422782 : nbdev_ch->rr_counter = 0;
775 : 422782 : }
776 : :
777 : : static struct nvme_io_path *
778 : 24 : _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
779 : : {
780 : : struct nvme_io_path *io_path;
781 : :
782 [ + + # # : 48 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
783 [ + + # # : 45 : if (io_path->nvme_ns == nvme_ns) {
# # ]
784 : 21 : break;
785 : : }
786 : 0 : }
787 : :
788 : 24 : return io_path;
789 : : }
790 : :
791 : : static struct nvme_io_path *
792 : 2491 : nvme_io_path_alloc(void)
793 : : {
794 : : struct nvme_io_path *io_path;
795 : :
796 : 2491 : io_path = calloc(1, sizeof(*io_path));
797 [ + + ]: 2491 : if (io_path == NULL) {
798 : 0 : SPDK_ERRLOG("Failed to alloc io_path.\n");
799 : 0 : return NULL;
800 : : }
801 : :
802 [ + + + + : 2491 : if (g_opts.io_path_stat) {
+ - ]
803 [ # # # # ]: 0 : io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
804 [ # # # # : 0 : if (io_path->stat == NULL) {
# # ]
805 : 0 : free(io_path);
806 : 0 : SPDK_ERRLOG("Failed to alloc io_path stat.\n");
807 : 0 : return NULL;
808 : : }
809 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
810 : 0 : }
811 : :
812 : 2491 : return io_path;
813 : 1 : }
814 : :
815 : : static void
816 : 2491 : nvme_io_path_free(struct nvme_io_path *io_path)
817 : : {
818 [ + - + - ]: 2491 : free(io_path->stat);
819 : 2491 : free(io_path);
820 : 2491 : }
821 : :
822 : : static int
823 : 2491 : _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
824 : : {
825 : : struct nvme_io_path *io_path;
826 : : struct spdk_io_channel *ch;
827 : : struct nvme_ctrlr_channel *ctrlr_ch;
828 : : struct nvme_qpair *nvme_qpair;
829 : :
830 : 2491 : io_path = nvme_io_path_alloc();
831 [ + + ]: 2491 : if (io_path == NULL) {
832 : 0 : return -ENOMEM;
833 : : }
834 : :
835 [ + - + - ]: 2491 : io_path->nvme_ns = nvme_ns;
836 : :
837 [ + - + - ]: 2491 : ch = spdk_get_io_channel(nvme_ns->ctrlr);
838 [ + + ]: 2491 : if (ch == NULL) {
839 : 0 : nvme_io_path_free(io_path);
840 : 0 : SPDK_ERRLOG("Failed to alloc io_channel.\n");
841 : 0 : return -ENOMEM;
842 : : }
843 : :
844 : 2491 : ctrlr_ch = spdk_io_channel_get_ctx(ch);
845 : :
846 [ + - + - ]: 2491 : nvme_qpair = ctrlr_ch->qpair;
847 [ + + # # ]: 2491 : assert(nvme_qpair != NULL);
848 : :
849 [ + - + - ]: 2491 : io_path->qpair = nvme_qpair;
850 [ + - + - : 2491 : TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
851 : :
852 [ + - + - ]: 2491 : io_path->nbdev_ch = nbdev_ch;
853 [ + - + - : 2491 : STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
854 : :
855 : 2491 : bdev_nvme_clear_current_io_path(nbdev_ch);
856 : :
857 : 2491 : return 0;
858 : 1 : }
859 : :
860 : : static void
861 : 2491 : bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch,
862 : : struct nvme_io_path *io_path)
863 : : {
864 : : struct nvme_bdev_io *bio;
865 : :
866 [ + + + - : 2494 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
+ - + - #
# # # #
# ]
867 [ + - # # : 3 : if (bio->io_path == io_path) {
# # ]
868 [ # # # # ]: 3 : bio->io_path = NULL;
869 : 0 : }
870 : 0 : }
871 : 2491 : }
872 : :
873 : : static void
874 : 2491 : _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path)
875 : : {
876 : : struct spdk_io_channel *ch;
877 : : struct nvme_qpair *nvme_qpair;
878 : : struct nvme_ctrlr_channel *ctrlr_ch;
879 : : struct nvme_bdev *nbdev;
880 : :
881 : 2491 : nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch));
882 : :
883 : : /* Add the statistics to nvme_ns before this path is destroyed. */
884 [ + + + - ]: 2491 : pthread_mutex_lock(&nbdev->mutex);
885 [ + + + + : 2491 : if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) {
+ - + - +
- + - + -
- + # # #
# # # ]
886 [ # # # # : 0 : spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat);
# # # # #
# # # ]
887 : 0 : }
888 [ + + + - ]: 2491 : pthread_mutex_unlock(&nbdev->mutex);
889 : :
890 : 2491 : bdev_nvme_clear_current_io_path(nbdev_ch);
891 : 2491 : bdev_nvme_clear_retry_io_path(nbdev_ch, io_path);
892 : :
893 [ + + + + : 2491 : STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq);
+ + + + +
- + - + -
+ - + - +
- + - + -
+ - - + +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
894 [ + - + - ]: 2491 : io_path->nbdev_ch = NULL;
895 : :
896 [ + - + - ]: 2491 : nvme_qpair = io_path->qpair;
897 [ + + # # ]: 2491 : assert(nvme_qpair != NULL);
898 : :
899 [ + - + - ]: 2491 : ctrlr_ch = nvme_qpair->ctrlr_ch;
900 [ + + # # ]: 2491 : assert(ctrlr_ch != NULL);
901 : :
902 : 2491 : ch = spdk_io_channel_from_ctx(ctrlr_ch);
903 : 2491 : spdk_put_io_channel(ch);
904 : :
905 : : /* After an io_path is removed, I/Os submitted to it may complete and update statistics
906 : : * of the io_path. To avoid heap-use-after-free error from this case, do not free the
907 : : * io_path here but free the io_path when the associated qpair is freed. It is ensured
908 : : * that all I/Os submitted to the io_path are completed when the associated qpair is freed.
909 : : */
910 : 2491 : }
911 : :
912 : : static void
913 : 2446 : _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch)
914 : : {
915 : : struct nvme_io_path *io_path, *tmp_io_path;
916 : :
917 [ + + + - : 4931 : STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) {
+ - + + +
- + - + -
+ + ]
918 : 2485 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
919 : 1 : }
920 : 2446 : }
921 : :
922 : : static int
923 : 2446 : bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf)
924 : : {
925 : 2446 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
926 : 2446 : struct nvme_bdev *nbdev = io_device;
927 : : struct nvme_ns *nvme_ns;
928 : : int rc;
929 : :
930 [ + - + - : 2446 : STAILQ_INIT(&nbdev_ch->io_path_list);
+ - + - +
- + - + -
+ - ]
931 [ + - + - : 2446 : TAILQ_INIT(&nbdev_ch->retry_io_list);
+ - + - +
- + - + -
+ - ]
932 : :
933 [ + + + - ]: 2446 : pthread_mutex_lock(&nbdev->mutex);
934 : :
935 [ + - + - : 2446 : nbdev_ch->mp_policy = nbdev->mp_policy;
+ - + - ]
936 [ + - + - : 2446 : nbdev_ch->mp_selector = nbdev->mp_selector;
+ - + - ]
937 [ + - + - : 2446 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
+ - + - ]
938 : :
939 [ + + + - : 4931 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
940 : 2485 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
941 [ + + ]: 2485 : if (rc != 0) {
942 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
943 : :
944 : 0 : _bdev_nvme_delete_io_paths(nbdev_ch);
945 : 0 : return rc;
946 : : }
947 : 1 : }
948 [ + + + - ]: 2446 : pthread_mutex_unlock(&nbdev->mutex);
949 : :
950 : 2446 : return 0;
951 : 1 : }
952 : :
953 : : /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'.
954 : : * If cpl == NULL, complete the bdev_io with bdev status based on 'status'.
955 : : */
956 : : static inline void
957 : 45259513 : __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status,
958 : : const struct spdk_nvme_cpl *cpl)
959 : : {
960 [ + + + + : 45259513 : spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx,
+ - + - +
- + - + -
- + # # ]
961 : : (uintptr_t)bdev_io);
962 [ + + ]: 45259513 : if (cpl) {
963 [ + - + - : 42703138 : spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
+ - + - +
- + - + -
+ - ]
964 : 3 : } else {
965 : 2556375 : spdk_bdev_io_complete(bdev_io, status);
966 : : }
967 : 45259513 : }
968 : :
969 : : static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch);
970 : :
971 : : static void
972 : 2446 : bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf)
973 : : {
974 : 2446 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
975 : :
976 : 2446 : bdev_nvme_abort_retry_ios(nbdev_ch);
977 : 2446 : _bdev_nvme_delete_io_paths(nbdev_ch);
978 : 2446 : }
979 : :
980 : : static inline bool
981 : 45680331 : bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type)
982 : : {
983 [ + + ]: 45680331 : switch (io_type) {
984 : 90 : case SPDK_BDEV_IO_TYPE_RESET:
985 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
986 : : case SPDK_BDEV_IO_TYPE_ABORT:
987 : 90 : return true;
988 : 45680238 : default:
989 : 45680241 : break;
990 : : }
991 : :
992 : 45680241 : return false;
993 : 3 : }
994 : :
995 : : static inline bool
996 : 10118374 : nvme_ns_is_active(struct nvme_ns *nvme_ns)
997 : : {
998 [ + + + + : 10118374 : if (spdk_unlikely(nvme_ns->ana_state_updating)) {
+ - - + ]
999 : 3327 : return false;
1000 : : }
1001 : :
1002 [ + + + - : 10115047 : if (spdk_unlikely(nvme_ns->ns == NULL)) {
- + ]
1003 : 0 : return false;
1004 : : }
1005 : :
1006 : 10115047 : return true;
1007 : 1 : }
1008 : :
1009 : : static inline bool
1010 : 10118338 : nvme_ns_is_accessible(struct nvme_ns *nvme_ns)
1011 : : {
1012 [ + + ]: 10118338 : if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) {
1013 : 3327 : return false;
1014 : : }
1015 : :
1016 [ + + + - : 10115011 : switch (nvme_ns->ana_state) {
+ - ]
1017 : 9331450 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1018 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1019 : 9331451 : return true;
1020 : 783560 : default:
1021 : 783560 : break;
1022 : : }
1023 : :
1024 : 783560 : return false;
1025 : 1 : }
1026 : :
1027 : : static inline bool
1028 : 11322684 : nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair)
1029 : : {
1030 [ + + + - : 11322684 : if (spdk_unlikely(nvme_qpair->qpair == NULL)) {
- + ]
1031 : 1182894 : return false;
1032 : : }
1033 : :
1034 [ + + + - : 10139790 : if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
- + ]
1035 : : SPDK_NVME_QPAIR_FAILURE_NONE)) {
1036 : 3014 : return false;
1037 : : }
1038 : :
1039 [ + + + - : 10136776 : if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) {
+ - + - -
+ ]
1040 : 9905 : return false;
1041 : : }
1042 : :
1043 : 10126871 : return true;
1044 : 1 : }
1045 : :
1046 : : static inline bool
1047 : 10907006 : nvme_io_path_is_available(struct nvme_io_path *io_path)
1048 : : {
1049 [ + + + - : 10907006 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
- + ]
1050 : 789148 : return false;
1051 : : }
1052 : :
1053 [ + + + - : 10117858 : if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) {
- + ]
1054 : 786767 : return false;
1055 : : }
1056 : :
1057 : 9331091 : return true;
1058 : 1 : }
1059 : :
1060 : : static inline bool
1061 : 406665 : nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr)
1062 : : {
1063 [ + + # # ]: 406665 : if (nvme_ctrlr->destruct) {
1064 : 512 : return true;
1065 : : }
1066 : :
1067 [ + + # # ]: 406153 : if (nvme_ctrlr->fast_io_fail_timedout) {
1068 : 87016 : return true;
1069 : : }
1070 : :
1071 [ + + # # ]: 319137 : if (nvme_ctrlr->resetting) {
1072 [ + + # # : 259812 : if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {
# # # # ]
1073 : 1039 : return false;
1074 : : } else {
1075 : 258773 : return true;
1076 : : }
1077 : : }
1078 : :
1079 [ + + # # ]: 59325 : if (nvme_ctrlr->reconnect_is_delayed) {
1080 : 4102 : return false;
1081 : : }
1082 : :
1083 [ - + # # ]: 55223 : if (nvme_ctrlr->disabled) {
1084 : 0 : return true;
1085 : : }
1086 : :
1087 [ + + # # : 55223 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1088 : 52727 : return true;
1089 : : } else {
1090 : 2496 : return false;
1091 : : }
1092 : 0 : }
1093 : :
1094 : : static bool
1095 : 15123 : nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr)
1096 : : {
1097 [ + + # # ]: 15123 : if (nvme_ctrlr->destruct) {
1098 : 990 : return false;
1099 : : }
1100 : :
1101 [ + + # # : 14133 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1102 : 9 : return false;
1103 : : }
1104 : :
1105 [ + + - + : 14124 : if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) {
# # # # ]
1106 : 6388 : return false;
1107 : : }
1108 : :
1109 [ - + # # ]: 7736 : if (nvme_ctrlr->disabled) {
1110 : 0 : return false;
1111 : : }
1112 : :
1113 : 7736 : return true;
1114 : 0 : }
1115 : :
1116 : : /* Simulate circular linked list. */
1117 : : static inline struct nvme_io_path *
1118 : 6906445 : nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path)
1119 : : {
1120 : : struct nvme_io_path *next_path;
1121 : :
1122 [ + + ]: 6906445 : if (prev_path != NULL) {
1123 [ # # # # : 5318921 : next_path = STAILQ_NEXT(prev_path, stailq);
# # ]
1124 [ + + ]: 5318921 : if (next_path != NULL) {
1125 : 2577642 : return next_path;
1126 : : }
1127 : 0 : }
1128 : :
1129 [ + - + - : 4328803 : return STAILQ_FIRST(&nbdev_ch->io_path_list);
+ - ]
1130 : 1 : }
1131 : :
1132 : : static struct nvme_io_path *
1133 : 2746486 : _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1134 : : {
1135 : 2746486 : struct nvme_io_path *io_path, *start, *non_optimized = NULL;
1136 : :
1137 [ + - + - ]: 2746486 : start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path);
1138 : :
1139 : 2746486 : io_path = start;
1140 : 1 : do {
1141 [ + + ]: 4787041 : if (spdk_likely(nvme_io_path_is_available(io_path))) {
1142 [ + + + - : 3211258 : switch (io_path->nvme_ns->ana_state) {
+ - + - +
- - ]
1143 : 627081 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1144 [ + - + - ]: 627082 : nbdev_ch->current_io_path = io_path;
1145 : 627082 : return io_path;
1146 : 2584176 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1147 [ + + ]: 2584176 : if (non_optimized == NULL) {
1148 : 2012083 : non_optimized = io_path;
1149 : 0 : }
1150 : 2584176 : break;
1151 : 0 : default:
1152 [ # # ]: 0 : assert(false);
1153 : : break;
1154 : : }
1155 : 0 : }
1156 : 4159959 : io_path = nvme_io_path_get_next(nbdev_ch, io_path);
1157 [ + + ]: 4159959 : } while (io_path != start);
1158 : :
1159 [ + + # # : 2119404 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
1160 : : /* We come here only if there is no optimized path. Cache even non_optimized
1161 : : * path for load balance across multiple non_optimized paths.
1162 : : */
1163 [ # # # # ]: 534456 : nbdev_ch->current_io_path = non_optimized;
1164 : 0 : }
1165 : :
1166 : 2119404 : return non_optimized;
1167 : 1 : }
1168 : :
1169 : : static struct nvme_io_path *
1170 : 12 : _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch)
1171 : : {
1172 : : struct nvme_io_path *io_path;
1173 : 12 : struct nvme_io_path *optimized = NULL, *non_optimized = NULL;
1174 : 12 : uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX;
1175 : : uint32_t num_outstanding_reqs;
1176 : :
1177 [ + + # # : 48 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1178 [ - + # # : 36 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
# # ]
1179 : : /* The device is currently resetting. */
1180 : 0 : continue;
1181 : : }
1182 : :
1183 [ - + # # : 36 : if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) {
# # ]
1184 : 0 : continue;
1185 : : }
1186 : :
1187 [ # # # # : 36 : num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair);
# # # # ]
1188 [ + + + # : 36 : switch (io_path->nvme_ns->ana_state) {
# # # # #
# # ]
1189 : 18 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1190 [ + + ]: 18 : if (num_outstanding_reqs < opt_min_qd) {
1191 : 15 : opt_min_qd = num_outstanding_reqs;
1192 : 15 : optimized = io_path;
1193 : 0 : }
1194 : 18 : break;
1195 : 9 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1196 [ + - ]: 9 : if (num_outstanding_reqs < non_opt_min_qd) {
1197 : 9 : non_opt_min_qd = num_outstanding_reqs;
1198 : 9 : non_optimized = io_path;
1199 : 0 : }
1200 : 9 : break;
1201 : 9 : default:
1202 : 9 : break;
1203 : : }
1204 : 0 : }
1205 : :
1206 : : /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */
1207 [ + + ]: 12 : if (optimized != NULL) {
1208 : 9 : return optimized;
1209 : : }
1210 : :
1211 : 3 : return non_optimized;
1212 : 0 : }
1213 : :
1214 : : static inline struct nvme_io_path *
1215 : 45275755 : bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1216 : : {
1217 [ + + + - : 45275755 : if (spdk_likely(nbdev_ch->current_io_path != NULL)) {
+ + ]
1218 [ + + + - : 43688219 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) {
- + ]
1219 [ + - + - ]: 42529248 : return nbdev_ch->current_io_path;
1220 [ + - # # : 1158971 : } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
1221 [ + + # # : 1158971 : if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) {
# # # # ]
1222 [ # # # # ]: 9 : return nbdev_ch->current_io_path;
1223 : : }
1224 [ # # # # ]: 1158962 : nbdev_ch->rr_counter = 0;
1225 : 0 : }
1226 : 0 : }
1227 : :
1228 [ + + + - : 2746498 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE ||
- + # # ]
1229 [ + + # # ]: 1159030 : nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
1230 : 2746486 : return _bdev_nvme_find_io_path(nbdev_ch);
1231 : : } else {
1232 : 12 : return _bdev_nvme_find_io_path_min_qd(nbdev_ch);
1233 : : }
1234 : 3 : }
1235 : :
1236 : : /* Return true if there is any io_path whose qpair is active or ctrlr is not failed,
1237 : : * or false otherwise.
1238 : : *
1239 : : * If any io_path has an active qpair but find_io_path() returned NULL, its namespace
1240 : : * is likely to be non-accessible now but may become accessible.
1241 : : *
1242 : : * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr
1243 : : * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed
1244 : : * when starting to reset it but it is set to failed when the reset failed. Hence, if
1245 : : * a ctrlr is unfailed, it is likely that it works fine or is resetting.
1246 : : */
1247 : : static bool
1248 : 415165 : any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch)
1249 : : {
1250 : : struct nvme_io_path *io_path;
1251 : :
1252 [ + + + + : 415165 : if (nbdev_ch->resetting) {
# # # # ]
1253 : 3 : return false;
1254 : : }
1255 : :
1256 [ + + # # : 815214 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1257 [ - + + + : 416186 : if (io_path->nvme_ns->ana_transition_timedout) {
# # # # #
# # # ]
1258 : 1024 : continue;
1259 : : }
1260 : :
1261 [ + + # # : 415162 : if (nvme_qpair_is_connected(io_path->qpair) ||
# # # # ]
1262 [ + + # # : 406665 : !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) {
# # # # ]
1263 : 16134 : return true;
1264 : : }
1265 : 0 : }
1266 : :
1267 : 399028 : return false;
1268 : 0 : }
1269 : :
1270 : : static void
1271 : 19723 : bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
1272 : : {
1273 [ # # ]: 19723 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1274 : : struct spdk_io_channel *ch;
1275 : :
1276 [ + + + - : 19723 : if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) {
# # # # #
# # # ]
1277 : 3592 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
1278 : 0 : } else {
1279 : 16131 : ch = spdk_io_channel_from_ctx(nbdev_ch);
1280 : 16131 : bdev_nvme_submit_request(ch, bdev_io);
1281 : : }
1282 : 19723 : }
1283 : :
1284 : : static int
1285 : 4091 : bdev_nvme_retry_ios(void *arg)
1286 : : {
1287 : 4091 : struct nvme_bdev_channel *nbdev_ch = arg;
1288 : : struct nvme_bdev_io *bio, *tmp_bio;
1289 : : uint64_t now, delay_us;
1290 : :
1291 : 4091 : now = spdk_get_ticks();
1292 : :
1293 [ + + # # : 23814 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
# # # # #
# # # # #
# # ]
1294 [ + + # # : 20103 : if (bio->retry_ticks > now) {
# # ]
1295 : 380 : break;
1296 : : }
1297 : :
1298 [ + + # # : 19723 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1299 : :
1300 : 19723 : bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio));
1301 : 0 : }
1302 : :
1303 [ # # ]: 4091 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1304 : :
1305 [ # # # # : 4091 : bio = TAILQ_FIRST(&nbdev_ch->retry_io_list);
# # ]
1306 [ + + ]: 4091 : if (bio != NULL) {
1307 [ - + # # : 389 : delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz();
# # ]
1308 : :
1309 [ # # # # ]: 389 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1310 : : delay_us);
1311 : 0 : }
1312 : :
1313 : 4091 : return SPDK_POLLER_BUSY;
1314 : : }
1315 : :
1316 : : static void
1317 : 19729 : bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch,
1318 : : struct nvme_bdev_io *bio, uint64_t delay_ms)
1319 : : {
1320 : : struct nvme_bdev_io *tmp_bio;
1321 : :
1322 [ # # # # : 19729 : bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL;
# # ]
1323 : :
1324 [ + + # # : 48815 : TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) {
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1325 [ + + # # : 45082 : if (tmp_bio->retry_ticks <= bio->retry_ticks) {
# # # # #
# ]
1326 [ + + # # : 15996 : TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1327 : : retry_link);
1328 : 15996 : return;
1329 : : }
1330 : 0 : }
1331 : :
1332 : : /* No earlier I/Os were found. This I/O must be the new head. */
1333 [ + + # # : 3733 : TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1334 : :
1335 [ # # ]: 3733 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1336 : :
1337 [ # # # # ]: 3733 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1338 : : delay_ms * 1000ULL);
1339 : 0 : }
1340 : :
1341 : : static void
1342 : 2581 : bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch)
1343 : : {
1344 : : struct nvme_bdev_io *bio, *tmp_bio;
1345 : :
1346 [ + + + - : 2584 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
+ - + - #
# # # # #
+ - ]
1347 [ - + # # : 3 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1348 : 3 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1349 : 0 : }
1350 : :
1351 [ + - ]: 2581 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1352 : 2581 : }
1353 : :
1354 : : static int
1355 : 7477 : bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch,
1356 : : struct nvme_bdev_io *bio_to_abort)
1357 : : {
1358 : : struct nvme_bdev_io *bio;
1359 : :
1360 [ + + # # : 7477 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
# # # # #
# # # #
# ]
1361 [ + - ]: 3 : if (bio == bio_to_abort) {
1362 [ - + # # : 3 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1363 : 3 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1364 : 3 : return 0;
1365 : : }
1366 : 0 : }
1367 : :
1368 : 7474 : return -ENOENT;
1369 : 0 : }
1370 : :
1371 : : static void
1372 : 10178 : bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl)
1373 : : {
1374 : : struct nvme_bdev *nbdev;
1375 : : uint16_t sct, sc;
1376 : :
1377 [ + + - + : 10178 : assert(spdk_nvme_cpl_is_error(cpl));
# # # # #
# # # # #
# # # # ]
1378 : :
1379 [ # # # # : 10178 : nbdev = bdev_io->bdev->ctxt;
# # # # ]
1380 : :
1381 [ + + # # : 10178 : if (nbdev->err_stat == NULL) {
# # ]
1382 : 6649 : return;
1383 : : }
1384 : :
1385 [ # # # # : 3529 : sct = cpl->status.sct;
# # ]
1386 [ # # # # : 3529 : sc = cpl->status.sc;
# # ]
1387 : :
1388 [ - + # # ]: 3529 : pthread_mutex_lock(&nbdev->mutex);
1389 : :
1390 [ # # # # : 3529 : nbdev->err_stat->status_type[sct]++;
# # # # #
# ]
1391 [ + - ]: 3529 : switch (sct) {
1392 : 3529 : case SPDK_NVME_SCT_GENERIC:
1393 : : case SPDK_NVME_SCT_COMMAND_SPECIFIC:
1394 : : case SPDK_NVME_SCT_MEDIA_ERROR:
1395 : : case SPDK_NVME_SCT_PATH:
1396 [ # # # # : 3529 : nbdev->err_stat->status[sct][sc]++;
# # # # #
# # # ]
1397 : 3529 : break;
1398 : 0 : default:
1399 : 0 : break;
1400 : : }
1401 : :
1402 [ - + # # ]: 3529 : pthread_mutex_unlock(&nbdev->mutex);
1403 : 0 : }
1404 : :
1405 : : static inline void
1406 : 42702975 : bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio)
1407 : : {
1408 : 42702975 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1409 [ + - + - : 42702975 : uint64_t num_blocks = bdev_io->u.bdev.num_blocks;
+ - + - ]
1410 [ + - + - : 42702975 : uint32_t blocklen = bdev_io->bdev->blocklen;
+ - + - ]
1411 : : struct spdk_bdev_io_stat *stat;
1412 : : uint64_t tsc_diff;
1413 : :
1414 [ + - + - : 42702975 : if (bio->io_path->stat == NULL) {
+ - + - +
- ]
1415 : 42702975 : return;
1416 : : }
1417 : :
1418 [ # # # # ]: 0 : tsc_diff = spdk_get_ticks() - bio->submit_tsc;
1419 [ # # # # : 0 : stat = bio->io_path->stat;
# # # # ]
1420 : :
1421 [ # # # # : 0 : switch (bdev_io->type) {
# # # # #
# ]
1422 : 0 : case SPDK_BDEV_IO_TYPE_READ:
1423 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1424 [ # # ]: 0 : stat->num_read_ops++;
1425 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1426 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1427 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1428 : 0 : }
1429 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1430 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1431 : 0 : }
1432 : 0 : break;
1433 : 0 : case SPDK_BDEV_IO_TYPE_WRITE:
1434 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1435 [ # # ]: 0 : stat->num_write_ops++;
1436 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1437 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1438 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1439 : 0 : }
1440 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1441 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1442 : 0 : }
1443 : 0 : break;
1444 : 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
1445 [ # # # # ]: 0 : stat->bytes_unmapped += num_blocks * blocklen;
1446 [ # # ]: 0 : stat->num_unmap_ops++;
1447 [ # # # # ]: 0 : stat->unmap_latency_ticks += tsc_diff;
1448 [ # # # # : 0 : if (stat->max_unmap_latency_ticks < tsc_diff) {
# # ]
1449 [ # # # # ]: 0 : stat->max_unmap_latency_ticks = tsc_diff;
1450 : 0 : }
1451 [ # # # # : 0 : if (stat->min_unmap_latency_ticks > tsc_diff) {
# # ]
1452 [ # # # # ]: 0 : stat->min_unmap_latency_ticks = tsc_diff;
1453 : 0 : }
1454 : 0 : break;
1455 : 0 : case SPDK_BDEV_IO_TYPE_ZCOPY:
1456 : : /* Track the data in the start phase only */
1457 [ # # # # : 0 : if (!bdev_io->u.bdev.zcopy.start) {
# # # # #
# ]
1458 : 0 : break;
1459 : : }
1460 [ # # # # : 0 : if (bdev_io->u.bdev.zcopy.populate) {
# # # # #
# ]
1461 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1462 [ # # ]: 0 : stat->num_read_ops++;
1463 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1464 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1465 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1466 : 0 : }
1467 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1468 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1469 : 0 : }
1470 : 0 : } else {
1471 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1472 [ # # ]: 0 : stat->num_write_ops++;
1473 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1474 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1475 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1476 : 0 : }
1477 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1478 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1479 : 0 : }
1480 : : }
1481 : 0 : break;
1482 : 0 : case SPDK_BDEV_IO_TYPE_COPY:
1483 [ # # # # ]: 0 : stat->bytes_copied += num_blocks * blocklen;
1484 [ # # ]: 0 : stat->num_copy_ops++;
1485 [ # # # # ]: 0 : stat->copy_latency_ticks += tsc_diff;
1486 [ # # # # : 0 : if (stat->max_copy_latency_ticks < tsc_diff) {
# # ]
1487 [ # # # # ]: 0 : stat->max_copy_latency_ticks = tsc_diff;
1488 : 0 : }
1489 [ # # # # : 0 : if (stat->min_copy_latency_ticks > tsc_diff) {
# # ]
1490 [ # # # # ]: 0 : stat->min_copy_latency_ticks = tsc_diff;
1491 : 0 : }
1492 : 0 : break;
1493 : 0 : default:
1494 : 0 : break;
1495 : : }
1496 : 3 : }
1497 : :
1498 : : static bool
1499 : 10102 : bdev_nvme_check_retry_io(struct nvme_bdev_io *bio,
1500 : : const struct spdk_nvme_cpl *cpl,
1501 : : struct nvme_bdev_channel *nbdev_ch,
1502 : : uint64_t *_delay_ms)
1503 : : {
1504 [ # # # # ]: 10102 : struct nvme_io_path *io_path = bio->io_path;
1505 [ # # # # : 10102 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
1506 : : const struct spdk_nvme_ctrlr_data *cdata;
1507 : :
1508 [ + + # # : 10102 : if (spdk_nvme_cpl_is_path_error(cpl) ||
# # # # #
# ]
1509 [ + + + + : 7726 : spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
# # # # #
# # # #
# ]
1510 [ + - ]: 3595 : !nvme_io_path_is_available(io_path) ||
1511 [ - + ]: 3595 : !nvme_ctrlr_is_available(nvme_ctrlr)) {
1512 : 6507 : bdev_nvme_clear_current_io_path(nbdev_ch);
1513 [ # # # # ]: 6507 : bio->io_path = NULL;
1514 [ + + + - : 6507 : if (spdk_nvme_cpl_is_ana_error(cpl)) {
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1515 [ + + ]: 2373 : if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) {
1516 [ # # # # : 19 : io_path->nvme_ns->ana_state_updating = true;
# # # # ]
1517 : 0 : }
1518 : 0 : }
1519 [ - + ]: 6507 : if (!any_io_path_may_become_available(nbdev_ch)) {
1520 : 0 : return false;
1521 : : }
1522 [ # # ]: 6507 : *_delay_ms = 0;
1523 : 0 : } else {
1524 [ # # # # ]: 3595 : bio->retry_count++;
1525 : :
1526 [ # # # # ]: 3595 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
1527 : :
1528 [ + + # # : 3595 : if (cpl->status.crd != 0) {
# # # # ]
1529 [ # # # # : 3 : *_delay_ms = cdata->crdt[cpl->status.crd] * 100;
# # # # #
# # # # #
# # # # ]
1530 : 0 : } else {
1531 [ # # ]: 3592 : *_delay_ms = 0;
1532 : : }
1533 : : }
1534 : :
1535 : 10102 : return true;
1536 : 0 : }
1537 : :
1538 : : static inline void
1539 : 42713153 : bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio,
1540 : : const struct spdk_nvme_cpl *cpl)
1541 : : {
1542 : 42713153 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1543 : : struct nvme_bdev_channel *nbdev_ch;
1544 : 15339842 : uint64_t delay_ms;
1545 : :
1546 [ + + + - : 42713153 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
+ - # # ]
1547 : :
1548 [ + + + + : 42713153 : if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
+ - - + +
- + - + -
+ - ]
1549 : 42702975 : bdev_nvme_update_io_path_stat(bio);
1550 : 42702975 : goto complete;
1551 : : }
1552 : :
1553 : : /* Update error counts before deciding if retry is needed.
1554 : : * Hence, error counts may be more than the number of I/O errors.
1555 : : */
1556 : 10178 : bdev_nvme_update_nvme_error_stat(bdev_io, cpl);
1557 : :
1558 [ + + + + : 10178 : if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) ||
+ + # # #
# # # # #
# # # # #
# # # #
# ]
1559 [ + + + + : 10123 : (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) {
# # # # #
# ]
1560 : 76 : goto complete;
1561 : : }
1562 : :
1563 : : /* At this point we don't know whether the sequence was successfully executed or not, so we
1564 : : * cannot retry the IO */
1565 [ - + # # : 10102 : if (bdev_io->u.bdev.accel_sequence != NULL) {
# # # # #
# ]
1566 : 0 : goto complete;
1567 : : }
1568 : :
1569 : 10102 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1570 : :
1571 [ + - ]: 10102 : if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) {
1572 : 10102 : bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
1573 : 10102 : return;
1574 : : }
1575 : :
1576 : 15339821 : complete:
1577 [ + - + - ]: 42703051 : bio->retry_count = 0;
1578 [ + - + - ]: 42703051 : bio->submit_tsc = 0;
1579 [ + - + - : 42703051 : bdev_io->u.bdev.accel_sequence = NULL;
+ - + - ]
1580 : 42703051 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
1581 : 3 : }
1582 : :
1583 : : static inline void
1584 : 2558430 : bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc)
1585 : : {
1586 : 2558430 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1587 : : struct nvme_bdev_channel *nbdev_ch;
1588 : : enum spdk_bdev_io_status io_status;
1589 : :
1590 [ - + # # : 2558430 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
1591 : :
1592 [ + + + - ]: 2558430 : switch (rc) {
1593 : 1980028 : case 0:
1594 : 1980028 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1595 : 1980028 : break;
1596 : 169744 : case -ENOMEM:
1597 : 169744 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1598 : 169744 : break;
1599 : 408658 : case -ENXIO:
1600 [ + + + - : 408658 : if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) {
# # # # #
# # # ]
1601 : 408658 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1602 : :
1603 : 408658 : bdev_nvme_clear_current_io_path(nbdev_ch);
1604 [ # # # # ]: 408658 : bio->io_path = NULL;
1605 : :
1606 [ + + ]: 408658 : if (any_io_path_may_become_available(nbdev_ch)) {
1607 : 9627 : bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1608 : 9627 : return;
1609 : : }
1610 : 0 : }
1611 : :
1612 : : /* fallthrough */
1613 : : default:
1614 [ # # # # : 399031 : spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence);
# # # # ]
1615 [ # # # # : 399031 : bdev_io->u.bdev.accel_sequence = NULL;
# # # # ]
1616 : 399031 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1617 : 399031 : break;
1618 : : }
1619 : :
1620 [ # # # # ]: 2548803 : bio->retry_count = 0;
1621 [ # # # # ]: 2548803 : bio->submit_tsc = 0;
1622 : 2548803 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1623 : 0 : }
1624 : :
1625 : : static inline void
1626 : 12 : bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc)
1627 : : {
1628 : 12 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1629 : : enum spdk_bdev_io_status io_status;
1630 : :
1631 [ + - + # ]: 12 : switch (rc) {
1632 : 3 : case 0:
1633 : 3 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1634 : 3 : break;
1635 : 0 : case -ENOMEM:
1636 : 0 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1637 : 0 : break;
1638 : 9 : case -ENXIO:
1639 : : /* fallthrough */
1640 : : default:
1641 : 9 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1642 : 9 : break;
1643 : : }
1644 : :
1645 : 12 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1646 : 12 : }
1647 : :
1648 : : static void
1649 : 1568 : bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr,
1650 : : void *ctx, int status)
1651 : : {
1652 [ - + # # ]: 1568 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1653 : :
1654 [ - + # # : 1568 : assert(nvme_ctrlr->io_path_cache_clearing == true);
# # ]
1655 [ # # ]: 1568 : nvme_ctrlr->io_path_cache_clearing = false;
1656 : :
1657 [ + - ]: 1568 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1658 [ - + # # ]: 1568 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1659 : 1568 : return;
1660 : : }
1661 : :
1662 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1663 : :
1664 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
1665 : 0 : }
1666 : :
1667 : : static void
1668 : 10078 : _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair)
1669 : : {
1670 : : struct nvme_io_path *io_path;
1671 : :
1672 [ + + + - : 17635 : TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) {
+ - + + +
- + - +
- ]
1673 [ + + + - : 7557 : if (io_path->nbdev_ch == NULL) {
- + ]
1674 : 4935 : continue;
1675 : : }
1676 [ # # # # ]: 2622 : bdev_nvme_clear_current_io_path(io_path->nbdev_ch);
1677 : 0 : }
1678 : 10078 : }
1679 : :
1680 : : static void
1681 : 1521 : bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i,
1682 : : struct nvme_ctrlr *nvme_ctrlr,
1683 : : struct nvme_ctrlr_channel *ctrlr_ch,
1684 : : void *ctx)
1685 : : {
1686 [ - + # # : 1521 : assert(ctrlr_ch->qpair != NULL);
# # # # ]
1687 : :
1688 [ # # # # ]: 1521 : _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair);
1689 : :
1690 : 1521 : nvme_ctrlr_for_each_channel_continue(i, 0);
1691 : 1521 : }
1692 : :
1693 : : static void
1694 : 8944 : bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr)
1695 : : {
1696 [ - + # # ]: 8944 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1697 [ + + + + ]: 8944 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
1698 [ # # ]: 0 : nvme_ctrlr->io_path_cache_clearing) {
1699 [ - + # # ]: 7376 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1700 : 7376 : return;
1701 : : }
1702 : :
1703 [ # # ]: 1568 : nvme_ctrlr->io_path_cache_clearing = true;
1704 [ - + # # ]: 1568 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1705 : :
1706 : 1568 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
1707 : : bdev_nvme_clear_io_path_cache,
1708 : : NULL,
1709 : : bdev_nvme_clear_io_path_caches_done);
1710 : 0 : }
1711 : :
1712 : : static struct nvme_qpair *
1713 : 2678 : nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
1714 : : {
1715 : : struct nvme_qpair *nvme_qpair;
1716 : :
1717 [ + - + - : 2834 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
+ - - + #
# # # #
# ]
1718 [ + + + - : 2834 : if (nvme_qpair->qpair == qpair) {
- + ]
1719 : 2678 : break;
1720 : : }
1721 : 0 : }
1722 : :
1723 : 2678 : return nvme_qpair;
1724 : : }
1725 : :
1726 : : static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair);
1727 : :
1728 : : static void
1729 : 2678 : bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
1730 : : {
1731 : 2678 : struct nvme_poll_group *group = poll_group_ctx;
1732 : : struct nvme_qpair *nvme_qpair;
1733 : : struct nvme_ctrlr *nvme_ctrlr;
1734 : : struct nvme_ctrlr_channel *ctrlr_ch;
1735 : : int status;
1736 : :
1737 : 2678 : nvme_qpair = nvme_poll_group_get_qpair(group, qpair);
1738 [ + + ]: 2678 : if (nvme_qpair == NULL) {
1739 : 0 : return;
1740 : : }
1741 : :
1742 [ + - + - : 2678 : if (nvme_qpair->qpair != NULL) {
- + ]
1743 [ + - + - ]: 2678 : spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair);
1744 [ + - + - ]: 2678 : nvme_qpair->qpair = NULL;
1745 : 1 : }
1746 : :
1747 : 2678 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1748 : :
1749 [ + - + - ]: 2678 : nvme_ctrlr = nvme_qpair->ctrlr;
1750 [ + - + - ]: 2678 : ctrlr_ch = nvme_qpair->ctrlr_ch;
1751 : :
1752 [ + + ]: 2678 : if (ctrlr_ch != NULL) {
1753 [ + + # # : 310 : if (ctrlr_ch->reset_iter != NULL) {
# # ]
1754 : : /* We are in a full reset sequence. */
1755 [ - + # # : 253 : if (ctrlr_ch->connect_poller != NULL) {
# # ]
1756 : : /* qpair was failed to connect. Abort the reset sequence. */
1757 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1758 : : "qpair %p was failed to connect. abort the reset ctrlr sequence.\n",
1759 : : qpair);
1760 [ # # ]: 0 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
1761 : 0 : status = -1;
1762 : 0 : } else {
1763 : : /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */
1764 [ - + - + : 253 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1765 : : "qpair %p was disconnected and freed in a reset ctrlr sequence.\n",
1766 : : qpair);
1767 : 253 : status = 0;
1768 : : }
1769 [ # # # # ]: 253 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status);
1770 [ # # # # ]: 253 : ctrlr_ch->reset_iter = NULL;
1771 : 0 : } else {
1772 : : /* qpair was disconnected unexpectedly. Reset controller for recovery. */
1773 [ - + - + : 57 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. reset controller.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1774 : : qpair);
1775 : 57 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1776 : : }
1777 : 0 : } else {
1778 : : /* In this case, ctrlr_channel is already deleted. */
1779 [ + + + + : 2368 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. delete nvme_qpair.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1780 : : qpair);
1781 : 2368 : nvme_qpair_delete(nvme_qpair);
1782 : : }
1783 : 1 : }
1784 : :
1785 : : static void
1786 : 14 : bdev_nvme_check_io_qpairs(struct nvme_poll_group *group)
1787 : : {
1788 : : struct nvme_qpair *nvme_qpair;
1789 : :
1790 [ + + # # : 28 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
# # # # #
# # # #
# ]
1791 [ + - - + : 14 : if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) {
# # # # #
# # # ]
1792 : 0 : continue;
1793 : : }
1794 : :
1795 [ + - # # : 14 : if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
# # ]
1796 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1797 : 14 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1798 : 0 : }
1799 : 0 : }
1800 : 14 : }
1801 : :
1802 : : static int
1803 : 1138582230 : bdev_nvme_poll(void *arg)
1804 : : {
1805 : 1138582230 : struct nvme_poll_group *group = arg;
1806 : : int64_t num_completions;
1807 : :
1808 [ + + + + : 1138582230 : if (group->collect_spin_stat && group->start_ticks == 0) {
+ - - + #
# # # #
# ]
1809 [ # # # # ]: 0 : group->start_ticks = spdk_get_ticks();
1810 : 0 : }
1811 : :
1812 [ + - + - ]: 1138582230 : num_completions = spdk_nvme_poll_group_process_completions(group->group, 0,
1813 : : bdev_nvme_disconnected_qpair_cb);
1814 [ + + + + : 1138582230 : if (group->collect_spin_stat) {
+ - + - ]
1815 [ # # ]: 0 : if (num_completions > 0) {
1816 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
1817 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
1818 [ # # # # ]: 0 : group->end_ticks = 0;
1819 : 0 : }
1820 [ # # # # ]: 0 : group->start_ticks = 0;
1821 : 0 : } else {
1822 [ # # # # ]: 0 : group->end_ticks = spdk_get_ticks();
1823 : : }
1824 : 0 : }
1825 : :
1826 [ + + ]: 1138582230 : if (spdk_unlikely(num_completions < 0)) {
1827 : 14 : bdev_nvme_check_io_qpairs(group);
1828 : 0 : }
1829 : :
1830 : 1138582230 : return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
1831 : : }
1832 : :
1833 : : static int bdev_nvme_poll_adminq(void *arg);
1834 : :
1835 : : static void
1836 : 1338 : bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us)
1837 : : {
1838 [ # # ]: 1338 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
1839 : :
1840 [ # # # # ]: 1338 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq,
1841 : : nvme_ctrlr, new_period_us);
1842 : 1338 : }
1843 : :
1844 : : static int
1845 : 927307 : bdev_nvme_poll_adminq(void *arg)
1846 : : {
1847 : : int32_t rc;
1848 : 927307 : struct nvme_ctrlr *nvme_ctrlr = arg;
1849 : : nvme_ctrlr_disconnected_cb disconnected_cb;
1850 : :
1851 [ + + # # ]: 927307 : assert(nvme_ctrlr != NULL);
1852 : :
1853 [ + - + - ]: 927307 : rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr);
1854 [ + + ]: 927307 : if (rc < 0) {
1855 [ # # # # ]: 1486 : disconnected_cb = nvme_ctrlr->disconnected_cb;
1856 [ # # # # ]: 1486 : nvme_ctrlr->disconnected_cb = NULL;
1857 : :
1858 [ + + ]: 1486 : if (disconnected_cb != NULL) {
1859 : 669 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr,
1860 [ # # ]: 0 : g_opts.nvme_adminq_poll_period_us);
1861 [ # # # # ]: 669 : disconnected_cb(nvme_ctrlr);
1862 : 0 : } else {
1863 : 817 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1864 : : }
1865 [ + + + - : 925821 : } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) !=
+ - ]
1866 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1867 : 8830 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
1868 : 0 : }
1869 : :
1870 : 927307 : return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
1871 : : }
1872 : :
1873 : : static void
1874 : 1453 : nvme_bdev_free(void *io_device)
1875 : : {
1876 : 1453 : struct nvme_bdev *nvme_disk = io_device;
1877 : :
1878 [ + + + - ]: 1453 : pthread_mutex_destroy(&nvme_disk->mutex);
1879 [ + - + - : 1453 : free(nvme_disk->disk.name);
+ - ]
1880 [ + - + - ]: 1453 : free(nvme_disk->err_stat);
1881 : 1453 : free(nvme_disk);
1882 : 1453 : }
1883 : :
1884 : : static int
1885 : 1448 : bdev_nvme_destruct(void *ctx)
1886 : : {
1887 : 1448 : struct nvme_bdev *nvme_disk = ctx;
1888 : : struct nvme_ns *nvme_ns, *tmp_nvme_ns;
1889 : :
1890 : 390 : SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid);
1891 : :
1892 [ + + + - : 2905 : TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) {
+ - + + +
- + - + -
+ + ]
1893 [ + + + - : 1457 : pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1894 : :
1895 [ + - + - ]: 1457 : nvme_ns->bdev = NULL;
1896 : :
1897 [ + + + - : 1457 : assert(nvme_ns->id > 0);
+ - # # ]
1898 : :
1899 [ + + + - : 1457 : if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) {
+ - + - +
- ]
1900 [ - + # # : 523 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
# # # # ]
1901 : :
1902 [ # # # # ]: 523 : nvme_ctrlr_release(nvme_ns->ctrlr);
1903 : 523 : nvme_ns_free(nvme_ns);
1904 : 0 : } else {
1905 [ + + + - : 934 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1906 : : }
1907 : 1 : }
1908 : :
1909 [ + + ]: 1448 : pthread_mutex_lock(&g_bdev_nvme_mutex);
1910 [ + + + - : 1448 : TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
1911 [ + + ]: 1448 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
1912 : :
1913 : 1448 : spdk_io_device_unregister(nvme_disk, nvme_bdev_free);
1914 : :
1915 : 1448 : return 0;
1916 : : }
1917 : :
1918 : : static int
1919 : 2681 : bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair)
1920 : : {
1921 : : struct nvme_ctrlr *nvme_ctrlr;
1922 : 1084 : struct spdk_nvme_io_qpair_opts opts;
1923 : : struct spdk_nvme_qpair *qpair;
1924 : : int rc;
1925 : :
1926 [ + - + - ]: 2681 : nvme_ctrlr = nvme_qpair->ctrlr;
1927 : :
1928 [ + - + - ]: 2681 : spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1929 [ + + + - : 2681 : opts.delay_cmd_submit = g_opts.delay_cmd_submit;
+ - + - ]
1930 [ + - ]: 2681 : opts.create_only = true;
1931 : 2681 : opts.async_mode = true;
1932 [ + - + - : 2681 : opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
- + # # +
- + - ]
1933 [ + - + - ]: 2681 : g_opts.io_queue_requests = opts.io_queue_requests;
1934 : :
1935 [ + - + - ]: 2681 : qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1936 [ + + ]: 2681 : if (qpair == NULL) {
1937 : 0 : return -1;
1938 : : }
1939 : :
1940 : 623 : SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name,
1941 : : spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread));
1942 : :
1943 [ + + + - : 2681 : assert(nvme_qpair->group != NULL);
+ - # # ]
1944 : :
1945 [ + - + - : 2681 : rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair);
+ - + - ]
1946 [ - + ]: 2681 : if (rc != 0) {
1947 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to begin polling on NVMe Channel.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1948 : 0 : goto err;
1949 : : }
1950 : :
1951 [ + - + - ]: 2681 : rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair);
1952 [ - + ]: 2681 : if (rc != 0) {
1953 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to connect I/O qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1954 : 0 : goto err;
1955 : : }
1956 : :
1957 [ + - + - ]: 2681 : nvme_qpair->qpair = qpair;
1958 : :
1959 [ + + + + : 2681 : if (!g_opts.disable_auto_failback) {
- + ]
1960 : 2582 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1961 : 1 : }
1962 : :
1963 [ + + + + : 2681 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Connecting qpair %p:%u started.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1964 : : qpair, spdk_nvme_qpair_get_id(qpair));
1965 : :
1966 : 2681 : return 0;
1967 : :
1968 : 0 : err:
1969 : 0 : spdk_nvme_ctrlr_free_io_qpair(qpair);
1970 : :
1971 : 0 : return rc;
1972 : 1 : }
1973 : :
1974 : : static void bdev_nvme_reset_io_continue(void *cb_arg, int rc);
1975 : :
1976 : : static void
1977 : 708 : bdev_nvme_complete_pending_resets(struct nvme_ctrlr_channel_iter *i,
1978 : : struct nvme_ctrlr *nvme_ctrlr,
1979 : : struct nvme_ctrlr_channel *ctrlr_ch,
1980 : : void *ctx)
1981 : : {
1982 : 708 : int rc = 0;
1983 : : struct nvme_bdev_io *bio;
1984 : :
1985 [ + + ]: 708 : if (ctx != NULL) {
1986 : 454 : rc = -1;
1987 : 0 : }
1988 : :
1989 [ + + # # : 741 : while (!TAILQ_EMPTY(&ctrlr_ch->pending_resets)) {
# # # # ]
1990 [ # # # # : 33 : bio = TAILQ_FIRST(&ctrlr_ch->pending_resets);
# # ]
1991 [ - + # # : 33 : TAILQ_REMOVE(&ctrlr_ch->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1992 : :
1993 : 33 : bdev_nvme_reset_io_continue(bio, rc);
1994 : : }
1995 : :
1996 : 708 : nvme_ctrlr_for_each_channel_continue(i, 0);
1997 : 708 : }
1998 : :
1999 : : /* This function marks the current trid as failed by storing the current ticks
2000 : : * and then sets the next trid to the active trid within a controller if exists.
2001 : : *
2002 : : * The purpose of the boolean return value is to request the caller to disconnect
2003 : : * the current trid now to try connecting the next trid.
2004 : : */
2005 : : static bool
2006 : 929 : bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start)
2007 : : {
2008 : : struct nvme_path_id *path_id, *next_path;
2009 : : int rc __attribute__((unused));
2010 : :
2011 [ # # # # : 929 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
2012 [ - + # # ]: 929 : assert(path_id);
2013 [ - + # # : 929 : assert(path_id == nvme_ctrlr->active_path_id);
# # # # ]
2014 [ # # # # : 929 : next_path = TAILQ_NEXT(path_id, link);
# # ]
2015 : :
2016 : : /* Update the last failed time. It means the trid is failed if its last
2017 : : * failed time is non-zero.
2018 : : */
2019 [ # # # # ]: 929 : path_id->last_failed_tsc = spdk_get_ticks();
2020 : :
2021 [ + + ]: 929 : if (next_path == NULL) {
2022 : : /* There is no alternate trid within a controller. */
2023 : 880 : return false;
2024 : : }
2025 : :
2026 [ + + + + : 49 : if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2027 : : /* Connect is not retried in a controller reset sequence. Connecting
2028 : : * the next trid will be done by the next bdev_nvme_failover_ctrlr() call.
2029 : : */
2030 : 9 : return false;
2031 : : }
2032 : :
2033 [ - + # # : 40 : assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE);
# # # # #
# ]
2034 : :
2035 [ + - # # : 40 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Start failover from %s:%s to %s:%s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2036 : : path_id->trid.traddr, path_id->trid.trsvcid,
2037 : : next_path->trid.traddr, next_path->trid.trsvcid);
2038 : :
2039 [ # # # # ]: 40 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2040 [ # # # # ]: 40 : nvme_ctrlr->active_path_id = next_path;
2041 [ # # # # : 40 : rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid);
# # ]
2042 [ - + # # ]: 40 : assert(rc == 0);
2043 [ + - # # : 40 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2044 [ + + # # ]: 40 : if (!remove) {
2045 : : /** Shuffle the old trid to the end of the list and use the new one.
2046 : : * Allows for round robin through multiple connections.
2047 : : */
2048 [ # # # # : 30 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2049 : 0 : } else {
2050 : 10 : free(path_id);
2051 : : }
2052 : :
2053 [ + + + + : 40 : if (start || next_path->last_failed_tsc == 0) {
# # # # #
# ]
2054 : : /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed
2055 : : * or used yet. Try the next trid now.
2056 : : */
2057 : 37 : return true;
2058 : : }
2059 : :
2060 [ # # # # : 3 : if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() *
# # ]
2061 [ - + # # : 3 : nvme_ctrlr->opts.reconnect_delay_sec) {
# # ]
2062 : : /* Enough backoff passed since the next trid failed. Try the next trid now. */
2063 : 0 : return true;
2064 : : }
2065 : :
2066 : : /* The next trid will be tried after reconnect_delay_sec seconds. */
2067 : 3 : return false;
2068 : 0 : }
2069 : :
2070 : : static bool
2071 : 681803 : bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr)
2072 : : {
2073 : : int32_t elapsed;
2074 : :
2075 [ + + # # : 681803 : if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||
# # # # #
# ]
2076 [ + + # # : 216743 : nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {
# # ]
2077 : 469697 : return false;
2078 : : }
2079 : :
2080 [ - + # # : 212106 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2081 [ + + # # : 212106 : if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {
# # # # ]
2082 : 64 : return true;
2083 : : } else {
2084 : 212042 : return false;
2085 : : }
2086 : 0 : }
2087 : :
2088 : : static bool
2089 : 81 : bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr)
2090 : : {
2091 : : uint32_t elapsed;
2092 : :
2093 [ + + # # : 81 : if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {
# # # # ]
2094 : 54 : return false;
2095 : : }
2096 : :
2097 [ - + # # : 27 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2098 [ + + # # : 27 : if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {
# # # # ]
2099 : 13 : return true;
2100 : : } else {
2101 : 14 : return false;
2102 : : }
2103 : 0 : }
2104 : :
2105 : : static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success);
2106 : :
2107 : : static void
2108 : 672 : nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn)
2109 : : {
2110 : : int rc;
2111 : :
2112 [ - + + + : 672 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2113 : :
2114 [ # # # # ]: 672 : rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
2115 [ + + ]: 672 : if (rc != 0) {
2116 [ + - # # : 3 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "disconnecting ctrlr failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2117 : :
2118 : : /* Disconnect fails if ctrlr is already resetting or removed. In this case,
2119 : : * fail the reset sequence immediately.
2120 : : */
2121 : 3 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2122 : 3 : return;
2123 : : }
2124 : :
2125 : : /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
2126 : : * Set callback here to execute the specified operation after ctrlr is really disconnected.
2127 : : */
2128 [ - + # # : 669 : assert(nvme_ctrlr->disconnected_cb == NULL);
# # # # ]
2129 [ # # # # ]: 669 : nvme_ctrlr->disconnected_cb = cb_fn;
2130 : :
2131 : : /* During disconnection, reduce the period to poll adminq more often. */
2132 : 669 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0);
2133 : 0 : }
2134 : :
2135 : : enum bdev_nvme_op_after_reset {
2136 : : OP_NONE,
2137 : : OP_COMPLETE_PENDING_DESTRUCT,
2138 : : OP_DESTRUCT,
2139 : : OP_DELAYED_RECONNECT,
2140 : : OP_FAILOVER,
2141 : : };
2142 : :
2143 : : typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset;
2144 : :
2145 : : static _bdev_nvme_op_after_reset
2146 : 669 : bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success)
2147 : : {
2148 [ + + ]: 669 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
2149 : : /* Complete pending destruct after reset completes. */
2150 : 15 : return OP_COMPLETE_PENDING_DESTRUCT;
2151 [ + + # # ]: 654 : } else if (nvme_ctrlr->pending_failover) {
2152 [ # # ]: 9 : nvme_ctrlr->pending_failover = false;
2153 [ # # # # ]: 9 : nvme_ctrlr->reset_start_tsc = 0;
2154 : 9 : return OP_FAILOVER;
2155 [ + + + + : 645 : } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2156 [ # # # # ]: 535 : nvme_ctrlr->reset_start_tsc = 0;
2157 : 535 : return OP_NONE;
2158 [ + + ]: 110 : } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2159 : 29 : return OP_DESTRUCT;
2160 : : } else {
2161 [ + + ]: 81 : if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) {
2162 [ # # ]: 13 : nvme_ctrlr->fast_io_fail_timedout = true;
2163 : 0 : }
2164 : 81 : return OP_DELAYED_RECONNECT;
2165 : : }
2166 : 0 : }
2167 : :
2168 : : static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug);
2169 : : static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
2170 : :
2171 : : static int
2172 : 72 : bdev_nvme_reconnect_delay_timer_expired(void *ctx)
2173 : : {
2174 : 72 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2175 : :
2176 : 36 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name);
2177 [ - + # # ]: 72 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2178 : :
2179 [ # # ]: 72 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2180 : :
2181 [ - + # # ]: 72 : if (!nvme_ctrlr->reconnect_is_delayed) {
2182 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2183 : 0 : return SPDK_POLLER_BUSY;
2184 : : }
2185 : :
2186 [ # # ]: 72 : nvme_ctrlr->reconnect_is_delayed = false;
2187 : :
2188 [ - + # # ]: 72 : if (nvme_ctrlr->destruct) {
2189 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2190 : 0 : return SPDK_POLLER_BUSY;
2191 : : }
2192 : :
2193 [ - + # # : 72 : assert(nvme_ctrlr->resetting == false);
# # ]
2194 [ # # ]: 72 : nvme_ctrlr->resetting = true;
2195 : :
2196 [ - + # # ]: 72 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2197 : :
2198 [ # # # # ]: 72 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2199 : :
2200 : 72 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2201 : 72 : return SPDK_POLLER_BUSY;
2202 : 0 : }
2203 : :
2204 : : static void
2205 : 81 : bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr)
2206 : : {
2207 [ # # # # ]: 81 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2208 : :
2209 [ - + # # : 81 : assert(nvme_ctrlr->reconnect_is_delayed == false);
# # ]
2210 [ # # ]: 81 : nvme_ctrlr->reconnect_is_delayed = true;
2211 : :
2212 [ - + # # : 81 : assert(nvme_ctrlr->reconnect_delay_timer == NULL);
# # # # ]
2213 [ # # # # : 81 : nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,
# # # # #
# ]
2214 : : nvme_ctrlr,
2215 : : nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);
2216 : 81 : }
2217 : :
2218 : : static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr);
2219 : :
2220 : : static void
2221 : 663 : _bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2222 : : {
2223 : 663 : bool success = (ctx == NULL);
2224 [ # # # # ]: 663 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2225 [ # # # # ]: 663 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2226 : : enum bdev_nvme_op_after_reset op_after_reset;
2227 : :
2228 [ - + # # : 663 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2229 : :
2230 [ # # # # ]: 663 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2231 [ # # # # ]: 663 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2232 : :
2233 [ + + # # ]: 663 : if (!success) {
2234 [ + - # # : 481 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Resetting controller failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2235 : 0 : } else {
2236 [ + + # # : 182 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Resetting controller successful.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2237 : : }
2238 : :
2239 [ - + # # ]: 663 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2240 [ # # ]: 663 : nvme_ctrlr->resetting = false;
2241 [ # # ]: 663 : nvme_ctrlr->dont_retry = false;
2242 [ # # ]: 663 : nvme_ctrlr->in_failover = false;
2243 : :
2244 [ # # ]: 663 : op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success);
2245 [ - + # # ]: 663 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2246 : :
2247 : : /* Delay callbacks when the next operation is a failover. */
2248 [ + + + - ]: 663 : if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) {
2249 [ + + # # : 101 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1);
# # ]
2250 : 0 : }
2251 : :
2252 [ + + + + : 663 : switch (op_after_reset) {
+ ]
2253 : 15 : case OP_COMPLETE_PENDING_DESTRUCT:
2254 : 15 : nvme_ctrlr_unregister(nvme_ctrlr);
2255 : 15 : break;
2256 : 29 : case OP_DESTRUCT:
2257 : 29 : bdev_nvme_delete_ctrlr(nvme_ctrlr, false);
2258 : 29 : remove_discovery_entry(nvme_ctrlr);
2259 : 29 : break;
2260 : 81 : case OP_DELAYED_RECONNECT:
2261 : 81 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer);
2262 : 81 : break;
2263 : 9 : case OP_FAILOVER:
2264 [ # # # # ]: 9 : nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn;
2265 [ # # # # ]: 9 : nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg;
2266 : 9 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
2267 : 9 : break;
2268 : 529 : default:
2269 : 529 : break;
2270 : : }
2271 : 663 : }
2272 : :
2273 : : static void
2274 : 669 : bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
2275 : : {
2276 [ - + # # ]: 669 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2277 [ + + # # ]: 669 : if (!success) {
2278 : : /* Connecting the active trid failed. Set the next alternate trid to the
2279 : : * active trid if it exists.
2280 : : */
2281 [ + + ]: 487 : if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) {
2282 : : /* The next alternate trid exists and is ready to try. Try it now. */
2283 [ - + # # ]: 6 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2284 : :
2285 [ - + - + : 6 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Try the next alternate trid %s:%s now.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2286 : : nvme_ctrlr->active_path_id->trid.traddr,
2287 : : nvme_ctrlr->active_path_id->trid.trsvcid);
2288 : :
2289 : 6 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2290 : 6 : return;
2291 : : }
2292 : :
2293 : : /* We came here if there is no alternate trid or if the next trid exists but
2294 : : * is not ready to try. We will try the active trid after reconnect_delay_sec
2295 : : * seconds if it is non-zero or at the next reset call otherwise.
2296 : : */
2297 : 0 : } else {
2298 : : /* Connecting the active trid succeeded. Clear the last failed time because it
2299 : : * means the trid is failed if its last failed time is non-zero.
2300 : : */
2301 [ # # # # : 182 : nvme_ctrlr->active_path_id->last_failed_tsc = 0;
# # # # ]
2302 : : }
2303 [ - + # # ]: 663 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2304 : :
2305 [ - + + + : 663 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Clear pending resets.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2306 : :
2307 : : /* Make sure we clear any pending resets before returning. */
2308 [ + + ]: 663 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2309 : : bdev_nvme_complete_pending_resets,
2310 [ # # ]: 0 : success ? NULL : (void *)0x1,
2311 : : _bdev_nvme_reset_ctrlr_complete);
2312 : 0 : }
2313 : :
2314 : : static void
2315 : 0 : bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2316 : : {
2317 : 0 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2318 : 0 : }
2319 : :
2320 : : static void
2321 : 647 : bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i,
2322 : : struct nvme_ctrlr *nvme_ctrlr,
2323 : : struct nvme_ctrlr_channel *ctrlr_ch, void *ctx)
2324 : : {
2325 : : struct nvme_qpair *nvme_qpair;
2326 : : struct spdk_nvme_qpair *qpair;
2327 : :
2328 [ # # # # ]: 647 : nvme_qpair = ctrlr_ch->qpair;
2329 [ - + # # ]: 647 : assert(nvme_qpair != NULL);
2330 : :
2331 : 647 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2332 : :
2333 [ # # # # ]: 647 : qpair = nvme_qpair->qpair;
2334 [ + + ]: 647 : if (qpair != NULL) {
2335 [ - + - + : 253 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting qpair %p:%u.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2336 : : qpair, spdk_nvme_qpair_get_id(qpair));
2337 : :
2338 [ + + # # : 253 : if (nvme_qpair->ctrlr->dont_retry) {
# # # # ]
2339 : 204 : spdk_nvme_qpair_set_abort_dnr(qpair, true);
2340 : 0 : }
2341 : 253 : spdk_nvme_ctrlr_disconnect_io_qpair(qpair);
2342 : :
2343 : : /* The current full reset sequence will move to the next
2344 : : * ctrlr_channel after the qpair is actually disconnected.
2345 : : */
2346 [ - + # # : 253 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2347 [ # # # # ]: 253 : ctrlr_ch->reset_iter = i;
2348 : 0 : } else {
2349 : 394 : nvme_ctrlr_for_each_channel_continue(i, 0);
2350 : : }
2351 : 647 : }
2352 : :
2353 : : static void
2354 : 182 : bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2355 : : {
2356 [ + - ]: 182 : if (status == 0) {
2357 [ - + - + : 182 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were created after ctrlr reset.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2358 : :
2359 : 182 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true);
2360 : 0 : } else {
2361 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were failed to create after ctrlr reset.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2362 : :
2363 : : /* Delete the added qpairs and quiesce ctrlr to make the states clean. */
2364 : 0 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2365 : : bdev_nvme_reset_destroy_qpair,
2366 : : NULL,
2367 : : bdev_nvme_reset_create_qpairs_failed);
2368 : : }
2369 : 182 : }
2370 : :
2371 : : static int
2372 : 13273 : bdev_nvme_reset_check_qpair_connected(void *ctx)
2373 : : {
2374 : 13273 : struct nvme_ctrlr_channel *ctrlr_ch = ctx;
2375 [ # # # # ]: 13273 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2376 : : struct spdk_nvme_qpair *qpair;
2377 : :
2378 [ - + # # : 13273 : if (ctrlr_ch->reset_iter == NULL) {
# # ]
2379 : : /* qpair was already failed to connect and the reset sequence is being aborted. */
2380 [ # # # # : 0 : assert(ctrlr_ch->connect_poller == NULL);
# # # # ]
2381 [ # # # # : 0 : assert(nvme_qpair->qpair == NULL);
# # # # ]
2382 : :
2383 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2384 : : "qpair was already failed to connect. reset is being aborted.\n");
2385 : 0 : return SPDK_POLLER_BUSY;
2386 : : }
2387 : :
2388 [ # # # # ]: 13273 : qpair = nvme_qpair->qpair;
2389 [ - + # # ]: 13273 : assert(qpair != NULL);
2390 : :
2391 [ + + ]: 13273 : if (!spdk_nvme_qpair_is_connected(qpair)) {
2392 : 13031 : return SPDK_POLLER_BUSY;
2393 : : }
2394 : :
2395 [ - + - + : 242 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr, "qpair %p:%u was connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2396 : : qpair, spdk_nvme_qpair_get_id(qpair));
2397 : :
2398 [ # # ]: 242 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
2399 : :
2400 : : /* qpair was completed to connect. Move to the next ctrlr_channel */
2401 [ # # # # ]: 242 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
2402 [ # # # # ]: 242 : ctrlr_ch->reset_iter = NULL;
2403 : :
2404 [ + + + + : 242 : if (!g_opts.disable_auto_failback) {
# # ]
2405 : 197 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2406 : 0 : }
2407 : :
2408 : 242 : return SPDK_POLLER_BUSY;
2409 : 0 : }
2410 : :
2411 : : static void
2412 : 242 : bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i,
2413 : : struct nvme_ctrlr *nvme_ctrlr,
2414 : : struct nvme_ctrlr_channel *ctrlr_ch,
2415 : : void *ctx)
2416 : : {
2417 [ # # # # ]: 242 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2418 : : struct spdk_nvme_qpair *qpair;
2419 : : int rc;
2420 : :
2421 : 242 : rc = bdev_nvme_create_qpair(nvme_qpair);
2422 [ + - ]: 242 : if (rc == 0) {
2423 [ # # # # ]: 242 : ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected,
2424 : : ctrlr_ch, 0);
2425 : :
2426 [ # # # # ]: 242 : qpair = nvme_qpair->qpair;
2427 : :
2428 [ - + - + : 242 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start checking qpair %p:%u to be connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2429 : : qpair, spdk_nvme_qpair_get_id(qpair));
2430 : :
2431 : : /* The current full reset sequence will move to the next
2432 : : * ctrlr_channel after the qpair is actually connected.
2433 : : */
2434 [ - + # # : 242 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2435 [ # # # # ]: 242 : ctrlr_ch->reset_iter = i;
2436 : 0 : } else {
2437 : 0 : nvme_ctrlr_for_each_channel_continue(i, rc);
2438 : : }
2439 : 242 : }
2440 : :
2441 : : static void
2442 : 182 : nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr)
2443 : : {
2444 [ # # # # ]: 182 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
2445 : : struct nvme_ns *nvme_ns;
2446 : :
2447 [ # # ]: 182 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
2448 [ + + ]: 328 : nvme_ns != NULL;
2449 : 146 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
2450 [ + + # # : 146 : if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
2451 [ - + - + : 3 : SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id);
# # # # #
# ]
2452 : : /* NS can be added again. Just nullify nvme_ns->ns. */
2453 [ # # # # ]: 3 : nvme_ns->ns = NULL;
2454 : 0 : }
2455 : 0 : }
2456 : 182 : }
2457 : :
2458 : :
2459 : : static int
2460 : 681678 : bdev_nvme_reconnect_ctrlr_poll(void *arg)
2461 : : {
2462 : 681678 : struct nvme_ctrlr *nvme_ctrlr = arg;
2463 : : struct spdk_nvme_transport_id *trid;
2464 : 681678 : int rc = -ETIMEDOUT;
2465 : :
2466 [ + + ]: 681678 : if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2467 : : /* Mark the ctrlr as failed. The next call to
2468 : : * spdk_nvme_ctrlr_reconnect_poll_async() will then
2469 : : * do the necessary cleanup and return failure.
2470 : : */
2471 [ # # # # ]: 29 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2472 : 0 : }
2473 : :
2474 [ # # # # ]: 681678 : rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr);
2475 [ + + ]: 681678 : if (rc == -EAGAIN) {
2476 : 681012 : return SPDK_POLLER_BUSY;
2477 : : }
2478 : :
2479 [ # # ]: 666 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
2480 [ + + ]: 666 : if (rc == 0) {
2481 [ # # # # : 182 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
2482 : :
2483 [ + + # # : 182 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
# # ]
2484 [ - + - + : 144 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected to %s:%s. Create qpairs.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2485 : : trid->traddr, trid->trsvcid);
2486 : 0 : } else {
2487 [ - + - + : 38 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected. Create qpairs.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2488 : : }
2489 : :
2490 : 182 : nvme_ctrlr_check_namespaces(nvme_ctrlr);
2491 : :
2492 : : /* Recreate all of the I/O queue pairs */
2493 : 182 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2494 : : bdev_nvme_reset_create_qpair,
2495 : : NULL,
2496 : : bdev_nvme_reset_create_qpairs_done);
2497 : 0 : } else {
2498 [ - + + + : 484 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr could not be connected.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2499 : :
2500 : 484 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2501 : : }
2502 : 666 : return SPDK_POLLER_BUSY;
2503 : 0 : }
2504 : :
2505 : : static void
2506 : 666 : bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2507 : : {
2508 [ - + + + : 666 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start reconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2509 : :
2510 [ # # # # ]: 666 : spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr);
2511 : :
2512 : 104 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name);
2513 [ - + # # : 666 : assert(nvme_ctrlr->reset_detach_poller == NULL);
# # # # ]
2514 [ # # # # ]: 666 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll,
2515 : : nvme_ctrlr, 0);
2516 : 666 : }
2517 : :
2518 : : static void
2519 : 582 : bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2520 : : {
2521 : 68 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name);
2522 [ - + # # ]: 582 : assert(status == 0);
2523 : :
2524 [ - + + + : 582 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were deleted.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2525 : :
2526 [ + + # # : 582 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2527 : 38 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2528 : 0 : } else {
2529 : 544 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2530 : : }
2531 : 582 : }
2532 : :
2533 : : static void
2534 : 582 : bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2535 : : {
2536 [ - + + + : 582 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Delete qpairs for reset.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2537 : :
2538 : 582 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2539 : : bdev_nvme_reset_destroy_qpair,
2540 : : NULL,
2541 : : bdev_nvme_reset_destroy_qpair_done);
2542 : 582 : }
2543 : :
2544 : : static void
2545 : 9 : bdev_nvme_reconnect_ctrlr_now(void *ctx)
2546 : : {
2547 : 9 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2548 : :
2549 [ - + # # : 9 : assert(nvme_ctrlr->resetting == true);
# # ]
2550 [ - + # # : 9 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2551 : :
2552 [ # # ]: 9 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2553 : :
2554 [ # # # # ]: 9 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2555 : :
2556 : 9 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2557 : 9 : }
2558 : :
2559 : : static void
2560 : 582 : _bdev_nvme_reset_ctrlr(void *ctx)
2561 : : {
2562 : 582 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2563 : :
2564 [ - + # # : 582 : assert(nvme_ctrlr->resetting == true);
# # ]
2565 [ - + # # : 582 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2566 : :
2567 [ + + # # : 582 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2568 : 38 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs);
2569 : 0 : } else {
2570 : 544 : bdev_nvme_reset_destroy_qpairs(nvme_ctrlr);
2571 : : }
2572 : 582 : }
2573 : :
2574 : : static int
2575 : 197 : bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2576 : : {
2577 : : spdk_msg_fn msg_fn;
2578 : :
2579 [ - + # # ]: 197 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2580 [ + + # # ]: 197 : if (nvme_ctrlr->destruct) {
2581 [ - + # # ]: 9 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2582 : 9 : return -ENXIO;
2583 : : }
2584 : :
2585 [ + + # # ]: 188 : if (nvme_ctrlr->resetting) {
2586 [ - + # # ]: 39 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2587 [ + - # # : 39 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2588 : 39 : return -EBUSY;
2589 : : }
2590 : :
2591 [ + + # # ]: 149 : if (nvme_ctrlr->disabled) {
2592 [ - + # # ]: 3 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2593 [ + - # # : 3 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset. Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2594 : 3 : return -EALREADY;
2595 : : }
2596 : :
2597 [ # # ]: 146 : nvme_ctrlr->resetting = true;
2598 [ # # ]: 146 : nvme_ctrlr->dont_retry = true;
2599 : :
2600 [ + + # # ]: 146 : if (nvme_ctrlr->reconnect_is_delayed) {
2601 [ - + - + : 3 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2602 : 3 : msg_fn = bdev_nvme_reconnect_ctrlr_now;
2603 [ # # ]: 3 : nvme_ctrlr->reconnect_is_delayed = false;
2604 : 0 : } else {
2605 : 143 : msg_fn = _bdev_nvme_reset_ctrlr;
2606 [ - + # # : 143 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
2607 : : }
2608 : :
2609 [ # # # # ]: 146 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2610 : :
2611 [ - + # # ]: 146 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2612 : :
2613 [ # # # # ]: 146 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2614 : 146 : return 0;
2615 : 0 : }
2616 : :
2617 : : static int
2618 : 9 : bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2619 : : {
2620 [ - + # # ]: 9 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2621 [ - + # # ]: 9 : if (nvme_ctrlr->destruct) {
2622 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2623 : 0 : return -ENXIO;
2624 : : }
2625 : :
2626 [ - + # # ]: 9 : if (nvme_ctrlr->resetting) {
2627 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2628 : 0 : return -EBUSY;
2629 : : }
2630 : :
2631 [ + + # # ]: 9 : if (!nvme_ctrlr->disabled) {
2632 [ - + # # ]: 3 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2633 : 3 : return -EALREADY;
2634 : : }
2635 : :
2636 [ # # ]: 6 : nvme_ctrlr->disabled = false;
2637 [ # # ]: 6 : nvme_ctrlr->resetting = true;
2638 : :
2639 [ # # # # ]: 6 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2640 : :
2641 [ - + # # ]: 6 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2642 : :
2643 [ # # # # ]: 6 : spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr);
2644 : 6 : return 0;
2645 : 0 : }
2646 : :
2647 : : static void
2648 : 6 : _bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2649 : : {
2650 [ # # # # ]: 6 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2651 [ # # # # ]: 6 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2652 : : enum bdev_nvme_op_after_reset op_after_disable;
2653 : :
2654 [ - + # # : 6 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2655 : :
2656 [ # # # # ]: 6 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2657 [ # # # # ]: 6 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2658 : :
2659 [ - + # # ]: 6 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2660 : :
2661 [ # # ]: 6 : nvme_ctrlr->resetting = false;
2662 [ # # ]: 6 : nvme_ctrlr->dont_retry = false;
2663 : :
2664 : 6 : op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true);
2665 : :
2666 [ # # ]: 6 : nvme_ctrlr->disabled = true;
2667 [ # # # # ]: 6 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2668 : :
2669 [ - + # # ]: 6 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2670 : :
2671 [ - + ]: 6 : if (ctrlr_op_cb_fn) {
2672 [ # # # # ]: 0 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0);
2673 : 0 : }
2674 : :
2675 [ - + ]: 6 : switch (op_after_disable) {
2676 : 0 : case OP_COMPLETE_PENDING_DESTRUCT:
2677 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
2678 : 0 : break;
2679 : 6 : default:
2680 : 6 : break;
2681 : : }
2682 : :
2683 : 6 : }
2684 : :
2685 : : static void
2686 : 6 : bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr)
2687 : : {
2688 : : /* Make sure we clear any pending resets before returning. */
2689 : 6 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2690 : : bdev_nvme_complete_pending_resets,
2691 : : NULL,
2692 : : _bdev_nvme_disable_ctrlr_complete);
2693 : 6 : }
2694 : :
2695 : : static void
2696 : 3 : bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2697 : : {
2698 [ - + # # ]: 3 : assert(status == 0);
2699 : :
2700 [ - + # # : 3 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2701 : 0 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2702 : 0 : } else {
2703 : 3 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete);
2704 : : }
2705 : 3 : }
2706 : :
2707 : : static void
2708 : 3 : bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2709 : : {
2710 : 3 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2711 : : bdev_nvme_reset_destroy_qpair,
2712 : : NULL,
2713 : : bdev_nvme_disable_destroy_qpairs_done);
2714 : 3 : }
2715 : :
2716 : : static void
2717 : 3 : _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx)
2718 : : {
2719 : 3 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2720 : :
2721 [ - + # # : 3 : assert(nvme_ctrlr->resetting == true);
# # ]
2722 [ - + # # : 3 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2723 : :
2724 [ # # ]: 3 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2725 : :
2726 : 3 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2727 : 3 : }
2728 : :
2729 : : static void
2730 : 3 : _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx)
2731 : : {
2732 : 3 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2733 : :
2734 [ - + # # : 3 : assert(nvme_ctrlr->resetting == true);
# # ]
2735 [ - + # # : 3 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2736 : :
2737 [ - + # # : 3 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2738 : 0 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs);
2739 : 0 : } else {
2740 : 3 : bdev_nvme_disable_destroy_qpairs(nvme_ctrlr);
2741 : : }
2742 : 3 : }
2743 : :
2744 : : static int
2745 : 15 : bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2746 : : {
2747 : : spdk_msg_fn msg_fn;
2748 : :
2749 [ - + # # ]: 15 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2750 [ + + # # ]: 15 : if (nvme_ctrlr->destruct) {
2751 [ - + # # ]: 3 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2752 : 3 : return -ENXIO;
2753 : : }
2754 : :
2755 [ + + # # ]: 12 : if (nvme_ctrlr->resetting) {
2756 [ - + # # ]: 3 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2757 : 3 : return -EBUSY;
2758 : : }
2759 : :
2760 [ + + # # ]: 9 : if (nvme_ctrlr->disabled) {
2761 [ - + # # ]: 3 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2762 : 3 : return -EALREADY;
2763 : : }
2764 : :
2765 [ # # ]: 6 : nvme_ctrlr->resetting = true;
2766 [ # # ]: 6 : nvme_ctrlr->dont_retry = true;
2767 : :
2768 [ + + # # ]: 6 : if (nvme_ctrlr->reconnect_is_delayed) {
2769 : 3 : msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr;
2770 [ # # ]: 3 : nvme_ctrlr->reconnect_is_delayed = false;
2771 : 0 : } else {
2772 : 3 : msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr;
2773 : : }
2774 : :
2775 [ # # # # ]: 6 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2776 : :
2777 [ - + # # ]: 6 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2778 : :
2779 [ # # # # ]: 6 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2780 : 6 : return 0;
2781 : 0 : }
2782 : :
2783 : : static int
2784 : 143 : nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2785 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2786 : : {
2787 : : int rc;
2788 : :
2789 [ + - - + ]: 143 : switch (op) {
2790 : 140 : case NVME_CTRLR_OP_RESET:
2791 : 140 : rc = bdev_nvme_reset_ctrlr(nvme_ctrlr);
2792 : 140 : break;
2793 : 0 : case NVME_CTRLR_OP_ENABLE:
2794 : 0 : rc = bdev_nvme_enable_ctrlr(nvme_ctrlr);
2795 : 0 : break;
2796 : 0 : case NVME_CTRLR_OP_DISABLE:
2797 : 0 : rc = bdev_nvme_disable_ctrlr(nvme_ctrlr);
2798 : 0 : break;
2799 : 3 : default:
2800 : 3 : rc = -EINVAL;
2801 : 3 : break;
2802 : : }
2803 : :
2804 [ + + ]: 143 : if (rc == 0) {
2805 [ - + # # : 98 : assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL);
# # # # ]
2806 [ - + # # : 98 : assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL);
# # # # ]
2807 [ # # # # ]: 98 : nvme_ctrlr->ctrlr_op_cb_fn = cb_fn;
2808 [ # # # # ]: 98 : nvme_ctrlr->ctrlr_op_cb_arg = cb_arg;
2809 : 0 : }
2810 : 143 : return rc;
2811 : : }
2812 : :
2813 : : struct nvme_ctrlr_op_rpc_ctx {
2814 : : struct nvme_ctrlr *nvme_ctrlr;
2815 : : struct spdk_thread *orig_thread;
2816 : : enum nvme_ctrlr_op op;
2817 : : int rc;
2818 : : bdev_nvme_ctrlr_op_cb cb_fn;
2819 : : void *cb_arg;
2820 : : };
2821 : :
2822 : : static void
2823 : 12 : _nvme_ctrlr_op_rpc_complete(void *_ctx)
2824 : : {
2825 : 12 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2826 : :
2827 [ - + # # ]: 12 : assert(ctx != NULL);
2828 [ - + # # : 12 : assert(ctx->cb_fn != NULL);
# # # # ]
2829 : :
2830 [ # # # # : 12 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2831 : :
2832 : 12 : free(ctx);
2833 : 12 : }
2834 : :
2835 : : static void
2836 : 12 : nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc)
2837 : : {
2838 : 12 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2839 : :
2840 [ # # # # ]: 12 : ctx->rc = rc;
2841 : :
2842 [ # # # # ]: 12 : spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx);
2843 : 12 : }
2844 : :
2845 : : void
2846 : 12 : nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2847 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2848 : : {
2849 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2850 : : int rc;
2851 : :
2852 [ - + # # ]: 12 : assert(cb_fn != NULL);
2853 : :
2854 : 12 : ctx = calloc(1, sizeof(*ctx));
2855 [ - + ]: 12 : if (ctx == NULL) {
2856 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2857 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2858 : 0 : return;
2859 : : }
2860 : :
2861 [ # # # # ]: 12 : ctx->orig_thread = spdk_get_thread();
2862 [ # # # # ]: 12 : ctx->cb_fn = cb_fn;
2863 [ # # # # ]: 12 : ctx->cb_arg = cb_arg;
2864 : :
2865 : 12 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx);
2866 [ + + ]: 12 : if (rc == 0) {
2867 : 3 : return;
2868 [ - + ]: 9 : } else if (rc == -EALREADY) {
2869 : 0 : rc = 0;
2870 : 0 : }
2871 : :
2872 : 9 : nvme_ctrlr_op_rpc_complete(ctx, rc);
2873 : 0 : }
2874 : :
2875 : : static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc);
2876 : :
2877 : : static void
2878 : 15 : _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx)
2879 : : {
2880 : 15 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2881 : : struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr;
2882 : : int rc;
2883 : :
2884 [ # # # # ]: 15 : prev_nvme_ctrlr = ctx->nvme_ctrlr;
2885 [ # # # # ]: 15 : ctx->nvme_ctrlr = NULL;
2886 : :
2887 [ - + # # : 15 : if (ctx->rc != 0) {
# # ]
2888 : 0 : goto complete;
2889 : : }
2890 : :
2891 [ # # # # : 15 : next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq);
# # ]
2892 [ + + ]: 15 : if (next_nvme_ctrlr == NULL) {
2893 : 12 : goto complete;
2894 : : }
2895 : :
2896 [ # # # # ]: 3 : rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2897 [ + - ]: 3 : if (rc == 0) {
2898 [ # # # # ]: 3 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2899 : 3 : return;
2900 [ # # ]: 0 : } else if (rc == -EALREADY) {
2901 [ # # # # ]: 0 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2902 : 0 : rc = 0;
2903 : 0 : }
2904 : :
2905 [ # # # # ]: 0 : ctx->rc = rc;
2906 : :
2907 : 12 : complete:
2908 [ # # # # : 12 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2909 : 12 : free(ctx);
2910 : 0 : }
2911 : :
2912 : : static void
2913 : 15 : nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc)
2914 : : {
2915 : 15 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2916 : :
2917 [ # # # # ]: 15 : ctx->rc = rc;
2918 : :
2919 [ # # # # ]: 15 : spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx);
2920 : 15 : }
2921 : :
2922 : : void
2923 : 12 : nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op,
2924 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2925 : : {
2926 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2927 : : struct nvme_ctrlr *nvme_ctrlr;
2928 : : int rc;
2929 : :
2930 [ - + # # ]: 12 : assert(cb_fn != NULL);
2931 : :
2932 : 12 : ctx = calloc(1, sizeof(*ctx));
2933 [ - + ]: 12 : if (ctx == NULL) {
2934 : 0 : SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
2935 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2936 : 0 : return;
2937 : : }
2938 : :
2939 [ # # # # ]: 12 : ctx->orig_thread = spdk_get_thread();
2940 [ # # # # ]: 12 : ctx->op = op;
2941 [ # # # # ]: 12 : ctx->cb_fn = cb_fn;
2942 [ # # # # ]: 12 : ctx->cb_arg = cb_arg;
2943 : :
2944 [ # # # # : 12 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # ]
2945 [ - + # # ]: 12 : assert(nvme_ctrlr != NULL);
2946 : :
2947 : 12 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2948 [ + - ]: 12 : if (rc == 0) {
2949 [ # # # # ]: 12 : ctx->nvme_ctrlr = nvme_ctrlr;
2950 : 12 : return;
2951 [ # # ]: 0 : } else if (rc == -EALREADY) {
2952 [ # # # # ]: 0 : ctx->nvme_ctrlr = nvme_ctrlr;
2953 : 0 : rc = 0;
2954 : 0 : }
2955 : :
2956 : 0 : nvme_bdev_ctrlr_op_rpc_continue(ctx, rc);
2957 : 0 : }
2958 : :
2959 : : static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
2960 : :
2961 : : static void
2962 : 86 : bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
2963 : : {
2964 : 86 : struct nvme_bdev_io *bio = ctx;
2965 : : enum spdk_bdev_io_status io_status;
2966 : :
2967 [ + + # # : 86 : if (bio->cpl.cdw0 == 0) {
# # # # ]
2968 : 74 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
2969 : 0 : } else {
2970 : 12 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
2971 : : }
2972 : :
2973 [ - + - + : 86 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p completed, status:%d\n", bio, io_status);
# # # # #
# # # ]
2974 : :
2975 : 86 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL);
2976 : 86 : }
2977 : :
2978 : : static void
2979 : 135 : bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i,
2980 : : struct nvme_bdev *nbdev,
2981 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
2982 : : {
2983 : 135 : bdev_nvme_abort_retry_ios(nbdev_ch);
2984 [ # # # # ]: 135 : nbdev_ch->resetting = false;
2985 : :
2986 : 135 : nvme_bdev_for_each_channel_continue(i, 0);
2987 : 135 : }
2988 : :
2989 : : static void
2990 : 86 : bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
2991 : : {
2992 : 86 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
2993 [ # # # # : 86 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
2994 : :
2995 : : /* Abort all queued I/Os for retry. */
2996 : 86 : nvme_bdev_for_each_channel(nbdev,
2997 : : bdev_nvme_unfreeze_bdev_channel,
2998 : 0 : bio,
2999 : : bdev_nvme_unfreeze_bdev_channel_done);
3000 : 86 : }
3001 : :
3002 : : static void
3003 : 116 : _bdev_nvme_reset_io_continue(void *ctx)
3004 : : {
3005 : 116 : struct nvme_bdev_io *bio = ctx;
3006 : : struct nvme_io_path *prev_io_path, *next_io_path;
3007 : : int rc;
3008 : :
3009 [ # # # # ]: 116 : prev_io_path = bio->io_path;
3010 [ # # # # ]: 116 : bio->io_path = NULL;
3011 : :
3012 [ # # # # : 116 : next_io_path = STAILQ_NEXT(prev_io_path, stailq);
# # ]
3013 [ + + ]: 116 : if (next_io_path == NULL) {
3014 : 86 : goto complete;
3015 : : }
3016 : :
3017 : 30 : rc = _bdev_nvme_reset_io(next_io_path, bio);
3018 [ + - ]: 30 : if (rc == 0) {
3019 : 30 : return;
3020 : : }
3021 : :
3022 : 0 : complete:
3023 : 86 : bdev_nvme_reset_io_complete(bio);
3024 : 0 : }
3025 : :
3026 : : static void
3027 : 116 : bdev_nvme_reset_io_continue(void *cb_arg, int rc)
3028 : : {
3029 : 116 : struct nvme_bdev_io *bio = cb_arg;
3030 : 116 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3031 [ # # # # : 116 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3032 : :
3033 [ - + - + : 116 : NVME_BDEV_INFOLOG(nbdev, "continue reset_io %p, rc:%d\n", bio, rc);
# # # # #
# # # ]
3034 : :
3035 : : /* Reset status is initialized as "failed". Set to "success" once we have at least one
3036 : : * successfully reset nvme_ctrlr.
3037 : : */
3038 [ + + ]: 116 : if (rc == 0) {
3039 [ # # # # : 86 : bio->cpl.cdw0 = 0;
# # ]
3040 : 0 : }
3041 : :
3042 : 116 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio);
3043 : 116 : }
3044 : :
3045 : : static int
3046 : 116 : _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio)
3047 : : {
3048 : 116 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3049 [ # # # # : 116 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3050 [ # # # # : 116 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
3051 : : struct nvme_ctrlr_channel *ctrlr_ch;
3052 : : int rc;
3053 : :
3054 [ - + # # : 116 : assert(bio->io_path == NULL);
# # # # ]
3055 [ # # # # ]: 116 : bio->io_path = io_path;
3056 : :
3057 : 116 : rc = nvme_ctrlr_op(nvme_ctrlr, NVME_CTRLR_OP_RESET,
3058 : 0 : bdev_nvme_reset_io_continue, bio);
3059 : :
3060 [ + + ]: 116 : if (rc == 0) {
3061 [ - + - + : 80 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started resetting ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3062 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3063 [ + + ]: 36 : } else if (rc == -EBUSY) {
3064 [ # # # # : 33 : ctrlr_ch = io_path->qpair->ctrlr_ch;
# # # # ]
3065 [ - + # # ]: 33 : assert(ctrlr_ch != NULL);
3066 : : /*
3067 : : * Reset call is queued only if it is from the app framework. This is on purpose so that
3068 : : * we don't interfere with the app framework reset strategy. i.e. we are deferring to the
3069 : : * upper level. If they are in the middle of a reset, we won't try to schedule another one.
3070 : : */
3071 [ # # # # : 33 : TAILQ_INSERT_TAIL(&ctrlr_ch->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3072 : :
3073 : 33 : rc = 0;
3074 : :
3075 [ - + - + : 33 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p was queued to ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3076 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3077 : 0 : } else {
3078 [ - + - + : 3 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p could not reset ctrlr [%s, %u], rc:%d\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3079 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr), rc);
3080 : : }
3081 : :
3082 : 116 : return rc;
3083 : : }
3084 : :
3085 : : static void
3086 : 86 : bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
3087 : : {
3088 : 86 : struct nvme_bdev_io *bio = ctx;
3089 : 86 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3090 : : struct nvme_bdev_channel *nbdev_ch;
3091 : : struct nvme_io_path *io_path;
3092 : : int rc;
3093 : :
3094 : 86 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
3095 : :
3096 : : /* Initialize with failed status. With multipath it is enough to have at least one successful
3097 : : * nvme_ctrlr reset. If there is none, reset status will remain failed.
3098 : : */
3099 [ # # # # : 86 : bio->cpl.cdw0 = 1;
# # ]
3100 : :
3101 : : /* Reset all nvme_ctrlrs of a bdev controller sequentially. */
3102 [ # # # # : 86 : io_path = STAILQ_FIRST(&nbdev_ch->io_path_list);
# # ]
3103 [ - + # # ]: 86 : assert(io_path != NULL);
3104 : :
3105 : 86 : rc = _bdev_nvme_reset_io(io_path, bio);
3106 [ + + ]: 86 : if (rc != 0) {
3107 : : /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */
3108 [ - + ]: 3 : rc = (rc == -EALREADY) ? 0 : rc;
3109 : :
3110 : 3 : bdev_nvme_reset_io_continue(bio, rc);
3111 : 0 : }
3112 : 86 : }
3113 : :
3114 : : static void
3115 : 135 : bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i,
3116 : : struct nvme_bdev *nbdev,
3117 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
3118 : : {
3119 [ # # # # ]: 135 : nbdev_ch->resetting = true;
3120 : :
3121 : 135 : nvme_bdev_for_each_channel_continue(i, 0);
3122 : 135 : }
3123 : :
3124 : : static void
3125 : 86 : bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio)
3126 : : {
3127 [ - + - + : 86 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started.\n", bio);
# # # # #
# # # ]
3128 : :
3129 : 86 : nvme_bdev_for_each_channel(nbdev,
3130 : : bdev_nvme_freeze_bdev_channel,
3131 : 0 : bio,
3132 : : bdev_nvme_freeze_bdev_channel_done);
3133 : 86 : }
3134 : :
3135 : : static int
3136 : 911 : bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove)
3137 : : {
3138 [ + + # # ]: 911 : if (nvme_ctrlr->destruct) {
3139 : : /* Don't bother resetting if the controller is in the process of being destructed. */
3140 : 418 : return -ENXIO;
3141 : : }
3142 : :
3143 [ + + # # ]: 493 : if (nvme_ctrlr->resetting) {
3144 [ + + # # ]: 51 : if (!nvme_ctrlr->in_failover) {
3145 [ + - # # : 9 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3146 : : "Reset is already in progress. Defer failover until reset completes.\n");
3147 : :
3148 : : /* Defer failover until reset completes. */
3149 [ # # ]: 9 : nvme_ctrlr->pending_failover = true;
3150 : 9 : return -EINPROGRESS;
3151 : : } else {
3152 [ + - # # : 42 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform failover, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3153 : 42 : return -EBUSY;
3154 : : }
3155 : : }
3156 : :
3157 [ # # ]: 442 : bdev_nvme_failover_trid(nvme_ctrlr, remove, true);
3158 : :
3159 [ + + # # ]: 442 : if (nvme_ctrlr->reconnect_is_delayed) {
3160 [ + - # # : 3 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3161 : :
3162 : : /* We rely on the next reconnect for the failover. */
3163 : 3 : return -EALREADY;
3164 : : }
3165 : :
3166 [ - + # # ]: 439 : if (nvme_ctrlr->disabled) {
3167 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3168 : :
3169 : : /* We rely on the enablement for the failover. */
3170 : 0 : return -EALREADY;
3171 : : }
3172 : :
3173 [ # # ]: 439 : nvme_ctrlr->resetting = true;
3174 [ # # ]: 439 : nvme_ctrlr->in_failover = true;
3175 : :
3176 [ - + # # : 439 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
3177 [ # # # # ]: 439 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
3178 : :
3179 : 439 : return 0;
3180 : 0 : }
3181 : :
3182 : : static int
3183 : 901 : bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
3184 : : {
3185 : : int rc;
3186 : :
3187 [ - + # # ]: 901 : pthread_mutex_lock(&nvme_ctrlr->mutex);
3188 : 901 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false);
3189 [ - + # # ]: 901 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
3190 : :
3191 [ + + ]: 901 : if (rc == 0) {
3192 [ # # # # ]: 432 : spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr);
3193 [ - + ]: 469 : } else if (rc == -EALREADY) {
3194 : 0 : rc = 0;
3195 : 0 : }
3196 : :
3197 : 901 : return rc;
3198 : : }
3199 : :
3200 : : static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3201 : : uint64_t num_blocks);
3202 : :
3203 : : static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3204 : : uint64_t num_blocks);
3205 : :
3206 : : static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks,
3207 : : uint64_t src_offset_blocks,
3208 : : uint64_t num_blocks);
3209 : :
3210 : : static void
3211 : 6112180 : bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
3212 : : bool success)
3213 : : {
3214 [ # # ]: 6112180 : struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3215 : : int ret;
3216 : :
3217 [ - + # # ]: 6112180 : if (!success) {
3218 : 0 : ret = -EINVAL;
3219 : 0 : goto exit;
3220 : : }
3221 : :
3222 [ - + # # : 6112180 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
3223 : 0 : ret = -ENXIO;
3224 : 0 : goto exit;
3225 : : }
3226 : :
3227 : 6112180 : ret = bdev_nvme_readv(bio,
3228 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3229 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3230 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3231 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3232 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3233 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3234 [ # # # # : 0 : bdev_io->u.bdev.memory_domain,
# # # # ]
3235 [ # # # # : 0 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3236 [ # # # # : 0 : bdev_io->u.bdev.accel_sequence);
# # # # ]
3237 : :
3238 : 6112180 : exit:
3239 [ + + ]: 6112180 : if (spdk_unlikely(ret != 0)) {
3240 : 81503 : bdev_nvme_io_complete(bio, ret);
3241 : 0 : }
3242 : 6112180 : }
3243 : :
3244 : : static inline void
3245 : 44870578 : _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
3246 : : {
3247 [ + - ]: 44870578 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3248 [ + - + - ]: 44870578 : struct spdk_bdev *bdev = bdev_io->bdev;
3249 : : struct nvme_bdev_io *nbdev_io_to_abort;
3250 : 44870578 : int rc = 0;
3251 : :
3252 [ + + + + : 44870578 : switch (bdev_io->type) {
+ + + + +
+ + + + -
- + + - -
- - - ]
3253 : 21289542 : case SPDK_BDEV_IO_TYPE_READ:
3254 [ + + + + : 21289545 : if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) {
+ - + - +
- + - + -
+ - + - +
- + - + -
- + ]
3255 : :
3256 : 15177368 : rc = bdev_nvme_readv(nbdev_io,
3257 [ - + - + : 3 : bdev_io->u.bdev.iovs,
- + - + ]
3258 [ - + - + : 3 : bdev_io->u.bdev.iovcnt,
- + - + ]
3259 [ - + - + : 3 : bdev_io->u.bdev.md_buf,
- + - + ]
3260 [ - + - + : 3 : bdev_io->u.bdev.num_blocks,
- + - + ]
3261 [ - + - + : 3 : bdev_io->u.bdev.offset_blocks,
- + - + ]
3262 [ - + - + : 3 : bdev_io->u.bdev.dif_check_flags,
- + - + ]
3263 [ - + - + : 3 : bdev_io->u.bdev.memory_domain,
- + - + ]
3264 [ - + - + : 3 : bdev_io->u.bdev.memory_domain_ctx,
- + - + ]
3265 [ - + - + : 3 : bdev_io->u.bdev.accel_sequence);
- + - + ]
3266 : 3 : } else {
3267 : 6112180 : spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb,
3268 [ # # # # : 6112180 : bdev_io->u.bdev.num_blocks * bdev->blocklen);
# # # # #
# # # ]
3269 : 6112180 : rc = 0;
3270 : : }
3271 : 21289545 : break;
3272 : 20365646 : case SPDK_BDEV_IO_TYPE_WRITE:
3273 : 20365646 : rc = bdev_nvme_writev(nbdev_io,
3274 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3275 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3276 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3277 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3278 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3279 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3280 [ # # # # : 0 : bdev_io->u.bdev.memory_domain,
# # # # ]
3281 [ # # # # : 0 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3282 [ # # # # : 0 : bdev_io->u.bdev.accel_sequence,
# # # # ]
3283 [ # # # # : 0 : bdev_io->u.bdev.nvme_cdw12,
# # ]
3284 [ # # # # : 0 : bdev_io->u.bdev.nvme_cdw13);
# # ]
3285 : 20365646 : break;
3286 : 53 : case SPDK_BDEV_IO_TYPE_COMPARE:
3287 : 53 : rc = bdev_nvme_comparev(nbdev_io,
3288 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3289 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3290 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3291 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3292 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3293 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3294 : 53 : break;
3295 : 51 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3296 : 51 : rc = bdev_nvme_comparev_and_writev(nbdev_io,
3297 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3298 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3299 [ # # # # : 0 : bdev_io->u.bdev.fused_iovs,
# # # # ]
3300 [ # # # # : 0 : bdev_io->u.bdev.fused_iovcnt,
# # # # ]
3301 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3302 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3303 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3304 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3305 : 51 : break;
3306 : 255667 : case SPDK_BDEV_IO_TYPE_UNMAP:
3307 : 255667 : rc = bdev_nvme_unmap(nbdev_io,
3308 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3309 [ # # # # : 0 : bdev_io->u.bdev.num_blocks);
# # # # ]
3310 : 255667 : break;
3311 : 714867 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3312 : 714867 : rc = bdev_nvme_write_zeroes(nbdev_io,
3313 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3314 [ # # # # : 0 : bdev_io->u.bdev.num_blocks);
# # # # ]
3315 : 714867 : break;
3316 : 86 : case SPDK_BDEV_IO_TYPE_RESET:
3317 [ # # # # ]: 86 : nbdev_io->io_path = NULL;
3318 [ # # # # ]: 86 : bdev_nvme_reset_io(bdev->ctxt, nbdev_io);
3319 : 86 : return;
3320 : :
3321 : 1980028 : case SPDK_BDEV_IO_TYPE_FLUSH:
3322 : 1980028 : bdev_nvme_io_complete(nbdev_io, 0);
3323 : 1980028 : return;
3324 : :
3325 : 256879 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3326 : 256879 : rc = bdev_nvme_zone_appendv(nbdev_io,
3327 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3328 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3329 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3330 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3331 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3332 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3333 : 256879 : break;
3334 : 1 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3335 : 1 : rc = bdev_nvme_get_zone_info(nbdev_io,
3336 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3337 [ # # # # : 0 : bdev_io->u.zone_mgmt.num_zones,
# # # # ]
3338 [ # # # # : 1 : bdev_io->u.zone_mgmt.buf);
# # # # ]
3339 : 1 : break;
3340 : 44 : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3341 : 44 : rc = bdev_nvme_zone_management(nbdev_io,
3342 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3343 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_action);
# # # # ]
3344 : 44 : break;
3345 : 90 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3346 [ # # # # ]: 90 : nbdev_io->io_path = NULL;
3347 : 90 : bdev_nvme_admin_passthru(nbdev_ch,
3348 : 0 : nbdev_io,
3349 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3350 [ # # # # : 0 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3351 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3352 : 90 : return;
3353 : :
3354 : 108 : case SPDK_BDEV_IO_TYPE_NVME_IO:
3355 : 108 : rc = bdev_nvme_io_passthru(nbdev_io,
3356 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3357 [ # # # # : 0 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3358 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3359 : 108 : break;
3360 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3361 : 0 : rc = bdev_nvme_io_passthru_md(nbdev_io,
3362 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3363 [ # # # # : 0 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3364 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3365 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3366 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3367 : 0 : break;
3368 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IOV_MD:
3369 : 0 : rc = bdev_nvme_iov_passthru_md(nbdev_io,
3370 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3371 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovs,
# # # # ]
3372 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovcnt,
# # # # ]
3373 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3374 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3375 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3376 : 0 : break;
3377 : 7477 : case SPDK_BDEV_IO_TYPE_ABORT:
3378 [ # # # # ]: 7477 : nbdev_io->io_path = NULL;
3379 [ # # # # : 7477 : nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx;
# # # # #
# ]
3380 : 7477 : bdev_nvme_abort(nbdev_ch,
3381 : 0 : nbdev_io,
3382 : 0 : nbdev_io_to_abort);
3383 : 7477 : return;
3384 : :
3385 : 36 : case SPDK_BDEV_IO_TYPE_COPY:
3386 : 36 : rc = bdev_nvme_copy(nbdev_io,
3387 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3388 [ # # # # : 0 : bdev_io->u.bdev.copy.src_offset_blocks,
# # # # #
# ]
3389 [ # # # # : 0 : bdev_io->u.bdev.num_blocks);
# # # # ]
3390 : 36 : break;
3391 : 0 : default:
3392 : 0 : rc = -EINVAL;
3393 : 0 : break;
3394 : : }
3395 : :
3396 [ + + ]: 42882897 : if (spdk_unlikely(rc != 0)) {
3397 : 88241 : bdev_nvme_io_complete(nbdev_io, rc);
3398 : 0 : }
3399 : 3 : }
3400 : :
3401 : : static void
3402 : 45275644 : bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
3403 : : {
3404 : 45275644 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
3405 [ + - ]: 45275644 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3406 : :
3407 [ + + + - : 45275644 : if (spdk_likely(nbdev_io->submit_tsc == 0)) {
- + ]
3408 [ + - + - ]: 45236705 : nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io);
3409 : 3 : } else {
3410 : : /* There are cases where submit_tsc != 0, i.e. retry I/O.
3411 : : * We need to update submit_tsc here.
3412 : : */
3413 [ # # # # ]: 38939 : nbdev_io->submit_tsc = spdk_get_ticks();
3414 : : }
3415 : :
3416 [ + + + + : 45275644 : spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io);
+ - + - +
- + - + -
+ - ]
3417 [ + - + - ]: 45275644 : nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch);
3418 [ + + + - : 45275644 : if (spdk_unlikely(!nbdev_io->io_path)) {
+ - ]
3419 [ + + # # : 408661 : if (!bdev_nvme_io_type_is_admin(bdev_io->type)) {
# # ]
3420 : 408658 : bdev_nvme_io_complete(nbdev_io, -ENXIO);
3421 : 408658 : return;
3422 : : }
3423 : :
3424 : : /* Admin commands do not use the optimal I/O path.
3425 : : * Simply fall through even if it is not found.
3426 : : */
3427 : 0 : }
3428 : :
3429 : 44866986 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
3430 : 3 : }
3431 : :
3432 : : static bool
3433 : 2737518 : bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi)
3434 : : {
3435 [ + + + ]: 2737518 : switch (csi) {
3436 : 2737485 : case SPDK_NVME_CSI_NVM:
3437 : 2737489 : return true;
3438 : 29 : case SPDK_NVME_CSI_ZNS:
3439 : 29 : return true;
3440 : 0 : default:
3441 : 0 : return false;
3442 : : }
3443 : 4 : }
3444 : :
3445 : : static bool
3446 : 2737518 : bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
3447 : : {
3448 : 2737518 : struct nvme_bdev *nbdev = ctx;
3449 : : struct nvme_ns *nvme_ns;
3450 : : struct spdk_nvme_ns *ns;
3451 : : struct spdk_nvme_ctrlr *ctrlr;
3452 : : const struct spdk_nvme_ctrlr_data *cdata;
3453 : :
3454 [ + - + - : 2737518 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
+ - ]
3455 [ + + # # ]: 2737518 : assert(nvme_ns != NULL);
3456 [ + - + - ]: 2737518 : ns = nvme_ns->ns;
3457 [ + + ]: 2737518 : if (ns == NULL) {
3458 : 0 : return false;
3459 : : }
3460 : :
3461 [ + + ]: 2737518 : if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) {
3462 [ # # # ]: 0 : switch (io_type) {
3463 : 0 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3464 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3465 : 0 : return true;
3466 : :
3467 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3468 : 0 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3469 : :
3470 : 0 : default:
3471 : 0 : return false;
3472 : : }
3473 : : }
3474 : :
3475 : 2737518 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
3476 : :
3477 [ + + + + : 2737518 : switch (io_type) {
+ + + + +
+ ]
3478 : 224147 : case SPDK_BDEV_IO_TYPE_READ:
3479 : : case SPDK_BDEV_IO_TYPE_WRITE:
3480 : : case SPDK_BDEV_IO_TYPE_RESET:
3481 : : case SPDK_BDEV_IO_TYPE_FLUSH:
3482 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3483 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3484 : : case SPDK_BDEV_IO_TYPE_ABORT:
3485 : 224148 : return true;
3486 : :
3487 : 3810 : case SPDK_BDEV_IO_TYPE_COMPARE:
3488 : 3810 : return spdk_nvme_ns_supports_compare(ns);
3489 : :
3490 : 1118 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3491 : 1118 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3492 : :
3493 : 82394 : case SPDK_BDEV_IO_TYPE_UNMAP:
3494 : 82394 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3495 [ # # # # ]: 82394 : return cdata->oncs.dsm;
3496 : :
3497 : 2389248 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3498 : 2389249 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3499 [ + - + - ]: 2389249 : return cdata->oncs.write_zeroes;
3500 : :
3501 : 3808 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3502 [ + + ]: 3808 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
3503 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) {
3504 : 239 : return true;
3505 : : }
3506 : 3569 : return false;
3507 : :
3508 : 7468 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3509 : : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3510 : 7468 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS;
3511 : :
3512 : 3735 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3513 [ + + ]: 3737 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
3514 [ + - ]: 2 : spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
3515 : :
3516 : 6827 : case SPDK_BDEV_IO_TYPE_COPY:
3517 : 6829 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3518 [ + - + - ]: 6829 : return cdata->oncs.copy;
3519 : :
3520 : 14959 : default:
3521 : 14959 : return false;
3522 : : }
3523 : 4 : }
3524 : :
3525 : : static int
3526 : 2439 : nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch)
3527 : : {
3528 : : struct nvme_qpair *nvme_qpair;
3529 : : struct spdk_io_channel *pg_ch;
3530 : : int rc;
3531 : :
3532 : 2439 : nvme_qpair = calloc(1, sizeof(*nvme_qpair));
3533 [ + + ]: 2439 : if (!nvme_qpair) {
3534 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to alloc nvme_qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3535 : 0 : return -1;
3536 : : }
3537 : :
3538 [ + - + - : 2439 : TAILQ_INIT(&nvme_qpair->io_path_list);
+ - + - +
- + - + -
+ - ]
3539 : :
3540 [ + - + - ]: 2439 : nvme_qpair->ctrlr = nvme_ctrlr;
3541 [ + - + - ]: 2439 : nvme_qpair->ctrlr_ch = ctrlr_ch;
3542 : :
3543 : 2439 : pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs);
3544 [ + + ]: 2439 : if (!pg_ch) {
3545 : 0 : free(nvme_qpair);
3546 : 0 : return -1;
3547 : : }
3548 : :
3549 [ + - + - ]: 2439 : nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch);
3550 : :
3551 : : #ifdef SPDK_CONFIG_VTUNE
3552 : : nvme_qpair->group->collect_spin_stat = true;
3553 : : #else
3554 [ + - + - : 2439 : nvme_qpair->group->collect_spin_stat = false;
+ - + - ]
3555 : : #endif
3556 : :
3557 [ + - - + ]: 2439 : if (!nvme_ctrlr->disabled) {
3558 : : /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will
3559 : : * be created when it's enabled.
3560 : : */
3561 : 2439 : rc = bdev_nvme_create_qpair(nvme_qpair);
3562 [ + + ]: 2439 : if (rc != 0) {
3563 : : /* nvme_ctrlr can't create IO qpair if connection is down.
3564 : : * If reconnect_delay_sec is non-zero, creating IO qpair is retried
3565 : : * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero,
3566 : : * submitted IO will be queued until IO qpair is successfully created.
3567 : : *
3568 : : * Hence, if both are satisfied, ignore the failure.
3569 : : */
3570 [ # # # # : 0 : if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) {
# # # # #
# # # ]
3571 : 0 : spdk_put_io_channel(pg_ch);
3572 : 0 : free(nvme_qpair);
3573 : 0 : return rc;
3574 : : }
3575 : 0 : }
3576 : 1 : }
3577 : :
3578 [ + - + - : 2439 : TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
3579 : :
3580 [ + - + - ]: 2439 : ctrlr_ch->qpair = nvme_qpair;
3581 : :
3582 [ + + + - : 2439 : pthread_mutex_lock(&nvme_qpair->ctrlr->mutex);
+ - + - ]
3583 [ + - + - : 2439 : nvme_qpair->ctrlr->ref++;
+ - + - ]
3584 [ + + + - : 2439 : pthread_mutex_unlock(&nvme_qpair->ctrlr->mutex);
+ - + - ]
3585 : :
3586 : 2439 : return 0;
3587 : 1 : }
3588 : :
3589 : : static int
3590 : 2439 : bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3591 : : {
3592 : 2439 : struct nvme_ctrlr *nvme_ctrlr = io_device;
3593 : 2439 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3594 : :
3595 [ + - + - : 2439 : TAILQ_INIT(&ctrlr_ch->pending_resets);
+ - + - +
- + - + -
+ - ]
3596 : :
3597 : 2439 : return nvme_qpair_create(nvme_ctrlr, ctrlr_ch);
3598 : : }
3599 : :
3600 : : static void
3601 : 2439 : nvme_qpair_delete(struct nvme_qpair *nvme_qpair)
3602 : : {
3603 : : struct nvme_io_path *io_path, *next;
3604 : :
3605 [ + + + - : 2439 : assert(nvme_qpair->group != NULL);
+ - # # ]
3606 : :
3607 [ + + + - : 4930 : TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) {
+ - + + +
- + - + -
+ + ]
3608 [ + + + - : 2491 : TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
3609 : 2491 : nvme_io_path_free(io_path);
3610 : 1 : }
3611 : :
3612 [ + + + - : 2439 : TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
3613 : :
3614 [ + - + - ]: 2439 : spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group));
3615 : :
3616 [ + - + - ]: 2439 : nvme_ctrlr_release(nvme_qpair->ctrlr);
3617 : :
3618 : 2439 : free(nvme_qpair);
3619 : 2439 : }
3620 : :
3621 : : static void
3622 : 2439 : bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3623 : : {
3624 : 2439 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3625 : : struct nvme_qpair *nvme_qpair;
3626 : :
3627 [ + - + - ]: 2439 : nvme_qpair = ctrlr_ch->qpair;
3628 [ + + # # ]: 2439 : assert(nvme_qpair != NULL);
3629 : :
3630 : 2439 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
3631 : :
3632 [ + + + - : 2439 : if (nvme_qpair->qpair != NULL) {
+ - ]
3633 [ + - + - : 2368 : if (ctrlr_ch->reset_iter == NULL) {
- + ]
3634 [ + - + - ]: 2368 : spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
3635 : 1 : } else {
3636 : : /* Skip current ctrlr_channel in a full reset sequence because
3637 : : * it is being deleted now. The qpair is already being disconnected.
3638 : : * We do not have to restart disconnecting it.
3639 : : */
3640 [ # # # # ]: 0 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
3641 : : }
3642 : :
3643 : : /* We cannot release a reference to the poll group now.
3644 : : * The qpair may be disconnected asynchronously later.
3645 : : * We need to poll it until it is actually disconnected.
3646 : : * Just detach the qpair from the deleting ctrlr_channel.
3647 : : */
3648 [ + - + - ]: 2368 : nvme_qpair->ctrlr_ch = NULL;
3649 : 1 : } else {
3650 [ - + # # : 71 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
3651 : :
3652 : 71 : nvme_qpair_delete(nvme_qpair);
3653 : : }
3654 : 2439 : }
3655 : :
3656 : : static inline struct spdk_io_channel *
3657 : 818417 : bdev_nvme_get_accel_channel(struct nvme_poll_group *group)
3658 : : {
3659 [ + + # # : 818417 : if (spdk_unlikely(!group->accel_channel)) {
# # ]
3660 [ # # # # ]: 53 : group->accel_channel = spdk_accel_get_io_channel();
3661 [ - + # # : 53 : if (!group->accel_channel) {
# # ]
3662 : 0 : SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
3663 : : group);
3664 : 0 : return NULL;
3665 : : }
3666 : 0 : }
3667 : :
3668 [ # # # # ]: 818417 : return group->accel_channel;
3669 : 0 : }
3670 : :
3671 : : static void
3672 : 818417 : bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
3673 : : {
3674 : 818417 : spdk_accel_sequence_finish(seq, cb_fn, cb_arg);
3675 : 818417 : }
3676 : :
3677 : : static void
3678 : 0 : bdev_nvme_abort_sequence(void *seq)
3679 : : {
3680 : 0 : spdk_accel_sequence_abort(seq);
3681 : 0 : }
3682 : :
3683 : : static void
3684 : 406034 : bdev_nvme_reverse_sequence(void *seq)
3685 : : {
3686 : 406034 : spdk_accel_sequence_reverse(seq);
3687 : 406034 : }
3688 : :
3689 : : static int
3690 : 818417 : bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
3691 : : struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed,
3692 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3693 : : {
3694 : : struct spdk_io_channel *ch;
3695 : 818417 : struct nvme_poll_group *group = ctx;
3696 : :
3697 : 818417 : ch = bdev_nvme_get_accel_channel(group);
3698 [ - + ]: 818417 : if (spdk_unlikely(ch == NULL)) {
3699 : 0 : return -ENOMEM;
3700 : : }
3701 : :
3702 : 818417 : return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt,
3703 : 0 : domain, domain_ctx, seed, cb_fn, cb_arg);
3704 : 0 : }
3705 : :
3706 : : static int
3707 : 0 : bdev_nvme_append_copy(void *ctx, void **seq, struct iovec *dst_iovs, uint32_t dst_iovcnt,
3708 : : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
3709 : : struct iovec *src_iovs, uint32_t src_iovcnt,
3710 : : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
3711 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3712 : : {
3713 : : struct spdk_io_channel *ch;
3714 : 0 : struct nvme_poll_group *group = ctx;
3715 : :
3716 : 0 : ch = bdev_nvme_get_accel_channel(group);
3717 [ # # ]: 0 : if (spdk_unlikely(ch == NULL)) {
3718 : 0 : return -ENOMEM;
3719 : : }
3720 : :
3721 : 0 : return spdk_accel_append_copy((struct spdk_accel_sequence **)seq, ch,
3722 : 0 : dst_iovs, dst_iovcnt, dst_domain, dst_domain_ctx,
3723 : 0 : src_iovs, src_iovcnt, src_domain, src_domain_ctx,
3724 : 0 : cb_fn, cb_arg);
3725 : 0 : }
3726 : :
3727 : : static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
3728 : : .table_size = sizeof(struct spdk_nvme_accel_fn_table),
3729 : : .append_crc32c = bdev_nvme_append_crc32c,
3730 : : .append_copy = bdev_nvme_append_copy,
3731 : : .finish_sequence = bdev_nvme_finish_sequence,
3732 : : .reverse_sequence = bdev_nvme_reverse_sequence,
3733 : : .abort_sequence = bdev_nvme_abort_sequence,
3734 : : };
3735 : :
3736 : : static int
3737 : 2283 : bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
3738 : : {
3739 : 2283 : struct nvme_poll_group *group = ctx_buf;
3740 : :
3741 [ + - + - : 2283 : TAILQ_INIT(&group->qpair_list);
+ - + - +
- + - + -
+ - ]
3742 : :
3743 [ + - + - ]: 2283 : group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
3744 [ + + + - : 2283 : if (group->group == NULL) {
+ - ]
3745 : 0 : return -1;
3746 : : }
3747 : :
3748 [ + - + - : 2283 : group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us);
+ - ]
3749 : :
3750 [ + + + - : 2283 : if (group->poller == NULL) {
+ - ]
3751 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3752 : 0 : return -1;
3753 : : }
3754 : :
3755 : 2283 : return 0;
3756 : 1 : }
3757 : :
3758 : : static void
3759 : 2283 : bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf)
3760 : : {
3761 : 2283 : struct nvme_poll_group *group = ctx_buf;
3762 : :
3763 [ + + + - : 2283 : assert(TAILQ_EMPTY(&group->qpair_list));
+ - + - #
# ]
3764 : :
3765 [ + + + - : 2283 : if (group->accel_channel) {
+ - ]
3766 [ # # # # ]: 53 : spdk_put_io_channel(group->accel_channel);
3767 : 0 : }
3768 : :
3769 [ + - ]: 2283 : spdk_poller_unregister(&group->poller);
3770 [ + + + - : 2283 : if (spdk_nvme_poll_group_destroy(group->group)) {
+ - ]
3771 : 0 : SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n");
3772 [ # # ]: 0 : assert(false);
3773 : : }
3774 : 2283 : }
3775 : :
3776 : : static struct spdk_io_channel *
3777 : 2384 : bdev_nvme_get_io_channel(void *ctx)
3778 : : {
3779 : 2384 : struct nvme_bdev *nvme_bdev = ctx;
3780 : :
3781 : 2384 : return spdk_get_io_channel(nvme_bdev);
3782 : : }
3783 : :
3784 : : static void *
3785 : 23 : bdev_nvme_get_module_ctx(void *ctx)
3786 : : {
3787 : 23 : struct nvme_bdev *nvme_bdev = ctx;
3788 : : struct nvme_ns *nvme_ns;
3789 : :
3790 [ + - - + : 23 : if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) {
# # # # #
# ]
3791 : 0 : return NULL;
3792 : : }
3793 : :
3794 [ # # # # : 23 : nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list);
# # ]
3795 [ - + ]: 23 : if (!nvme_ns) {
3796 : 0 : return NULL;
3797 : : }
3798 : :
3799 [ # # # # ]: 23 : return nvme_ns->ns;
3800 : 0 : }
3801 : :
3802 : : static const char *
3803 : 0 : _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state)
3804 : : {
3805 [ # # # # : 0 : switch (ana_state) {
# # ]
3806 : 0 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
3807 : 0 : return "optimized";
3808 : 0 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
3809 : 0 : return "non_optimized";
3810 : 0 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
3811 : 0 : return "inaccessible";
3812 : 0 : case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
3813 : 0 : return "persistent_loss";
3814 : 0 : case SPDK_NVME_ANA_CHANGE_STATE:
3815 : 0 : return "change";
3816 : 0 : default:
3817 : 0 : return NULL;
3818 : : }
3819 : 0 : }
3820 : :
3821 : : static int
3822 : 10631 : bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
3823 : : {
3824 : 10631 : struct spdk_memory_domain **_domains = NULL;
3825 : 10631 : struct nvme_bdev *nbdev = ctx;
3826 : : struct nvme_ns *nvme_ns;
3827 : 10631 : int i = 0, _array_size = array_size;
3828 : 10631 : int rc = 0;
3829 : :
3830 [ + + + - : 21332 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
3831 [ + + + + ]: 10701 : if (domains && array_size >= i) {
3832 [ # # ]: 267 : _domains = &domains[i];
3833 : 0 : } else {
3834 : 10434 : _domains = NULL;
3835 : : }
3836 [ + - + - : 10701 : rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size);
+ - + - ]
3837 [ + + ]: 10701 : if (rc > 0) {
3838 [ # # ]: 4256 : i += rc;
3839 [ + + ]: 4256 : if (_array_size >= rc) {
3840 [ # # ]: 261 : _array_size -= rc;
3841 : 0 : } else {
3842 : 3995 : _array_size = 0;
3843 : : }
3844 [ + + ]: 6445 : } else if (rc < 0) {
3845 : 0 : return rc;
3846 : : }
3847 : 4 : }
3848 : :
3849 : 10631 : return i;
3850 : 4 : }
3851 : :
3852 : : static const char *
3853 : 744 : nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr)
3854 : : {
3855 [ - + # # ]: 744 : if (nvme_ctrlr->destruct) {
3856 : 0 : return "deleting";
3857 [ - + # # : 744 : } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
3858 : 0 : return "failed";
3859 [ + + # # ]: 744 : } else if (nvme_ctrlr->resetting) {
3860 : 11 : return "resetting";
3861 [ + + # # ]: 733 : } else if (nvme_ctrlr->reconnect_is_delayed > 0) {
3862 : 7 : return "reconnect_is_delayed";
3863 [ - + # # ]: 726 : } else if (nvme_ctrlr->disabled) {
3864 : 0 : return "disabled";
3865 : : } else {
3866 : 726 : return "enabled";
3867 : : }
3868 : 0 : }
3869 : :
3870 : : void
3871 : 744 : nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr)
3872 : 744 : {
3873 : : struct spdk_nvme_transport_id *trid;
3874 : : const struct spdk_nvme_ctrlr_opts *opts;
3875 : : const struct spdk_nvme_ctrlr_data *cdata;
3876 : : struct nvme_path_id *path_id;
3877 : : int32_t numa_id;
3878 : :
3879 : 744 : spdk_json_write_object_begin(w);
3880 : :
3881 : 744 : spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr));
3882 : :
3883 : : #ifdef SPDK_CONFIG_NVME_CUSE
3884 : 744 : size_t cuse_name_size = 128;
3885 [ - + ]: 744 : char cuse_name[cuse_name_size];
3886 : :
3887 [ # # # # ]: 744 : int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size);
3888 [ + + ]: 744 : if (rc == 0) {
3889 : 3 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
3890 : 0 : }
3891 : : #endif
3892 [ # # # # : 744 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
3893 : 744 : spdk_json_write_named_object_begin(w, "trid");
3894 : 744 : nvme_bdev_dump_trid_json(trid, w);
3895 : 744 : spdk_json_write_object_end(w);
3896 : :
3897 [ # # # # : 744 : path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link);
# # # # #
# ]
3898 [ + + ]: 744 : if (path_id != NULL) {
3899 : 12 : spdk_json_write_named_array_begin(w, "alternate_trids");
3900 : 0 : do {
3901 [ # # ]: 16 : trid = &path_id->trid;
3902 : 16 : spdk_json_write_object_begin(w);
3903 : 16 : nvme_bdev_dump_trid_json(trid, w);
3904 : 16 : spdk_json_write_object_end(w);
3905 : :
3906 [ # # # # : 16 : path_id = TAILQ_NEXT(path_id, link);
# # ]
3907 [ + + ]: 16 : } while (path_id != NULL);
3908 : 12 : spdk_json_write_array_end(w);
3909 : 0 : }
3910 : :
3911 [ # # # # ]: 744 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
3912 [ # # # # ]: 744 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
3913 : :
3914 [ # # # # ]: 744 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
3915 : 744 : spdk_json_write_named_object_begin(w, "host");
3916 [ # # ]: 744 : spdk_json_write_named_string(w, "nqn", opts->hostnqn);
3917 [ # # ]: 744 : spdk_json_write_named_string(w, "addr", opts->src_addr);
3918 [ # # ]: 744 : spdk_json_write_named_string(w, "svcid", opts->src_svcid);
3919 : 744 : spdk_json_write_object_end(w);
3920 : :
3921 [ # # # # ]: 744 : numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr);
3922 [ + + ]: 744 : if (numa_id != SPDK_ENV_NUMA_ID_ANY) {
3923 : 358 : spdk_json_write_named_uint32(w, "numa_id", numa_id);
3924 : 0 : }
3925 : 744 : spdk_json_write_object_end(w);
3926 : 744 : }
3927 : :
3928 : : static void
3929 : 1128 : nvme_namespace_info_json(struct spdk_json_write_ctx *w,
3930 : : struct nvme_ns *nvme_ns)
3931 : 1128 : {
3932 : : struct spdk_nvme_ns *ns;
3933 : : struct spdk_nvme_ctrlr *ctrlr;
3934 : : const struct spdk_nvme_ctrlr_data *cdata;
3935 : : const struct spdk_nvme_transport_id *trid;
3936 : : union spdk_nvme_vs_register vs;
3937 : : const struct spdk_nvme_ns_data *nsdata;
3938 : 897 : char buf[128];
3939 : :
3940 [ # # # # ]: 1128 : ns = nvme_ns->ns;
3941 [ - + ]: 1128 : if (ns == NULL) {
3942 : 0 : return;
3943 : : }
3944 : :
3945 : 1128 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
3946 : :
3947 : 1128 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3948 : 1128 : trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
3949 : 1128 : vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
3950 : :
3951 : 1128 : spdk_json_write_object_begin(w);
3952 : :
3953 [ + + # # : 1128 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
3954 [ # # ]: 897 : spdk_json_write_named_string(w, "pci_address", trid->traddr);
3955 : 0 : }
3956 : :
3957 : 1128 : spdk_json_write_named_object_begin(w, "trid");
3958 : :
3959 : 1128 : nvme_bdev_dump_trid_json(trid, w);
3960 : :
3961 : 1128 : spdk_json_write_object_end(w);
3962 : :
3963 : : #ifdef SPDK_CONFIG_NVME_CUSE
3964 : 1128 : size_t cuse_name_size = 128;
3965 [ - + ]: 1128 : char cuse_name[cuse_name_size];
3966 : :
3967 : 1128 : int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns),
3968 : : cuse_name, &cuse_name_size);
3969 [ + + ]: 1128 : if (rc == 0) {
3970 : 4 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
3971 : 0 : }
3972 : : #endif
3973 : :
3974 : 1128 : spdk_json_write_named_object_begin(w, "ctrlr_data");
3975 : :
3976 [ # # # # ]: 1128 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
3977 : :
3978 [ # # # # ]: 1128 : spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);
3979 : :
3980 [ - + ]: 1128 : snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn);
3981 : 1128 : spdk_str_trim(buf);
3982 : 1128 : spdk_json_write_named_string(w, "model_number", buf);
3983 : :
3984 [ - + ]: 1128 : snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn);
3985 : 1128 : spdk_str_trim(buf);
3986 : 1128 : spdk_json_write_named_string(w, "serial_number", buf);
3987 : :
3988 [ - + ]: 1128 : snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr);
3989 : 1128 : spdk_str_trim(buf);
3990 : 1128 : spdk_json_write_named_string(w, "firmware_revision", buf);
3991 : :
3992 [ + + # # : 1128 : if (cdata->subnqn[0] != '\0') {
# # # # #
# ]
3993 [ # # ]: 1089 : spdk_json_write_named_string(w, "subnqn", cdata->subnqn);
3994 : 0 : }
3995 : :
3996 : 1128 : spdk_json_write_named_object_begin(w, "oacs");
3997 : :
3998 [ # # # # ]: 1128 : spdk_json_write_named_uint32(w, "security", cdata->oacs.security);
3999 [ # # # # ]: 1128 : spdk_json_write_named_uint32(w, "format", cdata->oacs.format);
4000 [ # # # # ]: 1128 : spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware);
4001 [ # # # # ]: 1128 : spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage);
4002 : :
4003 : 1128 : spdk_json_write_object_end(w);
4004 : :
4005 [ # # # # ]: 1128 : spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr);
4006 [ # # # # ]: 1128 : spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting);
4007 : :
4008 : 1128 : spdk_json_write_object_end(w);
4009 : :
4010 : 1128 : spdk_json_write_named_object_begin(w, "vs");
4011 : :
4012 : 1128 : spdk_json_write_name(w, "nvme_version");
4013 [ - + ]: 1128 : if (vs.bits.ter) {
4014 : 0 : spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter);
4015 : 0 : } else {
4016 : 1128 : spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr);
4017 : : }
4018 : :
4019 : 1128 : spdk_json_write_object_end(w);
4020 : :
4021 : 1128 : nsdata = spdk_nvme_ns_get_data(ns);
4022 : :
4023 : 1128 : spdk_json_write_named_object_begin(w, "ns_data");
4024 : :
4025 : 1128 : spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns));
4026 : :
4027 [ - + # # : 1128 : if (cdata->cmic.ana_reporting) {
# # ]
4028 : 0 : spdk_json_write_named_string(w, "ana_state",
4029 [ # # # # ]: 0 : _nvme_ana_state_str(nvme_ns->ana_state));
4030 : 0 : }
4031 : :
4032 [ # # # # ]: 1128 : spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share);
4033 : :
4034 : 1128 : spdk_json_write_object_end(w);
4035 : :
4036 [ + + # # : 1128 : if (cdata->oacs.security) {
# # ]
4037 : 714 : spdk_json_write_named_object_begin(w, "security");
4038 : :
4039 [ - + # # : 714 : spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal);
# # # # #
# ]
4040 : :
4041 : 714 : spdk_json_write_object_end(w);
4042 : 0 : }
4043 : :
4044 : 1128 : spdk_json_write_object_end(w);
4045 : 0 : }
4046 : :
4047 : : static const char *
4048 : 1118 : nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev)
4049 : : {
4050 [ + - - # : 1118 : switch (nbdev->mp_policy) {
# # # ]
4051 : 1118 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
4052 : 1118 : return "active_passive";
4053 : 0 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
4054 : 0 : return "active_active";
4055 : 0 : default:
4056 [ # # ]: 0 : assert(false);
4057 : : return "invalid";
4058 : : }
4059 : 0 : }
4060 : :
4061 : : static const char *
4062 : 0 : nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev)
4063 : : {
4064 [ # # # # : 0 : switch (nbdev->mp_selector) {
# # # ]
4065 : 0 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
4066 : 0 : return "round_robin";
4067 : 0 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
4068 : 0 : return "queue_depth";
4069 : 0 : default:
4070 [ # # ]: 0 : assert(false);
4071 : : return "invalid";
4072 : : }
4073 : 0 : }
4074 : :
4075 : : static int
4076 : 1118 : bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
4077 : : {
4078 : 1118 : struct nvme_bdev *nvme_bdev = ctx;
4079 : : struct nvme_ns *nvme_ns;
4080 : :
4081 [ - + # # ]: 1118 : pthread_mutex_lock(&nvme_bdev->mutex);
4082 : 1118 : spdk_json_write_named_array_begin(w, "nvme");
4083 [ + + # # : 2246 : TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
4084 : 1128 : nvme_namespace_info_json(w, nvme_ns);
4085 : 0 : }
4086 : 1118 : spdk_json_write_array_end(w);
4087 : 1118 : spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev));
4088 [ - + # # : 1118 : if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
4089 : 0 : spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev));
4090 [ # # # # : 0 : if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
4091 [ # # # # ]: 0 : spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io);
4092 : 0 : }
4093 : 0 : }
4094 [ - + # # ]: 1118 : pthread_mutex_unlock(&nvme_bdev->mutex);
4095 : :
4096 : 1118 : return 0;
4097 : : }
4098 : :
4099 : : static void
4100 : 166 : bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
4101 : : {
4102 : : /* No config per bdev needed */
4103 : 166 : }
4104 : :
4105 : : static uint64_t
4106 : 0 : bdev_nvme_get_spin_time(struct spdk_io_channel *ch)
4107 : : {
4108 : 0 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
4109 : : struct nvme_io_path *io_path;
4110 : : struct nvme_poll_group *group;
4111 : 0 : uint64_t spin_time = 0;
4112 : :
4113 [ # # # # : 0 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
4114 [ # # # # : 0 : group = io_path->qpair->group;
# # # # ]
4115 : :
4116 [ # # # # : 0 : if (!group || !group->collect_spin_stat) {
# # # # #
# ]
4117 : 0 : continue;
4118 : : }
4119 : :
4120 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
4121 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
4122 [ # # # # ]: 0 : group->end_ticks = 0;
4123 : 0 : }
4124 : :
4125 [ # # # # ]: 0 : spin_time += group->spin_ticks;
4126 [ # # # # ]: 0 : group->start_ticks = 0;
4127 [ # # # # ]: 0 : group->spin_ticks = 0;
4128 : 0 : }
4129 : :
4130 [ # # ]: 0 : return (spin_time * 1000000ULL) / spdk_get_ticks_hz();
4131 : : }
4132 : :
4133 : : static void
4134 : 0 : bdev_nvme_reset_device_stat(void *ctx)
4135 : : {
4136 : 0 : struct nvme_bdev *nbdev = ctx;
4137 : :
4138 [ # # # # : 0 : if (nbdev->err_stat != NULL) {
# # ]
4139 [ # # # # : 0 : memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat));
# # ]
4140 : 0 : }
4141 : 0 : }
4142 : :
4143 : : /* JSON string should be lowercases and underscore delimited string. */
4144 : : static void
4145 : 24 : bdev_nvme_format_nvme_status(char *dst, const char *src)
4146 : : {
4147 : 0 : char tmp[256];
4148 : :
4149 : 24 : spdk_strcpy_replace(dst, 256, src, " - ", "_");
4150 : 24 : spdk_strcpy_replace(tmp, 256, dst, "-", "_");
4151 : 24 : spdk_strcpy_replace(dst, 256, tmp, " ", "_");
4152 : 24 : spdk_strlwr(dst);
4153 : 24 : }
4154 : :
4155 : : static void
4156 : 29 : bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w)
4157 : : {
4158 : 29 : struct nvme_bdev *nbdev = ctx;
4159 : 29 : struct spdk_nvme_status status = {};
4160 : : uint16_t sct, sc;
4161 : 0 : char status_json[256];
4162 : : const char *status_str;
4163 : :
4164 [ + + # # : 29 : if (nbdev->err_stat == NULL) {
# # ]
4165 : 17 : return;
4166 : : }
4167 : :
4168 : 12 : spdk_json_write_named_object_begin(w, "nvme_error");
4169 : :
4170 : 12 : spdk_json_write_named_object_begin(w, "status_type");
4171 [ + + ]: 108 : for (sct = 0; sct < 8; sct++) {
4172 [ + + # # : 96 : if (nbdev->err_stat->status_type[sct] == 0) {
# # # # #
# # # #
# ]
4173 : 84 : continue;
4174 : : }
4175 : 12 : status.sct = sct;
4176 : :
4177 : 12 : status_str = spdk_nvme_cpl_get_status_type_string(&status);
4178 [ - + # # ]: 12 : assert(status_str != NULL);
4179 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4180 : :
4181 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]);
# # # # #
# # # ]
4182 : 0 : }
4183 : 12 : spdk_json_write_object_end(w);
4184 : :
4185 : 12 : spdk_json_write_named_object_begin(w, "status_code");
4186 [ + + ]: 60 : for (sct = 0; sct < 4; sct++) {
4187 : 48 : status.sct = sct;
4188 [ + + ]: 12336 : for (sc = 0; sc < 256; sc++) {
4189 [ + + # # : 12288 : if (nbdev->err_stat->status[sct][sc] == 0) {
# # # # #
# # # # #
# # ]
4190 : 12276 : continue;
4191 : : }
4192 : 12 : status.sc = sc;
4193 : :
4194 : 12 : status_str = spdk_nvme_cpl_get_status_string(&status);
4195 [ - + # # ]: 12 : assert(status_str != NULL);
4196 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4197 : :
4198 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]);
# # # # #
# # # #
# ]
4199 : 0 : }
4200 : 0 : }
4201 : 12 : spdk_json_write_object_end(w);
4202 : :
4203 : 12 : spdk_json_write_object_end(w);
4204 : 0 : }
4205 : :
4206 : : static bool
4207 : 162750 : bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
4208 : : {
4209 : 162750 : struct nvme_bdev *nbdev = ctx;
4210 : : struct spdk_nvme_ctrlr *ctrlr;
4211 : :
4212 [ + + + + ]: 162750 : if (!g_opts.allow_accel_sequence) {
4213 : 162666 : return false;
4214 : : }
4215 : :
4216 [ + + ]: 84 : switch (type) {
4217 : 8 : case SPDK_BDEV_IO_TYPE_WRITE:
4218 : : case SPDK_BDEV_IO_TYPE_READ:
4219 : 8 : break;
4220 : 76 : default:
4221 : 76 : return false;
4222 : : }
4223 : :
4224 [ # # ]: 8 : ctrlr = bdev_nvme_get_ctrlr(&nbdev->disk);
4225 [ - + # # ]: 8 : assert(ctrlr != NULL);
4226 : :
4227 : 8 : return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED;
4228 : 84 : }
4229 : :
4230 : : static const struct spdk_bdev_fn_table nvmelib_fn_table = {
4231 : : .destruct = bdev_nvme_destruct,
4232 : : .submit_request = bdev_nvme_submit_request,
4233 : : .io_type_supported = bdev_nvme_io_type_supported,
4234 : : .get_io_channel = bdev_nvme_get_io_channel,
4235 : : .dump_info_json = bdev_nvme_dump_info_json,
4236 : : .write_config_json = bdev_nvme_write_config_json,
4237 : : .get_spin_time = bdev_nvme_get_spin_time,
4238 : : .get_module_ctx = bdev_nvme_get_module_ctx,
4239 : : .get_memory_domains = bdev_nvme_get_memory_domains,
4240 : : .accel_sequence_supported = bdev_nvme_accel_sequence_supported,
4241 : : .reset_device_stat = bdev_nvme_reset_device_stat,
4242 : : .dump_device_stat_json = bdev_nvme_dump_device_stat_json,
4243 : : };
4244 : :
4245 : : typedef int (*bdev_nvme_parse_ana_log_page_cb)(
4246 : : const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
4247 : :
4248 : : static int
4249 : 550 : bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
4250 : : bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
4251 : : {
4252 : : struct spdk_nvme_ana_group_descriptor *copied_desc;
4253 : : uint8_t *orig_desc;
4254 : : uint32_t i, desc_size, copy_len;
4255 : 550 : int rc = 0;
4256 : :
4257 [ - + # # : 550 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
4258 : 0 : return -EINVAL;
4259 : : }
4260 : :
4261 [ # # # # ]: 550 : copied_desc = nvme_ctrlr->copied_ana_desc;
4262 : :
4263 [ # # # # : 550 : orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
# # ]
4264 [ # # # # ]: 550 : copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
4265 : :
4266 [ + + # # : 743 : for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) {
# # # # #
# ]
4267 [ - + - + ]: 625 : memcpy(copied_desc, orig_desc, copy_len);
4268 : :
4269 [ # # # # ]: 625 : rc = cb_fn(copied_desc, cb_arg);
4270 [ + + ]: 625 : if (rc != 0) {
4271 : 432 : break;
4272 : : }
4273 : :
4274 : 193 : desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
4275 [ # # # # ]: 193 : copied_desc->num_of_nsid * sizeof(uint32_t);
4276 [ # # ]: 193 : orig_desc += desc_size;
4277 : 193 : copy_len -= desc_size;
4278 : 0 : }
4279 : :
4280 : 550 : return rc;
4281 : 0 : }
4282 : :
4283 : : static int
4284 : 17 : nvme_ns_ana_transition_timedout(void *ctx)
4285 : : {
4286 : 17 : struct nvme_ns *nvme_ns = ctx;
4287 : :
4288 [ # # ]: 17 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4289 [ # # # # ]: 17 : nvme_ns->ana_transition_timedout = true;
4290 : :
4291 : 17 : return SPDK_POLLER_BUSY;
4292 : : }
4293 : :
4294 : : static void
4295 : 562 : _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns,
4296 : : const struct spdk_nvme_ana_group_descriptor *desc)
4297 : : {
4298 : : const struct spdk_nvme_ctrlr_data *cdata;
4299 : :
4300 [ # # # # : 562 : nvme_ns->ana_group_id = desc->ana_group_id;
# # # # ]
4301 [ # # # # : 562 : nvme_ns->ana_state = desc->ana_state;
# # ]
4302 [ # # # # ]: 562 : nvme_ns->ana_state_updating = false;
4303 : :
4304 [ + + + # : 562 : switch (nvme_ns->ana_state) {
# # # ]
4305 : 510 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
4306 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
4307 [ # # # # ]: 510 : nvme_ns->ana_transition_timedout = false;
4308 [ # # ]: 510 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4309 : 510 : break;
4310 : :
4311 : 49 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
4312 : : case SPDK_NVME_ANA_CHANGE_STATE:
4313 [ + + # # : 49 : if (nvme_ns->anatt_timer != NULL) {
# # ]
4314 : 16 : break;
4315 : : }
4316 : :
4317 [ # # # # : 33 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
4318 [ # # # # : 33 : nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout,
# # # # ]
4319 : : nvme_ns,
4320 : : cdata->anatt * SPDK_SEC_TO_USEC);
4321 : 33 : break;
4322 : 3 : default:
4323 : 3 : break;
4324 : : }
4325 : 562 : }
4326 : :
4327 : : static int
4328 : 501 : nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg)
4329 : : {
4330 : 501 : struct nvme_ns *nvme_ns = cb_arg;
4331 : : uint32_t i;
4332 : :
4333 [ - + # # : 501 : assert(nvme_ns->ns != NULL);
# # # # ]
4334 : :
4335 [ + + # # : 567 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
4336 [ + + # # : 498 : if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) {
# # # # #
# # # ]
4337 : 66 : continue;
4338 : : }
4339 : :
4340 : 432 : _nvme_ns_set_ana_state(nvme_ns, desc);
4341 : 432 : return 1;
4342 : : }
4343 : :
4344 : 69 : return 0;
4345 : 0 : }
4346 : :
4347 : : static int
4348 : 15 : nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid)
4349 : : {
4350 : 15 : int rc = 0;
4351 : 15 : struct spdk_uuid new_uuid, namespace_uuid;
4352 : 15 : char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'};
4353 : : /* This namespace UUID was generated using uuid_generate() method. */
4354 : 15 : const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"};
4355 : : int size;
4356 : :
4357 [ - + - + : 15 : assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN);
# # ]
4358 : :
4359 : 15 : spdk_uuid_set_null(&new_uuid);
4360 : 15 : spdk_uuid_set_null(&namespace_uuid);
4361 : :
4362 : 15 : size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid);
4363 [ + - - + ]: 15 : if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) {
4364 : 0 : return -EINVAL;
4365 : : }
4366 : :
4367 : 15 : spdk_uuid_parse(&namespace_uuid, namespace_str);
4368 : :
4369 : 15 : rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size);
4370 [ + - ]: 15 : if (rc == 0) {
4371 [ # # # # ]: 15 : memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid));
4372 : 0 : }
4373 : :
4374 : 15 : return rc;
4375 : 0 : }
4376 : :
4377 : : static int
4378 : 1453 : nvme_disk_create(struct spdk_bdev *disk, const char *base_name,
4379 : : struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns,
4380 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx)
4381 : : {
4382 : : const struct spdk_uuid *uuid;
4383 : : const uint8_t *nguid;
4384 : : const struct spdk_nvme_ctrlr_data *cdata;
4385 : : const struct spdk_nvme_ns_data *nsdata;
4386 : : const struct spdk_nvme_ctrlr_opts *opts;
4387 : : enum spdk_nvme_csi csi;
4388 : : uint32_t atomic_bs, phys_bs, bs;
4389 : 1453 : char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'};
4390 : : int rc;
4391 : :
4392 : 1453 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4393 : 1453 : csi = spdk_nvme_ns_get_csi(ns);
4394 : 1453 : opts = spdk_nvme_ctrlr_get_opts(ctrlr);
4395 : :
4396 [ + + - ]: 1453 : switch (csi) {
4397 : 1450 : case SPDK_NVME_CSI_NVM:
4398 [ + - + - ]: 1451 : disk->product_name = "NVMe disk";
4399 : 1451 : break;
4400 : 2 : case SPDK_NVME_CSI_ZNS:
4401 [ # # # # ]: 2 : disk->product_name = "NVMe ZNS disk";
4402 [ # # # # ]: 2 : disk->zoned = true;
4403 [ # # # # ]: 2 : disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
4404 [ # # # # : 2 : disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) /
# # # # ]
4405 : 2 : spdk_nvme_ns_get_extended_sector_size(ns);
4406 [ # # # # ]: 2 : disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns);
4407 [ # # # # ]: 2 : disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns);
4408 : 2 : break;
4409 : 0 : default:
4410 [ # # # # : 0 : if (bdev_opts->allow_unrecognized_csi) {
# # # # ]
4411 [ # # # # ]: 0 : disk->product_name = "NVMe Passthrough disk";
4412 : 0 : break;
4413 : : }
4414 : 0 : SPDK_ERRLOG("unsupported CSI: %u\n", csi);
4415 : 0 : return -ENOTSUP;
4416 : : }
4417 : :
4418 : 1453 : nguid = spdk_nvme_ns_get_nguid(ns);
4419 [ + + ]: 1453 : if (!nguid) {
4420 : 1070 : uuid = spdk_nvme_ns_get_uuid(ns);
4421 [ + + ]: 1070 : if (uuid) {
4422 [ # # ]: 348 : disk->uuid = *uuid;
4423 [ + + + + ]: 722 : } else if (g_opts.generate_uuids) {
4424 [ # # ]: 0 : spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0');
4425 [ # # ]: 0 : rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid);
4426 [ # # ]: 0 : if (rc < 0) {
4427 [ # # ]: 0 : SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc));
4428 : 0 : return rc;
4429 : : }
4430 : 0 : }
4431 : 1 : } else {
4432 [ # # # # : 383 : memcpy(&disk->uuid, nguid, sizeof(disk->uuid));
# # ]
4433 : : }
4434 : :
4435 [ + - + - ]: 1453 : disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns));
4436 [ + + + - : 1453 : if (!disk->name) {
- + ]
4437 : 0 : return -ENOMEM;
4438 : : }
4439 : :
4440 [ + - + - ]: 1453 : disk->write_cache = 0;
4441 [ + + + - : 1453 : if (cdata->vwc.present) {
+ - ]
4442 : : /* Enable if the Volatile Write Cache exists */
4443 [ # # # # ]: 1235 : disk->write_cache = 1;
4444 : 0 : }
4445 [ + + + - : 1453 : if (cdata->oncs.write_zeroes) {
+ - ]
4446 [ # # # # ]: 1254 : disk->max_write_zeroes = UINT16_MAX + 1;
4447 : 0 : }
4448 [ + - + - ]: 1453 : disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns);
4449 [ + - + - ]: 1453 : disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns);
4450 [ + - + - ]: 1453 : disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr);
4451 [ + - + - : 1453 : disk->ctratt.raw = cdata->ctratt.raw;
+ - + - +
- + - ]
4452 : : /* NVMe driver will split one request into multiple requests
4453 : : * based on MDTS and stripe boundary, the bdev layer will use
4454 : : * max_segment_size and max_num_segments to split one big IO
4455 : : * into multiple requests, then small request can't run out
4456 : : * of NVMe internal requests data structure.
4457 : : */
4458 [ + - + + : 1453 : if (opts && opts->io_queue_requests) {
+ - - + ]
4459 [ + - + - : 1339 : disk->max_num_segments = opts->io_queue_requests / 2;
+ - + - +
- ]
4460 : 1 : }
4461 [ + + ]: 1453 : if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
4462 : : /* The nvme driver will try to split I/O that have too many
4463 : : * SGEs, but it doesn't work if that last SGE doesn't end on
4464 : : * an aggregate total that is block aligned. The bdev layer has
4465 : : * a more robust splitting framework, so use that instead for
4466 : : * this case. (See issue #3269.)
4467 : : */
4468 : 1235 : uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr);
4469 : :
4470 [ - + # # : 1235 : if (disk->max_num_segments == 0) {
# # ]
4471 [ # # # # ]: 0 : disk->max_num_segments = max_sges;
4472 : 0 : } else {
4473 [ # # # # : 1235 : disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges);
# # # # #
# # # #
# ]
4474 : : }
4475 : 0 : }
4476 [ + - + - ]: 1453 : disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns);
4477 : :
4478 : 1453 : nsdata = spdk_nvme_ns_get_data(ns);
4479 : 1453 : bs = spdk_nvme_ns_get_sector_size(ns);
4480 : 1453 : atomic_bs = bs;
4481 : 1453 : phys_bs = bs;
4482 [ + - + - : 1453 : if (nsdata->nabo == 0) {
- + ]
4483 [ + + + + : 1453 : if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) {
- + # # #
# # # ]
4484 [ # # # # : 21 : atomic_bs = bs * (1 + nsdata->nawupf);
# # ]
4485 : 0 : } else {
4486 [ + - + - : 1432 : atomic_bs = bs * (1 + cdata->awupf);
+ - ]
4487 : : }
4488 : 1 : }
4489 [ + + + - : 1453 : if (nsdata->nsfeat.optperf) {
+ - ]
4490 [ # # # # : 1254 : phys_bs = bs * (1 + nsdata->npwg);
# # ]
4491 : 0 : }
4492 [ - + + - : 1453 : disk->phys_blocklen = spdk_min(phys_bs, atomic_bs);
+ - ]
4493 : :
4494 [ + - + - ]: 1453 : disk->md_len = spdk_nvme_ns_get_md_size(ns);
4495 [ + + + - : 1453 : if (disk->md_len != 0) {
- + ]
4496 [ # # # # : 52 : disk->md_interleave = nsdata->flbas.extended;
# # # # ]
4497 [ # # # # ]: 52 : disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns);
4498 [ - + # # : 52 : if (disk->dif_type != SPDK_DIF_DISABLE) {
# # ]
4499 [ # # # # : 0 : disk->dif_is_head_of_md = nsdata->dps.md_start;
# # # # ]
4500 [ # # # # : 0 : disk->dif_check_flags = bdev_opts->prchk_flags;
# # # # ]
4501 [ # # # # ]: 0 : disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns);
4502 : 0 : }
4503 : 0 : }
4504 : :
4505 [ + + ]: 1453 : if (!(spdk_nvme_ctrlr_get_flags(ctrlr) &
4506 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) {
4507 [ + - + - ]: 1089 : disk->acwu = 0;
4508 [ + - # # : 365 : } else if (nsdata->nsfeat.ns_atomic_write_unit) {
# # ]
4509 [ # # # # : 364 : disk->acwu = nsdata->nacwu + 1; /* 0-based */
# # # # #
# ]
4510 : 0 : } else {
4511 [ # # # # : 0 : disk->acwu = cdata->acwu + 1; /* 0-based */
# # # # #
# ]
4512 : : }
4513 : :
4514 [ + + + - : 1453 : if (cdata->oncs.copy) {
+ - ]
4515 : : /* For now bdev interface allows only single segment copy */
4516 [ # # # # : 923 : disk->max_copy = nsdata->mssrl;
# # # # ]
4517 : 0 : }
4518 : :
4519 [ + - + - ]: 1453 : disk->ctxt = ctx;
4520 [ + - + - ]: 1453 : disk->fn_table = &nvmelib_fn_table;
4521 [ + - + - ]: 1453 : disk->module = &nvme_if;
4522 : :
4523 [ + - + - ]: 1453 : disk->numa.id_valid = 1;
4524 [ + - + - ]: 1453 : disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr);
4525 : :
4526 : 1453 : return 0;
4527 : 1 : }
4528 : :
4529 : : static struct nvme_bdev *
4530 : 1453 : nvme_bdev_alloc(void)
4531 : : {
4532 : : struct nvme_bdev *bdev;
4533 : : int rc;
4534 : :
4535 : 1453 : bdev = calloc(1, sizeof(*bdev));
4536 [ + + ]: 1453 : if (!bdev) {
4537 : 0 : SPDK_ERRLOG("bdev calloc() failed\n");
4538 : 0 : return NULL;
4539 : : }
4540 : :
4541 [ + + + + ]: 1453 : if (g_opts.nvme_error_stat) {
4542 [ # # # # ]: 12 : bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat));
4543 [ - + # # : 12 : if (!bdev->err_stat) {
# # ]
4544 : 0 : SPDK_ERRLOG("err_stat calloc() failed\n");
4545 : 0 : free(bdev);
4546 : 0 : return NULL;
4547 : : }
4548 : 0 : }
4549 : :
4550 [ + + + - ]: 1453 : rc = pthread_mutex_init(&bdev->mutex, NULL);
4551 [ - + ]: 1453 : if (rc != 0) {
4552 [ # # # # ]: 0 : free(bdev->err_stat);
4553 : 0 : free(bdev);
4554 : 0 : return NULL;
4555 : : }
4556 : :
4557 [ - + - + ]: 1453 : bdev->ref = 1;
4558 [ - + - + ]: 1453 : bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE;
4559 [ - + - + ]: 1453 : bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN;
4560 [ - + - + ]: 1453 : bdev->rr_min_io = UINT32_MAX;
4561 [ - + - + : 1453 : TAILQ_INIT(&bdev->nvme_ns_list);
- + - + -
+ - + - +
- + ]
4562 : :
4563 : 1453 : return bdev;
4564 : 1 : }
4565 : :
4566 : : static int
4567 : 1453 : nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4568 : : {
4569 : : struct nvme_bdev *bdev;
4570 [ + - + - ]: 1453 : struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr;
4571 : : int rc;
4572 : :
4573 : 1453 : bdev = nvme_bdev_alloc();
4574 [ + + ]: 1453 : if (bdev == NULL) {
4575 : 0 : SPDK_ERRLOG("Failed to allocate NVMe bdev\n");
4576 : 0 : return -ENOMEM;
4577 : : }
4578 : :
4579 [ + - + - : 1453 : bdev->opal = nvme_ctrlr->opal_dev != NULL;
+ - + - ]
4580 : :
4581 [ + - + - : 1454 : rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr,
+ - + - +
- ]
4582 [ + - + - : 1 : nvme_ns->ns, &nvme_ctrlr->opts, bdev);
+ - ]
4583 [ - + ]: 1453 : if (rc != 0) {
4584 : 0 : SPDK_ERRLOG("Failed to create NVMe disk\n");
4585 : 0 : nvme_bdev_free(bdev);
4586 : 0 : return rc;
4587 : : }
4588 : :
4589 : 1454 : spdk_io_device_register(bdev,
4590 : : bdev_nvme_create_bdev_channel_cb,
4591 : : bdev_nvme_destroy_bdev_channel_cb,
4592 : : sizeof(struct nvme_bdev_channel),
4593 [ + - + - : 1453 : bdev->disk.name);
+ - ]
4594 : :
4595 [ + - + - ]: 1453 : nvme_ns->bdev = bdev;
4596 [ + - + - : 1453 : bdev->nsid = nvme_ns->id;
+ - + - ]
4597 [ + - + - : 1453 : TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4598 : :
4599 [ + - + - ]: 1453 : bdev->nbdev_ctrlr = nbdev_ctrlr;
4600 [ + - + - : 1453 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4601 : :
4602 [ + - ]: 1453 : rc = spdk_bdev_register(&bdev->disk);
4603 [ + + ]: 1453 : if (rc != 0) {
4604 : 5 : SPDK_ERRLOG("spdk_bdev_register() failed\n");
4605 : 5 : spdk_io_device_unregister(bdev, NULL);
4606 [ # # # # ]: 5 : nvme_ns->bdev = NULL;
4607 [ - + # # : 5 : TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4608 : 5 : nvme_bdev_free(bdev);
4609 : 5 : return rc;
4610 : : }
4611 : :
4612 : 1448 : return 0;
4613 : 1 : }
4614 : :
4615 : : static bool
4616 : 103 : bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
4617 : : {
4618 : : const struct spdk_nvme_ns_data *nsdata1, *nsdata2;
4619 : : const struct spdk_uuid *uuid1, *uuid2;
4620 : :
4621 : 103 : nsdata1 = spdk_nvme_ns_get_data(ns1);
4622 : 103 : nsdata2 = spdk_nvme_ns_get_data(ns2);
4623 : 103 : uuid1 = spdk_nvme_ns_get_uuid(ns1);
4624 : 103 : uuid2 = spdk_nvme_ns_get_uuid(ns2);
4625 : :
4626 [ - + - + : 125 : return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 &&
# # # # #
# ]
4627 [ + + + + : 100 : nsdata1->eui64 == nsdata2->eui64 &&
# # # # #
# ]
4628 [ + + + + ]: 97 : ((uuid1 == NULL && uuid2 == NULL) ||
4629 [ + + + - : 279 : (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) &&
+ + + + ]
4630 : 88 : spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2);
4631 : : }
4632 : :
4633 : : static bool
4634 : 56 : hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4635 : : struct spdk_nvme_ctrlr_opts *opts)
4636 : : {
4637 : : struct nvme_probe_skip_entry *entry;
4638 : :
4639 [ - + # # : 56 : TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) {
# # # # ]
4640 [ # # # # ]: 0 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
4641 : 0 : return false;
4642 : : }
4643 : 0 : }
4644 : :
4645 [ # # # # : 56 : opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst;
# # ]
4646 [ # # # # : 56 : opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight;
# # ]
4647 [ # # # # : 56 : opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight;
# # ]
4648 [ # # # # : 56 : opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight;
# # ]
4649 [ # # # # ]: 56 : opts->disable_read_ana_log_page = true;
4650 : :
4651 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr);
# # # # ]
4652 : :
4653 : 56 : return true;
4654 : 0 : }
4655 : :
4656 : : static void
4657 : 0 : nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
4658 : : {
4659 : 0 : struct nvme_ctrlr *nvme_ctrlr = ctx;
4660 : :
4661 [ # # # # : 0 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
4662 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Abort failed. Resetting controller. sc is %u, sct is %u.\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4663 : : cpl->status.sc, cpl->status.sct);
4664 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4665 [ # # # # : 0 : } else if (cpl->cdw0 & 0x1) {
# # ]
4666 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Specified command could not be aborted.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4667 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4668 : 0 : }
4669 : 0 : }
4670 : :
4671 : : static void
4672 : 0 : timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
4673 : : struct spdk_nvme_qpair *qpair, uint16_t cid)
4674 : : {
4675 : 0 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
4676 : : union spdk_nvme_csts_register csts;
4677 : : int rc;
4678 : :
4679 [ # # # # : 0 : assert(nvme_ctrlr->ctrlr == ctrlr);
# # # # ]
4680 : :
4681 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4682 : : ctrlr, qpair, cid);
4683 : :
4684 : : /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O
4685 : : * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we
4686 : : * would submit another fabrics cmd on the admin queue to read CSTS and check for its
4687 : : * completion recursively.
4688 : : */
4689 [ # # # # : 0 : if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) {
# # # # #
# # # #
# ]
4690 : 0 : csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr);
4691 [ # # ]: 0 : if (csts.bits.cfs) {
4692 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Controller Fatal Status, reset required\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4693 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4694 : 0 : return;
4695 : : }
4696 : 0 : }
4697 : :
4698 [ # # # # ]: 0 : switch (g_opts.action_on_timeout) {
4699 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
4700 [ # # ]: 0 : if (qpair) {
4701 : : /* Don't send abort to ctrlr when ctrlr is not available. */
4702 [ # # # # ]: 0 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4703 [ # # ]: 0 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
4704 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4705 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Quit abort. Ctrlr is not available.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4706 : 0 : return;
4707 : : }
4708 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4709 : :
4710 : 0 : rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
4711 : 0 : nvme_abort_cpl, nvme_ctrlr);
4712 [ # # ]: 0 : if (rc == 0) {
4713 : 0 : return;
4714 : : }
4715 : :
4716 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to send abort. Resetting, rc is %d.\n", rc);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4717 : 0 : }
4718 : :
4719 : : /* FALLTHROUGH */
4720 : : case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
4721 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4722 : 0 : break;
4723 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
4724 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "No action for nvme controller timeout.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4725 : 0 : break;
4726 : 0 : default:
4727 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "An invalid timeout action value is found.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4728 : 0 : break;
4729 : : }
4730 : 0 : }
4731 : :
4732 : : static struct nvme_ns *
4733 : 1510 : nvme_ns_alloc(void)
4734 : : {
4735 : : struct nvme_ns *nvme_ns;
4736 : :
4737 : 1510 : nvme_ns = calloc(1, sizeof(struct nvme_ns));
4738 [ + + ]: 1510 : if (nvme_ns == NULL) {
4739 : 0 : return NULL;
4740 : : }
4741 : :
4742 [ + + + + : 1510 : if (g_opts.io_path_stat) {
+ - ]
4743 [ # # # # ]: 0 : nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
4744 [ # # # # : 0 : if (nvme_ns->stat == NULL) {
# # ]
4745 : 0 : free(nvme_ns);
4746 : 0 : return NULL;
4747 : : }
4748 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
4749 : 0 : }
4750 : :
4751 : 1510 : return nvme_ns;
4752 : 1 : }
4753 : :
4754 : : static void
4755 : 1510 : nvme_ns_free(struct nvme_ns *nvme_ns)
4756 : : {
4757 [ + - + - ]: 1510 : free(nvme_ns->stat);
4758 : 1510 : free(nvme_ns);
4759 : 1510 : }
4760 : :
4761 : : static void
4762 : 1510 : nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc)
4763 : : {
4764 [ + - + - ]: 1510 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4765 [ + - + - ]: 1510 : struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx;
4766 : :
4767 [ + + ]: 1510 : if (rc == 0) {
4768 [ - + - + ]: 1502 : nvme_ns->probe_ctx = NULL;
4769 [ - + - + ]: 1502 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4770 [ - + - + ]: 1502 : nvme_ctrlr->ref++;
4771 [ - + - + ]: 1502 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4772 : 1 : } else {
4773 [ # # ]: 8 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4774 : 8 : nvme_ns_free(nvme_ns);
4775 : : }
4776 : :
4777 [ + + ]: 1510 : if (ctx) {
4778 [ + - ]: 1446 : ctx->populates_in_progress--;
4779 [ + + + - : 1446 : if (ctx->populates_in_progress == 0) {
+ - ]
4780 : 49 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
4781 : 0 : }
4782 : 1 : }
4783 : 1510 : }
4784 : :
4785 : : static void
4786 : 6 : bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i,
4787 : : struct nvme_bdev *nbdev,
4788 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4789 : : {
4790 : 6 : struct nvme_ns *nvme_ns = ctx;
4791 : : int rc;
4792 : :
4793 : 6 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
4794 [ - + ]: 6 : if (rc != 0) {
4795 : 0 : SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n");
4796 : 0 : }
4797 : :
4798 : 6 : nvme_bdev_for_each_channel_continue(i, rc);
4799 : 6 : }
4800 : :
4801 : : static void
4802 : 6 : bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i,
4803 : : struct nvme_bdev *nbdev,
4804 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4805 : : {
4806 : 6 : struct nvme_ns *nvme_ns = ctx;
4807 : : struct nvme_io_path *io_path;
4808 : :
4809 : 6 : io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns);
4810 [ + - ]: 6 : if (io_path != NULL) {
4811 : 6 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
4812 : 0 : }
4813 : :
4814 : 6 : nvme_bdev_for_each_channel_continue(i, 0);
4815 : 6 : }
4816 : :
4817 : : static void
4818 : 0 : bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status)
4819 : : {
4820 : 0 : struct nvme_ns *nvme_ns = ctx;
4821 : :
4822 : 0 : nvme_ctrlr_populate_namespace_done(nvme_ns, -1);
4823 : 0 : }
4824 : :
4825 : : static void
4826 : 54 : bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
4827 : : {
4828 : 54 : struct nvme_ns *nvme_ns = ctx;
4829 : :
4830 [ + - ]: 54 : if (status == 0) {
4831 : 54 : nvme_ctrlr_populate_namespace_done(nvme_ns, 0);
4832 : 0 : } else {
4833 : : /* Delete the added io_paths and fail populating the namespace. */
4834 : 0 : nvme_bdev_for_each_channel(nbdev,
4835 : : bdev_nvme_delete_io_path,
4836 : 0 : nvme_ns,
4837 : : bdev_nvme_add_io_path_failed);
4838 : : }
4839 : 54 : }
4840 : :
4841 : : static int
4842 : 57 : nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns)
4843 : : {
4844 : : struct nvme_ns *tmp_ns;
4845 : : const struct spdk_nvme_ns_data *nsdata;
4846 : :
4847 [ # # # # ]: 57 : nsdata = spdk_nvme_ns_get_data(nvme_ns->ns);
4848 [ - + # # : 57 : if (!nsdata->nmic.can_share) {
# # ]
4849 : 0 : SPDK_ERRLOG("Namespace cannot be shared.\n");
4850 : 0 : return -EINVAL;
4851 : : }
4852 : :
4853 [ - + # # ]: 57 : pthread_mutex_lock(&bdev->mutex);
4854 : :
4855 [ # # # # : 57 : tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list);
# # ]
4856 [ - + # # ]: 57 : assert(tmp_ns != NULL);
4857 : :
4858 [ + - + + : 57 : if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) {
# # # # #
# # # # #
# # ]
4859 [ - + # # ]: 3 : pthread_mutex_unlock(&bdev->mutex);
4860 : 3 : SPDK_ERRLOG("Namespaces are not identical.\n");
4861 : 3 : return -EINVAL;
4862 : : }
4863 : :
4864 [ # # # # ]: 54 : bdev->ref++;
4865 [ # # # # : 54 : TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4866 [ # # # # ]: 54 : nvme_ns->bdev = bdev;
4867 : :
4868 [ - + # # ]: 54 : pthread_mutex_unlock(&bdev->mutex);
4869 : :
4870 : : /* Add nvme_io_path to nvme_bdev_channels dynamically. */
4871 : 54 : nvme_bdev_for_each_channel(bdev,
4872 : : bdev_nvme_add_io_path,
4873 : 0 : nvme_ns,
4874 : : bdev_nvme_add_io_path_done);
4875 : :
4876 : 54 : return 0;
4877 : 0 : }
4878 : :
4879 : : static void
4880 : 1510 : nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4881 : : {
4882 : : struct spdk_nvme_ns *ns;
4883 : : struct nvme_bdev *bdev;
4884 : 1510 : int rc = 0;
4885 : :
4886 [ + - + - : 1510 : ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id);
+ - + - ]
4887 [ - + ]: 1510 : if (!ns) {
4888 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "Invalid NS %d\n", nvme_ns->id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4889 : 0 : rc = -EINVAL;
4890 : 0 : goto done;
4891 : : }
4892 : :
4893 [ + - + - ]: 1510 : nvme_ns->ns = ns;
4894 [ + - + - ]: 1510 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
4895 : :
4896 [ + + + - : 1510 : if (nvme_ctrlr->ana_log_page != NULL) {
+ - ]
4897 : 435 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns);
4898 : 0 : }
4899 : :
4900 [ + - + - : 1510 : bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id);
+ - + - ]
4901 [ + + ]: 1511 : if (bdev == NULL) {
4902 : 1453 : rc = nvme_bdev_create(nvme_ctrlr, nvme_ns);
4903 : 1 : } else {
4904 : 57 : rc = nvme_bdev_add_ns(bdev, nvme_ns);
4905 [ + + ]: 57 : if (rc == 0) {
4906 : 54 : return;
4907 : : }
4908 : : }
4909 : 3 : done:
4910 : 1456 : nvme_ctrlr_populate_namespace_done(nvme_ns, rc);
4911 : 1 : }
4912 : :
4913 : : static void
4914 : 1502 : nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns)
4915 : : {
4916 [ + - + - ]: 1502 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4917 : :
4918 [ + + # # ]: 1502 : assert(nvme_ctrlr != NULL);
4919 : :
4920 [ + + + - ]: 1502 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4921 : :
4922 [ + - ]: 1502 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4923 : :
4924 [ + + + - : 1502 : if (nvme_ns->bdev != NULL) {
- + ]
4925 [ - + # # ]: 523 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4926 : 523 : return;
4927 : : }
4928 : :
4929 : 979 : nvme_ns_free(nvme_ns);
4930 [ + + + - ]: 979 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4931 : :
4932 : 979 : nvme_ctrlr_release(nvme_ctrlr);
4933 : 1 : }
4934 : :
4935 : : static void
4936 : 45 : bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
4937 : : {
4938 : 45 : struct nvme_ns *nvme_ns = ctx;
4939 : :
4940 : 45 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
4941 : 45 : }
4942 : :
4943 : : static void
4944 : 1502 : nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4945 : : {
4946 : : struct nvme_bdev *bdev;
4947 : :
4948 [ + - ]: 1502 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4949 : :
4950 [ + - + - ]: 1502 : bdev = nvme_ns->bdev;
4951 [ + + ]: 1502 : if (bdev != NULL) {
4952 [ - + # # ]: 670 : pthread_mutex_lock(&bdev->mutex);
4953 : :
4954 [ - + # # : 670 : assert(bdev->ref > 0);
# # # # ]
4955 [ # # # # ]: 670 : bdev->ref--;
4956 [ + + # # : 670 : if (bdev->ref == 0) {
# # ]
4957 [ - + # # ]: 625 : pthread_mutex_unlock(&bdev->mutex);
4958 : :
4959 [ # # ]: 625 : spdk_bdev_unregister(&bdev->disk, NULL, NULL);
4960 : 0 : } else {
4961 : : /* spdk_bdev_unregister() is not called until the last nvme_ns is
4962 : : * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list
4963 : : * and clear nvme_ns->bdev here.
4964 : : */
4965 [ + + # # : 45 : TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4966 [ # # # # ]: 45 : nvme_ns->bdev = NULL;
4967 : :
4968 [ - + # # ]: 45 : pthread_mutex_unlock(&bdev->mutex);
4969 : :
4970 : : /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that,
4971 : : * we call depopulate_namespace_done() to avoid use-after-free.
4972 : : */
4973 : 45 : nvme_bdev_for_each_channel(bdev,
4974 : : bdev_nvme_delete_io_path,
4975 : 0 : nvme_ns,
4976 : : bdev_nvme_delete_io_path_done);
4977 : 45 : return;
4978 : : }
4979 : 0 : }
4980 : :
4981 : 1457 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
4982 : 1 : }
4983 : :
4984 : : static void
4985 : 1746 : nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
4986 : : struct nvme_async_probe_ctx *ctx)
4987 : : {
4988 [ + - + - ]: 1746 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
4989 : : struct nvme_ns *nvme_ns, *next;
4990 : : struct spdk_nvme_ns *ns;
4991 : : struct nvme_bdev *bdev;
4992 : : uint32_t nsid;
4993 : : int rc;
4994 : : uint64_t num_sectors;
4995 : :
4996 [ + + ]: 1746 : if (ctx) {
4997 : : /* Initialize this count to 1 to handle the populate functions
4998 : : * calling nvme_ctrlr_populate_namespace_done() immediately.
4999 : : */
5000 [ + - + - ]: 1616 : ctx->populates_in_progress = 1;
5001 : 1 : }
5002 : :
5003 : : /* First loop over our existing namespaces and see if they have been
5004 : : * removed. */
5005 : 1746 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5006 [ + + ]: 1776 : while (nvme_ns != NULL) {
5007 : 30 : next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
5008 : :
5009 [ + + # # : 30 : if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
5010 : : /* NS is still there or added again. Its attributes may have changed. */
5011 [ # # # # ]: 18 : ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id);
5012 [ + + # # : 18 : if (nvme_ns->ns != ns) {
# # ]
5013 [ - + # # : 3 : assert(nvme_ns->ns == NULL);
# # # # ]
5014 [ # # # # ]: 3 : nvme_ns->ns = ns;
5015 [ - + - + : 3 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "NSID %u was added\n", nvme_ns->id);
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5016 : 0 : }
5017 : :
5018 : 18 : num_sectors = spdk_nvme_ns_get_num_sectors(ns);
5019 [ # # # # ]: 18 : bdev = nvme_ns->bdev;
5020 [ - + # # ]: 18 : assert(bdev != NULL);
5021 [ + + # # : 18 : if (bdev->disk.blockcnt != num_sectors) {
# # # # ]
5022 [ + - # # : 3 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5023 : : "NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n",
5024 : : nvme_ns->id,
5025 : : bdev->disk.name,
5026 : : bdev->disk.blockcnt,
5027 : : num_sectors);
5028 [ # # ]: 3 : rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors);
5029 [ - + ]: 3 : if (rc != 0) {
5030 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5031 : : "Could not change num blocks for nvme bdev: name %s, errno: %d.\n",
5032 : : bdev->disk.name, rc);
5033 : 0 : }
5034 : 0 : }
5035 : 0 : } else {
5036 : : /* Namespace was removed */
5037 : 12 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5038 : : }
5039 : :
5040 : 30 : nvme_ns = next;
5041 : : }
5042 : :
5043 : : /* Loop through all of the namespaces at the nvme level and see if any of them are new */
5044 : 1746 : nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5045 [ + + ]: 3274 : while (nsid != 0) {
5046 : 1528 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5047 : :
5048 [ + + ]: 1528 : if (nvme_ns == NULL) {
5049 : : /* Found a new one */
5050 : 1510 : nvme_ns = nvme_ns_alloc();
5051 [ + + ]: 1510 : if (nvme_ns == NULL) {
5052 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate namespace\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5053 : : /* This just fails to attach the namespace. It may work on a future attempt. */
5054 : 0 : continue;
5055 : : }
5056 : :
5057 [ + - + - ]: 1510 : nvme_ns->id = nsid;
5058 [ + - + - ]: 1510 : nvme_ns->ctrlr = nvme_ctrlr;
5059 : :
5060 [ + - + - ]: 1510 : nvme_ns->bdev = NULL;
5061 : :
5062 [ + + ]: 1510 : if (ctx) {
5063 [ + - ]: 1446 : ctx->populates_in_progress++;
5064 : 1 : }
5065 [ + - + - ]: 1510 : nvme_ns->probe_ctx = ctx;
5066 : :
5067 [ + - ]: 1510 : RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
5068 : :
5069 : 1510 : nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns);
5070 : 1 : }
5071 : :
5072 : 1528 : nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid);
5073 : : }
5074 : :
5075 [ + + ]: 1746 : if (ctx) {
5076 : : /* Decrement this count now that the loop is over to account
5077 : : * for the one we started with. If the count is then 0, we
5078 : : * know any populate_namespace functions completed immediately,
5079 : : * so we'll kick the callback here.
5080 : : */
5081 [ + - ]: 1616 : ctx->populates_in_progress--;
5082 [ + + + - : 1616 : if (ctx->populates_in_progress == 0) {
- + ]
5083 : 1567 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
5084 : 1 : }
5085 : 1 : }
5086 : :
5087 : 1746 : }
5088 : :
5089 : : static void
5090 : 1717 : nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr)
5091 : : {
5092 : : struct nvme_ns *nvme_ns, *tmp;
5093 : :
5094 [ + + + + : 3207 : RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) {
+ + ]
5095 : 1490 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5096 : 1 : }
5097 : 1717 : }
5098 : :
5099 : : static uint32_t
5100 : 2902 : nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr)
5101 : : {
5102 [ # # # # ]: 2902 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5103 : : const struct spdk_nvme_ctrlr_data *cdata;
5104 : 2902 : uint32_t nsid, ns_count = 0;
5105 : :
5106 : 2902 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5107 : :
5108 [ # # ]: 2902 : for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5109 [ + + ]: 5828 : nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
5110 : 2926 : ns_count++;
5111 : 0 : }
5112 : :
5113 [ # # # # ]: 2902 : return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5114 : 2902 : sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count *
5115 : : sizeof(uint32_t);
5116 : : }
5117 : :
5118 : : static int
5119 : 124 : nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
5120 : : void *cb_arg)
5121 : : {
5122 : 124 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
5123 : : struct nvme_ns *nvme_ns;
5124 : : uint32_t i, nsid;
5125 : :
5126 [ + + # # : 245 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
5127 [ # # # # : 121 : nsid = desc->nsid[i];
# # ]
5128 [ - + ]: 121 : if (nsid == 0) {
5129 : 0 : continue;
5130 : : }
5131 : :
5132 : 121 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5133 : :
5134 [ + + ]: 121 : if (nvme_ns == NULL) {
5135 : : /* Target told us that an inactive namespace had an ANA change */
5136 : 3 : continue;
5137 : : }
5138 : :
5139 : 118 : _nvme_ns_set_ana_state(nvme_ns, desc);
5140 : 0 : }
5141 : :
5142 : 124 : return 0;
5143 : : }
5144 : :
5145 : : static void
5146 : 2 : bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5147 : : {
5148 : : struct nvme_ns *nvme_ns;
5149 : :
5150 [ # # # # ]: 2 : spdk_free(nvme_ctrlr->ana_log_page);
5151 [ # # # # ]: 2 : nvme_ctrlr->ana_log_page = NULL;
5152 : :
5153 [ # # ]: 2 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5154 [ + + ]: 4 : nvme_ns != NULL;
5155 : 2 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
5156 [ # # # # ]: 2 : nvme_ns->ana_state_updating = false;
5157 [ # # # # ]: 2 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
5158 : 0 : }
5159 : 2 : }
5160 : :
5161 : : static void
5162 : 114 : nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl)
5163 : : {
5164 : 114 : struct nvme_ctrlr *nvme_ctrlr = ctx;
5165 : :
5166 [ + - + + : 114 : if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) {
+ - # # #
# # # # #
# # # # ]
5167 : 112 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states,
5168 : 0 : nvme_ctrlr);
5169 : 0 : } else {
5170 : 2 : bdev_nvme_disable_read_ana_log_page(nvme_ctrlr);
5171 : : }
5172 : :
5173 [ - + # # ]: 114 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5174 : :
5175 [ - + # # : 114 : assert(nvme_ctrlr->ana_log_page_updating == true);
# # ]
5176 [ # # ]: 114 : nvme_ctrlr->ana_log_page_updating = false;
5177 : :
5178 [ - + ]: 114 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
5179 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5180 : :
5181 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
5182 : 0 : } else {
5183 [ - + # # ]: 114 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5184 : :
5185 : 114 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
5186 : : }
5187 : 114 : }
5188 : :
5189 : : static int
5190 : 2488 : nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5191 : : {
5192 : : uint32_t ana_log_page_size;
5193 : : int rc;
5194 : :
5195 [ - + # # : 2488 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5196 : 0 : return -EINVAL;
5197 : : }
5198 : :
5199 : 2488 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5200 : :
5201 [ - + # # : 2488 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5202 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5203 : : "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
5204 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5205 : 0 : return -EINVAL;
5206 : : }
5207 : :
5208 [ - + # # ]: 2488 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5209 [ + + + + ]: 2488 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
5210 [ # # ]: 0 : nvme_ctrlr->ana_log_page_updating) {
5211 [ - + # # ]: 2374 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5212 : 2374 : return -EBUSY;
5213 : : }
5214 : :
5215 [ # # ]: 114 : nvme_ctrlr->ana_log_page_updating = true;
5216 [ - + # # ]: 114 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5217 : :
5218 [ # # # # ]: 114 : rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr,
5219 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5220 : : SPDK_NVME_GLOBAL_NS_TAG,
5221 [ # # # # ]: 114 : nvme_ctrlr->ana_log_page,
5222 : 0 : ana_log_page_size, 0,
5223 : : nvme_ctrlr_read_ana_log_page_done,
5224 : 0 : nvme_ctrlr);
5225 [ - + ]: 114 : if (rc != 0) {
5226 : 0 : nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL);
5227 : 0 : }
5228 : :
5229 : 114 : return rc;
5230 : 0 : }
5231 : :
5232 : : static void
5233 : 0 : dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
5234 : : {
5235 : 0 : }
5236 : :
5237 : : struct bdev_nvme_set_preferred_path_ctx {
5238 : : struct spdk_bdev_desc *desc;
5239 : : struct nvme_ns *nvme_ns;
5240 : : bdev_nvme_set_preferred_path_cb cb_fn;
5241 : : void *cb_arg;
5242 : : };
5243 : :
5244 : : static void
5245 : 9 : bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5246 : : {
5247 : 9 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5248 : :
5249 [ - + # # ]: 9 : assert(ctx != NULL);
5250 [ - + # # : 9 : assert(ctx->desc != NULL);
# # # # ]
5251 [ - + # # : 9 : assert(ctx->cb_fn != NULL);
# # # # ]
5252 : :
5253 [ # # # # ]: 9 : spdk_bdev_close(ctx->desc);
5254 : :
5255 [ # # # # : 9 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5256 : :
5257 : 9 : free(ctx);
5258 : 9 : }
5259 : :
5260 : : static void
5261 : 6 : _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i,
5262 : : struct nvme_bdev *nbdev,
5263 : : struct nvme_bdev_channel *nbdev_ch, void *_ctx)
5264 : : {
5265 : 6 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5266 : : struct nvme_io_path *io_path, *prev;
5267 : :
5268 : 6 : prev = NULL;
5269 [ + - # # : 9 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
5270 [ + + # # : 9 : if (io_path->nvme_ns == ctx->nvme_ns) {
# # # # #
# ]
5271 : 6 : break;
5272 : : }
5273 : 3 : prev = io_path;
5274 : 0 : }
5275 : :
5276 [ + - ]: 6 : if (io_path != NULL) {
5277 [ + + ]: 6 : if (prev != NULL) {
5278 [ - + # # : 3 : STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5279 [ - + # # : 3 : STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5280 : 0 : }
5281 : :
5282 : : /* We can set io_path to nbdev_ch->current_io_path directly here.
5283 : : * However, it needs to be conditional. To simplify the code,
5284 : : * just clear nbdev_ch->current_io_path and let find_io_path()
5285 : : * fill it.
5286 : : *
5287 : : * Automatic failback may be disabled. Hence even if the io_path is
5288 : : * already at the head, clear nbdev_ch->current_io_path.
5289 : : */
5290 : 6 : bdev_nvme_clear_current_io_path(nbdev_ch);
5291 : 0 : }
5292 : :
5293 : 6 : nvme_bdev_for_each_channel_continue(i, 0);
5294 : 6 : }
5295 : :
5296 : : static struct nvme_ns *
5297 : 9 : bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid)
5298 : : {
5299 : : struct nvme_ns *nvme_ns, *prev;
5300 : : const struct spdk_nvme_ctrlr_data *cdata;
5301 : :
5302 : 9 : prev = NULL;
5303 [ + - # # : 18 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
5304 [ # # # # : 18 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
5305 : :
5306 [ + + # # : 18 : if (cdata->cntlid == cntlid) {
# # ]
5307 : 9 : break;
5308 : : }
5309 : 9 : prev = nvme_ns;
5310 : 0 : }
5311 : :
5312 [ + - + + ]: 9 : if (nvme_ns != NULL && prev != NULL) {
5313 [ + + # # : 6 : TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5314 [ + - # # : 6 : TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5315 : 0 : }
5316 : :
5317 : 9 : return nvme_ns;
5318 : : }
5319 : :
5320 : : /* This function supports only multipath mode. There is only a single I/O path
5321 : : * for each NVMe-oF controller. Hence, just move the matched I/O path to the
5322 : : * head of the I/O path list for each NVMe bdev channel.
5323 : : *
5324 : : * NVMe bdev channel may be acquired after completing this function. move the
5325 : : * matched namespace to the head of the namespace list for the NVMe bdev too.
5326 : : */
5327 : : void
5328 : 9 : bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
5329 : : bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg)
5330 : : {
5331 : : struct bdev_nvme_set_preferred_path_ctx *ctx;
5332 : : struct spdk_bdev *bdev;
5333 : : struct nvme_bdev *nbdev;
5334 : 9 : int rc = 0;
5335 : :
5336 [ - + # # ]: 9 : assert(cb_fn != NULL);
5337 : :
5338 : 9 : ctx = calloc(1, sizeof(*ctx));
5339 [ - + ]: 9 : if (ctx == NULL) {
5340 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5341 : 0 : rc = -ENOMEM;
5342 : 0 : goto err_alloc;
5343 : : }
5344 : :
5345 [ # # # # ]: 9 : ctx->cb_fn = cb_fn;
5346 [ # # # # ]: 9 : ctx->cb_arg = cb_arg;
5347 : :
5348 [ # # ]: 9 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5349 [ - + ]: 9 : if (rc != 0) {
5350 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5351 : 0 : goto err_open;
5352 : : }
5353 : :
5354 [ # # # # ]: 9 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5355 : :
5356 [ - + # # : 9 : if (bdev->module != &nvme_if) {
# # ]
5357 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5358 : 0 : rc = -ENODEV;
5359 : 0 : goto err_bdev;
5360 : : }
5361 : :
5362 : 9 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5363 : :
5364 [ - + # # ]: 9 : pthread_mutex_lock(&nbdev->mutex);
5365 : :
5366 [ # # # # ]: 9 : ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid);
5367 [ - + # # : 9 : if (ctx->nvme_ns == NULL) {
# # ]
5368 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
5369 : :
5370 : 0 : SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid);
5371 : 0 : rc = -ENODEV;
5372 : 0 : goto err_bdev;
5373 : : }
5374 : :
5375 [ - + # # ]: 9 : pthread_mutex_unlock(&nbdev->mutex);
5376 : :
5377 : 9 : nvme_bdev_for_each_channel(nbdev,
5378 : : _bdev_nvme_set_preferred_path,
5379 : 0 : ctx,
5380 : : bdev_nvme_set_preferred_path_done);
5381 : 9 : return;
5382 : :
5383 : 0 : err_bdev:
5384 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5385 : 0 : err_open:
5386 : 0 : free(ctx);
5387 : 0 : err_alloc:
5388 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5389 : 0 : }
5390 : :
5391 : : struct bdev_nvme_set_multipath_policy_ctx {
5392 : : struct spdk_bdev_desc *desc;
5393 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn;
5394 : : void *cb_arg;
5395 : : };
5396 : :
5397 : : static void
5398 : 13 : bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5399 : : {
5400 : 13 : struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx;
5401 : :
5402 [ - + # # ]: 13 : assert(ctx != NULL);
5403 [ - + # # : 13 : assert(ctx->desc != NULL);
# # # # ]
5404 [ - + # # : 13 : assert(ctx->cb_fn != NULL);
# # # # ]
5405 : :
5406 [ # # # # ]: 13 : spdk_bdev_close(ctx->desc);
5407 : :
5408 [ # # # # : 13 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5409 : :
5410 : 13 : free(ctx);
5411 : 13 : }
5412 : :
5413 : : static void
5414 : 7 : _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i,
5415 : : struct nvme_bdev *nbdev,
5416 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
5417 : : {
5418 [ # # # # : 7 : nbdev_ch->mp_policy = nbdev->mp_policy;
# # # # ]
5419 [ # # # # : 7 : nbdev_ch->mp_selector = nbdev->mp_selector;
# # # # ]
5420 [ # # # # : 7 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
# # # # ]
5421 : 7 : bdev_nvme_clear_current_io_path(nbdev_ch);
5422 : :
5423 : 7 : nvme_bdev_for_each_channel_continue(i, 0);
5424 : 7 : }
5425 : :
5426 : : void
5427 : 13 : spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy,
5428 : : enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io,
5429 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg)
5430 : : {
5431 : : struct bdev_nvme_set_multipath_policy_ctx *ctx;
5432 : : struct spdk_bdev *bdev;
5433 : : struct nvme_bdev *nbdev;
5434 : : int rc;
5435 : :
5436 [ - + # # ]: 13 : assert(cb_fn != NULL);
5437 : :
5438 [ + + - ]: 13 : switch (policy) {
5439 : 3 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
5440 : 3 : break;
5441 [ + + - ]: 10 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
5442 [ + - - ]: 4 : switch (selector) {
5443 : 7 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
5444 [ + + ]: 7 : if (rr_min_io == UINT32_MAX) {
5445 : 4 : rr_min_io = 1;
5446 [ - + ]: 3 : } else if (rr_min_io == 0) {
5447 : 0 : rc = -EINVAL;
5448 : 0 : goto exit;
5449 : : }
5450 : 7 : break;
5451 : 3 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
5452 : 3 : break;
5453 : 0 : default:
5454 : 0 : rc = -EINVAL;
5455 : 0 : goto exit;
5456 : : }
5457 : 10 : break;
5458 : 0 : default:
5459 : 0 : rc = -EINVAL;
5460 : 0 : goto exit;
5461 : : }
5462 : :
5463 : 13 : ctx = calloc(1, sizeof(*ctx));
5464 [ - + ]: 13 : if (ctx == NULL) {
5465 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5466 : 0 : rc = -ENOMEM;
5467 : 0 : goto exit;
5468 : : }
5469 : :
5470 [ # # # # ]: 13 : ctx->cb_fn = cb_fn;
5471 [ # # # # ]: 13 : ctx->cb_arg = cb_arg;
5472 : :
5473 [ # # ]: 13 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5474 [ - + ]: 13 : if (rc != 0) {
5475 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5476 : 0 : rc = -ENODEV;
5477 : 0 : goto err_open;
5478 : : }
5479 : :
5480 [ # # # # ]: 13 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5481 [ - + # # : 13 : if (bdev->module != &nvme_if) {
# # ]
5482 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5483 : 0 : rc = -ENODEV;
5484 : 0 : goto err_module;
5485 : : }
5486 : 13 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5487 : :
5488 [ - + # # ]: 13 : pthread_mutex_lock(&nbdev->mutex);
5489 [ # # # # ]: 13 : nbdev->mp_policy = policy;
5490 [ # # # # ]: 13 : nbdev->mp_selector = selector;
5491 [ # # # # ]: 13 : nbdev->rr_min_io = rr_min_io;
5492 [ - + # # ]: 13 : pthread_mutex_unlock(&nbdev->mutex);
5493 : :
5494 : 13 : nvme_bdev_for_each_channel(nbdev,
5495 : : _bdev_nvme_set_multipath_policy,
5496 : 0 : ctx,
5497 : : bdev_nvme_set_multipath_policy_done);
5498 : 13 : return;
5499 : :
5500 : 0 : err_module:
5501 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5502 : 0 : err_open:
5503 : 0 : free(ctx);
5504 : 0 : exit:
5505 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5506 : 0 : }
5507 : :
5508 : : static void
5509 : 151 : aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
5510 : : {
5511 : 151 : struct nvme_ctrlr *nvme_ctrlr = arg;
5512 : : union spdk_nvme_async_event_completion event;
5513 : :
5514 [ + + - + : 151 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5515 : 16 : SPDK_WARNLOG("AER request execute failed\n");
5516 : 16 : return;
5517 : : }
5518 : :
5519 [ # # # # ]: 135 : event.raw = cpl->cdw0;
5520 [ + - # # ]: 135 : if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5521 [ + + ]: 135 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
5522 : 32 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL);
5523 [ + - # # ]: 103 : } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5524 [ + - ]: 103 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
5525 : 103 : nvme_ctrlr_read_ana_log_page(nvme_ctrlr);
5526 : 0 : }
5527 : 0 : }
5528 : :
5529 : : static void
5530 : 1714 : free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx)
5531 : : {
5532 [ + - + - : 1714 : spdk_keyring_put_key(ctx->drv_opts.tls_psk);
+ - ]
5533 [ + - + - : 1714 : spdk_keyring_put_key(ctx->drv_opts.dhchap_key);
+ - ]
5534 [ + - + - : 1714 : spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key);
+ - ]
5535 : 1714 : free(ctx);
5536 : 1714 : }
5537 : :
5538 : : static void
5539 : 1705 : populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc)
5540 : : {
5541 [ + - + - : 1705 : if (ctx->cb_fn) {
+ - ]
5542 [ + - + - : 1705 : ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc);
- + + - +
- + - + -
+ - ]
5543 : 1 : }
5544 : :
5545 [ + - + - ]: 1705 : ctx->namespaces_populated = true;
5546 [ + + + + : 1705 : if (ctx->probe_done) {
+ - + - ]
5547 : : /* The probe was already completed, so we need to free the context
5548 : : * here. This can happen for cases like OCSSD, where we need to
5549 : : * send additional commands to the SSD after attach.
5550 : : */
5551 : 479 : free_nvme_async_probe_ctx(ctx);
5552 : 0 : }
5553 : 1705 : }
5554 : :
5555 : : static int
5556 : 66637 : bdev_nvme_remove_poller(void *ctx)
5557 : : {
5558 : 31479 : struct spdk_nvme_transport_id trid_pcie;
5559 : :
5560 [ + + ]: 66637 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
5561 : 865 : spdk_poller_unregister(&g_hotplug_poller);
5562 : 865 : return SPDK_POLLER_IDLE;
5563 : : }
5564 : :
5565 [ + + ]: 65772 : memset(&trid_pcie, 0, sizeof(trid_pcie));
5566 : 65772 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
5567 : :
5568 [ + + ]: 65772 : if (spdk_nvme_scan_attached(&trid_pcie)) {
5569 [ # # # # ]: 0 : SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n");
5570 : 0 : }
5571 : :
5572 : 65772 : return SPDK_POLLER_BUSY;
5573 : 29 : }
5574 : :
5575 : : static void
5576 : 1714 : nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr,
5577 : : struct nvme_async_probe_ctx *ctx)
5578 : : {
5579 [ + - + - : 1714 : struct spdk_nvme_transport_id *trid = &nvme_ctrlr->active_path_id->trid;
+ - ]
5580 : :
5581 [ + + + - : 1714 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
- + ]
5582 [ - + + + : 1146 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created to %s:%s\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5583 : : trid->traddr, trid->trsvcid);
5584 : 0 : } else {
5585 [ + + + + : 568 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5586 : : }
5587 : :
5588 : 1715 : spdk_io_device_register(nvme_ctrlr,
5589 : : bdev_nvme_create_ctrlr_channel_cb,
5590 : : bdev_nvme_destroy_ctrlr_channel_cb,
5591 : : sizeof(struct nvme_ctrlr_channel),
5592 [ + - + - : 1714 : nvme_ctrlr->nbdev_ctrlr->name);
+ - + - ]
5593 : :
5594 : 1714 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx);
5595 : :
5596 [ + + ]: 1714 : if (g_hotplug_poller == NULL) {
5597 : 1271 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL,
5598 : : NVME_HOTPLUG_POLL_PERIOD_DEFAULT);
5599 : 1 : }
5600 : 1714 : }
5601 : :
5602 : : static void
5603 : 414 : nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl)
5604 : : {
5605 : 414 : struct nvme_ctrlr *nvme_ctrlr = _ctx;
5606 [ # # # # ]: 414 : struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx;
5607 : :
5608 [ # # # # ]: 414 : nvme_ctrlr->probe_ctx = NULL;
5609 : :
5610 [ + - - + : 414 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5611 : 0 : nvme_ctrlr_delete(nvme_ctrlr);
5612 : :
5613 [ # # ]: 0 : if (ctx != NULL) {
5614 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
5615 : 0 : populate_namespaces_cb(ctx, -1);
5616 : 0 : }
5617 : 0 : return;
5618 : : }
5619 : :
5620 : 414 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
5621 : 0 : }
5622 : :
5623 : : static int
5624 : 414 : nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
5625 : : struct nvme_async_probe_ctx *ctx)
5626 : : {
5627 [ # # # # ]: 414 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5628 : : const struct spdk_nvme_ctrlr_data *cdata;
5629 : : uint32_t ana_log_page_size;
5630 : :
5631 : 414 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5632 : :
5633 : : /* Set buffer size enough to include maximum number of allowed namespaces. */
5634 [ # # # # ]: 414 : ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5635 [ # # # # ]: 414 : sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan *
5636 : : sizeof(uint32_t);
5637 : :
5638 [ # # # # ]: 414 : nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL,
5639 : : SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA);
5640 [ - + # # : 414 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5641 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate ANA log page buffer\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5642 : 0 : return -ENXIO;
5643 : : }
5644 : :
5645 : : /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned.
5646 : : * Hence copy each descriptor to a temporary area when parsing it.
5647 : : *
5648 : : * Allocate a buffer whose size is as large as ANA log page buffer because
5649 : : * we do not know the size of a descriptor until actually reading it.
5650 : : */
5651 [ # # # # ]: 414 : nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size);
5652 [ - + # # : 414 : if (nvme_ctrlr->copied_ana_desc == NULL) {
# # ]
5653 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate a buffer to parse ANA descriptor\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5654 : 0 : return -ENOMEM;
5655 : : }
5656 : :
5657 [ # # # # ]: 414 : nvme_ctrlr->max_ana_log_page_size = ana_log_page_size;
5658 : :
5659 [ # # # # ]: 414 : nvme_ctrlr->probe_ctx = ctx;
5660 : :
5661 : : /* Then, set the read size only to include the current active namespaces. */
5662 : 414 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5663 : :
5664 [ - + # # : 414 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5665 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5666 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5667 : 0 : return -EINVAL;
5668 : : }
5669 : :
5670 : 414 : return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
5671 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5672 : : SPDK_NVME_GLOBAL_NS_TAG,
5673 [ # # # # ]: 414 : nvme_ctrlr->ana_log_page,
5674 : 0 : ana_log_page_size, 0,
5675 : : nvme_ctrlr_init_ana_log_page_done,
5676 : 0 : nvme_ctrlr);
5677 : 0 : }
5678 : :
5679 : : /* hostnqn and subnqn were already verified before attaching a controller.
5680 : : * Hence check only the multipath capability and cntlid here.
5681 : : */
5682 : : static bool
5683 : 61 : bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr)
5684 : : {
5685 : : struct nvme_ctrlr *tmp;
5686 : : const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata;
5687 : :
5688 : 61 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5689 : :
5690 [ - + # # : 61 : if (!cdata->cmic.multi_ctrlr) {
# # ]
5691 [ # # # # ]: 0 : SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
5692 : 0 : return false;
5693 : : }
5694 : :
5695 [ + + # # : 125 : TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5696 [ # # # # ]: 67 : tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr);
5697 : :
5698 [ - + # # : 67 : if (!tmp_cdata->cmic.multi_ctrlr) {
# # ]
5699 [ # # # # : 0 : NVME_CTRLR_ERRLOG(tmp, "Ctrlr%u does not support multipath.\n", cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5700 : 0 : return false;
5701 : : }
5702 [ + + # # : 67 : if (cdata->cntlid == tmp_cdata->cntlid) {
# # # # #
# ]
5703 [ + - # # : 3 : NVME_CTRLR_ERRLOG(tmp, "cntlid %u are duplicated.\n", tmp_cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5704 : 3 : return false;
5705 : : }
5706 : 0 : }
5707 : :
5708 : 58 : return true;
5709 : 0 : }
5710 : :
5711 : :
5712 : : static int
5713 : 1717 : nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr)
5714 : : {
5715 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
5716 [ + - + - ]: 1717 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5717 : : struct nvme_ctrlr *nctrlr;
5718 : 1717 : int rc = 0;
5719 : :
5720 [ + + ]: 1717 : pthread_mutex_lock(&g_bdev_nvme_mutex);
5721 : :
5722 : 1717 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
5723 [ + + ]: 1717 : if (nbdev_ctrlr != NULL) {
5724 [ + + ]: 61 : if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) {
5725 : 3 : rc = -EINVAL;
5726 : 3 : goto exit;
5727 : : }
5728 [ + + # # : 122 : TAILQ_FOREACH(nctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5729 [ - + - + : 64 : if (nctrlr->opts.multipath != nvme_ctrlr->opts.multipath) {
- + # # #
# # # # #
# # # # ]
5730 : : /* All controllers with the same name must be configured the same
5731 : : * way, either for multipath or failover. If the configuration doesn't
5732 : : * match - report error.
5733 : : */
5734 : 0 : rc = -EINVAL;
5735 : 0 : goto exit;
5736 : : }
5737 : 0 : }
5738 : 0 : } else {
5739 : 1656 : nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr));
5740 [ + + ]: 1656 : if (nbdev_ctrlr == NULL) {
5741 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5742 : 0 : rc = -ENOMEM;
5743 : 0 : goto exit;
5744 : : }
5745 [ + + + - : 1656 : nbdev_ctrlr->name = strdup(name);
+ - ]
5746 [ + + + - : 1656 : if (nbdev_ctrlr->name == NULL) {
+ - ]
5747 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate name of nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5748 : 0 : free(nbdev_ctrlr);
5749 : 0 : goto exit;
5750 : : }
5751 [ + - + - : 1656 : TAILQ_INIT(&nbdev_ctrlr->ctrlrs);
+ - + - +
- + - + -
+ - ]
5752 [ + - + - : 1656 : TAILQ_INIT(&nbdev_ctrlr->bdevs);
+ - + - +
- + - + -
+ - ]
5753 [ + - + - : 1656 : TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
5754 : : }
5755 [ + - + - ]: 1714 : nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr;
5756 [ + - + - : 1714 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5757 : 1716 : exit:
5758 [ + + ]: 1717 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
5759 : 1717 : return rc;
5760 : : }
5761 : :
5762 : : static int
5763 : 1717 : nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
5764 : : const char *name,
5765 : : const struct spdk_nvme_transport_id *trid,
5766 : : struct nvme_async_probe_ctx *ctx)
5767 : : {
5768 : : struct nvme_ctrlr *nvme_ctrlr;
5769 : : struct nvme_path_id *path_id;
5770 : : const struct spdk_nvme_ctrlr_data *cdata;
5771 : : int rc;
5772 : :
5773 : 1717 : nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
5774 [ + + ]: 1717 : if (nvme_ctrlr == NULL) {
5775 : 0 : SPDK_ERRLOG("Failed to allocate device struct\n");
5776 : 0 : return -ENOMEM;
5777 : : }
5778 : :
5779 [ + + + - ]: 1717 : rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL);
5780 [ - + ]: 1717 : if (rc != 0) {
5781 : 0 : free(nvme_ctrlr);
5782 : 0 : return rc;
5783 : : }
5784 : :
5785 [ + - + - : 1717 : TAILQ_INIT(&nvme_ctrlr->trids);
+ - + - +
- + - + -
+ - ]
5786 [ + - + - : 1717 : RB_INIT(&nvme_ctrlr->namespaces);
+ - ]
5787 : :
5788 : : /* Get another reference to the key, so the first one can be released from probe_ctx */
5789 [ + + ]: 1717 : if (ctx != NULL) {
5790 [ + + + - : 1619 : if (ctx->drv_opts.tls_psk != NULL) {
+ - + - ]
5791 [ # # # # ]: 42 : nvme_ctrlr->psk = spdk_keyring_get_key(
5792 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.tls_psk));
# # ]
5793 [ - + # # : 42 : if (nvme_ctrlr->psk == NULL) {
# # ]
5794 : : /* Could only happen if the key was removed in the meantime */
5795 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5796 : : spdk_key_get_name(ctx->drv_opts.tls_psk));
5797 : 0 : rc = -ENOKEY;
5798 : 0 : goto err;
5799 : : }
5800 : 0 : }
5801 : :
5802 [ + + + - : 1619 : if (ctx->drv_opts.dhchap_key != NULL) {
+ - + - ]
5803 [ # # # # ]: 628 : nvme_ctrlr->dhchap_key = spdk_keyring_get_key(
5804 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_key));
# # ]
5805 [ - + # # : 628 : if (nvme_ctrlr->dhchap_key == NULL) {
# # ]
5806 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5807 : : spdk_key_get_name(ctx->drv_opts.dhchap_key));
5808 : 0 : rc = -ENOKEY;
5809 : 0 : goto err;
5810 : : }
5811 : 0 : }
5812 : :
5813 [ + + + - : 1619 : if (ctx->drv_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
5814 [ # # # # ]: 484 : nvme_ctrlr->dhchap_ctrlr_key =
5815 : 484 : spdk_keyring_get_key(
5816 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
# # ]
5817 [ - + # # : 484 : if (nvme_ctrlr->dhchap_ctrlr_key == NULL) {
# # ]
5818 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5819 : : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
5820 : 0 : rc = -ENOKEY;
5821 : 0 : goto err;
5822 : : }
5823 : 0 : }
5824 : 1 : }
5825 : :
5826 : 1717 : path_id = calloc(1, sizeof(*path_id));
5827 [ + + ]: 1717 : if (path_id == NULL) {
5828 : 0 : SPDK_ERRLOG("Failed to allocate trid entry pointer\n");
5829 : 0 : rc = -ENOMEM;
5830 : 0 : goto err;
5831 : : }
5832 : :
5833 [ + - ]: 1717 : path_id->trid = *trid;
5834 [ + + ]: 1717 : if (ctx != NULL) {
5835 [ + + + + : 1619 : memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr));
+ - + - +
- + - ]
5836 [ + + + + : 1619 : memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid));
+ - + - +
- + - ]
5837 : 1 : }
5838 [ + - + - ]: 1717 : nvme_ctrlr->active_path_id = path_id;
5839 [ + + + - : 1717 : TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link);
+ - + - +
- + - - +
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5840 : :
5841 [ + - + - ]: 1717 : nvme_ctrlr->thread = spdk_get_thread();
5842 [ + - + - ]: 1717 : nvme_ctrlr->ctrlr = ctrlr;
5843 [ + - + - ]: 1717 : nvme_ctrlr->ref = 1;
5844 : :
5845 [ - + ]: 1717 : if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) {
5846 : 0 : SPDK_ERRLOG("OCSSDs are not supported");
5847 : 0 : rc = -ENOTSUP;
5848 : 0 : goto err;
5849 : : }
5850 : :
5851 [ + + ]: 1717 : if (ctx != NULL) {
5852 [ + + + + : 1619 : memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));
+ - + - ]
5853 : 1 : } else {
5854 [ # # ]: 98 : spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
5855 : : }
5856 : :
5857 [ + - + - : 1717 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
+ - ]
5858 : : g_opts.nvme_adminq_poll_period_us);
5859 : :
5860 [ + + + - ]: 1717 : if (g_opts.timeout_us > 0) {
5861 : : /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
5862 : : /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */
5863 [ # # # # ]: 0 : uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ?
5864 [ # # # # ]: 0 : g_opts.timeout_us : g_opts.timeout_admin_us;
5865 [ # # ]: 0 : spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
5866 : 0 : adm_timeout_us, timeout_cb, nvme_ctrlr);
5867 : 0 : }
5868 : :
5869 : 1717 : spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr);
5870 : 1717 : spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr);
5871 : :
5872 [ + + ]: 1717 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
5873 : : SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
5874 [ # # # # ]: 32 : nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr);
5875 : 0 : }
5876 : :
5877 : 1717 : rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr);
5878 [ + + ]: 1717 : if (rc != 0) {
5879 : 3 : goto err;
5880 : : }
5881 : :
5882 : 1714 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5883 : :
5884 [ + + + - : 1714 : if (cdata->cmic.ana_reporting) {
- + ]
5885 : 414 : rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx);
5886 [ + - ]: 414 : if (rc == 0) {
5887 : 414 : return 0;
5888 : : }
5889 : 0 : } else {
5890 : 1300 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
5891 : 1300 : return 0;
5892 : : }
5893 : :
5894 : 3 : err:
5895 : 3 : nvme_ctrlr_delete(nvme_ctrlr);
5896 : 3 : return rc;
5897 : 1 : }
5898 : :
5899 : : void
5900 : 1684 : spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts)
5901 : : {
5902 [ + - + - ]: 1684 : opts->prchk_flags = 0;
5903 [ + - + - : 1684 : opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec;
+ - ]
5904 [ + - + - : 1684 : opts->reconnect_delay_sec = g_opts.reconnect_delay_sec;
+ - ]
5905 [ + - + - : 1684 : opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec;
+ - ]
5906 [ + - + - ]: 1684 : opts->multipath = true;
5907 : 1684 : }
5908 : :
5909 : : static void
5910 : 56 : attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
5911 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts)
5912 : : {
5913 : : char *name;
5914 : :
5915 [ # # ]: 56 : name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++);
5916 [ - + ]: 56 : if (!name) {
5917 : 0 : SPDK_ERRLOG("Failed to assign name to NVMe device\n");
5918 : 0 : return;
5919 : : }
5920 : :
5921 [ + - ]: 56 : if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) {
5922 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name);
# # # # ]
5923 : 0 : } else {
5924 [ # # ]: 0 : SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name);
5925 : : }
5926 : :
5927 : 56 : free(name);
5928 : 0 : }
5929 : :
5930 : : static void
5931 : 1714 : _nvme_ctrlr_destruct(void *ctx)
5932 : : {
5933 : 1714 : struct nvme_ctrlr *nvme_ctrlr = ctx;
5934 : :
5935 : 1714 : nvme_ctrlr_depopulate_namespaces(nvme_ctrlr);
5936 : 1714 : nvme_ctrlr_release(nvme_ctrlr);
5937 : 1714 : }
5938 : :
5939 : : static int
5940 : 974 : bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
5941 : : {
5942 : : struct nvme_probe_skip_entry *entry;
5943 : :
5944 : : /* The controller's destruction was already started */
5945 [ - + # # ]: 974 : if (nvme_ctrlr->destruct) {
5946 : 0 : return -EALREADY;
5947 : : }
5948 : :
5949 [ + + # # : 974 : if (!hotplug &&
# # ]
5950 [ + + # # : 926 : nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # # # #
# ]
5951 : 56 : entry = calloc(1, sizeof(*entry));
5952 [ - + ]: 56 : if (!entry) {
5953 : 0 : return -ENOMEM;
5954 : : }
5955 [ # # # # : 56 : entry->trid = nvme_ctrlr->active_path_id->trid;
# # # # ]
5956 [ # # # # : 56 : TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
5957 : 0 : }
5958 : :
5959 [ # # ]: 974 : nvme_ctrlr->destruct = true;
5960 : 974 : return 0;
5961 : 0 : }
5962 : :
5963 : : static int
5964 : 77 : bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
5965 : : {
5966 : : int rc;
5967 : :
5968 [ - + # # ]: 77 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5969 [ # # ]: 77 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug);
5970 [ - + # # ]: 77 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5971 : :
5972 [ + - ]: 77 : if (rc == 0) {
5973 : 77 : _nvme_ctrlr_destruct(nvme_ctrlr);
5974 [ # # ]: 0 : } else if (rc == -EALREADY) {
5975 : 0 : rc = 0;
5976 : 0 : }
5977 : :
5978 : 77 : return rc;
5979 : : }
5980 : :
5981 : : static void
5982 : 48 : remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
5983 : : {
5984 : 48 : struct nvme_ctrlr *nvme_ctrlr = cb_ctx;
5985 : :
5986 : 48 : bdev_nvme_delete_ctrlr(nvme_ctrlr, true);
5987 : 48 : }
5988 : :
5989 : : static int
5990 : 19910 : bdev_nvme_hotplug_probe(void *arg)
5991 : : {
5992 [ + + ]: 19910 : if (g_hotplug_probe_ctx == NULL) {
5993 : 2 : spdk_poller_unregister(&g_hotplug_probe_poller);
5994 : 2 : return SPDK_POLLER_IDLE;
5995 : : }
5996 : :
5997 [ + + ]: 19908 : if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) {
5998 : 3866 : g_hotplug_probe_ctx = NULL;
5999 : 3866 : spdk_poller_unregister(&g_hotplug_probe_poller);
6000 : 0 : }
6001 : :
6002 : 19908 : return SPDK_POLLER_BUSY;
6003 : 0 : }
6004 : :
6005 : : static int
6006 : 4015 : bdev_nvme_hotplug(void *arg)
6007 : : {
6008 : 1941 : struct spdk_nvme_transport_id trid_pcie;
6009 : :
6010 [ + + ]: 4015 : if (g_hotplug_probe_ctx) {
6011 : 147 : return SPDK_POLLER_BUSY;
6012 : : }
6013 : :
6014 [ - + ]: 3868 : memset(&trid_pcie, 0, sizeof(trid_pcie));
6015 : 3868 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
6016 : :
6017 : 3868 : g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL,
6018 : : hotplug_probe_cb, attach_cb, NULL);
6019 : :
6020 [ + - ]: 3868 : if (g_hotplug_probe_ctx) {
6021 [ - + # # ]: 3868 : assert(g_hotplug_probe_poller == NULL);
6022 : 3868 : g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000);
6023 : 0 : }
6024 : :
6025 : 3868 : return SPDK_POLLER_BUSY;
6026 : 0 : }
6027 : :
6028 : : void
6029 : 1022 : bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts)
6030 : : {
6031 : 1022 : *opts = g_opts;
6032 : 1022 : }
6033 : :
6034 : : static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6035 : : uint32_t reconnect_delay_sec,
6036 : : uint32_t fast_io_fail_timeout_sec);
6037 : :
6038 : : static int
6039 : 1022 : bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts)
6040 : : {
6041 [ + + + + : 1022 : if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) {
+ - + - +
- + - ]
6042 : : /* Can't set timeout_admin_us without also setting timeout_us */
6043 : 0 : SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n");
6044 : 0 : return -EINVAL;
6045 : : }
6046 : :
6047 [ + + + - : 1022 : if (opts->bdev_retry_count < -1) {
- + ]
6048 : 0 : SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n");
6049 : 0 : return -EINVAL;
6050 : : }
6051 : :
6052 [ + + + - : 1049 : if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec,
+ - ]
6053 [ + - + - ]: 1022 : opts->reconnect_delay_sec,
6054 [ + - + - ]: 1022 : opts->fast_io_fail_timeout_sec)) {
6055 : 0 : return -EINVAL;
6056 : : }
6057 : :
6058 : 1022 : return 0;
6059 : 27 : }
6060 : :
6061 : : int
6062 : 1022 : bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
6063 : : {
6064 : : int ret;
6065 : :
6066 : 1022 : ret = bdev_nvme_validate_opts(opts);
6067 [ - + ]: 1022 : if (ret) {
6068 : 0 : SPDK_WARNLOG("Failed to set nvme opts.\n");
6069 : 0 : return ret;
6070 : : }
6071 : :
6072 [ + + ]: 1022 : if (g_bdev_nvme_init_thread != NULL) {
6073 [ - + ]: 652 : if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
6074 : 0 : return -EPERM;
6075 : : }
6076 : 0 : }
6077 : :
6078 [ + - + - : 1049 : if (opts->rdma_srq_size != 0 ||
+ - - + ]
6079 [ + - + - : 1022 : opts->rdma_max_cq_size != 0 ||
+ - ]
6080 [ - + - + ]: 1022 : opts->rdma_cm_event_timeout_ms != 0) {
6081 : 0 : struct spdk_nvme_transport_opts drv_opts;
6082 : :
6083 : 0 : spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts));
6084 [ # # # # : 0 : if (opts->rdma_srq_size != 0) {
# # ]
6085 [ # # # # ]: 0 : drv_opts.rdma_srq_size = opts->rdma_srq_size;
6086 : 0 : }
6087 [ # # # # : 0 : if (opts->rdma_max_cq_size != 0) {
# # ]
6088 [ # # # # : 0 : drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size;
# # ]
6089 : 0 : }
6090 [ # # # # : 0 : if (opts->rdma_cm_event_timeout_ms != 0) {
# # ]
6091 [ # # # # : 0 : drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms;
# # ]
6092 : 0 : }
6093 : :
6094 : 0 : ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts));
6095 [ # # ]: 0 : if (ret) {
6096 : 0 : SPDK_ERRLOG("Failed to set NVMe transport opts.\n");
6097 : 0 : return ret;
6098 : : }
6099 : 0 : }
6100 : :
6101 : 1022 : g_opts = *opts;
6102 : :
6103 : 1022 : return 0;
6104 : 27 : }
6105 : :
6106 : : struct set_nvme_hotplug_ctx {
6107 : : uint64_t period_us;
6108 : : bool enabled;
6109 : : spdk_msg_fn fn;
6110 : : void *fn_ctx;
6111 : : };
6112 : :
6113 : : static void
6114 : 301 : set_nvme_hotplug_period_cb(void *_ctx)
6115 : : {
6116 : 301 : struct set_nvme_hotplug_ctx *ctx = _ctx;
6117 : :
6118 : 301 : spdk_poller_unregister(&g_hotplug_poller);
6119 [ + + + + : 301 : if (ctx->enabled) {
+ - - + ]
6120 [ # # # # ]: 12 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us);
6121 : 0 : } else {
6122 : 289 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL,
6123 : : NVME_HOTPLUG_POLL_PERIOD_DEFAULT);
6124 : : }
6125 : :
6126 [ + - + - ]: 301 : g_nvme_hotplug_poll_period_us = ctx->period_us;
6127 [ + + + - : 301 : g_nvme_hotplug_enabled = ctx->enabled;
+ - ]
6128 [ + - + - : 301 : if (ctx->fn) {
+ - ]
6129 [ + - + - : 301 : ctx->fn(ctx->fn_ctx);
- + + - +
- + - ]
6130 : 26 : }
6131 : :
6132 : 301 : free(ctx);
6133 : 301 : }
6134 : :
6135 : : int
6136 : 301 : bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx)
6137 : : {
6138 : : struct set_nvme_hotplug_ctx *ctx;
6139 : :
6140 [ + + - + : 301 : if (enabled == true && !spdk_process_is_primary()) {
# # ]
6141 : 0 : return -EPERM;
6142 : : }
6143 : :
6144 : 301 : ctx = calloc(1, sizeof(*ctx));
6145 [ + + ]: 301 : if (ctx == NULL) {
6146 : 0 : return -ENOMEM;
6147 : : }
6148 : :
6149 [ + + ]: 301 : period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
6150 [ + - - + : 301 : ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
- + ]
6151 [ - + - + : 301 : ctx->enabled = enabled;
- + ]
6152 [ - + - + ]: 301 : ctx->fn = cb;
6153 [ - + - + ]: 301 : ctx->fn_ctx = cb_ctx;
6154 : :
6155 : 301 : spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
6156 : 301 : return 0;
6157 : 26 : }
6158 : :
6159 : : static void
6160 : 1616 : nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
6161 : : struct nvme_async_probe_ctx *ctx)
6162 : : {
6163 : : struct nvme_ns *nvme_ns;
6164 : : struct nvme_bdev *nvme_bdev;
6165 : : size_t j;
6166 : :
6167 [ + + # # ]: 1616 : assert(nvme_ctrlr != NULL);
6168 : :
6169 [ + + + - : 1616 : if (ctx->names == NULL) {
+ - ]
6170 [ # # # # ]: 37 : ctx->reported_bdevs = 0;
6171 : 37 : populate_namespaces_cb(ctx, 0);
6172 : 37 : return;
6173 : : }
6174 : :
6175 : : /*
6176 : : * Report the new bdevs that were created in this call.
6177 : : * There can be more than one bdev per NVMe controller.
6178 : : */
6179 : 1579 : j = 0;
6180 : 1579 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6181 [ + + ]: 2965 : while (nvme_ns != NULL) {
6182 [ + - + - ]: 1386 : nvme_bdev = nvme_ns->bdev;
6183 [ + - + - : 1386 : if (j < ctx->max_bdevs) {
+ - ]
6184 [ + - + - : 1386 : ctx->names[j] = nvme_bdev->disk.name;
+ - + - +
- + - +
- ]
6185 : 1386 : j++;
6186 : 1 : } else {
6187 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6188 : : "Maximum number of namespaces supported per NVMe controller is %du. "
6189 : : "Unable to return all names of created bdevs\n",
6190 : : ctx->max_bdevs);
6191 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
6192 : 0 : populate_namespaces_cb(ctx, -ERANGE);
6193 : 0 : return;
6194 : : }
6195 : :
6196 : 1386 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6197 : : }
6198 : :
6199 [ + - + - ]: 1579 : ctx->reported_bdevs = j;
6200 : 1579 : populate_namespaces_cb(ctx, 0);
6201 : 1 : }
6202 : :
6203 : : static int
6204 : 43 : bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6205 : : struct spdk_nvme_ctrlr *new_ctrlr,
6206 : : struct spdk_nvme_transport_id *trid)
6207 : : {
6208 : : struct nvme_path_id *tmp_trid;
6209 : :
6210 [ - + # # : 43 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
6211 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "PCIe failover is not supported.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6212 : 0 : return -ENOTSUP;
6213 : : }
6214 : :
6215 : : /* Currently we only support failover to the same transport type. */
6216 [ - + # # : 43 : if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) {
# # # # #
# # # # #
# # ]
6217 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6218 : : "Failover from trtype: %s to a different trtype: %s is not supported currently\n",
6219 : : spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype),
6220 : : spdk_nvme_transport_id_trtype_str(trid->trtype));
6221 : 0 : return -EINVAL;
6222 : : }
6223 : :
6224 : :
6225 : : /* Currently we only support failover to the same NQN. */
6226 [ - + - + : 43 : if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
- + # # #
# # # # #
# # ]
6227 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6228 : : "Failover from subnqn: %s to a different subnqn: %s is not supported currently\n",
6229 : : nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn);
6230 : 0 : return -EINVAL;
6231 : : }
6232 : :
6233 : : /* Skip all the other checks if we've already registered this path. */
6234 [ + + # # : 103 : TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # #
# ]
6235 [ - + # # ]: 60 : if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) {
6236 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "This path (traddr: %s subnqn: %s) is already registered\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6237 : : trid->traddr, trid->subnqn);
6238 : 0 : return -EALREADY;
6239 : : }
6240 : 0 : }
6241 : :
6242 : 43 : return 0;
6243 : 0 : }
6244 : :
6245 : : static int
6246 : 43 : bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr,
6247 : : struct spdk_nvme_ctrlr *new_ctrlr)
6248 : : {
6249 : : struct nvme_ns *nvme_ns;
6250 : : struct spdk_nvme_ns *new_ns;
6251 : :
6252 : 43 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6253 [ + + ]: 59 : while (nvme_ns != NULL) {
6254 [ # # # # ]: 16 : new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id);
6255 [ - + # # ]: 16 : assert(new_ns != NULL);
6256 : :
6257 [ - + # # : 16 : if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) {
# # ]
6258 : 0 : return -EINVAL;
6259 : : }
6260 : :
6261 : 16 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6262 : : }
6263 : :
6264 : 43 : return 0;
6265 : 0 : }
6266 : :
6267 : : static int
6268 : 43 : _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6269 : : struct spdk_nvme_transport_id *trid)
6270 : : {
6271 : : struct nvme_path_id *active_id, *new_trid, *tmp_trid;
6272 : :
6273 : 43 : new_trid = calloc(1, sizeof(*new_trid));
6274 [ - + ]: 43 : if (new_trid == NULL) {
6275 : 0 : return -ENOMEM;
6276 : : }
6277 [ # # ]: 43 : new_trid->trid = *trid;
6278 : :
6279 [ # # # # ]: 43 : active_id = nvme_ctrlr->active_path_id;
6280 [ - + # # ]: 43 : assert(active_id != NULL);
6281 [ - + # # : 43 : assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
6282 : :
6283 : : /* Skip the active trid not to replace it until it is failed. */
6284 [ # # # # : 43 : tmp_trid = TAILQ_NEXT(active_id, link);
# # ]
6285 [ + + ]: 43 : if (tmp_trid == NULL) {
6286 : 26 : goto add_tail;
6287 : : }
6288 : :
6289 : : /* It means the trid is faled if its last failed time is non-zero.
6290 : : * Insert the new alternate trid before any failed trid.
6291 : : */
6292 [ - + + + : 27 : TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # # #
# # ]
6293 [ + + # # : 17 : if (tmp_trid->last_failed_tsc != 0) {
# # ]
6294 [ # # # # : 7 : TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6295 : 7 : return 0;
6296 : : }
6297 : 0 : }
6298 : :
6299 : 10 : add_tail:
6300 [ # # # # : 36 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6301 : 36 : return 0;
6302 : 0 : }
6303 : :
6304 : : /* This is the case that a secondary path is added to an existing
6305 : : * nvme_ctrlr for failover. After checking if it can access the same
6306 : : * namespaces as the primary path, it is disconnected until failover occurs.
6307 : : */
6308 : : static int
6309 : 43 : bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6310 : : struct spdk_nvme_ctrlr *new_ctrlr,
6311 : : struct spdk_nvme_transport_id *trid)
6312 : : {
6313 : : int rc;
6314 : :
6315 [ - + # # ]: 43 : assert(nvme_ctrlr != NULL);
6316 : :
6317 [ - + # # ]: 43 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6318 : :
6319 : 43 : rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid);
6320 [ - + ]: 43 : if (rc != 0) {
6321 : 0 : goto exit;
6322 : : }
6323 : :
6324 : 43 : rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr);
6325 [ - + ]: 43 : if (rc != 0) {
6326 : 0 : goto exit;
6327 : : }
6328 : :
6329 : 43 : rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid);
6330 : :
6331 : 43 : exit:
6332 [ - + # # ]: 43 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6333 : :
6334 : 43 : spdk_nvme_detach(new_ctrlr);
6335 : :
6336 : 43 : return rc;
6337 : : }
6338 : :
6339 : : static void
6340 : 1619 : connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6341 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
6342 : : {
6343 : 1619 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6344 : : struct nvme_async_probe_ctx *ctx;
6345 : : int rc;
6346 : :
6347 : 1619 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6348 [ + - + - ]: 1619 : ctx->ctrlr_attached = true;
6349 : :
6350 [ + - + - : 1619 : rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx);
+ - ]
6351 [ + + ]: 1619 : if (rc != 0) {
6352 [ # # # # ]: 3 : ctx->reported_bdevs = 0;
6353 : 3 : populate_namespaces_cb(ctx, rc);
6354 : 0 : }
6355 : 1619 : }
6356 : :
6357 : :
6358 : : static void
6359 : 28 : connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6360 : : struct spdk_nvme_ctrlr *ctrlr,
6361 : : const struct spdk_nvme_ctrlr_opts *opts)
6362 : : {
6363 : 28 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6364 : : struct nvme_ctrlr *nvme_ctrlr;
6365 : : struct nvme_async_probe_ctx *ctx;
6366 : : int rc;
6367 : :
6368 : 28 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6369 [ # # # # ]: 28 : ctx->ctrlr_attached = true;
6370 : :
6371 [ # # # # ]: 28 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6372 [ + - ]: 28 : if (nvme_ctrlr) {
6373 [ # # ]: 28 : rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid);
6374 : 0 : } else {
6375 : 0 : rc = -ENODEV;
6376 : : }
6377 : :
6378 [ # # # # ]: 28 : ctx->reported_bdevs = 0;
6379 : 28 : populate_namespaces_cb(ctx, rc);
6380 : 28 : }
6381 : :
6382 : : static int
6383 : 278825 : bdev_nvme_async_poll(void *arg)
6384 : : {
6385 : 278825 : struct nvme_async_probe_ctx *ctx = arg;
6386 : : int rc;
6387 : :
6388 [ + - + - ]: 278825 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
6389 [ + + ]: 278825 : if (spdk_unlikely(rc != -EAGAIN)) {
6390 [ + - + - ]: 1705 : ctx->probe_done = true;
6391 [ + - ]: 1705 : spdk_poller_unregister(&ctx->poller);
6392 [ + + + + : 1705 : if (!ctx->ctrlr_attached) {
+ - + - ]
6393 : : /* The probe is done, but no controller was attached.
6394 : : * That means we had a failure, so report -EIO back to
6395 : : * the caller (usually the RPC). populate_namespaces_cb()
6396 : : * will take care of freeing the nvme_async_probe_ctx.
6397 : : */
6398 [ # # # # ]: 58 : ctx->reported_bdevs = 0;
6399 : 58 : populate_namespaces_cb(ctx, -EIO);
6400 [ + + + + : 1647 : } else if (ctx->namespaces_populated) {
+ - - + ]
6401 : : /* The namespaces for the attached controller were all
6402 : : * populated and the response was already sent to the
6403 : : * caller (usually the RPC). So free the context here.
6404 : : */
6405 : 1226 : free_nvme_async_probe_ctx(ctx);
6406 : 1 : }
6407 : 1 : }
6408 : :
6409 : 278825 : return SPDK_POLLER_BUSY;
6410 : : }
6411 : :
6412 : : static bool
6413 : 2793 : bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6414 : : uint32_t reconnect_delay_sec,
6415 : : uint32_t fast_io_fail_timeout_sec)
6416 : : {
6417 [ + + ]: 2793 : if (ctrlr_loss_timeout_sec < -1) {
6418 : 3 : SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n");
6419 : 3 : return false;
6420 [ + + ]: 2790 : } else if (ctrlr_loss_timeout_sec == -1) {
6421 [ + + ]: 54 : if (reconnect_delay_sec == 0) {
6422 : 3 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6423 : 3 : return false;
6424 [ + + + + ]: 51 : } else if (fast_io_fail_timeout_sec != 0 &&
6425 : 0 : fast_io_fail_timeout_sec < reconnect_delay_sec) {
6426 : 3 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n");
6427 : 3 : return false;
6428 : : }
6429 [ + + ]: 2736 : } else if (ctrlr_loss_timeout_sec != 0) {
6430 [ + + ]: 61 : if (reconnect_delay_sec == 0) {
6431 : 3 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6432 : 3 : return false;
6433 [ + + ]: 58 : } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6434 : 3 : SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n");
6435 : 3 : return false;
6436 [ + + ]: 55 : } else if (fast_io_fail_timeout_sec != 0) {
6437 [ + + ]: 26 : if (fast_io_fail_timeout_sec < reconnect_delay_sec) {
6438 : 3 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n");
6439 : 3 : return false;
6440 [ + + ]: 23 : } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6441 : 3 : SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n");
6442 : 3 : return false;
6443 : : }
6444 : 0 : }
6445 [ + + + + ]: 2675 : } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) {
6446 : 6 : SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n");
6447 : 6 : return false;
6448 : : }
6449 : :
6450 : 2766 : return true;
6451 : 28 : }
6452 : :
6453 : : int
6454 : 1714 : spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
6455 : : const char *base_name,
6456 : : const char **names,
6457 : : uint32_t count,
6458 : : spdk_bdev_nvme_create_cb cb_fn,
6459 : : void *cb_ctx,
6460 : : struct spdk_nvme_ctrlr_opts *drv_opts,
6461 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts)
6462 : : {
6463 : : struct nvme_probe_skip_entry *entry, *tmp;
6464 : : struct nvme_async_probe_ctx *ctx;
6465 : : spdk_nvme_attach_cb attach_cb;
6466 : : struct nvme_ctrlr *nvme_ctrlr;
6467 : : int len;
6468 : :
6469 : : /* TODO expand this check to include both the host and target TRIDs.
6470 : : * Only if both are the same should we fail.
6471 : : */
6472 [ + + - + ]: 1714 : if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) {
6473 [ # # # # ]: 0 : SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) "
6474 : : "already exists.\n", trid->traddr, drv_opts->hostnqn);
6475 : 0 : return -EEXIST;
6476 : : }
6477 : :
6478 [ + + ]: 1714 : len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX);
6479 : :
6480 [ + - - + ]: 1714 : if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) {
6481 : 0 : SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1);
6482 : 0 : return -EINVAL;
6483 : : }
6484 : :
6485 [ + - + - ]: 1715 : if (bdev_opts != NULL &&
6486 [ + + + - ]: 1715 : !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec,
6487 [ + - + - ]: 1 : bdev_opts->reconnect_delay_sec,
6488 [ + - + - ]: 1 : bdev_opts->fast_io_fail_timeout_sec)) {
6489 : 0 : return -EINVAL;
6490 : : }
6491 : :
6492 : 1714 : ctx = calloc(1, sizeof(*ctx));
6493 [ - + ]: 1714 : if (!ctx) {
6494 : 0 : return -ENOMEM;
6495 : : }
6496 [ + - + - ]: 1714 : ctx->base_name = base_name;
6497 [ + - + - ]: 1714 : ctx->names = names;
6498 [ + - + - ]: 1714 : ctx->max_bdevs = count;
6499 [ + - + - ]: 1714 : ctx->cb_fn = cb_fn;
6500 [ + - + - ]: 1714 : ctx->cb_ctx = cb_ctx;
6501 [ + - ]: 1714 : ctx->trid = *trid;
6502 : :
6503 [ + - ]: 1714 : if (bdev_opts) {
6504 [ + + + + : 1714 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
+ - ]
6505 : 1 : } else {
6506 [ # # ]: 0 : spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts);
6507 : : }
6508 : :
6509 [ + + + - : 1714 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
- + ]
6510 [ + + # # : 512 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
# # # # +
- ]
6511 [ + - # # ]: 2 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
6512 [ - + # # : 2 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6513 : 2 : free(entry);
6514 : 2 : break;
6515 : : }
6516 : 0 : }
6517 : 1 : }
6518 : :
6519 [ + + + + : 1714 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
+ - ]
6520 [ + - + - : 1714 : ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count;
+ - + - ]
6521 [ + - + - : 1714 : ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout;
+ - + - ]
6522 [ + - + - : 1714 : ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms;
+ - + - ]
6523 [ + - + - : 1714 : ctx->drv_opts.disable_read_ana_log_page = true;
+ - ]
6524 [ + - + - : 1714 : ctx->drv_opts.transport_tos = g_opts.transport_tos;
+ - + - ]
6525 : :
6526 [ + + + - : 1714 : if (ctx->bdev_opts.psk != NULL) {
+ - + - ]
6527 [ # # # # : 66 : ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk);
# # # # #
# # # ]
6528 [ + + # # : 66 : if (ctx->drv_opts.tls_psk == NULL) {
# # # # ]
6529 [ # # # # : 6 : SPDK_ERRLOG("Could not load PSK: %s\n", ctx->bdev_opts.psk);
# # ]
6530 : 6 : free_nvme_async_probe_ctx(ctx);
6531 : 6 : return -ENOKEY;
6532 : : }
6533 : 0 : }
6534 : :
6535 [ + + + - : 1708 : if (ctx->bdev_opts.dhchap_key != NULL) {
+ - + - ]
6536 [ # # # # : 664 : ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key);
# # # # #
# # # ]
6537 [ - + # # : 664 : if (ctx->drv_opts.dhchap_key == NULL) {
# # # # ]
6538 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n",
# # ]
6539 : : ctx->bdev_opts.dhchap_key);
6540 : 0 : free_nvme_async_probe_ctx(ctx);
6541 : 0 : return -ENOKEY;
6542 : : }
6543 : :
6544 [ # # # # : 664 : ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests;
# # # # ]
6545 [ # # # # : 664 : ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups;
# # # # ]
6546 : 0 : }
6547 [ + + + - : 1708 : if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
6548 [ # # # # : 500 : ctx->drv_opts.dhchap_ctrlr_key =
# # ]
6549 [ # # # # : 500 : spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key);
# # ]
6550 [ - + # # : 500 : if (ctx->drv_opts.dhchap_ctrlr_key == NULL) {
# # # # ]
6551 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n",
# # ]
6552 : : ctx->bdev_opts.dhchap_ctrlr_key);
6553 : 0 : free_nvme_async_probe_ctx(ctx);
6554 : 0 : return -ENOKEY;
6555 : : }
6556 : 0 : }
6557 : :
6558 [ + + + + : 1708 : if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || ctx->bdev_opts.multipath) {
+ + # # #
# # # ]
6559 : 1680 : attach_cb = connect_attach_cb;
6560 : 1 : } else {
6561 : 28 : attach_cb = connect_set_failover_cb;
6562 : : }
6563 : :
6564 [ + - + - ]: 1708 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6565 [ + + - + : 1708 : if (nvme_ctrlr && nvme_ctrlr->opts.multipath != ctx->bdev_opts.multipath) {
- + - + #
# # # # #
# # # # #
# ]
6566 : : /* All controllers with the same name must be configured the same
6567 : : * way, either for multipath or failover. If the configuration doesn't
6568 : : * match - report error.
6569 : : */
6570 : 0 : free_nvme_async_probe_ctx(ctx);
6571 : 0 : return -EINVAL;
6572 : : }
6573 : :
6574 [ + - + - : 1708 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb);
+ - ]
6575 [ + + + - : 1708 : if (ctx->probe_ctx == NULL) {
+ - ]
6576 [ # # ]: 3 : SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr);
6577 : 3 : free_nvme_async_probe_ctx(ctx);
6578 : 3 : return -ENODEV;
6579 : : }
6580 [ + - + - ]: 1705 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000);
6581 : :
6582 : 1705 : return 0;
6583 : 1 : }
6584 : :
6585 : : struct bdev_nvme_delete_ctx {
6586 : : char *name;
6587 : : struct nvme_path_id path_id;
6588 : : bdev_nvme_delete_done_fn delete_done;
6589 : : void *delete_done_ctx;
6590 : : uint64_t timeout_ticks;
6591 : : struct spdk_poller *poller;
6592 : : };
6593 : :
6594 : : static void
6595 : 712 : free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx)
6596 : : {
6597 [ + + ]: 712 : if (ctx != NULL) {
6598 [ # # # # ]: 709 : free(ctx->name);
6599 : 709 : free(ctx);
6600 : 0 : }
6601 : 712 : }
6602 : :
6603 : : static bool
6604 : 40233 : nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id)
6605 : : {
6606 [ + + # # : 40233 : if (path_id->trid.trtype != 0) {
# # # # ]
6607 [ - + # # : 205 : if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
# # # # ]
6608 [ # # # # : 0 : if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) {
# # # # #
# # # #
# ]
6609 : 0 : return false;
6610 : : }
6611 : 0 : } else {
6612 [ - + # # : 205 : if (path_id->trid.trtype != p->trid.trtype) {
# # # # #
# # # #
# ]
6613 : 0 : return false;
6614 : : }
6615 : : }
6616 : 0 : }
6617 : :
6618 [ + + # # : 40233 : if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) {
# # ]
6619 [ + + - + : 205 : if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) {
+ + # # #
# # # #
# ]
6620 : 33 : return false;
6621 : : }
6622 : 0 : }
6623 : :
6624 [ + + # # : 40200 : if (path_id->trid.adrfam != 0) {
# # # # ]
6625 [ - + # # : 142 : if (path_id->trid.adrfam != p->trid.adrfam) {
# # # # #
# # # #
# ]
6626 : 0 : return false;
6627 : : }
6628 : 0 : }
6629 : :
6630 [ + + # # : 40200 : if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) {
# # ]
6631 [ - + - + : 172 : if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) {
+ + # # #
# # # #
# ]
6632 : 49 : return false;
6633 : : }
6634 : 0 : }
6635 : :
6636 [ + + # # : 40151 : if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) {
# # ]
6637 [ - + - + : 123 : if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) {
- + # # #
# # # #
# ]
6638 : 0 : return false;
6639 : : }
6640 : 0 : }
6641 : :
6642 [ - + # # : 40151 : if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) {
# # ]
6643 [ # # # # : 0 : if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) {
# # # # #
# # # #
# ]
6644 : 0 : return false;
6645 : : }
6646 : 0 : }
6647 : :
6648 [ - + # # : 40151 : if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) {
# # ]
6649 [ # # # # : 0 : if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) {
# # # # #
# # # #
# ]
6650 : 0 : return false;
6651 : : }
6652 : 0 : }
6653 : :
6654 : 40151 : return true;
6655 : 0 : }
6656 : :
6657 : : static bool
6658 : 39931 : nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id)
6659 : : {
6660 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
6661 : : struct nvme_ctrlr *ctrlr;
6662 : : struct nvme_path_id *p;
6663 : :
6664 [ - + ]: 39931 : pthread_mutex_lock(&g_bdev_nvme_mutex);
6665 : 39931 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
6666 [ + + ]: 39931 : if (!nbdev_ctrlr) {
6667 [ - + ]: 699 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6668 : 699 : return false;
6669 : : }
6670 : :
6671 [ + + # # : 39258 : TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
6672 [ - + # # ]: 39248 : pthread_mutex_lock(&ctrlr->mutex);
6673 [ + + # # : 39278 : TAILQ_FOREACH(p, &ctrlr->trids, link) {
# # # # #
# # # #
# ]
6674 [ + + ]: 39252 : if (nvme_path_id_compare(p, path_id)) {
6675 [ - + # # ]: 39222 : pthread_mutex_unlock(&ctrlr->mutex);
6676 [ - + ]: 39222 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6677 : 39222 : return true;
6678 : : }
6679 : 0 : }
6680 [ - + # # ]: 26 : pthread_mutex_unlock(&ctrlr->mutex);
6681 : 0 : }
6682 [ - + ]: 10 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6683 : :
6684 : 10 : return false;
6685 : 0 : }
6686 : :
6687 : : static int
6688 : 39931 : bdev_nvme_delete_complete_poll(void *arg)
6689 : : {
6690 : 39931 : struct bdev_nvme_delete_ctx *ctx = arg;
6691 : 39931 : int rc = 0;
6692 : :
6693 [ + + # # : 39931 : if (nvme_path_id_exists(ctx->name, &ctx->path_id)) {
# # # # ]
6694 [ + - # # : 39222 : if (ctx->timeout_ticks > spdk_get_ticks()) {
# # ]
6695 : 39222 : return SPDK_POLLER_BUSY;
6696 : : }
6697 : :
6698 [ # # # # ]: 0 : SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name);
6699 : 0 : rc = -ETIMEDOUT;
6700 : 0 : }
6701 : :
6702 [ # # ]: 709 : spdk_poller_unregister(&ctx->poller);
6703 : :
6704 [ # # # # : 709 : ctx->delete_done(ctx->delete_done_ctx, rc);
# # # # #
# # # ]
6705 : 709 : free_bdev_nvme_delete_ctx(ctx);
6706 : :
6707 : 709 : return SPDK_POLLER_BUSY;
6708 : 0 : }
6709 : :
6710 : : static int
6711 : 939 : _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)
6712 : : {
6713 : : struct nvme_path_id *p, *t;
6714 : : spdk_msg_fn msg_fn;
6715 : 939 : int rc = -ENXIO;
6716 : :
6717 [ - + # # ]: 939 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6718 : :
6719 [ + - # # : 981 : TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6720 [ + + # # : 981 : if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) {
# # # # ]
6721 : 939 : break;
6722 : : }
6723 : :
6724 [ + + ]: 42 : if (!nvme_path_id_compare(p, path_id)) {
6725 : 17 : continue;
6726 : : }
6727 : :
6728 : : /* We are not using the specified path. */
6729 [ + + # # : 25 : TAILQ_REMOVE(&nvme_ctrlr->trids, p, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6730 : 25 : free(p);
6731 : 25 : rc = 0;
6732 : 0 : }
6733 : :
6734 [ + - + + ]: 939 : if (p == NULL || !nvme_path_id_compare(p, path_id)) {
6735 [ - + # # ]: 35 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6736 : 35 : return rc;
6737 : : }
6738 : :
6739 : : /* If we made it here, then this path is a match! Now we need to remove it. */
6740 : :
6741 : : /* This is the active path in use right now. The active path is always the first in the list. */
6742 [ - + # # : 904 : assert(p == nvme_ctrlr->active_path_id);
# # # # ]
6743 : :
6744 [ + + # # : 904 : if (!TAILQ_NEXT(p, link)) {
# # # # ]
6745 : : /* The current path is the only path. */
6746 : 897 : msg_fn = _nvme_ctrlr_destruct;
6747 : 897 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false);
6748 : 0 : } else {
6749 : : /* There is an alternative path. */
6750 : 7 : msg_fn = _bdev_nvme_reset_ctrlr;
6751 : 7 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true);
6752 : : }
6753 : :
6754 [ - + # # ]: 904 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6755 : :
6756 [ + - ]: 904 : if (rc == 0) {
6757 [ # # # # ]: 904 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
6758 [ # # ]: 0 : } else if (rc == -EALREADY) {
6759 : 0 : rc = 0;
6760 : 0 : }
6761 : :
6762 : 904 : return rc;
6763 : 0 : }
6764 : :
6765 : : int
6766 : 887 : bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,
6767 : : bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx)
6768 : : {
6769 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
6770 : : struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr;
6771 : 887 : struct bdev_nvme_delete_ctx *ctx = NULL;
6772 : 887 : int rc = -ENXIO, _rc;
6773 : :
6774 [ + - - + ]: 887 : if (name == NULL || path_id == NULL) {
6775 : 0 : rc = -EINVAL;
6776 : 0 : goto exit;
6777 : : }
6778 : :
6779 [ - + ]: 887 : pthread_mutex_lock(&g_bdev_nvme_mutex);
6780 : :
6781 : 887 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
6782 [ - + ]: 887 : if (nbdev_ctrlr == NULL) {
6783 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6784 : :
6785 : 0 : SPDK_ERRLOG("Failed to find NVMe bdev controller\n");
6786 : 0 : rc = -ENODEV;
6787 : 0 : goto exit;
6788 : : }
6789 : :
6790 [ + + # # : 1826 : TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {
# # # # #
# # # # #
# # ]
6791 : 939 : _rc = _bdev_nvme_delete(nvme_ctrlr, path_id);
6792 [ + + - + ]: 939 : if (_rc < 0 && _rc != -ENXIO) {
6793 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6794 : 0 : rc = _rc;
6795 : 0 : goto exit;
6796 [ + + ]: 939 : } else if (_rc == 0) {
6797 : : /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr
6798 : : * was deleted successfully. To remember the successful deletion,
6799 : : * overwrite rc only if _rc is zero.
6800 : : */
6801 : 914 : rc = 0;
6802 : 0 : }
6803 : 0 : }
6804 : :
6805 [ - + ]: 887 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6806 : :
6807 [ + + + + ]: 887 : if (rc != 0 || delete_done == NULL) {
6808 : 178 : goto exit;
6809 : : }
6810 : :
6811 : 709 : ctx = calloc(1, sizeof(*ctx));
6812 [ - + ]: 709 : if (ctx == NULL) {
6813 : 0 : SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n");
6814 : 0 : rc = -ENOMEM;
6815 : 0 : goto exit;
6816 : : }
6817 : :
6818 [ - + # # : 709 : ctx->name = strdup(name);
# # ]
6819 [ - + # # : 709 : if (ctx->name == NULL) {
# # ]
6820 : 0 : SPDK_ERRLOG("Failed to copy controller name for deletion\n");
6821 : 0 : rc = -ENOMEM;
6822 : 0 : goto exit;
6823 : : }
6824 : :
6825 [ # # # # ]: 709 : ctx->delete_done = delete_done;
6826 [ # # # # ]: 709 : ctx->delete_done_ctx = delete_done_ctx;
6827 [ # # ]: 709 : ctx->path_id = *path_id;
6828 [ # # # # ]: 709 : ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz();
6829 [ # # # # ]: 709 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000);
6830 [ + - # # : 709 : if (ctx->poller == NULL) {
# # ]
6831 : 0 : SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n");
6832 : 0 : rc = -ENOMEM;
6833 : 0 : goto exit;
6834 : : }
6835 : :
6836 : 709 : exit:
6837 [ + + ]: 887 : if (rc != 0) {
6838 : 3 : free_bdev_nvme_delete_ctx(ctx);
6839 : 0 : }
6840 : :
6841 : 887 : return rc;
6842 : : }
6843 : :
6844 : : #define DISCOVERY_INFOLOG(ctx, format, ...) \
6845 : : SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
6846 : :
6847 : : #define DISCOVERY_ERRLOG(ctx, format, ...) \
6848 : : SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
6849 : :
6850 : : struct discovery_entry_ctx {
6851 : : char name[128];
6852 : : struct spdk_nvme_transport_id trid;
6853 : : struct spdk_nvme_ctrlr_opts drv_opts;
6854 : : struct spdk_nvmf_discovery_log_page_entry entry;
6855 : : TAILQ_ENTRY(discovery_entry_ctx) tailq;
6856 : : struct discovery_ctx *ctx;
6857 : : };
6858 : :
6859 : : struct discovery_ctx {
6860 : : char *name;
6861 : : spdk_bdev_nvme_start_discovery_fn start_cb_fn;
6862 : : spdk_bdev_nvme_stop_discovery_fn stop_cb_fn;
6863 : : void *cb_ctx;
6864 : : struct spdk_nvme_probe_ctx *probe_ctx;
6865 : : struct spdk_nvme_detach_ctx *detach_ctx;
6866 : : struct spdk_nvme_ctrlr *ctrlr;
6867 : : struct spdk_nvme_transport_id trid;
6868 : : struct discovery_entry_ctx *entry_ctx_in_use;
6869 : : struct spdk_poller *poller;
6870 : : struct spdk_nvme_ctrlr_opts drv_opts;
6871 : : struct spdk_bdev_nvme_ctrlr_opts bdev_opts;
6872 : : struct spdk_nvmf_discovery_log_page *log_page;
6873 : : TAILQ_ENTRY(discovery_ctx) tailq;
6874 : : TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs;
6875 : : TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs;
6876 : : int rc;
6877 : : bool wait_for_attach;
6878 : : uint64_t timeout_ticks;
6879 : : /* Denotes that the discovery service is being started. We're waiting
6880 : : * for the initial connection to the discovery controller to be
6881 : : * established and attach discovered NVM ctrlrs.
6882 : : */
6883 : : bool initializing;
6884 : : /* Denotes if a discovery is currently in progress for this context.
6885 : : * That includes connecting to newly discovered subsystems. Used to
6886 : : * ensure we do not start a new discovery until an existing one is
6887 : : * complete.
6888 : : */
6889 : : bool in_progress;
6890 : :
6891 : : /* Denotes if another discovery is needed after the one in progress
6892 : : * completes. Set when we receive an AER completion while a discovery
6893 : : * is already in progress.
6894 : : */
6895 : : bool pending;
6896 : :
6897 : : /* Signal to the discovery context poller that it should stop the
6898 : : * discovery service, including detaching from the current discovery
6899 : : * controller.
6900 : : */
6901 : : bool stop;
6902 : :
6903 : : struct spdk_thread *calling_thread;
6904 : : uint32_t index;
6905 : : uint32_t attach_in_progress;
6906 : : char *hostnqn;
6907 : :
6908 : : /* Denotes if the discovery service was started by the mdns discovery.
6909 : : */
6910 : : bool from_mdns_discovery_service;
6911 : : };
6912 : :
6913 : : TAILQ_HEAD(discovery_ctxs, discovery_ctx);
6914 : : static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs);
6915 : :
6916 : : static void get_discovery_log_page(struct discovery_ctx *ctx);
6917 : :
6918 : : static void
6919 : 34 : free_discovery_ctx(struct discovery_ctx *ctx)
6920 : : {
6921 [ # # # # ]: 34 : free(ctx->log_page);
6922 [ # # # # ]: 34 : free(ctx->hostnqn);
6923 [ # # # # ]: 34 : free(ctx->name);
6924 : 34 : free(ctx);
6925 : 34 : }
6926 : :
6927 : : static void
6928 : 49 : discovery_complete(struct discovery_ctx *ctx)
6929 : : {
6930 [ # # # # ]: 49 : ctx->initializing = false;
6931 [ # # # # ]: 49 : ctx->in_progress = false;
6932 [ - + + + : 49 : if (ctx->pending) {
# # # # ]
6933 [ # # # # ]: 5 : ctx->pending = false;
6934 : 5 : get_discovery_log_page(ctx);
6935 : 0 : }
6936 : 49 : }
6937 : :
6938 : : static void
6939 : 161 : build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid,
6940 : : struct spdk_nvmf_discovery_log_page_entry *entry)
6941 : : {
6942 : : char *space;
6943 : :
6944 [ # # # # : 161 : trid->trtype = entry->trtype;
# # # # ]
6945 [ # # # # : 161 : trid->adrfam = entry->adrfam;
# # # # ]
6946 [ - + - + : 161 : memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr));
# # # # ]
6947 [ - + - + : 161 : memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid));
# # # # ]
6948 : : /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and
6949 : : * before call to this function trid->subnqn is zeroed out, we need
6950 : : * to copy sizeof(trid->subnqn) minus one byte to make sure the last character
6951 : : * remains 0. Then we can shorten the string (replace ' ' with 0) if required
6952 : : */
6953 [ - + - + : 161 : memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1);
# # # # ]
6954 : :
6955 : : /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated.
6956 : : * But the log page entries typically pad them with spaces, not zeroes.
6957 : : * So add a NULL terminator to each of these fields at the appropriate
6958 : : * location.
6959 : : */
6960 [ - + # # ]: 161 : space = strchr(trid->traddr, ' ');
6961 [ + - ]: 161 : if (space) {
6962 [ # # ]: 161 : *space = 0;
6963 : 0 : }
6964 [ - + # # ]: 161 : space = strchr(trid->trsvcid, ' ');
6965 [ + - ]: 161 : if (space) {
6966 [ # # ]: 161 : *space = 0;
6967 : 0 : }
6968 [ - + # # ]: 161 : space = strchr(trid->subnqn, ' ');
6969 [ - + ]: 161 : if (space) {
6970 [ # # ]: 0 : *space = 0;
6971 : 0 : }
6972 : 161 : }
6973 : :
6974 : : static void
6975 : 34 : _stop_discovery(void *_ctx)
6976 : : {
6977 : 34 : struct discovery_ctx *ctx = _ctx;
6978 : :
6979 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
6980 : 0 : spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx);
6981 : 0 : return;
6982 : : }
6983 : :
6984 [ # # # # ]: 34 : ctx->stop = true;
6985 : :
6986 [ + + # # : 63 : while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {
# # # # ]
6987 : : struct discovery_entry_ctx *entry_ctx;
6988 : 29 : struct nvme_path_id path = {};
6989 : :
6990 [ # # # # : 29 : entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);
# # ]
6991 [ # # ]: 29 : path.trid = entry_ctx->trid;
6992 [ # # ]: 29 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
6993 [ - + # # : 29 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6994 : 29 : free(entry_ctx);
6995 : : }
6996 : :
6997 [ + + # # : 77 : while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) {
# # # # ]
6998 : : struct discovery_entry_ctx *entry_ctx;
6999 : :
7000 [ # # # # : 43 : entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # ]
7001 [ + + # # : 43 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7002 : 43 : free(entry_ctx);
7003 : : }
7004 : :
7005 [ # # # # ]: 34 : free(ctx->entry_ctx_in_use);
7006 [ # # # # ]: 34 : ctx->entry_ctx_in_use = NULL;
7007 : 0 : }
7008 : :
7009 : : static void
7010 : 34 : stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7011 : : {
7012 [ # # # # ]: 34 : ctx->stop_cb_fn = cb_fn;
7013 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7014 : :
7015 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
7016 [ # # # # : 0 : DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n",
# # # # #
# # # # #
# # # # ]
7017 : : ctx->attach_in_progress);
7018 : 0 : }
7019 : :
7020 : 34 : _stop_discovery(ctx);
7021 : 34 : }
7022 : :
7023 : : static void
7024 : 29 : remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr)
7025 : : {
7026 : : struct discovery_ctx *d_ctx;
7027 : : struct nvme_path_id *path_id;
7028 : 29 : struct spdk_nvme_transport_id trid = {};
7029 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7030 : :
7031 [ # # # # : 29 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
7032 : :
7033 [ + + # # : 32 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7034 [ + + # # : 6 : TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7035 [ # # ]: 3 : build_trid_from_log_page_entry(&trid, &entry_ctx->entry);
7036 [ - + # # ]: 3 : if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) {
7037 : 0 : continue;
7038 : : }
7039 : :
7040 [ - + # # : 3 : TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7041 : 3 : free(entry_ctx);
7042 [ - + + - : 3 : DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n",
# # # # #
# # # #
# ]
7043 : : trid.subnqn, trid.traddr, trid.trsvcid);
7044 : :
7045 : : /* Fail discovery ctrlr to force reattach attempt */
7046 [ # # # # ]: 3 : spdk_nvme_ctrlr_fail(d_ctx->ctrlr);
7047 : 0 : }
7048 : 0 : }
7049 : 29 : }
7050 : :
7051 : : static void
7052 : 49 : discovery_remove_controllers(struct discovery_ctx *ctx)
7053 : : {
7054 [ # # # # ]: 49 : struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page;
7055 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7056 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7057 : 49 : struct spdk_nvme_transport_id old_trid = {};
7058 : : uint64_t numrec, i;
7059 : : bool found;
7060 : :
7061 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7062 [ + + # # : 109 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7063 : 60 : found = false;
7064 [ # # ]: 60 : old_entry = &entry_ctx->entry;
7065 : 60 : build_trid_from_log_page_entry(&old_trid, old_entry);
7066 [ + + ]: 143 : for (i = 0; i < numrec; i++) {
7067 [ # # # # ]: 138 : new_entry = &log_page->entries[i];
7068 [ - + - + : 138 : if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) {
+ + ]
7069 [ - + + + : 55 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n",
# # # # #
# # # #
# ]
7070 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7071 : 55 : found = true;
7072 : 55 : break;
7073 : : }
7074 : 0 : }
7075 [ + + # # ]: 60 : if (!found) {
7076 : 5 : struct nvme_path_id path = {};
7077 : :
7078 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",
# # # # #
# # # #
# ]
7079 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7080 : :
7081 [ # # ]: 5 : path.trid = entry_ctx->trid;
7082 [ # # ]: 5 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
7083 [ + - # # : 5 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7084 : 5 : free(entry_ctx);
7085 : 0 : }
7086 : 0 : }
7087 : 49 : free(log_page);
7088 [ # # # # ]: 49 : ctx->log_page = NULL;
7089 : 49 : discovery_complete(ctx);
7090 : 49 : }
7091 : :
7092 : : static void
7093 : 42 : complete_discovery_start(struct discovery_ctx *ctx, int status)
7094 : : {
7095 [ # # # # ]: 42 : ctx->timeout_ticks = 0;
7096 [ # # # # ]: 42 : ctx->rc = status;
7097 [ + + # # : 42 : if (ctx->start_cb_fn) {
# # ]
7098 [ # # # # : 27 : ctx->start_cb_fn(ctx->cb_ctx, status);
# # # # #
# # # ]
7099 [ # # # # ]: 27 : ctx->start_cb_fn = NULL;
7100 [ # # # # ]: 27 : ctx->cb_ctx = NULL;
7101 : 0 : }
7102 : 42 : }
7103 : :
7104 : : static void
7105 : 37 : discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc)
7106 : : {
7107 : 37 : struct discovery_entry_ctx *entry_ctx = cb_ctx;
7108 [ # # # # ]: 37 : struct discovery_ctx *ctx = entry_ctx->ctx;
7109 : :
7110 [ - + + + : 37 : DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name);
# # # # #
# # # # #
# # ]
7111 [ # # ]: 37 : ctx->attach_in_progress--;
7112 [ + - # # : 37 : if (ctx->attach_in_progress == 0) {
# # ]
7113 [ # # # # ]: 37 : complete_discovery_start(ctx, ctx->rc);
7114 [ - + + + : 37 : if (ctx->initializing && ctx->rc != 0) {
- + # # #
# # # #
# ]
7115 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc);
# # # # #
# # # ]
7116 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7117 : 0 : } else {
7118 : 37 : discovery_remove_controllers(ctx);
7119 : : }
7120 : 0 : }
7121 : 37 : }
7122 : :
7123 : : static struct discovery_entry_ctx *
7124 : 95 : create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid)
7125 : : {
7126 : : struct discovery_entry_ctx *new_ctx;
7127 : :
7128 : 95 : new_ctx = calloc(1, sizeof(*new_ctx));
7129 [ - + ]: 95 : if (new_ctx == NULL) {
7130 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7131 : 0 : return NULL;
7132 : : }
7133 : :
7134 [ # # # # ]: 95 : new_ctx->ctx = ctx;
7135 [ - + - + : 95 : memcpy(&new_ctx->trid, trid, sizeof(*trid));
# # ]
7136 [ # # ]: 95 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7137 [ - + # # : 95 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7138 : 95 : return new_ctx;
7139 : 0 : }
7140 : :
7141 : : static void
7142 : 49 : discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl,
7143 : : struct spdk_nvmf_discovery_log_page *log_page)
7144 : : {
7145 : 49 : struct discovery_ctx *ctx = cb_arg;
7146 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7147 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7148 : : uint64_t numrec, i;
7149 : : bool found;
7150 : :
7151 [ + - + - : 49 : if (rc || spdk_nvme_cpl_is_error(cpl)) {
- + # # #
# # # # #
# # # # ]
7152 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7153 : 0 : return;
7154 : : }
7155 : :
7156 [ # # # # ]: 49 : ctx->log_page = log_page;
7157 [ - + # # : 49 : assert(ctx->attach_in_progress == 0);
# # # # ]
7158 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7159 [ + + # # : 71 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7160 [ + + # # : 22 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7161 : 22 : free(entry_ctx);
7162 : 0 : }
7163 [ + + ]: 165 : for (i = 0; i < numrec; i++) {
7164 : 116 : found = false;
7165 [ # # # # ]: 116 : new_entry = &log_page->entries[i];
7166 [ + + # # : 116 : if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT ||
# # # # ]
7167 [ - + # # ]: 55 : new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
7168 : : struct discovery_entry_ctx *new_ctx;
7169 : 61 : struct spdk_nvme_transport_id trid = {};
7170 : :
7171 : 61 : build_trid_from_log_page_entry(&trid, new_entry);
7172 : 61 : new_ctx = create_discovery_entry_ctx(ctx, &trid);
7173 [ - + ]: 61 : if (new_ctx == NULL) {
7174 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7175 : 0 : break;
7176 : : }
7177 : :
7178 [ # # # # : 61 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7179 : 61 : continue;
7180 : : }
7181 [ + + # # : 67 : TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7182 [ # # ]: 30 : old_entry = &entry_ctx->entry;
7183 [ - + - + : 30 : if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) {
+ + ]
7184 : 18 : found = true;
7185 : 18 : break;
7186 : : }
7187 : 0 : }
7188 [ + + # # ]: 55 : if (!found) {
7189 : 37 : struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx;
7190 : : struct discovery_ctx *d_ctx;
7191 : :
7192 [ + + # # : 77 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7193 [ + + # # : 52 : TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7194 [ - + - + : 12 : if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn,
+ + # # #
# # # ]
7195 : : sizeof(new_entry->subnqn))) {
7196 : 5 : break;
7197 : : }
7198 : 0 : }
7199 [ + + ]: 45 : if (subnqn_ctx) {
7200 : 5 : break;
7201 : : }
7202 : 0 : }
7203 : :
7204 : 37 : new_ctx = calloc(1, sizeof(*new_ctx));
7205 [ - + ]: 37 : if (new_ctx == NULL) {
7206 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7207 : 0 : break;
7208 : : }
7209 : :
7210 [ # # # # ]: 37 : new_ctx->ctx = ctx;
7211 [ - + - + : 37 : memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry));
# # ]
7212 [ # # ]: 37 : build_trid_from_log_page_entry(&new_ctx->trid, new_entry);
7213 [ + + ]: 37 : if (subnqn_ctx) {
7214 [ # # # # ]: 5 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name);
7215 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7216 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7217 : : new_ctx->name);
7218 : 0 : } else {
7219 [ # # # # : 32 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++);
# # # # ]
7220 [ - + + + : 32 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7221 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7222 : : new_ctx->name);
7223 : : }
7224 [ # # ]: 37 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7225 [ # # # # : 37 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7226 [ # # # # ]: 37 : rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,
7227 : 0 : discovery_attach_controller_done, new_ctx,
7228 [ # # # # ]: 0 : &new_ctx->drv_opts, &ctx->bdev_opts);
7229 [ + - ]: 37 : if (rc == 0) {
7230 [ # # # # : 37 : TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7231 [ # # ]: 37 : ctx->attach_in_progress++;
7232 : 0 : } else {
7233 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc));
# # # # #
# ]
7234 : : }
7235 : 0 : }
7236 : 0 : }
7237 : :
7238 [ + + # # : 49 : if (ctx->attach_in_progress == 0) {
# # ]
7239 : 12 : discovery_remove_controllers(ctx);
7240 : 0 : }
7241 : 0 : }
7242 : :
7243 : : static void
7244 : 49 : get_discovery_log_page(struct discovery_ctx *ctx)
7245 : : {
7246 : : int rc;
7247 : :
7248 [ - + - + : 49 : assert(ctx->in_progress == false);
# # # # #
# ]
7249 [ # # # # ]: 49 : ctx->in_progress = true;
7250 [ # # # # ]: 49 : rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx);
7251 [ - + ]: 49 : if (rc != 0) {
7252 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7253 : 0 : }
7254 [ - + + + : 49 : DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n");
# # # # #
# # # #
# ]
7255 : 49 : }
7256 : :
7257 : : static void
7258 : 16 : discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
7259 : : {
7260 : 16 : struct discovery_ctx *ctx = arg;
7261 [ # # # # : 16 : uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
# # ]
7262 : :
7263 [ + - - + : 16 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7264 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "aer failed\n");
# # # # ]
7265 : 0 : return;
7266 : : }
7267 : :
7268 [ - + ]: 16 : if (log_page_id != SPDK_NVME_LOG_DISCOVERY) {
7269 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id);
# # # # ]
7270 : 0 : return;
7271 : : }
7272 : :
7273 [ - + + - : 16 : DISCOVERY_INFOLOG(ctx, "got aer\n");
# # # # #
# # # #
# ]
7274 [ - + + + : 16 : if (ctx->in_progress) {
# # # # ]
7275 [ # # # # ]: 5 : ctx->pending = true;
7276 : 5 : return;
7277 : : }
7278 : :
7279 : 11 : get_discovery_log_page(ctx);
7280 : 0 : }
7281 : :
7282 : : static void
7283 : 33 : discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
7284 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
7285 : : {
7286 : 33 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
7287 : : struct discovery_ctx *ctx;
7288 : :
7289 : 33 : ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts);
7290 : :
7291 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n");
# # # # #
# # # #
# ]
7292 [ # # # # ]: 33 : ctx->probe_ctx = NULL;
7293 [ # # # # ]: 33 : ctx->ctrlr = ctrlr;
7294 : :
7295 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7296 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n",
# # # # #
# # # ]
7297 : : ctx->rc);
7298 : 0 : return;
7299 : : }
7300 : :
7301 [ # # # # ]: 33 : spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx);
7302 : 0 : }
7303 : :
7304 : : static int
7305 : 107853 : discovery_poller(void *arg)
7306 : : {
7307 : 107853 : struct discovery_ctx *ctx = arg;
7308 : : struct spdk_nvme_transport_id *trid;
7309 : : int rc;
7310 : :
7311 [ + + # # : 107853 : if (ctx->detach_ctx) {
# # ]
7312 [ # # # # ]: 243 : rc = spdk_nvme_detach_poll_async(ctx->detach_ctx);
7313 [ + + ]: 243 : if (rc != -EAGAIN) {
7314 [ # # # # ]: 33 : ctx->detach_ctx = NULL;
7315 [ # # # # ]: 33 : ctx->ctrlr = NULL;
7316 : 0 : }
7317 [ - + + + : 107610 : } else if (ctx->stop) {
# # # # ]
7318 [ + + # # : 60 : if (ctx->ctrlr != NULL) {
# # ]
7319 [ # # # # : 30 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7320 [ + - ]: 30 : if (rc == 0) {
7321 : 30 : return SPDK_POLLER_BUSY;
7322 : : }
7323 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7324 : 0 : }
7325 [ # # ]: 30 : spdk_poller_unregister(&ctx->poller);
7326 [ + + # # : 30 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7327 [ - + # # : 30 : assert(ctx->start_cb_fn == NULL);
# # # # ]
7328 [ + + # # : 30 : if (ctx->stop_cb_fn != NULL) {
# # ]
7329 [ # # # # : 25 : ctx->stop_cb_fn(ctx->cb_ctx);
# # # # #
# # # ]
7330 : 0 : }
7331 : 30 : free_discovery_ctx(ctx);
7332 [ + + + + : 107550 : } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) {
# # # # #
# # # ]
7333 [ + + + + : 50 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7334 [ # # # # : 4 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7335 [ - + - + : 4 : assert(ctx->initializing);
# # # # #
# ]
7336 [ # # ]: 4 : spdk_poller_unregister(&ctx->poller);
7337 [ - + # # : 4 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7338 : 4 : complete_discovery_start(ctx, -ETIMEDOUT);
7339 : 4 : stop_discovery(ctx, NULL, NULL);
7340 : 4 : free_discovery_ctx(ctx);
7341 : 4 : return SPDK_POLLER_BUSY;
7342 : : }
7343 : :
7344 [ - + # # : 46 : assert(ctx->entry_ctx_in_use == NULL);
# # # # ]
7345 [ # # # # : 46 : ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # # # #
# ]
7346 [ + + # # : 46 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7347 [ # # # # : 46 : trid = &ctx->entry_ctx_in_use->trid;
# # ]
7348 : :
7349 : : /* All controllers must be configured explicitely either for multipath or failover.
7350 : : * While discovery use multipath mode, we need to set this in bdev options as well.
7351 : : */
7352 [ # # # # : 46 : ctx->bdev_opts.multipath = true;
# # ]
7353 : :
7354 [ # # # # : 46 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb);
# # ]
7355 [ + + # # : 46 : if (ctx->probe_ctx) {
# # ]
7356 [ # # ]: 33 : spdk_poller_unregister(&ctx->poller);
7357 [ # # # # ]: 33 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000);
7358 : 0 : } else {
7359 [ # # # # : 13 : DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n");
# # # # ]
7360 [ # # # # : 13 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7361 [ # # # # ]: 13 : ctx->entry_ctx_in_use = NULL;
7362 : : }
7363 [ + + # # : 107500 : } else if (ctx->probe_ctx) {
# # ]
7364 [ + + - + : 33 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7365 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7366 : 0 : complete_discovery_start(ctx, -ETIMEDOUT);
7367 : 0 : return SPDK_POLLER_BUSY;
7368 : : }
7369 : :
7370 [ # # # # ]: 33 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
7371 [ + - ]: 33 : if (rc != -EAGAIN) {
7372 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7373 [ # # # # : 0 : assert(ctx->initializing);
# # # # #
# ]
7374 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7375 : 0 : } else {
7376 [ - + # # ]: 33 : assert(rc == 0);
7377 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n");
# # # # #
# # # #
# ]
7378 [ # # # # ]: 33 : ctx->rc = rc;
7379 : 33 : get_discovery_log_page(ctx);
7380 : : }
7381 : 0 : }
7382 : 0 : } else {
7383 [ + + + + : 107467 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7384 [ # # # # : 1 : DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n");
# # # # ]
7385 : 1 : complete_discovery_start(ctx, -ETIMEDOUT);
7386 : : /* We need to wait until all NVM ctrlrs are attached before we stop the
7387 : : * discovery service to make sure we don't detach a ctrlr that is still
7388 : : * being attached.
7389 : : */
7390 [ + - # # : 1 : if (ctx->attach_in_progress == 0) {
# # ]
7391 [ # # # # ]: 1 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7392 : 1 : return SPDK_POLLER_BUSY;
7393 : : }
7394 : 0 : }
7395 : :
7396 [ # # # # ]: 107466 : rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr);
7397 [ + + ]: 107466 : if (rc < 0) {
7398 [ # # ]: 3 : spdk_poller_unregister(&ctx->poller);
7399 [ # # # # ]: 3 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7400 [ # # # # : 3 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7401 [ # # # # ]: 3 : ctx->entry_ctx_in_use = NULL;
7402 : :
7403 [ # # # # : 3 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7404 [ - + ]: 3 : if (rc != 0) {
7405 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7406 [ # # # # ]: 0 : ctx->ctrlr = NULL;
7407 : 0 : }
7408 : 0 : }
7409 : : }
7410 : :
7411 : 107818 : return SPDK_POLLER_BUSY;
7412 : 0 : }
7413 : :
7414 : : static void
7415 : 34 : start_discovery_poller(void *arg)
7416 : : {
7417 : 34 : struct discovery_ctx *ctx = arg;
7418 : :
7419 [ # # # # : 34 : TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
7420 [ # # # # ]: 34 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7421 : 34 : }
7422 : :
7423 : : int
7424 : 40 : bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid,
7425 : : const char *base_name,
7426 : : struct spdk_nvme_ctrlr_opts *drv_opts,
7427 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts,
7428 : : uint64_t attach_timeout,
7429 : : bool from_mdns,
7430 : : spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx)
7431 : : {
7432 : : struct discovery_ctx *ctx;
7433 : : struct discovery_entry_ctx *discovery_entry_ctx;
7434 : :
7435 [ # # ]: 40 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
7436 [ + + # # : 50 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7437 [ - + - + : 16 : if (strcmp(ctx->name, base_name) == 0) {
+ + # # #
# ]
7438 : 3 : return -EEXIST;
7439 : : }
7440 : :
7441 [ + + # # : 13 : if (ctx->entry_ctx_in_use != NULL) {
# # ]
7442 [ + + # # : 11 : if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) {
# # # # ]
7443 : 3 : return -EEXIST;
7444 : : }
7445 : 0 : }
7446 : :
7447 [ + + # # : 20 : TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7448 [ - + # # ]: 10 : if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) {
7449 : 0 : return -EEXIST;
7450 : : }
7451 : 0 : }
7452 : 0 : }
7453 : :
7454 : 34 : ctx = calloc(1, sizeof(*ctx));
7455 [ - + ]: 34 : if (ctx == NULL) {
7456 : 0 : return -ENOMEM;
7457 : : }
7458 : :
7459 [ - + # # : 34 : ctx->name = strdup(base_name);
# # ]
7460 [ - + # # : 34 : if (ctx->name == NULL) {
# # ]
7461 : 0 : free_discovery_ctx(ctx);
7462 : 0 : return -ENOMEM;
7463 : : }
7464 [ - + - + : 34 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
# # ]
7465 [ - + - + : 34 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
# # ]
7466 [ # # # # : 34 : ctx->from_mdns_discovery_service = from_mdns;
# # ]
7467 [ # # # # : 34 : ctx->bdev_opts.from_discovery_service = true;
# # ]
7468 [ # # # # ]: 34 : ctx->calling_thread = spdk_get_thread();
7469 [ # # # # ]: 34 : ctx->start_cb_fn = cb_fn;
7470 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7471 [ # # # # ]: 34 : ctx->initializing = true;
7472 [ + + # # : 34 : if (ctx->start_cb_fn) {
# # ]
7473 : : /* We can use this when dumping json to denote if this RPC parameter
7474 : : * was specified or not.
7475 : : */
7476 [ # # # # ]: 27 : ctx->wait_for_attach = true;
7477 : 0 : }
7478 [ + + ]: 34 : if (attach_timeout != 0) {
7479 [ # # # # ]: 42 : ctx->timeout_ticks = spdk_get_ticks() + attach_timeout *
7480 [ # # # # ]: 21 : spdk_get_ticks_hz() / 1000ull;
7481 : 0 : }
7482 [ # # # # : 34 : TAILQ_INIT(&ctx->nvm_entry_ctxs);
# # # # #
# # # # #
# # ]
7483 [ # # # # : 34 : TAILQ_INIT(&ctx->discovery_entry_ctxs);
# # # # #
# # # # #
# # ]
7484 [ - + - + : 34 : memcpy(&ctx->trid, trid, sizeof(*trid));
# # ]
7485 : : /* Even if user did not specify hostnqn, we can still strdup("\0"); */
7486 [ - + # # : 34 : ctx->hostnqn = strdup(ctx->drv_opts.hostnqn);
# # # # #
# ]
7487 [ - + # # : 34 : if (ctx->hostnqn == NULL) {
# # ]
7488 : 0 : free_discovery_ctx(ctx);
7489 : 0 : return -ENOMEM;
7490 : : }
7491 : 34 : discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid);
7492 [ - + ]: 34 : if (discovery_entry_ctx == NULL) {
7493 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7494 : 0 : free_discovery_ctx(ctx);
7495 : 0 : return -ENOMEM;
7496 : : }
7497 : :
7498 [ # # # # : 34 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7499 : 34 : spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx);
7500 : 34 : return 0;
7501 : 0 : }
7502 : :
7503 : : int
7504 : 23 : bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7505 : : {
7506 : : struct discovery_ctx *ctx;
7507 : :
7508 [ + - # # : 27 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7509 [ - + - + : 27 : if (strcmp(name, ctx->name) == 0) {
+ + # # #
# ]
7510 [ - + - + : 23 : if (ctx->stop) {
# # # # ]
7511 : 0 : return -EALREADY;
7512 : : }
7513 : : /* If we're still starting the discovery service and ->rc is non-zero, we're
7514 : : * going to stop it as soon as we can
7515 : : */
7516 [ - + - + : 23 : if (ctx->initializing && ctx->rc != 0) {
- - # # #
# # # #
# ]
7517 : 0 : return -EALREADY;
7518 : : }
7519 : 23 : stop_discovery(ctx, cb_fn, cb_ctx);
7520 : 23 : return 0;
7521 : : }
7522 : 0 : }
7523 : :
7524 : 0 : return -ENOENT;
7525 : 0 : }
7526 : :
7527 : : static int
7528 : 1943 : bdev_nvme_library_init(void)
7529 : : {
7530 : 1943 : g_bdev_nvme_init_thread = spdk_get_thread();
7531 : :
7532 : 1943 : spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb,
7533 : : bdev_nvme_destroy_poll_group_cb,
7534 : : sizeof(struct nvme_poll_group), "nvme_poll_groups");
7535 : :
7536 : 1943 : return 0;
7537 : : }
7538 : :
7539 : : static void
7540 : 1943 : bdev_nvme_fini_destruct_ctrlrs(void)
7541 : : {
7542 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
7543 : : struct nvme_ctrlr *nvme_ctrlr;
7544 : :
7545 [ + + ]: 1943 : pthread_mutex_lock(&g_bdev_nvme_mutex);
7546 [ + + + - : 2668 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ - + - ]
7547 [ + + + - : 1456 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
+ - + + +
- + - +
- ]
7548 [ + + + - ]: 731 : pthread_mutex_lock(&nvme_ctrlr->mutex);
7549 [ + + - + ]: 731 : if (nvme_ctrlr->destruct) {
7550 : : /* This controller's destruction was already started
7551 : : * before the application started shutting down
7552 : : */
7553 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7554 : 0 : continue;
7555 : : }
7556 [ - + ]: 731 : nvme_ctrlr->destruct = true;
7557 [ - + - + ]: 731 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7558 : :
7559 [ - + - + ]: 731 : spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct,
7560 : 1 : nvme_ctrlr);
7561 : 1 : }
7562 : 1 : }
7563 : :
7564 : 1943 : g_bdev_nvme_module_finish = true;
7565 [ + + ]: 1943 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
7566 [ - + ]: 1402 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7567 : 1402 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
7568 : 1402 : spdk_bdev_module_fini_done();
7569 : 1402 : return;
7570 : : }
7571 : :
7572 [ + + ]: 541 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7573 : 54 : }
7574 : :
7575 : : static void
7576 : 6 : check_discovery_fini(void *arg)
7577 : : {
7578 [ + - ]: 6 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7579 : 6 : bdev_nvme_fini_destruct_ctrlrs();
7580 : 0 : }
7581 : 6 : }
7582 : :
7583 : : static void
7584 : 1943 : bdev_nvme_library_fini(void)
7585 : : {
7586 : : struct nvme_probe_skip_entry *entry, *entry_tmp;
7587 : : struct discovery_ctx *ctx;
7588 : :
7589 : 1943 : spdk_poller_unregister(&g_hotplug_poller);
7590 : 1943 : free(g_hotplug_probe_ctx);
7591 : 1943 : g_hotplug_probe_ctx = NULL;
7592 : :
7593 [ + + # # : 1997 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) {
# # # # -
+ ]
7594 [ + + # # : 54 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7595 : 54 : free(entry);
7596 : 0 : }
7597 : :
7598 [ + + # # ]: 1943 : assert(spdk_get_thread() == g_bdev_nvme_init_thread);
7599 [ + + ]: 1943 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7600 : 1937 : bdev_nvme_fini_destruct_ctrlrs();
7601 : 54 : } else {
7602 [ + + # # : 12 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7603 : 6 : stop_discovery(ctx, check_discovery_fini, NULL);
7604 : 0 : }
7605 : : }
7606 : 1943 : }
7607 : :
7608 : : static void
7609 : 0 : bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio)
7610 : : {
7611 : 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7612 [ # # # # ]: 0 : struct spdk_bdev *bdev = bdev_io->bdev;
7613 : 0 : struct spdk_dif_ctx dif_ctx;
7614 : 0 : struct spdk_dif_error err_blk = {};
7615 : : int rc;
7616 : 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
7617 : :
7618 : 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
7619 [ # # # # : 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
# # ]
7620 : 0 : rc = spdk_dif_ctx_init(&dif_ctx,
7621 [ # # # # : 0 : bdev->blocklen, bdev->md_len, bdev->md_interleave,
# # # # #
# # # #
# ]
7622 [ # # # # : 0 : bdev->dif_is_head_of_md, bdev->dif_type,
# # # # #
# ]
7623 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
7624 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts);
# # # # ]
7625 [ # # ]: 0 : if (rc != 0) {
7626 : 0 : SPDK_ERRLOG("Initialization of DIF context failed\n");
7627 : 0 : return;
7628 : : }
7629 : :
7630 [ # # # # : 0 : if (bdev->md_interleave) {
# # # # ]
7631 [ # # # # : 0 : rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7632 [ # # # # : 0 : bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7633 : 0 : } else {
7634 : 0 : struct iovec md_iov = {
7635 [ # # # # : 0 : .iov_base = bdev_io->u.bdev.md_buf,
# # # # ]
7636 [ # # # # : 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
# # # # #
# # # ]
7637 : : };
7638 : :
7639 [ # # # # : 0 : rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7640 [ # # # # : 0 : &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7641 : : }
7642 : :
7643 [ # # ]: 0 : if (rc != 0) {
7644 [ # # ]: 0 : SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
7645 : : err_blk.err_type, err_blk.err_offset);
7646 : 0 : } else {
7647 : 0 : SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n");
7648 : : }
7649 : 0 : }
7650 : :
7651 : : static void
7652 : 0 : bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7653 : : {
7654 : 0 : struct nvme_bdev_io *bio = ref;
7655 : :
7656 [ # # # # : 0 : if (spdk_nvme_cpl_is_success(cpl)) {
# # # # #
# # # # #
# # ]
7657 : : /* Run PI verification for read data buffer. */
7658 : 0 : bdev_nvme_verify_pi_error(bio);
7659 : 0 : }
7660 : :
7661 : : /* Return original completion status */
7662 [ # # ]: 0 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
7663 : 0 : }
7664 : :
7665 : : static void
7666 : 21207884 : bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7667 : : {
7668 : 21207884 : struct nvme_bdev_io *bio = ref;
7669 : 21207884 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7670 : : int ret;
7671 : :
7672 [ + + + - : 21207884 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
+ - + - -
- - - # #
# # # # #
# # # # #
# # # # #
# + - ]
7673 [ # # # # : 0 : SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7674 : : cpl->status.sct, cpl->status.sc);
7675 : :
7676 : : /* Save completion status to use after verifying PI error. */
7677 [ # # ]: 0 : bio->cpl = *cpl;
7678 : :
7679 [ # # # # : 0 : if (spdk_likely(nvme_io_path_is_available(bio->io_path))) {
# # ]
7680 : : /* Read without PI checking to verify PI error. */
7681 : 0 : ret = bdev_nvme_no_pi_readv(bio,
7682 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
7683 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
7684 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
7685 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
7686 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks);
# # # # ]
7687 [ # # ]: 0 : if (ret == 0) {
7688 : 0 : return;
7689 : : }
7690 : 0 : }
7691 : 0 : }
7692 : :
7693 : 21207884 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7694 : 3 : }
7695 : :
7696 : : static void
7697 : 20280517 : bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7698 : : {
7699 : 20280517 : struct nvme_bdev_io *bio = ref;
7700 : :
7701 [ - + - - : 20280517 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
- - - - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # ]
7702 [ # # # # : 0 : SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7703 : : cpl->status.sct, cpl->status.sc);
7704 : : /* Run PI verification for write data buffer if PI error is detected. */
7705 : 0 : bdev_nvme_verify_pi_error(bio);
7706 : 0 : }
7707 : :
7708 : 20280517 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7709 : 20280517 : }
7710 : :
7711 : : static void
7712 : 256879 : bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7713 : : {
7714 : 256879 : struct nvme_bdev_io *bio = ref;
7715 : 256879 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7716 : :
7717 : : /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks.
7718 : : * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error().
7719 : : */
7720 [ # # # # : 256879 : bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0;
# # # # #
# # # ]
7721 : :
7722 [ - + - - : 256879 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7723 [ # # # # : 0 : SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7724 : : cpl->status.sct, cpl->status.sc);
7725 : : /* Run PI verification for zone append data buffer if PI error is detected. */
7726 : 0 : bdev_nvme_verify_pi_error(bio);
7727 : 0 : }
7728 : :
7729 : 256879 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7730 : 256879 : }
7731 : :
7732 : : static void
7733 : 53 : bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7734 : : {
7735 : 53 : struct nvme_bdev_io *bio = ref;
7736 : :
7737 [ + + + - : 53 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7738 [ # # # # : 0 : SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7739 : : cpl->status.sct, cpl->status.sc);
7740 : : /* Run PI verification for compare data buffer if PI error is detected. */
7741 : 0 : bdev_nvme_verify_pi_error(bio);
7742 : 0 : }
7743 : :
7744 : 53 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7745 : 53 : }
7746 : :
7747 : : static void
7748 : 102 : bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7749 : : {
7750 : 102 : struct nvme_bdev_io *bio = ref;
7751 : :
7752 : : /* Compare operation completion */
7753 [ + + + + : 102 : if (!bio->first_fused_completed) {
# # # # ]
7754 : : /* Save compare result for write callback */
7755 [ # # ]: 51 : bio->cpl = *cpl;
7756 [ # # # # ]: 51 : bio->first_fused_completed = true;
7757 : 51 : return;
7758 : : }
7759 : :
7760 : : /* Write operation completion */
7761 [ + + - + : 51 : if (spdk_nvme_cpl_is_error(&bio->cpl)) {
# # # # #
# # # # #
# # # # #
# ]
7762 : : /* If bio->cpl is already an error, it means the compare operation failed. In that case,
7763 : : * complete the IO with the compare operation's status.
7764 : : */
7765 [ + + + - : 39 : if (!spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7766 : 3 : SPDK_ERRLOG("Unexpected write success after compare failure.\n");
7767 : 0 : }
7768 : :
7769 [ # # ]: 39 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
7770 : 0 : } else {
7771 : 12 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7772 : : }
7773 : 0 : }
7774 : :
7775 : : static void
7776 : 967724 : bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl)
7777 : : {
7778 : 967724 : struct nvme_bdev_io *bio = ref;
7779 : :
7780 : 967724 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7781 : 967724 : }
7782 : :
7783 : : static int
7784 : 40 : fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc)
7785 : : {
7786 [ + - # # ]: 40 : switch (desc->zt) {
7787 : 40 : case SPDK_NVME_ZONE_TYPE_SEQWR:
7788 [ # # # # ]: 40 : info->type = SPDK_BDEV_ZONE_TYPE_SEQWR;
7789 : 40 : break;
7790 : 0 : default:
7791 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt);
7792 : 0 : return -EIO;
7793 : : }
7794 : :
7795 [ + - - - : 40 : switch (desc->zs) {
- - - - #
# ]
7796 : 40 : case SPDK_NVME_ZONE_STATE_EMPTY:
7797 [ # # # # ]: 40 : info->state = SPDK_BDEV_ZONE_STATE_EMPTY;
7798 : 40 : break;
7799 : 0 : case SPDK_NVME_ZONE_STATE_IOPEN:
7800 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN;
7801 : 0 : break;
7802 : 0 : case SPDK_NVME_ZONE_STATE_EOPEN:
7803 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN;
7804 : 0 : break;
7805 : 0 : case SPDK_NVME_ZONE_STATE_CLOSED:
7806 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_CLOSED;
7807 : 0 : break;
7808 : 0 : case SPDK_NVME_ZONE_STATE_RONLY:
7809 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY;
7810 : 0 : break;
7811 : 0 : case SPDK_NVME_ZONE_STATE_FULL:
7812 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_FULL;
7813 : 0 : break;
7814 : 0 : case SPDK_NVME_ZONE_STATE_OFFLINE:
7815 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_OFFLINE;
7816 : 0 : break;
7817 : 0 : default:
7818 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs);
7819 : 0 : return -EIO;
7820 : : }
7821 : :
7822 [ # # # # : 40 : info->zone_id = desc->zslba;
# # # # ]
7823 [ # # # # : 40 : info->write_pointer = desc->wp;
# # # # ]
7824 [ # # # # : 40 : info->capacity = desc->zcap;
# # # # ]
7825 : :
7826 : 40 : return 0;
7827 : 0 : }
7828 : :
7829 : : static void
7830 : 1 : bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl)
7831 : : {
7832 : 1 : struct nvme_bdev_io *bio = ref;
7833 : 1 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7834 [ # # # # : 1 : uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id;
# # # # ]
7835 [ # # # # : 1 : uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones;
# # # # ]
7836 [ # # # # : 1 : struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf;
# # # # ]
7837 : : uint64_t max_zones_per_buf, i;
7838 : : uint32_t zone_report_bufsize;
7839 : : struct spdk_nvme_ns *ns;
7840 : : struct spdk_nvme_qpair *qpair;
7841 : : int ret;
7842 : :
7843 [ + - - + : 1 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7844 : 0 : goto out_complete_io_nvme_cpl;
7845 : : }
7846 : :
7847 [ - + # # : 1 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
7848 : 0 : ret = -ENXIO;
7849 : 0 : goto out_complete_io_ret;
7850 : : }
7851 : :
7852 [ # # # # : 1 : ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
7853 [ # # # # : 1 : qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
7854 : :
7855 : 1 : zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
7856 [ # # ]: 1 : max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) /
7857 : : sizeof(bio->zone_report_buf->descs[0]);
7858 : :
7859 [ - + # # : 1 : if (bio->zone_report_buf->nr_zones > max_zones_per_buf) {
# # # # #
# ]
7860 : 0 : ret = -EINVAL;
7861 : 0 : goto out_complete_io_ret;
7862 : : }
7863 : :
7864 [ - + # # : 1 : if (!bio->zone_report_buf->nr_zones) {
# # # # #
# ]
7865 : 0 : ret = -EINVAL;
7866 : 0 : goto out_complete_io_ret;
7867 : : }
7868 : :
7869 [ + + + - : 41 : for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) {
# # # # #
# # # # #
# # ]
7870 [ # # # # : 40 : ret = fill_zone_from_report(&info[bio->handled_zones],
# # ]
7871 [ # # # # : 40 : &bio->zone_report_buf->descs[i]);
# # # # ]
7872 [ - + ]: 40 : if (ret) {
7873 : 0 : goto out_complete_io_ret;
7874 : : }
7875 [ # # ]: 40 : bio->handled_zones++;
7876 : 0 : }
7877 : :
7878 [ - + # # : 1 : if (bio->handled_zones < zones_to_copy) {
# # ]
7879 : 0 : uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
7880 [ # # # # ]: 0 : uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones);
7881 : :
7882 [ # # # # : 0 : memset(bio->zone_report_buf, 0, zone_report_bufsize);
# # ]
7883 : 0 : ret = spdk_nvme_zns_report_zones(ns, qpair,
7884 [ # # # # ]: 0 : bio->zone_report_buf, zone_report_bufsize,
7885 : 0 : slba, SPDK_NVME_ZRA_LIST_ALL, true,
7886 : 0 : bdev_nvme_get_zone_info_done, bio);
7887 [ # # ]: 0 : if (!ret) {
7888 : 0 : return;
7889 : : } else {
7890 : 0 : goto out_complete_io_ret;
7891 : : }
7892 : : }
7893 : :
7894 : 1 : out_complete_io_nvme_cpl:
7895 [ # # # # ]: 1 : free(bio->zone_report_buf);
7896 [ # # # # ]: 1 : bio->zone_report_buf = NULL;
7897 : 1 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7898 : 1 : return;
7899 : :
7900 : 0 : out_complete_io_ret:
7901 [ # # # # ]: 0 : free(bio->zone_report_buf);
7902 [ # # # # ]: 0 : bio->zone_report_buf = NULL;
7903 : 0 : bdev_nvme_io_complete(bio, ret);
7904 : 0 : }
7905 : :
7906 : : static void
7907 : 44 : bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl)
7908 : : {
7909 : 44 : struct nvme_bdev_io *bio = ref;
7910 : :
7911 : 44 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7912 : 44 : }
7913 : :
7914 : : static void
7915 : 87 : bdev_nvme_admin_passthru_complete_nvme_status(void *ctx)
7916 : : {
7917 : 87 : struct nvme_bdev_io *bio = ctx;
7918 : 87 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7919 [ # # ]: 87 : const struct spdk_nvme_cpl *cpl = &bio->cpl;
7920 : :
7921 [ - + # # : 87 : assert(bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
7922 : :
7923 : 87 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
7924 : 87 : }
7925 : :
7926 : : static void
7927 : 7468 : bdev_nvme_abort_complete(void *ctx)
7928 : : {
7929 : 7468 : struct nvme_bdev_io *bio = ctx;
7930 : 7468 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7931 : :
7932 [ + - + - : 7468 : if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) {
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7933 : 9 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL);
7934 : 0 : } else {
7935 : 7459 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL);
7936 : : }
7937 : 7468 : }
7938 : :
7939 : : static void
7940 : 7468 : bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl)
7941 : : {
7942 : 7468 : struct nvme_bdev_io *bio = ref;
7943 : 7468 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7944 : :
7945 [ # # ]: 7468 : bio->cpl = *cpl;
7946 : 7468 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio);
7947 : 7468 : }
7948 : :
7949 : : static void
7950 : 87 : bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl)
7951 : : {
7952 : 87 : struct nvme_bdev_io *bio = ref;
7953 : 87 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7954 : :
7955 [ # # ]: 87 : bio->cpl = *cpl;
7956 : 87 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io),
7957 : 0 : bdev_nvme_admin_passthru_complete_nvme_status, bio);
7958 : 87 : }
7959 : :
7960 : : static void
7961 : 7740819 : bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset)
7962 : : {
7963 : 7740819 : struct nvme_bdev_io *bio = ref;
7964 : : struct iovec *iov;
7965 : :
7966 [ # # # # ]: 7740819 : bio->iov_offset = sgl_offset;
7967 [ + - # # : 15343154 : for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) {
# # # # #
# # # # #
# # # # ]
7968 [ # # # # : 15343154 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
7969 [ + + # # : 15343154 : if (bio->iov_offset < iov->iov_len) {
# # # # #
# ]
7970 : 7740819 : break;
7971 : : }
7972 : :
7973 [ # # # # : 7602335 : bio->iov_offset -= iov->iov_len;
# # # # ]
7974 : 0 : }
7975 : 7740819 : }
7976 : :
7977 : : static int
7978 : 33413497 : bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length)
7979 : : {
7980 : 33413497 : struct nvme_bdev_io *bio = ref;
7981 : : struct iovec *iov;
7982 : :
7983 [ - + # # : 33413497 : assert(bio->iovpos < bio->iovcnt);
# # # # #
# # # ]
7984 : :
7985 [ # # # # : 33413497 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
7986 : :
7987 [ # # # # : 33413497 : *address = iov->iov_base;
# # ]
7988 [ # # # # : 33413497 : *length = iov->iov_len;
# # ]
7989 : :
7990 [ + + # # : 33413497 : if (bio->iov_offset) {
# # ]
7991 [ - + # # : 110474 : assert(bio->iov_offset <= iov->iov_len);
# # # # #
# # # ]
7992 [ # # # # : 110474 : *address += bio->iov_offset;
# # ]
7993 [ # # # # : 110474 : *length -= bio->iov_offset;
# # ]
7994 : 0 : }
7995 : :
7996 [ # # # # : 33413497 : bio->iov_offset += *length;
# # ]
7997 [ + - # # : 33413497 : if (bio->iov_offset == iov->iov_len) {
# # # # #
# ]
7998 [ # # # # ]: 33413497 : bio->iovpos++;
7999 [ # # # # ]: 33413497 : bio->iov_offset = 0;
8000 : 0 : }
8001 : :
8002 : 33413497 : return 0;
8003 : : }
8004 : :
8005 : : static void
8006 : 90 : bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset)
8007 : : {
8008 : 90 : struct nvme_bdev_io *bio = ref;
8009 : : struct iovec *iov;
8010 : :
8011 [ # # # # ]: 90 : bio->fused_iov_offset = sgl_offset;
8012 [ + - # # : 90 : for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) {
# # # # #
# # # # #
# # # # ]
8013 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8014 [ + - # # : 90 : if (bio->fused_iov_offset < iov->iov_len) {
# # # # #
# ]
8015 : 90 : break;
8016 : : }
8017 : :
8018 [ # # # # : 0 : bio->fused_iov_offset -= iov->iov_len;
# # # # ]
8019 : 0 : }
8020 : 90 : }
8021 : :
8022 : : static int
8023 : 90 : bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length)
8024 : : {
8025 : 90 : struct nvme_bdev_io *bio = ref;
8026 : : struct iovec *iov;
8027 : :
8028 [ - + # # : 90 : assert(bio->fused_iovpos < bio->fused_iovcnt);
# # # # #
# # # ]
8029 : :
8030 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8031 : :
8032 [ # # # # : 90 : *address = iov->iov_base;
# # ]
8033 [ # # # # : 90 : *length = iov->iov_len;
# # ]
8034 : :
8035 [ - + # # : 90 : if (bio->fused_iov_offset) {
# # ]
8036 [ # # # # : 0 : assert(bio->fused_iov_offset <= iov->iov_len);
# # # # #
# # # ]
8037 [ # # # # : 0 : *address += bio->fused_iov_offset;
# # ]
8038 [ # # # # : 0 : *length -= bio->fused_iov_offset;
# # ]
8039 : 0 : }
8040 : :
8041 [ # # # # : 90 : bio->fused_iov_offset += *length;
# # ]
8042 [ + - # # : 90 : if (bio->fused_iov_offset == iov->iov_len) {
# # # # #
# ]
8043 [ # # # # ]: 90 : bio->fused_iovpos++;
8044 [ # # # # ]: 90 : bio->fused_iov_offset = 0;
8045 : 0 : }
8046 : :
8047 : 90 : return 0;
8048 : : }
8049 : :
8050 : : static int
8051 : 0 : bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8052 : : void *md, uint64_t lba_count, uint64_t lba)
8053 : : {
8054 : : int rc;
8055 : :
8056 [ # # # # : 0 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n",
# # ]
8057 : : lba_count, lba);
8058 : :
8059 [ # # # # ]: 0 : bio->iovs = iov;
8060 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8061 [ # # # # ]: 0 : bio->iovpos = 0;
8062 [ # # # # ]: 0 : bio->iov_offset = 0;
8063 : :
8064 [ # # # # : 0 : rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8065 [ # # # # : 0 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8066 : 0 : lba, lba_count,
8067 : 0 : bdev_nvme_no_pi_readv_done, bio, 0,
8068 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8069 : 0 : md, 0, 0);
8070 : :
8071 [ # # # # ]: 0 : if (rc != 0 && rc != -ENOMEM) {
8072 : 0 : SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc);
8073 : 0 : }
8074 : 0 : return rc;
8075 : : }
8076 : :
8077 : : static int
8078 : 21289545 : bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8079 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8080 : : struct spdk_memory_domain *domain, void *domain_ctx,
8081 : : struct spdk_accel_sequence *seq)
8082 : : {
8083 [ + - + - : 21289545 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
+ - + - +
- + - ]
8084 [ + - + - : 21289545 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
+ - + - +
- + - ]
8085 : : int rc;
8086 : :
8087 [ + + + + : 21289545 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n",
+ - ]
8088 : : lba_count, lba);
8089 : :
8090 [ + - + - ]: 21289545 : bio->iovs = iov;
8091 [ + - + - ]: 21289545 : bio->iovcnt = iovcnt;
8092 [ + - + - ]: 21289545 : bio->iovpos = 0;
8093 [ + - + - ]: 21289545 : bio->iov_offset = 0;
8094 : :
8095 [ + + + + ]: 21289545 : if (domain != NULL || seq != NULL) {
8096 [ # # # # : 365815 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8097 [ # # # # : 365815 : bio->ext_opts.memory_domain = domain;
# # ]
8098 [ # # # # : 365815 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8099 [ # # # # : 365815 : bio->ext_opts.io_flags = flags;
# # ]
8100 [ # # # # : 365815 : bio->ext_opts.metadata = md;
# # ]
8101 [ # # # # : 365815 : bio->ext_opts.accel_sequence = seq;
# # ]
8102 : :
8103 [ + - ]: 365815 : if (iovcnt == 1) {
8104 [ # # # # : 365815 : rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done,
# # ]
8105 [ # # ]: 0 : bio, &bio->ext_opts);
8106 : 0 : } else {
8107 : 0 : rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count,
8108 : 0 : bdev_nvme_readv_done, bio,
8109 : : bdev_nvme_queued_reset_sgl,
8110 : : bdev_nvme_queued_next_sge,
8111 [ # # ]: 0 : &bio->ext_opts);
8112 : : }
8113 [ + + ]: 20923730 : } else if (iovcnt == 1) {
8114 [ - + - + : 20157047 : rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base,
- + ]
8115 : 3 : md, lba, lba_count, bdev_nvme_readv_done,
8116 : 3 : bio, flags, 0, 0);
8117 : 3 : } else {
8118 : 766683 : rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count,
8119 : 0 : bdev_nvme_readv_done, bio, flags,
8120 : : bdev_nvme_queued_reset_sgl,
8121 : 0 : bdev_nvme_queued_next_sge, md, 0, 0);
8122 : : }
8123 : :
8124 [ + + + + ]: 21289545 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8125 : 0 : SPDK_ERRLOG("readv failed: rc = %d\n", rc);
8126 : 0 : }
8127 : 21289545 : return rc;
8128 : : }
8129 : :
8130 : : static int
8131 : 20365646 : bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8132 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8133 : : struct spdk_memory_domain *domain, void *domain_ctx,
8134 : : struct spdk_accel_sequence *seq,
8135 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13)
8136 : : {
8137 [ # # # # : 20365646 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8138 [ # # # # : 20365646 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8139 : : int rc;
8140 : :
8141 [ - + - + : 20365646 : SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8142 : : lba_count, lba);
8143 : :
8144 [ # # # # ]: 20365646 : bio->iovs = iov;
8145 [ # # # # ]: 20365646 : bio->iovcnt = iovcnt;
8146 [ # # # # ]: 20365646 : bio->iovpos = 0;
8147 [ # # # # ]: 20365646 : bio->iov_offset = 0;
8148 : :
8149 [ + + - + ]: 20365646 : if (domain != NULL || seq != NULL) {
8150 [ # # # # : 201265 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8151 [ # # # # : 201265 : bio->ext_opts.memory_domain = domain;
# # ]
8152 [ # # # # : 201265 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8153 [ - + # # : 201265 : bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype);
# # # # #
# ]
8154 [ # # # # : 201265 : bio->ext_opts.cdw13 = cdw13.raw;
# # ]
8155 [ # # # # : 201265 : bio->ext_opts.metadata = md;
# # ]
8156 [ # # # # : 201265 : bio->ext_opts.accel_sequence = seq;
# # ]
8157 : :
8158 [ + - ]: 201265 : if (iovcnt == 1) {
8159 [ # # # # : 201265 : rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done,
# # ]
8160 [ # # ]: 0 : bio, &bio->ext_opts);
8161 : 0 : } else {
8162 : 0 : rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count,
8163 : 0 : bdev_nvme_writev_done, bio,
8164 : : bdev_nvme_queued_reset_sgl,
8165 : : bdev_nvme_queued_next_sge,
8166 [ # # ]: 0 : &bio->ext_opts);
8167 : : }
8168 [ + + ]: 20164381 : } else if (iovcnt == 1) {
8169 [ # # # # : 18647718 : rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base,
# # ]
8170 : 0 : md, lba, lba_count, bdev_nvme_writev_done,
8171 : 0 : bio, flags, 0, 0);
8172 : 0 : } else {
8173 : 1516663 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8174 : 0 : bdev_nvme_writev_done, bio, flags,
8175 : : bdev_nvme_queued_reset_sgl,
8176 : 0 : bdev_nvme_queued_next_sge, md, 0, 0);
8177 : : }
8178 : :
8179 [ + + - + ]: 20365646 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8180 : 0 : SPDK_ERRLOG("writev failed: rc = %d\n", rc);
8181 : 0 : }
8182 : 20365646 : return rc;
8183 : : }
8184 : :
8185 : : static int
8186 : 256879 : bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8187 : : void *md, uint64_t lba_count, uint64_t zslba,
8188 : : uint32_t flags)
8189 : : {
8190 [ # # # # : 256879 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8191 [ # # # # : 256879 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8192 : : int rc;
8193 : :
8194 [ - + # # : 256879 : SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n",
# # ]
8195 : : lba_count, zslba);
8196 : :
8197 [ # # # # ]: 256879 : bio->iovs = iov;
8198 [ # # # # ]: 256879 : bio->iovcnt = iovcnt;
8199 [ # # # # ]: 256879 : bio->iovpos = 0;
8200 [ # # # # ]: 256879 : bio->iov_offset = 0;
8201 : :
8202 [ + - ]: 256879 : if (iovcnt == 1) {
8203 [ # # # # : 256879 : rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba,
# # ]
8204 : 0 : lba_count,
8205 : 0 : bdev_nvme_zone_appendv_done, bio,
8206 : 0 : flags,
8207 : : 0, 0);
8208 : 0 : } else {
8209 : 0 : rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count,
8210 : 0 : bdev_nvme_zone_appendv_done, bio, flags,
8211 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8212 : 0 : md, 0, 0);
8213 : : }
8214 : :
8215 [ - + - - ]: 256879 : if (rc != 0 && rc != -ENOMEM) {
8216 : 0 : SPDK_ERRLOG("zone append failed: rc = %d\n", rc);
8217 : 0 : }
8218 : 256879 : return rc;
8219 : : }
8220 : :
8221 : : static int
8222 : 53 : bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8223 : : void *md, uint64_t lba_count, uint64_t lba,
8224 : : uint32_t flags)
8225 : : {
8226 : : int rc;
8227 : :
8228 [ - + - + : 53 : SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8229 : : lba_count, lba);
8230 : :
8231 [ # # # # ]: 53 : bio->iovs = iov;
8232 [ # # # # ]: 53 : bio->iovcnt = iovcnt;
8233 [ # # # # ]: 53 : bio->iovpos = 0;
8234 [ # # # # ]: 53 : bio->iov_offset = 0;
8235 : :
8236 [ # # # # : 53 : rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8237 [ # # # # : 53 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8238 : 0 : lba, lba_count,
8239 : 0 : bdev_nvme_comparev_done, bio, flags,
8240 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8241 : 0 : md, 0, 0);
8242 : :
8243 [ - + - - ]: 53 : if (rc != 0 && rc != -ENOMEM) {
8244 : 0 : SPDK_ERRLOG("comparev failed: rc = %d\n", rc);
8245 : 0 : }
8246 : 53 : return rc;
8247 : : }
8248 : :
8249 : : static int
8250 : 51 : bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt,
8251 : : struct iovec *write_iov, int write_iovcnt,
8252 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags)
8253 : : {
8254 [ # # # # : 51 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8255 [ # # # # : 51 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8256 : 51 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8257 : : int rc;
8258 : :
8259 [ - + - + : 51 : SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8260 : : lba_count, lba);
8261 : :
8262 [ # # # # ]: 51 : bio->iovs = cmp_iov;
8263 [ # # # # ]: 51 : bio->iovcnt = cmp_iovcnt;
8264 [ # # # # ]: 51 : bio->iovpos = 0;
8265 [ # # # # ]: 51 : bio->iov_offset = 0;
8266 [ # # # # ]: 51 : bio->fused_iovs = write_iov;
8267 [ # # # # ]: 51 : bio->fused_iovcnt = write_iovcnt;
8268 [ # # # # ]: 51 : bio->fused_iovpos = 0;
8269 [ # # # # ]: 51 : bio->fused_iov_offset = 0;
8270 : :
8271 [ + - # # : 51 : if (bdev_io->num_retries == 0) {
# # ]
8272 [ # # # # ]: 51 : bio->first_fused_submitted = false;
8273 [ # # # # ]: 51 : bio->first_fused_completed = false;
8274 : 0 : }
8275 : :
8276 [ + + + - : 51 : if (!bio->first_fused_submitted) {
# # # # ]
8277 [ # # # # ]: 51 : flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8278 [ - + # # ]: 51 : memset(&bio->cpl, 0, sizeof(bio->cpl));
8279 : :
8280 : 51 : rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count,
8281 : 0 : bdev_nvme_comparev_and_writev_done, bio, flags,
8282 : 0 : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0);
8283 [ + - ]: 51 : if (rc == 0) {
8284 [ # # # # ]: 51 : bio->first_fused_submitted = true;
8285 [ # # # # ]: 51 : flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8286 : 0 : } else {
8287 [ # # ]: 0 : if (rc != -ENOMEM) {
8288 : 0 : SPDK_ERRLOG("compare failed: rc = %d\n", rc);
8289 : 0 : }
8290 : 0 : return rc;
8291 : : }
8292 : 0 : }
8293 : :
8294 [ # # # # ]: 51 : flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND;
8295 : :
8296 : 51 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8297 : 0 : bdev_nvme_comparev_and_writev_done, bio, flags,
8298 : 0 : bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0);
8299 [ - + - - ]: 51 : if (rc != 0 && rc != -ENOMEM) {
8300 : 0 : SPDK_ERRLOG("write failed: rc = %d\n", rc);
8301 : 0 : rc = 0;
8302 : 0 : }
8303 : :
8304 : 51 : return rc;
8305 : 0 : }
8306 : :
8307 : : static int
8308 : 255667 : bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8309 : : {
8310 : 163994 : struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES];
8311 : : struct spdk_nvme_dsm_range *range;
8312 : : uint64_t offset, remaining;
8313 : : uint64_t num_ranges_u64;
8314 : : uint16_t num_ranges;
8315 : : int rc;
8316 : :
8317 [ # # ]: 255667 : num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) /
8318 : : SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8319 [ - + ]: 255667 : if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) {
8320 : 0 : SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks);
8321 : 0 : return -EINVAL;
8322 : : }
8323 : 255667 : num_ranges = (uint16_t)num_ranges_u64;
8324 : :
8325 : 255667 : offset = offset_blocks;
8326 : 255667 : remaining = num_blocks;
8327 [ # # # # ]: 255667 : range = &dsm_ranges[0];
8328 : :
8329 : : /* Fill max-size ranges until the remaining blocks fit into one range */
8330 [ + + ]: 255669 : while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
8331 [ # # # # : 2 : range->attributes.raw = 0;
# # ]
8332 [ # # # # ]: 2 : range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8333 [ # # # # ]: 2 : range->starting_lba = offset;
8334 : :
8335 : 2 : offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8336 : 2 : remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8337 [ # # ]: 2 : range++;
8338 : : }
8339 : :
8340 : : /* Final range describes the remaining blocks */
8341 [ # # # # : 255667 : range->attributes.raw = 0;
# # ]
8342 [ # # # # ]: 255667 : range->length = remaining;
8343 [ # # # # ]: 255667 : range->starting_lba = offset;
8344 : :
8345 [ # # # # : 255667 : rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8346 [ # # # # : 255667 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8347 : : SPDK_NVME_DSM_ATTR_DEALLOCATE,
8348 : 0 : dsm_ranges, num_ranges,
8349 : 0 : bdev_nvme_queued_done, bio);
8350 : :
8351 : 255667 : return rc;
8352 : 0 : }
8353 : :
8354 : : static int
8355 : 714867 : bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8356 : : {
8357 [ - + ]: 714867 : if (num_blocks > UINT16_MAX + 1) {
8358 : 0 : SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n");
8359 : 0 : return -EINVAL;
8360 : : }
8361 : :
8362 [ # # # # : 714867 : return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8363 [ # # # # : 714867 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8364 : 0 : offset_blocks, num_blocks,
8365 : 0 : bdev_nvme_queued_done, bio,
8366 : : 0);
8367 : 0 : }
8368 : :
8369 : : static int
8370 : 1 : bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones,
8371 : : struct spdk_bdev_zone_info *info)
8372 : : {
8373 [ # # # # : 1 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8374 [ # # # # : 1 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8375 : 1 : uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
8376 : 1 : uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
8377 : 1 : uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns);
8378 : :
8379 [ - + # # ]: 1 : if (zone_id % zone_size != 0) {
8380 : 0 : return -EINVAL;
8381 : : }
8382 : :
8383 [ + - - + ]: 1 : if (num_zones > total_zones || !num_zones) {
8384 : 0 : return -EINVAL;
8385 : : }
8386 : :
8387 [ - + # # : 1 : assert(!bio->zone_report_buf);
# # # # ]
8388 [ # # # # ]: 1 : bio->zone_report_buf = calloc(1, zone_report_bufsize);
8389 [ - + # # : 1 : if (!bio->zone_report_buf) {
# # ]
8390 : 0 : return -ENOMEM;
8391 : : }
8392 : :
8393 [ # # # # ]: 1 : bio->handled_zones = 0;
8394 : :
8395 [ # # # # ]: 1 : return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize,
8396 : 0 : zone_id, SPDK_NVME_ZRA_LIST_ALL, true,
8397 : 0 : bdev_nvme_get_zone_info_done, bio);
8398 : 0 : }
8399 : :
8400 : : static int
8401 : 44 : bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
8402 : : enum spdk_bdev_zone_action action)
8403 : : {
8404 [ # # # # : 44 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8405 [ # # # # : 44 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8406 : :
8407 [ - - - + : 44 : switch (action) {
- - ]
8408 : 0 : case SPDK_BDEV_ZONE_CLOSE:
8409 : 0 : return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false,
8410 : 0 : bdev_nvme_zone_management_done, bio);
8411 : 0 : case SPDK_BDEV_ZONE_FINISH:
8412 : 0 : return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false,
8413 : 0 : bdev_nvme_zone_management_done, bio);
8414 : 0 : case SPDK_BDEV_ZONE_OPEN:
8415 : 0 : return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false,
8416 : 0 : bdev_nvme_zone_management_done, bio);
8417 : 44 : case SPDK_BDEV_ZONE_RESET:
8418 : 44 : return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false,
8419 : 0 : bdev_nvme_zone_management_done, bio);
8420 : 0 : case SPDK_BDEV_ZONE_OFFLINE:
8421 : 0 : return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false,
8422 : 0 : bdev_nvme_zone_management_done, bio);
8423 : 0 : default:
8424 : 0 : return -EINVAL;
8425 : : }
8426 : 0 : }
8427 : :
8428 : : static void
8429 : 90 : bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8430 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes)
8431 : : {
8432 : : struct nvme_io_path *io_path;
8433 : : struct nvme_ctrlr *nvme_ctrlr;
8434 : : uint32_t max_xfer_size;
8435 : 90 : int rc = -ENXIO;
8436 : :
8437 : : /* Choose the first ctrlr which is not failed. */
8438 [ + + # # : 99 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8439 [ # # # # : 96 : nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
8440 : :
8441 : : /* We should skip any unavailable nvme_ctrlr rather than checking
8442 : : * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO.
8443 : : */
8444 [ + + ]: 96 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
8445 : 9 : continue;
8446 : : }
8447 : :
8448 [ # # # # ]: 87 : max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr);
8449 : :
8450 [ - + ]: 87 : if (nbytes > max_xfer_size) {
8451 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8452 : 0 : rc = -EINVAL;
8453 : 0 : goto err;
8454 : : }
8455 : :
8456 [ # # # # ]: 87 : rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes,
8457 : 0 : bdev_nvme_admin_passthru_done, bio);
8458 [ + - ]: 87 : if (rc == 0) {
8459 : 87 : return;
8460 : : }
8461 : 0 : }
8462 : :
8463 : 3 : err:
8464 : 3 : bdev_nvme_admin_complete(bio, rc);
8465 : 0 : }
8466 : :
8467 : : static int
8468 : 108 : bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8469 : : void *buf, size_t nbytes)
8470 : : {
8471 [ # # # # : 108 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8472 [ # # # # : 108 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8473 : 108 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8474 : 108 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8475 : :
8476 [ - + ]: 108 : if (nbytes > max_xfer_size) {
8477 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8478 : 0 : return -EINVAL;
8479 : : }
8480 : :
8481 : : /*
8482 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8483 : : * so fill it out automatically.
8484 : : */
8485 [ # # # # ]: 108 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8486 : :
8487 : 108 : return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf,
8488 : 0 : (uint32_t)nbytes, bdev_nvme_queued_done, bio);
8489 : 0 : }
8490 : :
8491 : : static int
8492 : 0 : bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8493 : : void *buf, size_t nbytes, void *md_buf, size_t md_len)
8494 : : {
8495 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8496 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8497 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8498 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8499 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8500 : :
8501 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8502 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8503 : 0 : return -EINVAL;
8504 : : }
8505 : :
8506 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8507 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8508 : 0 : return -EINVAL;
8509 : : }
8510 : :
8511 : : /*
8512 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8513 : : * so fill it out automatically.
8514 : : */
8515 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8516 : :
8517 : 0 : return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf,
8518 : 0 : (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio);
8519 : 0 : }
8520 : :
8521 : : static int
8522 : 0 : bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio,
8523 : : struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt,
8524 : : size_t nbytes, void *md_buf, size_t md_len)
8525 : : {
8526 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8527 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8528 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8529 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8530 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8531 : :
8532 [ # # # # ]: 0 : bio->iovs = iov;
8533 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8534 [ # # # # ]: 0 : bio->iovpos = 0;
8535 [ # # # # ]: 0 : bio->iov_offset = 0;
8536 : :
8537 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8538 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8539 : 0 : return -EINVAL;
8540 : : }
8541 : :
8542 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8543 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8544 : 0 : return -EINVAL;
8545 : : }
8546 : :
8547 : : /*
8548 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands
8549 : : * require a nsid, so fill it out automatically.
8550 : : */
8551 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8552 : :
8553 : 0 : return spdk_nvme_ctrlr_cmd_iov_raw_with_md(
8554 : 0 : ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio,
8555 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
8556 : 0 : }
8557 : :
8558 : : static void
8559 : 7477 : bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8560 : : struct nvme_bdev_io *bio_to_abort)
8561 : : {
8562 : : struct nvme_io_path *io_path;
8563 : 7477 : int rc = 0;
8564 : :
8565 : 7477 : rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort);
8566 [ + + ]: 7477 : if (rc == 0) {
8567 : 3 : bdev_nvme_admin_complete(bio, 0);
8568 : 3 : return;
8569 : : }
8570 : :
8571 [ # # # # ]: 7474 : io_path = bio_to_abort->io_path;
8572 [ + + ]: 7474 : if (io_path != NULL) {
8573 [ # # # # : 7468 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8574 [ # # # # : 7468 : io_path->qpair->qpair,
# # # # ]
8575 : 0 : bio_to_abort,
8576 : 0 : bdev_nvme_abort_done, bio);
8577 : 0 : } else {
8578 [ + + # # : 9 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8579 [ # # # # : 6 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8580 : : NULL,
8581 : 0 : bio_to_abort,
8582 : 0 : bdev_nvme_abort_done, bio);
8583 : :
8584 [ + + ]: 6 : if (rc != -ENOENT) {
8585 : 3 : break;
8586 : : }
8587 : 0 : }
8588 : : }
8589 : :
8590 [ + + ]: 7474 : if (rc != 0) {
8591 : : /* If no command was found or there was any error, complete the abort
8592 : : * request with failure.
8593 : : */
8594 : 6 : bdev_nvme_admin_complete(bio, rc);
8595 : 0 : }
8596 : 0 : }
8597 : :
8598 : : static int
8599 : 36 : bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks,
8600 : : uint64_t num_blocks)
8601 : : {
8602 : 36 : struct spdk_nvme_scc_source_range range = {
8603 : 0 : .slba = src_offset_blocks,
8604 : 36 : .nlb = num_blocks - 1
8605 : : };
8606 : :
8607 [ # # # # : 44 : return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8608 [ # # # # : 36 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8609 : 0 : &range, 1, dst_offset_blocks,
8610 : 0 : bdev_nvme_queued_done, bio);
8611 : : }
8612 : :
8613 : : static void
8614 : 165 : bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w)
8615 : : {
8616 : : const char *action;
8617 : : uint32_t i;
8618 : :
8619 [ - + ]: 165 : if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
8620 : 0 : action = "reset";
8621 [ + + ]: 165 : } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
8622 : 6 : action = "abort";
8623 : 0 : } else {
8624 : 159 : action = "none";
8625 : : }
8626 : :
8627 : 165 : spdk_json_write_object_begin(w);
8628 : :
8629 : 165 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_options");
8630 : :
8631 : 165 : spdk_json_write_named_object_begin(w, "params");
8632 : 165 : spdk_json_write_named_string(w, "action_on_timeout", action);
8633 [ + - ]: 165 : spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
8634 [ + - ]: 165 : spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us);
8635 [ + - ]: 165 : spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms);
8636 [ + - ]: 165 : spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst);
8637 [ + - ]: 165 : spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight);
8638 [ + - ]: 165 : spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight);
8639 [ + - ]: 165 : spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight);
8640 [ + - ]: 165 : spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
8641 [ + - ]: 165 : spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
8642 [ + - ]: 165 : spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
8643 [ + + + - ]: 165 : spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit);
8644 [ + - ]: 165 : spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count);
8645 [ + - ]: 165 : spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count);
8646 [ + - ]: 165 : spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout);
8647 [ + - ]: 165 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec);
8648 [ + - ]: 165 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec);
8649 [ + - ]: 165 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec);
8650 [ + + + - ]: 165 : spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback);
8651 [ + + ]: 165 : spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids);
8652 [ + - ]: 165 : spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos);
8653 [ + + ]: 165 : spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat);
8654 [ + - ]: 165 : spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size);
8655 [ + + + - ]: 165 : spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat);
8656 [ + + ]: 165 : spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence);
8657 [ + - ]: 165 : spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size);
8658 [ + - ]: 165 : spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms);
8659 : 165 : spdk_json_write_named_array_begin(w, "dhchap_digests");
8660 [ + + ]: 5445 : for (i = 0; i < 32; ++i) {
8661 [ + + + + : 5280 : if (g_opts.dhchap_digests & SPDK_BIT(i)) {
+ + ]
8662 : 495 : spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i));
8663 : 3 : }
8664 : 32 : }
8665 : 165 : spdk_json_write_array_end(w);
8666 : 165 : spdk_json_write_named_array_begin(w, "dhchap_dhgroups");
8667 [ + + ]: 5445 : for (i = 0; i < 32; ++i) {
8668 [ + + + + : 5280 : if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) {
+ + ]
8669 : 990 : spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i));
8670 : 6 : }
8671 : 32 : }
8672 : :
8673 : 165 : spdk_json_write_array_end(w);
8674 : 165 : spdk_json_write_object_end(w);
8675 : :
8676 : 165 : spdk_json_write_object_end(w);
8677 : 165 : }
8678 : :
8679 : : static void
8680 : 0 : bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx)
8681 : : {
8682 : 0 : struct spdk_nvme_transport_id trid;
8683 : :
8684 : 0 : spdk_json_write_object_begin(w);
8685 : :
8686 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery");
8687 : :
8688 : 0 : spdk_json_write_named_object_begin(w, "params");
8689 [ # # # # ]: 0 : spdk_json_write_named_string(w, "name", ctx->name);
8690 [ # # # # ]: 0 : spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn);
8691 : :
8692 [ # # ]: 0 : trid = ctx->trid;
8693 [ # # ]: 0 : memset(trid.subnqn, 0, sizeof(trid.subnqn));
8694 : 0 : nvme_bdev_dump_trid_json(&trid, w);
8695 : :
8696 [ # # # # : 0 : spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach);
# # ]
8697 [ # # # # : 0 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec);
# # ]
8698 [ # # # # : 0 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec);
# # ]
8699 : 0 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8700 [ # # # # : 0 : ctx->bdev_opts.fast_io_fail_timeout_sec);
# # ]
8701 : 0 : spdk_json_write_object_end(w);
8702 : :
8703 : 0 : spdk_json_write_object_end(w);
8704 : 0 : }
8705 : :
8706 : : #ifdef SPDK_CONFIG_NVME_CUSE
8707 : : static void
8708 : 118 : nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w,
8709 : : struct nvme_ctrlr *nvme_ctrlr)
8710 : 118 : {
8711 : 118 : size_t cuse_name_size = 128;
8712 [ - + ]: 118 : char cuse_name[cuse_name_size];
8713 : :
8714 [ + - # # : 118 : if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr,
# # # # ]
8715 : 0 : cuse_name, &cuse_name_size) != 0) {
8716 : 118 : return;
8717 : : }
8718 : :
8719 : 0 : spdk_json_write_object_begin(w);
8720 : :
8721 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register");
8722 : :
8723 : 0 : spdk_json_write_named_object_begin(w, "params");
8724 [ # # # # : 0 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8725 : 0 : spdk_json_write_object_end(w);
8726 : :
8727 : 0 : spdk_json_write_object_end(w);
8728 [ # # ]: 0 : }
8729 : : #endif
8730 : :
8731 : : static void
8732 : 118 : nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,
8733 : : struct nvme_ctrlr *nvme_ctrlr,
8734 : : struct nvme_path_id *path_id)
8735 : : {
8736 : : struct spdk_nvme_transport_id *trid;
8737 : : const struct spdk_nvme_ctrlr_opts *opts;
8738 : :
8739 [ - + - + : 118 : if (nvme_ctrlr->opts.from_discovery_service) {
# # # # #
# ]
8740 : : /* Do not emit an RPC for this - it will be implicitly
8741 : : * covered by a separate bdev_nvme_start_discovery or
8742 : : * bdev_nvme_start_mdns_discovery RPC.
8743 : : */
8744 : 0 : return;
8745 : : }
8746 : :
8747 [ # # ]: 118 : trid = &path_id->trid;
8748 : :
8749 : 118 : spdk_json_write_object_begin(w);
8750 : :
8751 : 118 : spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller");
8752 : :
8753 : 118 : spdk_json_write_named_object_begin(w, "params");
8754 [ # # # # : 118 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8755 : 118 : nvme_bdev_dump_trid_json(trid, w);
8756 : 118 : spdk_json_write_named_bool(w, "prchk_reftag",
8757 [ # # # # : 118 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);
# # # # ]
8758 : 118 : spdk_json_write_named_bool(w, "prchk_guard",
8759 [ # # # # : 118 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);
# # # # ]
8760 [ # # # # : 118 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);
# # ]
8761 [ # # # # : 118 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);
# # ]
8762 : 118 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8763 [ # # # # : 0 : nvme_ctrlr->opts.fast_io_fail_timeout_sec);
# # ]
8764 [ + + # # : 118 : if (nvme_ctrlr->psk != NULL) {
# # ]
8765 [ # # # # ]: 9 : spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk));
8766 : 0 : }
8767 [ - + # # : 118 : if (nvme_ctrlr->dhchap_key != NULL) {
# # ]
8768 : 0 : spdk_json_write_named_string(w, "dhchap_key",
8769 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_key));
8770 : 0 : }
8771 [ - + # # : 118 : if (nvme_ctrlr->dhchap_ctrlr_key != NULL) {
# # ]
8772 : 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
8773 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_ctrlr_key));
8774 : 0 : }
8775 [ # # # # ]: 118 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
8776 [ # # ]: 118 : spdk_json_write_named_string(w, "hostnqn", opts->hostnqn);
8777 [ - + # # : 118 : spdk_json_write_named_bool(w, "hdgst", opts->header_digest);
# # ]
8778 [ - + # # : 118 : spdk_json_write_named_bool(w, "ddgst", opts->data_digest);
# # ]
8779 [ - + # # : 118 : if (opts->src_addr[0] != '\0') {
# # # # #
# ]
8780 [ # # ]: 0 : spdk_json_write_named_string(w, "hostaddr", opts->src_addr);
8781 : 0 : }
8782 [ - + # # : 118 : if (opts->src_svcid[0] != '\0') {
# # # # #
# ]
8783 [ # # ]: 0 : spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid);
8784 : 0 : }
8785 : :
8786 [ + + + - : 118 : if (nvme_ctrlr->opts.multipath) {
# # # # #
# ]
8787 : 118 : spdk_json_write_named_string(w, "multipath", "multipath");
8788 : 0 : }
8789 : 118 : spdk_json_write_object_end(w);
8790 : :
8791 : 118 : spdk_json_write_object_end(w);
8792 : 0 : }
8793 : :
8794 : : static void
8795 : 165 : bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w)
8796 : : {
8797 : 165 : spdk_json_write_object_begin(w);
8798 : 165 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug");
8799 : :
8800 : 165 : spdk_json_write_named_object_begin(w, "params");
8801 : 165 : spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
8802 [ + + ]: 165 : spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
8803 : 165 : spdk_json_write_object_end(w);
8804 : :
8805 : 165 : spdk_json_write_object_end(w);
8806 : 165 : }
8807 : :
8808 : : static int
8809 : 165 : bdev_nvme_config_json(struct spdk_json_write_ctx *w)
8810 : : {
8811 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
8812 : : struct nvme_ctrlr *nvme_ctrlr;
8813 : : struct discovery_ctx *ctx;
8814 : : struct nvme_path_id *path_id;
8815 : :
8816 : 165 : bdev_nvme_opts_config_json(w);
8817 : :
8818 [ + + ]: 165 : pthread_mutex_lock(&g_bdev_nvme_mutex);
8819 : :
8820 [ + + # # : 283 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
8821 [ + + # # : 236 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
8822 [ # # # # ]: 118 : path_id = nvme_ctrlr->active_path_id;
8823 [ - + # # : 118 : assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
8824 : 118 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
8825 : :
8826 [ # # # # : 118 : path_id = TAILQ_NEXT(path_id, link);
# # ]
8827 [ - + ]: 118 : while (path_id != NULL) {
8828 : 0 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
8829 [ # # # # : 0 : path_id = TAILQ_NEXT(path_id, link);
# # ]
8830 : : }
8831 : :
8832 : : #ifdef SPDK_CONFIG_NVME_CUSE
8833 : 118 : nvme_ctrlr_cuse_config_json(w, nvme_ctrlr);
8834 : : #endif
8835 : 0 : }
8836 : 0 : }
8837 : :
8838 [ - + # # : 165 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
8839 [ # # # # : 0 : if (!ctx->from_mdns_discovery_service) {
# # # # ]
8840 : 0 : bdev_nvme_discovery_config_json(w, ctx);
8841 : 0 : }
8842 : 0 : }
8843 : :
8844 : 165 : bdev_nvme_mdns_discovery_config_json(w);
8845 : :
8846 : : /* Dump as last parameter to give all NVMe bdevs chance to be constructed
8847 : : * before enabling hotplug poller.
8848 : : */
8849 : 165 : bdev_nvme_hotplug_config_json(w);
8850 : :
8851 [ + + ]: 165 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
8852 : 165 : return 0;
8853 : : }
8854 : :
8855 : : struct spdk_nvme_ctrlr *
8856 : 17 : bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
8857 : : {
8858 : : struct nvme_bdev *nbdev;
8859 : : struct nvme_ns *nvme_ns;
8860 : :
8861 [ + - - + : 17 : if (!bdev || bdev->module != &nvme_if) {
# # # # ]
8862 : 0 : return NULL;
8863 : : }
8864 : :
8865 : 17 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
8866 [ # # # # : 17 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
8867 [ - + # # ]: 17 : assert(nvme_ns != NULL);
8868 : :
8869 [ # # # # : 17 : return nvme_ns->ctrlr->ctrlr;
# # # # ]
8870 : 0 : }
8871 : :
8872 : : static bool
8873 : 516 : nvme_io_path_is_current(struct nvme_io_path *io_path)
8874 : : {
8875 : : const struct nvme_bdev_channel *nbdev_ch;
8876 : : bool current;
8877 : :
8878 [ + + ]: 516 : if (!nvme_io_path_is_available(io_path)) {
8879 : 132 : return false;
8880 : : }
8881 : :
8882 [ # # # # ]: 384 : nbdev_ch = io_path->nbdev_ch;
8883 [ + + ]: 384 : if (nbdev_ch == NULL) {
8884 : 3 : current = false;
8885 [ + + # # : 381 : } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
8886 : 177 : struct nvme_io_path *optimized_io_path = NULL;
8887 : :
8888 [ + + # # : 378 : STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8889 [ + + # # : 303 : if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) {
# # # # #
# ]
8890 : 102 : break;
8891 : : }
8892 : 0 : }
8893 : :
8894 : : /* A non-optimized path is only current if there are no optimized paths. */
8895 [ + + + + : 177 : current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) ||
# # # # #
# ]
8896 : 0 : (optimized_io_path == NULL);
8897 : 0 : } else {
8898 [ + + # # : 204 : if (nbdev_ch->current_io_path) {
# # ]
8899 [ # # # # ]: 123 : current = (io_path == nbdev_ch->current_io_path);
8900 : 0 : } else {
8901 : : struct nvme_io_path *first_path;
8902 : :
8903 : : /* We arrived here as there are no optimized paths for active-passive
8904 : : * mode. Check if this io_path is the first one available on the list.
8905 : : */
8906 : 81 : current = false;
8907 [ + - # # : 81 : STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8908 [ + - ]: 81 : if (nvme_io_path_is_available(first_path)) {
8909 : 81 : current = (io_path == first_path);
8910 : 81 : break;
8911 : : }
8912 : 0 : }
8913 : : }
8914 : : }
8915 : :
8916 [ # # ]: 384 : return current;
8917 : 0 : }
8918 : :
8919 : : static struct nvme_ctrlr *
8920 : 40 : bdev_nvme_next_ctrlr_unsafe(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct nvme_ctrlr *prev)
8921 : : {
8922 : : struct nvme_ctrlr *next;
8923 : :
8924 : : /* Must be called under g_bdev_nvme_mutex */
8925 [ + + # # : 40 : next = prev != NULL ? TAILQ_NEXT(prev, tailq) : TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # # # #
# # # #
# ]
8926 [ + + ]: 40 : while (next != NULL) {
8927 : : /* ref can be 0 when the ctrlr was released, but hasn't been detached yet */
8928 [ - + # # ]: 28 : pthread_mutex_lock(&next->mutex);
8929 [ + - # # : 28 : if (next->ref > 0) {
# # ]
8930 [ # # # # ]: 28 : next->ref++;
8931 [ - + # # ]: 28 : pthread_mutex_unlock(&next->mutex);
8932 : 28 : return next;
8933 : : }
8934 : :
8935 [ # # # # ]: 0 : pthread_mutex_unlock(&next->mutex);
8936 [ # # # # : 0 : next = TAILQ_NEXT(next, tailq);
# # ]
8937 : : }
8938 : :
8939 : 12 : return NULL;
8940 : 0 : }
8941 : :
8942 : : struct bdev_nvme_set_keys_ctx {
8943 : : struct nvme_ctrlr *nctrlr;
8944 : : struct spdk_key *dhchap_key;
8945 : : struct spdk_key *dhchap_ctrlr_key;
8946 : : struct spdk_thread *thread;
8947 : : bdev_nvme_set_keys_cb cb_fn;
8948 : : void *cb_ctx;
8949 : : int status;
8950 : : };
8951 : :
8952 : : static void
8953 : 28 : bdev_nvme_free_set_keys_ctx(struct bdev_nvme_set_keys_ctx *ctx)
8954 : : {
8955 [ - + ]: 28 : if (ctx == NULL) {
8956 : 0 : return;
8957 : : }
8958 : :
8959 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_key);
8960 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_ctrlr_key);
8961 : 28 : free(ctx);
8962 : 0 : }
8963 : :
8964 : : static void
8965 : 28 : _bdev_nvme_set_keys_done(void *_ctx)
8966 : : {
8967 : 28 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
8968 : :
8969 [ # # # # : 28 : ctx->cb_fn(ctx->cb_ctx, ctx->status);
# # # # #
# # # # #
# # ]
8970 : :
8971 [ + + # # : 28 : if (ctx->nctrlr != NULL) {
# # ]
8972 [ # # # # ]: 16 : nvme_ctrlr_release(ctx->nctrlr);
8973 : 0 : }
8974 : 28 : bdev_nvme_free_set_keys_ctx(ctx);
8975 : 28 : }
8976 : :
8977 : : static void
8978 : 28 : bdev_nvme_set_keys_done(struct bdev_nvme_set_keys_ctx *ctx, int status)
8979 : : {
8980 [ # # # # ]: 28 : ctx->status = status;
8981 [ # # # # ]: 28 : spdk_thread_exec_msg(ctx->thread, _bdev_nvme_set_keys_done, ctx);
8982 : 28 : }
8983 : :
8984 : : static void bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx);
8985 : :
8986 : : static void
8987 : 12 : bdev_nvme_authenticate_ctrlr_continue(struct bdev_nvme_set_keys_ctx *ctx)
8988 : : {
8989 : : struct nvme_ctrlr *next;
8990 : :
8991 [ - + ]: 12 : pthread_mutex_lock(&g_bdev_nvme_mutex);
8992 [ # # # # ]: 12 : next = bdev_nvme_next_ctrlr_unsafe(NULL, ctx->nctrlr);
8993 [ - + ]: 12 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
8994 : :
8995 [ # # # # ]: 12 : nvme_ctrlr_release(ctx->nctrlr);
8996 [ # # # # ]: 12 : ctx->nctrlr = next;
8997 : :
8998 [ + - ]: 12 : if (next == NULL) {
8999 : 12 : bdev_nvme_set_keys_done(ctx, 0);
9000 : 0 : } else {
9001 : 0 : bdev_nvme_authenticate_ctrlr(ctx);
9002 : : }
9003 : 12 : }
9004 : :
9005 : : static void
9006 : 8 : bdev_nvme_authenticate_qpairs_done(struct spdk_io_channel_iter *i, int status)
9007 : : {
9008 : 8 : struct bdev_nvme_set_keys_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
9009 : :
9010 [ - + ]: 8 : if (status != 0) {
9011 : 0 : bdev_nvme_set_keys_done(ctx, status);
9012 : 0 : return;
9013 : : }
9014 : 8 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9015 : 0 : }
9016 : :
9017 : : static void
9018 : 0 : bdev_nvme_authenticate_qpair_done(void *ctx, int status)
9019 : : {
9020 : 0 : spdk_for_each_channel_continue(ctx, status);
9021 : 0 : }
9022 : :
9023 : : static void
9024 : 0 : bdev_nvme_authenticate_qpair(struct spdk_io_channel_iter *i)
9025 : : {
9026 : 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
9027 : 0 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
9028 [ # # # # ]: 0 : struct nvme_qpair *qpair = ctrlr_ch->qpair;
9029 : : int rc;
9030 : :
9031 [ # # ]: 0 : if (!nvme_qpair_is_connected(qpair)) {
9032 : 0 : spdk_for_each_channel_continue(i, 0);
9033 : 0 : return;
9034 : : }
9035 : :
9036 [ # # # # ]: 0 : rc = spdk_nvme_qpair_authenticate(qpair->qpair, bdev_nvme_authenticate_qpair_done, i);
9037 [ # # ]: 0 : if (rc != 0) {
9038 : 0 : spdk_for_each_channel_continue(i, rc);
9039 : 0 : }
9040 : 0 : }
9041 : :
9042 : : static void
9043 : 20 : bdev_nvme_authenticate_ctrlr_done(void *_ctx, int status)
9044 : : {
9045 : 20 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
9046 : :
9047 [ + + ]: 20 : if (status != 0) {
9048 : 12 : bdev_nvme_set_keys_done(ctx, status);
9049 : 12 : return;
9050 : : }
9051 : :
9052 [ # # # # ]: 8 : spdk_for_each_channel(ctx->nctrlr, bdev_nvme_authenticate_qpair, ctx,
9053 : : bdev_nvme_authenticate_qpairs_done);
9054 : 0 : }
9055 : :
9056 : : static void
9057 : 28 : bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx)
9058 : : {
9059 : 28 : struct spdk_nvme_ctrlr_key_opts opts = {};
9060 [ # # # # ]: 28 : struct nvme_ctrlr *nctrlr = ctx->nctrlr;
9061 : : int rc;
9062 : :
9063 : 28 : opts.size = SPDK_SIZEOF(&opts, dhchap_ctrlr_key);
9064 [ # # # # : 28 : opts.dhchap_key = ctx->dhchap_key;
# # ]
9065 [ # # # # : 28 : opts.dhchap_ctrlr_key = ctx->dhchap_ctrlr_key;
# # ]
9066 [ # # # # ]: 28 : rc = spdk_nvme_ctrlr_set_keys(nctrlr->ctrlr, &opts);
9067 [ - + ]: 28 : if (rc != 0) {
9068 : 0 : bdev_nvme_set_keys_done(ctx, rc);
9069 : 0 : return;
9070 : : }
9071 : :
9072 [ + + # # : 28 : if (ctx->dhchap_key != NULL) {
# # ]
9073 [ # # # # ]: 24 : rc = spdk_nvme_ctrlr_authenticate(nctrlr->ctrlr,
9074 : 0 : bdev_nvme_authenticate_ctrlr_done, ctx);
9075 [ + + ]: 24 : if (rc != 0) {
9076 : 4 : bdev_nvme_set_keys_done(ctx, rc);
9077 : 0 : }
9078 : 0 : } else {
9079 : 4 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9080 : : }
9081 : 0 : }
9082 : :
9083 : : int
9084 : 28 : bdev_nvme_set_keys(const char *name, const char *dhchap_key, const char *dhchap_ctrlr_key,
9085 : : bdev_nvme_set_keys_cb cb_fn, void *cb_ctx)
9086 : : {
9087 : : struct bdev_nvme_set_keys_ctx *ctx;
9088 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
9089 : : struct nvme_ctrlr *nctrlr;
9090 : :
9091 : 28 : ctx = calloc(1, sizeof(*ctx));
9092 [ - + ]: 28 : if (ctx == NULL) {
9093 : 0 : return -ENOMEM;
9094 : : }
9095 : :
9096 [ + + ]: 28 : if (dhchap_key != NULL) {
9097 [ # # # # ]: 24 : ctx->dhchap_key = spdk_keyring_get_key(dhchap_key);
9098 [ - + # # : 24 : if (ctx->dhchap_key == NULL) {
# # ]
9099 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_key, name);
9100 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9101 : 0 : return -ENOKEY;
9102 : : }
9103 : 0 : }
9104 [ + + ]: 28 : if (dhchap_ctrlr_key != NULL) {
9105 [ # # # # ]: 24 : ctx->dhchap_ctrlr_key = spdk_keyring_get_key(dhchap_ctrlr_key);
9106 [ - + # # : 24 : if (ctx->dhchap_ctrlr_key == NULL) {
# # ]
9107 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_ctrlr_key, name);
9108 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9109 : 0 : return -ENOKEY;
9110 : : }
9111 : 0 : }
9112 : :
9113 [ - + ]: 28 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9114 : 28 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
9115 [ - + ]: 28 : if (nbdev_ctrlr == NULL) {
9116 : 0 : SPDK_ERRLOG("Could not find bdev_ctrlr %s\n", name);
9117 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9118 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9119 : 0 : return -ENODEV;
9120 : : }
9121 : 28 : nctrlr = bdev_nvme_next_ctrlr_unsafe(nbdev_ctrlr, NULL);
9122 [ - + ]: 28 : if (nctrlr == NULL) {
9123 : 0 : SPDK_ERRLOG("Could not find any nvme_ctrlrs on bdev_ctrlr %s\n", name);
9124 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9125 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9126 : 0 : return -ENODEV;
9127 : : }
9128 [ - + ]: 28 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9129 : :
9130 [ # # # # ]: 28 : ctx->nctrlr = nctrlr;
9131 [ # # # # ]: 28 : ctx->cb_fn = cb_fn;
9132 [ # # # # ]: 28 : ctx->cb_ctx = cb_ctx;
9133 [ # # # # ]: 28 : ctx->thread = spdk_get_thread();
9134 : :
9135 : 28 : bdev_nvme_authenticate_ctrlr(ctx);
9136 : :
9137 : 28 : return 0;
9138 : 0 : }
9139 : :
9140 : : void
9141 : 480 : nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
9142 : : {
9143 [ # # # # ]: 480 : struct nvme_ns *nvme_ns = io_path->nvme_ns;
9144 [ # # # # : 480 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
9145 : : const struct spdk_nvme_ctrlr_data *cdata;
9146 : : const struct spdk_nvme_transport_id *trid;
9147 : : const char *adrfam_str;
9148 : :
9149 : 480 : spdk_json_write_object_begin(w);
9150 : :
9151 [ # # # # : 480 : spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
# # # # #
# ]
9152 : :
9153 [ # # # # ]: 480 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
9154 [ # # # # ]: 480 : trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr);
9155 : :
9156 [ # # # # ]: 480 : spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
9157 : 480 : spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path));
9158 [ # # # # ]: 480 : spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair));
9159 : 480 : spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
9160 : :
9161 : 480 : spdk_json_write_named_object_begin(w, "transport");
9162 [ # # ]: 480 : spdk_json_write_named_string(w, "trtype", trid->trstring);
9163 [ # # ]: 480 : spdk_json_write_named_string(w, "traddr", trid->traddr);
9164 [ + - # # : 480 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
9165 [ # # ]: 480 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
9166 : 0 : }
9167 [ # # # # ]: 480 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
9168 [ + - ]: 480 : if (adrfam_str) {
9169 : 480 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
9170 : 0 : }
9171 : 480 : spdk_json_write_object_end(w);
9172 : :
9173 : 480 : spdk_json_write_object_end(w);
9174 : 480 : }
9175 : :
9176 : : void
9177 : 77 : bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w)
9178 : : {
9179 : : struct discovery_ctx *ctx;
9180 : : struct discovery_entry_ctx *entry_ctx;
9181 : :
9182 : 77 : spdk_json_write_array_begin(w);
9183 [ + + # # : 150 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
9184 : 73 : spdk_json_write_object_begin(w);
9185 [ # # # # ]: 73 : spdk_json_write_named_string(w, "name", ctx->name);
9186 : :
9187 : 73 : spdk_json_write_named_object_begin(w, "trid");
9188 [ # # ]: 73 : nvme_bdev_dump_trid_json(&ctx->trid, w);
9189 : 73 : spdk_json_write_object_end(w);
9190 : :
9191 : 73 : spdk_json_write_named_array_begin(w, "referrals");
9192 [ + + # # : 171 : TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
9193 : 98 : spdk_json_write_object_begin(w);
9194 : 98 : spdk_json_write_named_object_begin(w, "trid");
9195 [ # # ]: 98 : nvme_bdev_dump_trid_json(&entry_ctx->trid, w);
9196 : 98 : spdk_json_write_object_end(w);
9197 : 98 : spdk_json_write_object_end(w);
9198 : 0 : }
9199 : 73 : spdk_json_write_array_end(w);
9200 : :
9201 : 73 : spdk_json_write_object_end(w);
9202 : 0 : }
9203 : 77 : spdk_json_write_array_end(w);
9204 : 77 : }
9205 : :
9206 : 2127 : SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)
9207 : :
9208 : : static void
9209 : 1964 : bdev_nvme_trace(void)
9210 : : {
9211 : 1964 : struct spdk_trace_tpoint_opts opts[] = {
9212 : : {
9213 : : "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START,
9214 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1,
9215 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9216 : : },
9217 : : {
9218 : : "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE,
9219 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0,
9220 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9221 : : }
9222 : : };
9223 : :
9224 : :
9225 : 1964 : spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N');
9226 : 1964 : spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
9227 : 1964 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9228 : 1964 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9229 : 1964 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9230 : 1964 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9231 : 1964 : }
9232 : 2127 : SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME)
|