Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
6 : : */
7 : :
8 : : #include "spdk/stdinc.h"
9 : :
10 : : #include "bdev_nvme.h"
11 : :
12 : : #include "spdk/accel.h"
13 : : #include "spdk/config.h"
14 : : #include "spdk/endian.h"
15 : : #include "spdk/bdev.h"
16 : : #include "spdk/json.h"
17 : : #include "spdk/keyring.h"
18 : : #include "spdk/likely.h"
19 : : #include "spdk/nvme.h"
20 : : #include "spdk/nvme_ocssd.h"
21 : : #include "spdk/nvme_zns.h"
22 : : #include "spdk/opal.h"
23 : : #include "spdk/thread.h"
24 : : #include "spdk/trace.h"
25 : : #include "spdk/string.h"
26 : : #include "spdk/util.h"
27 : : #include "spdk/uuid.h"
28 : :
29 : : #include "spdk/bdev_module.h"
30 : : #include "spdk/log.h"
31 : :
32 : : #include "spdk_internal/usdt.h"
33 : : #include "spdk_internal/trace_defs.h"
34 : :
35 : : #define CTRLR_STRING(nvme_ctrlr) \
36 : : (spdk_nvme_trtype_is_fabrics(nvme_ctrlr->active_path_id->trid.trtype) ? \
37 : : nvme_ctrlr->active_path_id->trid.subnqn : nvme_ctrlr->active_path_id->trid.traddr)
38 : :
39 : : #define CTRLR_ID(nvme_ctrlr) (spdk_nvme_ctrlr_get_id(nvme_ctrlr->ctrlr))
40 : :
41 : : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
42 : : SPDK_ERRLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
43 : :
44 : : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
45 : : SPDK_WARNLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
46 : :
47 : : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
48 : : SPDK_NOTICELOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
49 : :
50 : : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
51 : : SPDK_INFOLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
52 : :
53 : : #ifdef DEBUG
54 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
55 : : SPDK_DEBUGLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
56 : : #else
57 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
58 : : #endif
59 : :
60 : : #define BDEV_STRING(nbdev) (nbdev->disk.name)
61 : :
62 : : #define NVME_BDEV_ERRLOG(nbdev, format, ...) \
63 : : SPDK_ERRLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
64 : :
65 : : #define NVME_BDEV_WARNLOG(nbdev, format, ...) \
66 : : SPDK_WARNLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
67 : :
68 : : #define NVME_BDEV_NOTICELOG(nbdev, format, ...) \
69 : : SPDK_NOTICELOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
70 : :
71 : : #define NVME_BDEV_INFOLOG(nbdev, format, ...) \
72 : : SPDK_INFOLOG(bdev_nvme, "[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
73 : :
74 : : #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true
75 : : #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000)
76 : :
77 : : #define NSID_STR_LEN 10
78 : :
79 : : #define SPDK_CONTROLLER_NAME_MAX 512
80 : :
81 : : static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
82 : :
83 : : struct nvme_bdev_io {
84 : : /** array of iovecs to transfer. */
85 : : struct iovec *iovs;
86 : :
87 : : /** Number of iovecs in iovs array. */
88 : : int iovcnt;
89 : :
90 : : /** Current iovec position. */
91 : : int iovpos;
92 : :
93 : : /** Offset in current iovec. */
94 : : uint32_t iov_offset;
95 : :
96 : : /** Offset in current iovec. */
97 : : uint32_t fused_iov_offset;
98 : :
99 : : /** array of iovecs to transfer. */
100 : : struct iovec *fused_iovs;
101 : :
102 : : /** Number of iovecs in iovs array. */
103 : : int fused_iovcnt;
104 : :
105 : : /** Current iovec position. */
106 : : int fused_iovpos;
107 : :
108 : : /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
109 : : * being reset in a reset I/O.
110 : : */
111 : : struct nvme_io_path *io_path;
112 : :
113 : : /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
114 : : struct spdk_nvme_cpl cpl;
115 : :
116 : : /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
117 : : struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
118 : :
119 : : /** Keeps track if first of fused commands was submitted */
120 : : bool first_fused_submitted;
121 : :
122 : : /** Keeps track if first of fused commands was completed */
123 : : bool first_fused_completed;
124 : :
125 : : /* How many times the current I/O was retried. */
126 : : int32_t retry_count;
127 : :
128 : : /** Expiration value in ticks to retry the current I/O. */
129 : : uint64_t retry_ticks;
130 : :
131 : : /** Temporary pointer to zone report buffer */
132 : : struct spdk_nvme_zns_zone_report *zone_report_buf;
133 : :
134 : : /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
135 : : uint64_t handled_zones;
136 : :
137 : : /* Current tsc at submit time. */
138 : : uint64_t submit_tsc;
139 : :
140 : : /* Used to put nvme_bdev_io into the list */
141 : : TAILQ_ENTRY(nvme_bdev_io) retry_link;
142 : : };
143 : :
144 : : struct nvme_probe_skip_entry {
145 : : struct spdk_nvme_transport_id trid;
146 : : TAILQ_ENTRY(nvme_probe_skip_entry) tailq;
147 : : };
148 : : /* All the controllers deleted by users via RPC are skipped by hotplug monitor */
149 : : static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(
150 : : g_skipped_nvme_ctrlrs);
151 : :
152 : : #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \
153 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \
154 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512))
155 : :
156 : : #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \
157 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \
158 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \
159 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \
160 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \
161 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192))
162 : :
163 : : static struct spdk_bdev_nvme_opts g_opts = {
164 : : .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
165 : : .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS,
166 : : .timeout_us = 0,
167 : : .timeout_admin_us = 0,
168 : : .transport_retry_count = 4,
169 : : .arbitration_burst = 0,
170 : : .low_priority_weight = 0,
171 : : .medium_priority_weight = 0,
172 : : .high_priority_weight = 0,
173 : : .io_queue_requests = 0,
174 : : .nvme_adminq_poll_period_us = 10000ULL,
175 : : .nvme_ioq_poll_period_us = 0,
176 : : .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT,
177 : : .bdev_retry_count = 3,
178 : : .ctrlr_loss_timeout_sec = 0,
179 : : .reconnect_delay_sec = 0,
180 : : .fast_io_fail_timeout_sec = 0,
181 : : .transport_ack_timeout = 0,
182 : : .disable_auto_failback = false,
183 : : .generate_uuids = false,
184 : : .transport_tos = 0,
185 : : .nvme_error_stat = false,
186 : : .io_path_stat = false,
187 : : .allow_accel_sequence = false,
188 : : .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS,
189 : : .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS,
190 : : };
191 : :
192 : : #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL
193 : : #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL
194 : :
195 : : static int g_hot_insert_nvme_controller_index = 0;
196 : : static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
197 : : static bool g_nvme_hotplug_enabled = false;
198 : : struct spdk_thread *g_bdev_nvme_init_thread;
199 : : static struct spdk_poller *g_hotplug_poller;
200 : : static struct spdk_poller *g_hotplug_probe_poller;
201 : : static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
202 : :
203 : : static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
204 : : struct nvme_async_probe_ctx *ctx);
205 : : static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
206 : : struct nvme_async_probe_ctx *ctx);
207 : : static int bdev_nvme_library_init(void);
208 : : static void bdev_nvme_library_fini(void);
209 : : static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch,
210 : : struct spdk_bdev_io *bdev_io);
211 : : static void bdev_nvme_submit_request(struct spdk_io_channel *ch,
212 : : struct spdk_bdev_io *bdev_io);
213 : : static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
214 : : void *md, uint64_t lba_count, uint64_t lba,
215 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
216 : : struct spdk_accel_sequence *seq);
217 : : static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
218 : : void *md, uint64_t lba_count, uint64_t lba);
219 : : static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
220 : : void *md, uint64_t lba_count, uint64_t lba,
221 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
222 : : struct spdk_accel_sequence *seq,
223 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13);
224 : : static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
225 : : void *md, uint64_t lba_count,
226 : : uint64_t zslba, uint32_t flags);
227 : : static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
228 : : void *md, uint64_t lba_count, uint64_t lba,
229 : : uint32_t flags);
230 : : static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio,
231 : : struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov,
232 : : int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba,
233 : : uint32_t flags);
234 : : static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id,
235 : : uint32_t num_zones, struct spdk_bdev_zone_info *info);
236 : : static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
237 : : enum spdk_bdev_zone_action action);
238 : : static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch,
239 : : struct nvme_bdev_io *bio,
240 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes);
241 : : static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
242 : : void *buf, size_t nbytes);
243 : : static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
244 : : void *buf, size_t nbytes, void *md_buf, size_t md_len);
245 : : static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
246 : : struct iovec *iov, int iovcnt, size_t nbytes,
247 : : void *md_buf, size_t md_len);
248 : : static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch,
249 : : struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort);
250 : : static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio);
251 : : static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
252 : : static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
253 : : static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
254 : : static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
255 : :
256 : : static struct nvme_ns *nvme_ns_alloc(void);
257 : : static void nvme_ns_free(struct nvme_ns *ns);
258 : :
259 : : static int
260 : 2045 : nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
261 : : {
262 [ + + + - : 2045 : return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
+ - + - -
+ + - + -
+ - + - ]
263 : : }
264 : :
265 [ + + + + : 22819 : RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + - +
- + + - -
- - + # #
# # # # -
+ + + + -
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # +
- + + + +
+ + + - +
+ + + - +
+ + + + +
+ + + + -
# # + - +
- + - + -
+ - + + +
- + - + +
+ - + - +
- - + # #
# # - + +
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
# # # # #
# # # - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
- + + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
266 : :
267 : : struct spdk_nvme_qpair *
268 : 4 : bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
269 : : {
270 : : struct nvme_ctrlr_channel *ctrlr_ch;
271 : :
272 [ + + # # ]: 4 : assert(ctrlr_io_ch != NULL);
273 : :
274 : 4 : ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch);
275 : :
276 [ # # # # : 4 : return ctrlr_ch->qpair->qpair;
# # # # ]
277 : : }
278 : :
279 : : static int
280 : 1960 : bdev_nvme_get_ctx_size(void)
281 : : {
282 : 1960 : return sizeof(struct nvme_bdev_io);
283 : : }
284 : :
285 : : static struct spdk_bdev_module nvme_if = {
286 : : .name = "nvme",
287 : : .async_fini = true,
288 : : .module_init = bdev_nvme_library_init,
289 : : .module_fini = bdev_nvme_library_fini,
290 : : .config_json = bdev_nvme_config_json,
291 : : .get_ctx_size = bdev_nvme_get_ctx_size,
292 : :
293 : : };
294 : 2149 : SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if)
295 : :
296 : : struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs);
297 : : pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
298 : : bool g_bdev_nvme_module_finish;
299 : :
300 : : struct nvme_bdev_ctrlr *
301 : 50794 : nvme_bdev_ctrlr_get_by_name(const char *name)
302 : : {
303 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
304 : :
305 [ + + # # : 52708 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
306 [ + + + + : 45364 : if (strcmp(name, nbdev_ctrlr->name) == 0) {
+ + + - -
+ ]
307 : 43450 : break;
308 : : }
309 : 0 : }
310 : :
311 : 50794 : return nbdev_ctrlr;
312 : : }
313 : :
314 : : static struct nvme_ctrlr *
315 : 731 : nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr,
316 : : const struct spdk_nvme_transport_id *trid, const char *hostnqn)
317 : : {
318 : : const struct spdk_nvme_ctrlr_opts *opts;
319 : : struct nvme_ctrlr *nvme_ctrlr;
320 : :
321 [ + + # # : 1390 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
322 [ # # # # ]: 795 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
323 [ + + + - : 795 : if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 &&
# # # # #
# ]
324 [ + + - + : 139 : strcmp(hostnqn, opts->hostnqn) == 0) {
+ + ]
325 : 136 : break;
326 : : }
327 : 41 : }
328 : :
329 : 731 : return nvme_ctrlr;
330 : : }
331 : :
332 : : struct nvme_ctrlr *
333 : 0 : nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr,
334 : : uint16_t cntlid)
335 : : {
336 : : struct nvme_ctrlr *nvme_ctrlr;
337 : : const struct spdk_nvme_ctrlr_data *cdata;
338 : :
339 [ # # # # : 0 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
340 [ # # # # ]: 0 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
341 [ # # # # : 0 : if (cdata->cntlid == cntlid) {
# # ]
342 : 0 : break;
343 : : }
344 : 0 : }
345 : :
346 : 0 : return nvme_ctrlr;
347 : : }
348 : :
349 : : static struct nvme_bdev *
350 : 1643 : nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid)
351 : : {
352 : : struct nvme_bdev *bdev;
353 : :
354 [ + + ]: 1643 : pthread_mutex_lock(&g_bdev_nvme_mutex);
355 [ + + + - : 1941 : TAILQ_FOREACH(bdev, &nbdev_ctrlr->bdevs, tailq) {
+ - + - #
# # # #
# ]
356 [ + + # # : 456 : if (bdev->nsid == nsid) {
# # ]
357 : 158 : break;
358 : : }
359 : 34 : }
360 [ + + ]: 1643 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
361 : :
362 : 1643 : return bdev;
363 : : }
364 : :
365 : : struct nvme_ns *
366 : 3365 : nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid)
367 : : {
368 : 1142 : struct nvme_ns ns;
369 : :
370 [ + + # # ]: 3365 : assert(nsid > 0);
371 : :
372 : 3365 : ns.id = nsid;
373 [ + - ]: 3365 : return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns);
374 : : }
375 : :
376 : : struct nvme_ns *
377 : 3729 : nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr)
378 : : {
379 [ + - ]: 3729 : return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces);
380 : : }
381 : :
382 : : struct nvme_ns *
383 : 1646 : nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
384 : : {
385 [ - + ]: 1646 : if (ns == NULL) {
386 : 0 : return NULL;
387 : : }
388 : :
389 : 1646 : return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
390 : 87 : }
391 : :
392 : : static struct nvme_ctrlr *
393 : 1756 : nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn)
394 : : {
395 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
396 : 1756 : struct nvme_ctrlr *nvme_ctrlr = NULL;
397 : :
398 [ + + ]: 1756 : pthread_mutex_lock(&g_bdev_nvme_mutex);
399 [ + + # # : 2327 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
400 : 571 : nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn);
401 [ - + ]: 571 : if (nvme_ctrlr != NULL) {
402 : 0 : break;
403 : : }
404 : 19 : }
405 [ + + ]: 1756 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
406 : :
407 : 1756 : return nvme_ctrlr;
408 : : }
409 : :
410 : : struct nvme_ctrlr *
411 : 3603 : nvme_ctrlr_get_by_name(const char *name)
412 : : {
413 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
414 : 3603 : struct nvme_ctrlr *nvme_ctrlr = NULL;
415 : :
416 [ + + ]: 3603 : if (name == NULL) {
417 : 0 : return NULL;
418 : : }
419 : :
420 [ + + ]: 3603 : pthread_mutex_lock(&g_bdev_nvme_mutex);
421 : 3603 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
422 [ + + ]: 3603 : if (nbdev_ctrlr != NULL) {
423 [ + - + - : 347 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
+ - ]
424 : 63 : }
425 [ + + ]: 3603 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
426 : :
427 : 3603 : return nvme_ctrlr;
428 : 151 : }
429 : :
430 : : void
431 : 764 : nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx)
432 : : {
433 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
434 : :
435 [ - + ]: 764 : pthread_mutex_lock(&g_bdev_nvme_mutex);
436 [ + + # # : 1483 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
437 [ # # # # ]: 719 : fn(nbdev_ctrlr, ctx);
438 : 1 : }
439 [ - + ]: 764 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
440 : 764 : }
441 : :
442 : : struct nvme_ctrlr_channel_iter {
443 : : nvme_ctrlr_for_each_channel_msg fn;
444 : : nvme_ctrlr_for_each_channel_done cpl;
445 : : struct spdk_io_channel_iter *i;
446 : : void *ctx;
447 : : };
448 : :
449 : : void
450 : 2533 : nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status)
451 : : {
452 [ # # # # ]: 2533 : spdk_for_each_channel_continue(iter->i, status);
453 : 2533 : }
454 : :
455 : : static void
456 : 2533 : nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i)
457 : : {
458 : 2533 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
459 : 2533 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
460 : 2533 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
461 : 2533 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
462 : :
463 [ # # # # ]: 2533 : iter->i = i;
464 [ # # # # : 2533 : iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx);
# # # # #
# # # ]
465 : 2533 : }
466 : :
467 : : static void
468 : 2389 : nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
469 : : {
470 : 2389 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
471 : 2389 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
472 : :
473 [ # # # # ]: 2389 : iter->i = i;
474 [ # # # # : 2389 : iter->cpl(nvme_ctrlr, iter->ctx, status);
# # # # #
# # # ]
475 : :
476 : 2389 : free(iter);
477 : 2389 : }
478 : :
479 : : void
480 : 2389 : nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr,
481 : : nvme_ctrlr_for_each_channel_msg fn, void *ctx,
482 : : nvme_ctrlr_for_each_channel_done cpl)
483 : : {
484 : : struct nvme_ctrlr_channel_iter *iter;
485 : :
486 [ + - + - ]: 2389 : assert(nvme_ctrlr != NULL && fn != NULL);
487 : :
488 : 2389 : iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter));
489 [ - + ]: 2389 : if (iter == NULL) {
490 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
491 [ # # ]: 0 : assert(false);
492 : : return;
493 : : }
494 : :
495 [ # # # # ]: 2389 : iter->fn = fn;
496 [ # # # # ]: 2389 : iter->cpl = cpl;
497 [ # # # # ]: 2389 : iter->ctx = ctx;
498 : :
499 : 2490 : spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg,
500 : 101 : iter, nvme_ctrlr_each_channel_cpl);
501 : 101 : }
502 : :
503 : : struct nvme_bdev_channel_iter {
504 : : nvme_bdev_for_each_channel_msg fn;
505 : : nvme_bdev_for_each_channel_done cpl;
506 : : struct spdk_io_channel_iter *i;
507 : : void *ctx;
508 : : };
509 : :
510 : : void
511 : 360 : nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status)
512 : : {
513 [ # # # # ]: 360 : spdk_for_each_channel_continue(iter->i, status);
514 : 360 : }
515 : :
516 : : static void
517 : 360 : nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i)
518 : : {
519 : 360 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
520 : 360 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
521 : 360 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
522 : 360 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
523 : :
524 [ # # # # ]: 360 : iter->i = i;
525 [ # # # # : 360 : iter->fn(iter, nbdev, nbdev_ch, iter->ctx);
# # # # #
# # # ]
526 : 360 : }
527 : :
528 : : static void
529 : 354 : nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
530 : : {
531 : 354 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
532 : 354 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
533 : :
534 [ # # # # ]: 354 : iter->i = i;
535 [ # # # # : 354 : iter->cpl(nbdev, iter->ctx, status);
# # # # #
# # # ]
536 : :
537 : 354 : free(iter);
538 : 354 : }
539 : :
540 : : void
541 : 354 : nvme_bdev_for_each_channel(struct nvme_bdev *nbdev,
542 : : nvme_bdev_for_each_channel_msg fn, void *ctx,
543 : : nvme_bdev_for_each_channel_done cpl)
544 : : {
545 : : struct nvme_bdev_channel_iter *iter;
546 : :
547 [ + - + - ]: 354 : assert(nbdev != NULL && fn != NULL);
548 : :
549 : 354 : iter = calloc(1, sizeof(struct nvme_bdev_channel_iter));
550 [ - + ]: 354 : if (iter == NULL) {
551 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
552 [ # # ]: 0 : assert(false);
553 : : return;
554 : : }
555 : :
556 [ # # # # ]: 354 : iter->fn = fn;
557 [ # # # # ]: 354 : iter->cpl = cpl;
558 [ # # # # ]: 354 : iter->ctx = ctx;
559 : :
560 : 354 : spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter,
561 : : nvme_bdev_each_channel_cpl);
562 : 62 : }
563 : :
564 : : void
565 : 2171 : nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w)
566 : : {
567 : : const char *trtype_str;
568 : : const char *adrfam_str;
569 : :
570 [ # # # # ]: 2171 : trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype);
571 [ + - ]: 2171 : if (trtype_str) {
572 : 2171 : spdk_json_write_named_string(w, "trtype", trtype_str);
573 : 8 : }
574 : :
575 [ # # # # ]: 2171 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
576 [ + + ]: 2171 : if (adrfam_str) {
577 : 1151 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
578 : 0 : }
579 : :
580 [ + - # # : 2171 : if (trid->traddr[0] != '\0') {
# # # # #
# ]
581 [ # # ]: 2171 : spdk_json_write_named_string(w, "traddr", trid->traddr);
582 : 8 : }
583 : :
584 [ + + # # : 2171 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
585 [ # # ]: 1151 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
586 : 0 : }
587 : :
588 [ + + # # : 2171 : if (trid->subnqn[0] != '\0') {
# # # # #
# ]
589 [ # # ]: 1151 : spdk_json_write_named_string(w, "subnqn", trid->subnqn);
590 : 0 : }
591 : 2171 : }
592 : :
593 : : static void
594 : 1764 : nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr,
595 : : struct nvme_ctrlr *nvme_ctrlr)
596 : : {
597 : 514 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name);
598 [ + + ]: 1764 : pthread_mutex_lock(&g_bdev_nvme_mutex);
599 : :
600 [ + + + - : 1764 : TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
601 [ + + + - : 1764 : if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) {
+ - - + ]
602 [ - + ]: 73 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
603 : :
604 : 73 : return;
605 : : }
606 [ + + + - : 1691 : TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
607 : :
608 [ + + ]: 1691 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
609 : :
610 [ + + + - : 1691 : assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs));
+ - + - #
# ]
611 : :
612 [ + - + - ]: 1691 : free(nbdev_ctrlr->name);
613 : 1691 : free(nbdev_ctrlr);
614 : 72 : }
615 : :
616 : : static void
617 : 1768 : _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
618 : : {
619 : : struct nvme_path_id *path_id, *tmp_path;
620 : : struct nvme_ns *ns, *tmp_ns;
621 : :
622 [ + - + - ]: 1768 : free(nvme_ctrlr->copied_ana_desc);
623 [ + - + - ]: 1768 : spdk_free(nvme_ctrlr->ana_log_page);
624 : :
625 [ + + + - : 1768 : if (nvme_ctrlr->opal_dev) {
+ - ]
626 [ # # # # ]: 36 : spdk_opal_dev_destruct(nvme_ctrlr->opal_dev);
627 [ # # # # ]: 36 : nvme_ctrlr->opal_dev = NULL;
628 : 0 : }
629 : :
630 [ + + + - : 1768 : if (nvme_ctrlr->nbdev_ctrlr) {
- + ]
631 [ + - + - ]: 1764 : nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr);
632 : 72 : }
633 : :
634 [ + + + + : 1768 : RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) {
- + ]
635 [ # # ]: 0 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
636 : 0 : nvme_ns_free(ns);
637 : 0 : }
638 : :
639 [ + + + + : 3544 : TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) {
+ - + + +
- + - + -
+ + ]
640 [ + + + - : 1776 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
641 : 1776 : free(path_id);
642 : 73 : }
643 : :
644 [ + + + - ]: 1768 : pthread_mutex_destroy(&nvme_ctrlr->mutex);
645 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->psk);
646 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->dhchap_key);
647 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key);
648 : 1768 : free(nvme_ctrlr);
649 : :
650 [ + + ]: 1768 : pthread_mutex_lock(&g_bdev_nvme_mutex);
651 [ + + + + : 1768 : if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
+ + ]
652 [ + + ]: 536 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
653 : 536 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
654 : 536 : spdk_bdev_module_fini_done();
655 : 536 : return;
656 : : }
657 [ - + ]: 1232 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
658 : 73 : }
659 : :
660 : : static int
661 : 184527 : nvme_detach_poller(void *arg)
662 : : {
663 : 184527 : struct nvme_ctrlr *nvme_ctrlr = arg;
664 : : int rc;
665 : :
666 [ + - + - ]: 184527 : rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx);
667 [ + + ]: 184527 : if (rc != -EAGAIN) {
668 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
669 : 1768 : _nvme_ctrlr_delete(nvme_ctrlr);
670 : 73 : }
671 : :
672 : 184527 : return SPDK_POLLER_BUSY;
673 : : }
674 : :
675 : : static void
676 : 1768 : nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
677 : : {
678 : : int rc;
679 : :
680 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
681 : :
682 [ + + ]: 1768 : if (spdk_interrupt_mode_is_enabled()) {
683 [ # # ]: 1 : spdk_interrupt_unregister(&nvme_ctrlr->intr);
684 : 0 : }
685 : :
686 : : /* First, unregister the adminq poller, as the driver will poll adminq if necessary */
687 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
688 : :
689 : : /* If we got here, the reset/detach poller cannot be active */
690 [ + + + - : 1768 : assert(nvme_ctrlr->reset_detach_poller == NULL);
+ - # # ]
691 [ + - + - ]: 1768 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller,
692 : : nvme_ctrlr, 1000);
693 [ + + + - : 1768 : if (nvme_ctrlr->reset_detach_poller == NULL) {
+ - ]
694 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to register detach poller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
695 : 0 : goto error;
696 : : }
697 : :
698 [ + - + - : 1768 : rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx);
+ - ]
699 [ - + ]: 1768 : if (rc != 0) {
700 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to detach the NVMe controller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
701 : 0 : goto error;
702 : : }
703 : :
704 : 1768 : return;
705 : 0 : error:
706 : : /* We don't have a good way to handle errors here, so just do what we can and delete the
707 : : * controller without detaching the underlying NVMe device.
708 : : */
709 [ # # ]: 0 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
710 : 0 : _nvme_ctrlr_delete(nvme_ctrlr);
711 : 73 : }
712 : :
713 : : static void
714 : 1764 : nvme_ctrlr_unregister_cb(void *io_device)
715 : : {
716 : 1764 : struct nvme_ctrlr *nvme_ctrlr = io_device;
717 : :
718 : 1764 : nvme_ctrlr_delete(nvme_ctrlr);
719 : 1764 : }
720 : :
721 : : static void
722 : 1764 : nvme_ctrlr_unregister(void *ctx)
723 : : {
724 : 1764 : struct nvme_ctrlr *nvme_ctrlr = ctx;
725 : :
726 : 1764 : spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);
727 : 1764 : }
728 : :
729 : : static bool
730 : 8112 : nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr)
731 : : {
732 [ + + + + ]: 8112 : if (!nvme_ctrlr->destruct) {
733 : 4726 : return false;
734 : : }
735 : :
736 [ + + + - : 3386 : if (nvme_ctrlr->ref > 0) {
+ + ]
737 : 1607 : return false;
738 : : }
739 : :
740 [ + + - + ]: 1779 : if (nvme_ctrlr->resetting) {
741 : 15 : return false;
742 : : }
743 : :
744 [ + + - + ]: 1764 : if (nvme_ctrlr->ana_log_page_updating) {
745 : 0 : return false;
746 : : }
747 : :
748 [ + + - + ]: 1764 : if (nvme_ctrlr->io_path_cache_clearing) {
749 : 0 : return false;
750 : : }
751 : :
752 : 1764 : return true;
753 : 294 : }
754 : :
755 : : static void
756 : 5730 : nvme_ctrlr_put_ref(struct nvme_ctrlr *nvme_ctrlr)
757 : : {
758 [ + + + - ]: 5730 : pthread_mutex_lock(&nvme_ctrlr->mutex);
759 : 1519 : SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref);
760 : :
761 [ + + + - : 5730 : assert(nvme_ctrlr->ref > 0);
+ - # # ]
762 [ + - + - ]: 5730 : nvme_ctrlr->ref--;
763 : :
764 [ + + ]: 5730 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
765 [ + + + - ]: 3981 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
766 : 3981 : return;
767 : : }
768 : :
769 [ + + + - ]: 1749 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
770 : :
771 [ + - + - ]: 1749 : spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr);
772 : 215 : }
773 : :
774 : : static void
775 : 3938 : nvme_ctrlr_get_ref(struct nvme_ctrlr *nvme_ctrlr)
776 : : {
777 [ + + + - ]: 3938 : pthread_mutex_lock(&nvme_ctrlr->mutex);
778 [ + - + - ]: 3938 : nvme_ctrlr->ref++;
779 [ + + + - ]: 3938 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
780 : 3938 : }
781 : :
782 : : static void
783 : 400139 : bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch)
784 : : {
785 [ + - + - ]: 400139 : nbdev_ch->current_io_path = NULL;
786 [ + - + - ]: 400139 : nbdev_ch->rr_counter = 0;
787 : 400139 : }
788 : :
789 : : static struct nvme_io_path *
790 : 32 : _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
791 : : {
792 : : struct nvme_io_path *io_path;
793 : :
794 [ + + # # : 64 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
795 [ + + # # : 60 : if (io_path->nvme_ns == nvme_ns) {
# # ]
796 : 28 : break;
797 : : }
798 : 8 : }
799 : :
800 : 32 : return io_path;
801 : : }
802 : :
803 : : static struct nvme_io_path *
804 : 2431 : nvme_io_path_alloc(void)
805 : : {
806 : : struct nvme_io_path *io_path;
807 : :
808 : 2431 : io_path = calloc(1, sizeof(*io_path));
809 [ + + ]: 2431 : if (io_path == NULL) {
810 : 0 : SPDK_ERRLOG("Failed to alloc io_path.\n");
811 : 0 : return NULL;
812 : : }
813 : :
814 [ + + + + ]: 2431 : if (g_opts.io_path_stat) {
815 [ # # # # ]: 0 : io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
816 [ # # # # : 0 : if (io_path->stat == NULL) {
# # ]
817 : 0 : free(io_path);
818 : 0 : SPDK_ERRLOG("Failed to alloc io_path stat.\n");
819 : 0 : return NULL;
820 : : }
821 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
822 : 0 : }
823 : :
824 : 2431 : return io_path;
825 : 60 : }
826 : :
827 : : static void
828 : 2431 : nvme_io_path_free(struct nvme_io_path *io_path)
829 : : {
830 [ + - + - ]: 2431 : free(io_path->stat);
831 : 2431 : free(io_path);
832 : 2431 : }
833 : :
834 : : static int
835 : 2431 : _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
836 : : {
837 : : struct nvme_io_path *io_path;
838 : : struct spdk_io_channel *ch;
839 : : struct nvme_ctrlr_channel *ctrlr_ch;
840 : : struct nvme_qpair *nvme_qpair;
841 : :
842 : 2431 : io_path = nvme_io_path_alloc();
843 [ + + ]: 2431 : if (io_path == NULL) {
844 : 0 : return -ENOMEM;
845 : : }
846 : :
847 [ + - + - ]: 2431 : io_path->nvme_ns = nvme_ns;
848 : :
849 [ + - + - ]: 2431 : ch = spdk_get_io_channel(nvme_ns->ctrlr);
850 [ + + ]: 2431 : if (ch == NULL) {
851 : 0 : nvme_io_path_free(io_path);
852 : 0 : SPDK_ERRLOG("Failed to alloc io_channel.\n");
853 : 0 : return -ENOMEM;
854 : : }
855 : :
856 : 2431 : ctrlr_ch = spdk_io_channel_get_ctx(ch);
857 : :
858 [ + - + - ]: 2431 : nvme_qpair = ctrlr_ch->qpair;
859 [ + + # # ]: 2431 : assert(nvme_qpair != NULL);
860 : :
861 [ + - + - ]: 2431 : io_path->qpair = nvme_qpair;
862 [ + - + - : 2431 : TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
863 : :
864 [ + - + - ]: 2431 : io_path->nbdev_ch = nbdev_ch;
865 [ + - + - : 2431 : STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
866 : :
867 : 2431 : bdev_nvme_clear_current_io_path(nbdev_ch);
868 : :
869 : 2431 : return 0;
870 : 60 : }
871 : :
872 : : static void
873 : 2431 : bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch,
874 : : struct nvme_io_path *io_path)
875 : : {
876 : : struct nvme_bdev_io *bio;
877 : :
878 [ + + + - : 2435 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
+ - + - #
# # # #
# ]
879 [ + + # # : 4 : if (bio->io_path == io_path) {
# # ]
880 [ # # # # ]: 4 : bio->io_path = NULL;
881 : 1 : }
882 : 1 : }
883 : 2431 : }
884 : :
885 : : static void
886 : 2431 : _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path)
887 : : {
888 : : struct spdk_io_channel *ch;
889 : : struct nvme_qpair *nvme_qpair;
890 : : struct nvme_ctrlr_channel *ctrlr_ch;
891 : : struct nvme_bdev *nbdev;
892 : :
893 : 2431 : nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch));
894 : :
895 : : /* Add the statistics to nvme_ns before this path is destroyed. */
896 [ + + + - ]: 2431 : pthread_mutex_lock(&nbdev->mutex);
897 [ + + + + : 2431 : if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) {
+ - + - +
- + - + -
- + # # #
# # # ]
898 [ # # # # : 0 : spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat);
# # # # #
# # # ]
899 : 0 : }
900 [ + + + - ]: 2431 : pthread_mutex_unlock(&nbdev->mutex);
901 : :
902 : 2431 : bdev_nvme_clear_current_io_path(nbdev_ch);
903 : 2431 : bdev_nvme_clear_retry_io_path(nbdev_ch, io_path);
904 : :
905 [ + + + + : 2433 : STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq);
+ + + + +
- + - + -
+ - + - +
- + - + -
+ - - + +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
906 [ + - + - ]: 2431 : io_path->nbdev_ch = NULL;
907 : :
908 [ + - + - ]: 2431 : nvme_qpair = io_path->qpair;
909 [ + + # # ]: 2431 : assert(nvme_qpair != NULL);
910 : :
911 [ + - + - ]: 2431 : ctrlr_ch = nvme_qpair->ctrlr_ch;
912 [ + + # # ]: 2431 : assert(ctrlr_ch != NULL);
913 : :
914 : 2431 : ch = spdk_io_channel_from_ctx(ctrlr_ch);
915 : 2431 : spdk_put_io_channel(ch);
916 : :
917 : : /* After an io_path is removed, I/Os submitted to it may complete and update statistics
918 : : * of the io_path. To avoid heap-use-after-free error from this case, do not free the
919 : : * io_path here but free the io_path when the associated qpair is freed. It is ensured
920 : : * that all I/Os submitted to the io_path are completed when the associated qpair is freed.
921 : : */
922 : 2431 : }
923 : :
924 : : static void
925 : 2373 : _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch)
926 : : {
927 : : struct nvme_io_path *io_path, *tmp_io_path;
928 : :
929 [ + + + + : 4796 : STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) {
+ - + + +
- + - + -
+ + ]
930 : 2423 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
931 : 58 : }
932 : 2373 : }
933 : :
934 : : static int
935 : 2373 : bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf)
936 : : {
937 : 2373 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
938 : 2373 : struct nvme_bdev *nbdev = io_device;
939 : : struct nvme_ns *nvme_ns;
940 : : int rc;
941 : :
942 [ + - + - : 2373 : STAILQ_INIT(&nbdev_ch->io_path_list);
+ - + - +
- + - + -
+ - ]
943 [ + - + - : 2373 : TAILQ_INIT(&nbdev_ch->retry_io_list);
+ - + - +
- + - + -
+ - ]
944 : :
945 [ + + + - ]: 2373 : pthread_mutex_lock(&nbdev->mutex);
946 : :
947 [ + - + - : 2373 : nbdev_ch->mp_policy = nbdev->mp_policy;
+ - + - ]
948 [ + - + - : 2373 : nbdev_ch->mp_selector = nbdev->mp_selector;
+ - + - ]
949 [ + - + - : 2373 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
+ - + - ]
950 : :
951 [ + + + - : 4796 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
952 : 2423 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
953 [ + + ]: 2423 : if (rc != 0) {
954 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
955 : :
956 : 0 : _bdev_nvme_delete_io_paths(nbdev_ch);
957 : 0 : return rc;
958 : : }
959 : 58 : }
960 [ + + + - ]: 2373 : pthread_mutex_unlock(&nbdev->mutex);
961 : :
962 : 2373 : return 0;
963 : 47 : }
964 : :
965 : : /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'.
966 : : * If cpl == NULL, complete the bdev_io with bdev status based on 'status'.
967 : : */
968 : : static inline void
969 : 29350344 : __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status,
970 : : const struct spdk_nvme_cpl *cpl)
971 : : {
972 [ + + + + : 29350344 : spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx,
+ - + - +
- + - + -
- + # # ]
973 : : (uintptr_t)bdev_io);
974 [ + + ]: 29350344 : if (cpl) {
975 [ + - + - : 27678388 : spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
+ - + - +
- + - + -
+ - ]
976 : 619961 : } else {
977 : 1671956 : spdk_bdev_io_complete(bdev_io, status);
978 : : }
979 : 29350344 : }
980 : :
981 : : static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch);
982 : :
983 : : static void
984 : 2373 : bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf)
985 : : {
986 : 2373 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
987 : :
988 : 2373 : bdev_nvme_abort_retry_ios(nbdev_ch);
989 : 2373 : _bdev_nvme_delete_io_paths(nbdev_ch);
990 : 2373 : }
991 : :
992 : : static inline bool
993 : 29748048 : bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type)
994 : : {
995 [ + + ]: 29748048 : switch (io_type) {
996 : 136 : case SPDK_BDEV_IO_TYPE_RESET:
997 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
998 : : case SPDK_BDEV_IO_TYPE_ABORT:
999 : 142 : return true;
1000 : 29127720 : default:
1001 : 29747906 : break;
1002 : : }
1003 : :
1004 : 29747906 : return false;
1005 : 620192 : }
1006 : :
1007 : : static inline bool
1008 : 10560370 : nvme_ns_is_active(struct nvme_ns *nvme_ns)
1009 : : {
1010 [ + + + + : 10560370 : if (spdk_unlikely(nvme_ns->ana_state_updating)) {
+ - - + ]
1011 : 3585 : return false;
1012 : : }
1013 : :
1014 [ + + + - : 10556785 : if (spdk_unlikely(nvme_ns->ns == NULL)) {
- + ]
1015 : 0 : return false;
1016 : : }
1017 : :
1018 : 10556785 : return true;
1019 : 273968 : }
1020 : :
1021 : : static inline bool
1022 : 10560316 : nvme_ns_is_accessible(struct nvme_ns *nvme_ns)
1023 : : {
1024 [ + + ]: 10560316 : if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) {
1025 : 3585 : return false;
1026 : : }
1027 : :
1028 [ + + + - : 10556731 : switch (nvme_ns->ana_state) {
+ - ]
1029 : 9525492 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1030 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1031 : 9799432 : return true;
1032 : 757290 : default:
1033 : 757299 : break;
1034 : : }
1035 : :
1036 : 757299 : return false;
1037 : 273950 : }
1038 : :
1039 : : static inline bool
1040 : 11722129 : nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair)
1041 : : {
1042 [ + + + - : 11722129 : if (spdk_unlikely(nvme_qpair->qpair == NULL)) {
- + ]
1043 : 1139791 : return false;
1044 : : }
1045 : :
1046 [ + + + - : 10582338 : if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
- + ]
1047 : : SPDK_NVME_QPAIR_FAILURE_NONE)) {
1048 : 3272 : return false;
1049 : : }
1050 : :
1051 [ + + + - : 10579066 : if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) {
+ - + - -
+ ]
1052 : 10252 : return false;
1053 : : }
1054 : :
1055 : 10568814 : return true;
1056 : 273998 : }
1057 : :
1058 : : static inline bool
1059 : 11329072 : nvme_io_path_is_available(struct nvme_io_path *io_path)
1060 : : {
1061 [ + + + - : 11329072 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
- + ]
1062 : 769234 : return false;
1063 : : }
1064 : :
1065 [ + + + - : 10559838 : if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) {
- + ]
1066 : 760764 : return false;
1067 : : }
1068 : :
1069 : 9799074 : return true;
1070 : 273968 : }
1071 : :
1072 : : static inline bool
1073 : 384081 : nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr)
1074 : : {
1075 [ + + # # ]: 384081 : if (nvme_ctrlr->destruct) {
1076 : 512 : return true;
1077 : : }
1078 : :
1079 [ + + # # ]: 383569 : if (nvme_ctrlr->fast_io_fail_timedout) {
1080 : 85348 : return true;
1081 : : }
1082 : :
1083 [ + + # # ]: 298221 : if (nvme_ctrlr->resetting) {
1084 [ + + # # : 227277 : if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {
# # # # ]
1085 : 1044 : return false;
1086 : : } else {
1087 : 226233 : return true;
1088 : : }
1089 : : }
1090 : :
1091 [ + + # # ]: 70944 : if (nvme_ctrlr->reconnect_is_delayed) {
1092 : 4104 : return false;
1093 : : }
1094 : :
1095 [ - + # # ]: 66840 : if (nvme_ctrlr->disabled) {
1096 : 0 : return true;
1097 : : }
1098 : :
1099 [ + + # # : 66840 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1100 : 64664 : return true;
1101 : : } else {
1102 : 2176 : return false;
1103 : : }
1104 : 9 : }
1105 : :
1106 : : static bool
1107 : 13856 : nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr)
1108 : : {
1109 [ + + # # ]: 13856 : if (nvme_ctrlr->destruct) {
1110 : 957 : return false;
1111 : : }
1112 : :
1113 [ + + # # : 12899 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1114 : 12 : return false;
1115 : : }
1116 : :
1117 [ + + - + : 12887 : if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) {
# # # # ]
1118 : 5237 : return false;
1119 : : }
1120 : :
1121 [ + + # # ]: 7650 : if (nvme_ctrlr->disabled) {
1122 : 0 : return false;
1123 : : }
1124 : :
1125 : 7650 : return true;
1126 : 33 : }
1127 : :
1128 : : /* Simulate circular linked list. */
1129 : : static inline struct nvme_io_path *
1130 : 6787309 : nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path)
1131 : : {
1132 : : struct nvme_io_path *next_path;
1133 : :
1134 [ + + ]: 6787309 : if (prev_path != NULL) {
1135 [ # # # # : 5242106 : next_path = STAILQ_NEXT(prev_path, stailq);
# # ]
1136 [ + + ]: 5242106 : if (next_path != NULL) {
1137 : 2542526 : return next_path;
1138 : : }
1139 : 25 : }
1140 : :
1141 [ + - + - : 4244783 : return STAILQ_FIRST(&nbdev_ch->io_path_list);
+ - ]
1142 : 120 : }
1143 : :
1144 : : static struct nvme_io_path *
1145 : 2722642 : _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1146 : : {
1147 : 2722642 : struct nvme_io_path *io_path, *start, *non_optimized = NULL;
1148 : :
1149 [ + - + - ]: 2722642 : start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path);
1150 : :
1151 : 2722642 : io_path = start;
1152 : 88 : do {
1153 [ + + ]: 4740973 : if (spdk_likely(nvme_io_path_is_available(io_path))) {
1154 [ + + + - : 3211111 : switch (io_path->nvme_ns->ana_state) {
+ - + - +
- - ]
1155 : 676238 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1156 [ + - + - ]: 676306 : nbdev_ch->current_io_path = io_path;
1157 : 676306 : return io_path;
1158 : 2534795 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1159 [ + + ]: 2534805 : if (non_optimized == NULL) {
1160 : 1985200 : non_optimized = io_path;
1161 : 7 : }
1162 : 2534805 : break;
1163 : 0 : default:
1164 [ # # ]: 0 : assert(false);
1165 : : break;
1166 : : }
1167 : 10 : }
1168 : 4064667 : io_path = nvme_io_path_get_next(nbdev_ch, io_path);
1169 [ + + ]: 4064667 : } while (io_path != start);
1170 : :
1171 [ + + # # : 2046336 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
1172 : : /* We come here only if there is no optimized path. Cache even non_optimized
1173 : : * path for load balance across multiple non_optimized paths.
1174 : : */
1175 [ # # # # ]: 503693 : nbdev_ch->current_io_path = non_optimized;
1176 : 1 : }
1177 : :
1178 : 2046336 : return non_optimized;
1179 : 88 : }
1180 : :
1181 : : static struct nvme_io_path *
1182 : 16 : _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch)
1183 : : {
1184 : : struct nvme_io_path *io_path;
1185 : 16 : struct nvme_io_path *optimized = NULL, *non_optimized = NULL;
1186 : 16 : uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX;
1187 : : uint32_t num_outstanding_reqs;
1188 : :
1189 [ + + # # : 64 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1190 [ - + # # : 48 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
# # ]
1191 : : /* The device is currently resetting. */
1192 : 0 : continue;
1193 : : }
1194 : :
1195 [ - + # # : 48 : if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) {
# # ]
1196 : 0 : continue;
1197 : : }
1198 : :
1199 [ # # # # : 48 : num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair);
# # # # ]
1200 [ + + + # : 48 : switch (io_path->nvme_ns->ana_state) {
# # # # #
# # ]
1201 : 18 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1202 [ + + ]: 24 : if (num_outstanding_reqs < opt_min_qd) {
1203 : 20 : opt_min_qd = num_outstanding_reqs;
1204 : 20 : optimized = io_path;
1205 : 5 : }
1206 : 24 : break;
1207 : 9 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1208 [ + + ]: 12 : if (num_outstanding_reqs < non_opt_min_qd) {
1209 : 12 : non_opt_min_qd = num_outstanding_reqs;
1210 : 12 : non_optimized = io_path;
1211 : 3 : }
1212 : 12 : break;
1213 : 9 : default:
1214 : 12 : break;
1215 : : }
1216 : 12 : }
1217 : :
1218 : : /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */
1219 [ + + ]: 16 : if (optimized != NULL) {
1220 : 12 : return optimized;
1221 : : }
1222 : :
1223 : 4 : return non_optimized;
1224 : 4 : }
1225 : :
1226 : : static inline struct nvme_io_path *
1227 : 29365240 : bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1228 : : {
1229 [ + + + - : 29365240 : if (spdk_likely(nbdev_ch->current_io_path != NULL)) {
+ + ]
1230 [ + + + - : 27820021 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) {
- + ]
1231 [ + - + - ]: 26642570 : return nbdev_ch->current_io_path;
1232 [ + + # # : 1177451 : } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
1233 [ + + # # : 1177451 : if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) {
# # # # ]
1234 [ # # # # ]: 12 : return nbdev_ch->current_io_path;
1235 : : }
1236 [ # # # # ]: 1177439 : nbdev_ch->rr_counter = 0;
1237 : 7 : }
1238 : 7 : }
1239 : :
1240 [ + + + + : 2722658 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE ||
- + # # ]
1241 [ + + # # ]: 1177537 : nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
1242 : 2722642 : return _bdev_nvme_find_io_path(nbdev_ch);
1243 : : } else {
1244 : 16 : return _bdev_nvme_find_io_path_min_qd(nbdev_ch);
1245 : : }
1246 : 619026 : }
1247 : :
1248 : : /* Return true if there is any io_path whose qpair is active or ctrlr is not failed,
1249 : : * or false otherwise.
1250 : : *
1251 : : * If any io_path has an active qpair but find_io_path() returned NULL, its namespace
1252 : : * is likely to be non-accessible now but may become accessible.
1253 : : *
1254 : : * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr
1255 : : * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed
1256 : : * when starting to reset it but it is set to failed when the reset failed. Hence, if
1257 : : * a ctrlr is unfailed, it is likely that it works fine or is resetting.
1258 : : */
1259 : : static bool
1260 : 392529 : any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch)
1261 : : {
1262 : : struct nvme_io_path *io_path;
1263 : :
1264 [ + + + + : 392529 : if (nbdev_ch->resetting) {
# # # # ]
1265 : 4 : return false;
1266 : : }
1267 : :
1268 [ + + # # : 770178 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1269 [ - + + + : 393421 : if (io_path->nvme_ns->ana_transition_timedout) {
# # # # #
# # # ]
1270 : 896 : continue;
1271 : : }
1272 : :
1273 [ + + + + : 392525 : if (nvme_qpair_is_connected(io_path->qpair) ||
# # # # ]
1274 [ + + # # : 384081 : !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) {
# # # # ]
1275 : 15768 : return true;
1276 : : }
1277 : 2 : }
1278 : :
1279 : 376757 : return false;
1280 : 15 : }
1281 : :
1282 : : static void
1283 : 19093 : bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
1284 : : {
1285 [ # # ]: 19093 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1286 : : struct spdk_io_channel *ch;
1287 : :
1288 [ + + + - : 19093 : if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) {
# # # # #
# # # ]
1289 : 3329 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
1290 : 3 : } else {
1291 : 15764 : ch = spdk_io_channel_from_ctx(nbdev_ch);
1292 : 15764 : bdev_nvme_submit_request(ch, bdev_io);
1293 : : }
1294 : 19093 : }
1295 : :
1296 : : static int
1297 : 4177 : bdev_nvme_retry_ios(void *arg)
1298 : : {
1299 : 4177 : struct nvme_bdev_channel *nbdev_ch = arg;
1300 : : struct nvme_bdev_io *bio, *tmp_bio;
1301 : : uint64_t now, delay_us;
1302 : :
1303 : 4177 : now = spdk_get_ticks();
1304 : :
1305 [ + + + + : 23270 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
# # # # #
# # # # #
# # ]
1306 [ + + # # : 19801 : if (bio->retry_ticks > now) {
# # ]
1307 : 708 : break;
1308 : : }
1309 : :
1310 [ + + # # : 19093 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1311 : :
1312 : 19093 : bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio));
1313 : 14 : }
1314 : :
1315 [ # # ]: 4177 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1316 : :
1317 [ # # # # : 4177 : bio = TAILQ_FIRST(&nbdev_ch->retry_io_list);
# # ]
1318 [ + + ]: 4177 : if (bio != NULL) {
1319 [ - + # # : 720 : delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz();
# # ]
1320 : :
1321 [ # # # # ]: 720 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1322 : : delay_us);
1323 : 4 : }
1324 : :
1325 : 4177 : return SPDK_POLLER_BUSY;
1326 : : }
1327 : :
1328 : : static void
1329 : 19101 : bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch,
1330 : : struct nvme_bdev_io *bio, uint64_t delay_ms)
1331 : : {
1332 : : struct nvme_bdev_io *tmp_bio;
1333 : :
1334 [ # # # # : 19101 : bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL;
# # ]
1335 : :
1336 [ + + # # : 49176 : TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) {
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1337 [ + + # # : 45682 : if (tmp_bio->retry_ticks <= bio->retry_ticks) {
# # # # #
# ]
1338 [ + + # # : 15607 : TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1339 : : retry_link);
1340 : 15607 : return;
1341 : : }
1342 : 0 : }
1343 : :
1344 : : /* No earlier I/Os were found. This I/O must be the new head. */
1345 [ + + # # : 3494 : TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1346 : :
1347 [ # # ]: 3494 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1348 : :
1349 [ # # # # ]: 3494 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1350 : : delay_ms * 1000ULL);
1351 : 16 : }
1352 : :
1353 : : static void
1354 : 2541 : bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch)
1355 : : {
1356 : : struct nvme_bdev_io *bio, *tmp_bio;
1357 : :
1358 [ + + + + : 2545 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
+ - + - #
# # # # #
+ - ]
1359 [ - + # # : 4 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1360 : 4 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1361 : 1 : }
1362 : :
1363 [ + - ]: 2541 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1364 : 2541 : }
1365 : :
1366 : : static int
1367 : 7176 : bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch,
1368 : : struct nvme_bdev_io *bio_to_abort)
1369 : : {
1370 : : struct nvme_bdev_io *bio;
1371 : :
1372 [ + + # # : 7176 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
# # # # #
# # # #
# ]
1373 [ + + ]: 4 : if (bio == bio_to_abort) {
1374 [ - + # # : 4 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1375 : 4 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1376 : 4 : return 0;
1377 : : }
1378 : 0 : }
1379 : :
1380 : 7172 : return -ENOENT;
1381 : 6 : }
1382 : :
1383 : : static void
1384 : 10252 : bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl)
1385 : : {
1386 : : struct nvme_bdev *nbdev;
1387 : : uint16_t sct, sc;
1388 : :
1389 [ + + + + : 10252 : assert(spdk_nvme_cpl_is_error(cpl));
# # # # #
# # # # #
# # # # ]
1390 : :
1391 [ # # # # : 10252 : nbdev = bdev_io->bdev->ctxt;
# # # # ]
1392 : :
1393 [ + + # # : 10252 : if (nbdev->err_stat == NULL) {
# # ]
1394 : 6989 : return;
1395 : : }
1396 : :
1397 [ # # # # : 3263 : sct = cpl->status.sct;
# # ]
1398 [ # # # # : 3263 : sc = cpl->status.sc;
# # ]
1399 : :
1400 [ - + # # ]: 3263 : pthread_mutex_lock(&nbdev->mutex);
1401 : :
1402 [ # # # # : 3263 : nbdev->err_stat->status_type[sct]++;
# # # # #
# ]
1403 [ + - ]: 3263 : switch (sct) {
1404 : 3263 : case SPDK_NVME_SCT_GENERIC:
1405 : : case SPDK_NVME_SCT_COMMAND_SPECIFIC:
1406 : : case SPDK_NVME_SCT_MEDIA_ERROR:
1407 : : case SPDK_NVME_SCT_PATH:
1408 [ # # # # : 3263 : nbdev->err_stat->status[sct][sc]++;
# # # # #
# # # ]
1409 : 3263 : break;
1410 : 0 : default:
1411 : 0 : break;
1412 : : }
1413 : :
1414 [ - + # # ]: 3263 : pthread_mutex_unlock(&nbdev->mutex);
1415 : 14 : }
1416 : :
1417 : : static inline void
1418 : 27677583 : bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio)
1419 : : {
1420 : 27677583 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1421 [ + - + - : 27677583 : uint64_t num_blocks = bdev_io->u.bdev.num_blocks;
+ - + - ]
1422 [ + - + - : 27677583 : uint32_t blocklen = bdev_io->bdev->blocklen;
+ - + - ]
1423 : : struct spdk_bdev_io_stat *stat;
1424 : : uint64_t tsc_diff;
1425 : :
1426 [ + + + - : 27677583 : if (bio->io_path->stat == NULL) {
+ - + - +
- ]
1427 : 27677583 : return;
1428 : : }
1429 : :
1430 [ # # # # ]: 0 : tsc_diff = spdk_get_ticks() - bio->submit_tsc;
1431 [ # # # # : 0 : stat = bio->io_path->stat;
# # # # ]
1432 : :
1433 [ # # # # : 0 : switch (bdev_io->type) {
# # # # #
# ]
1434 : 0 : case SPDK_BDEV_IO_TYPE_READ:
1435 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1436 [ # # ]: 0 : stat->num_read_ops++;
1437 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1438 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1439 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1440 : 0 : }
1441 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1442 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1443 : 0 : }
1444 : 0 : break;
1445 : 0 : case SPDK_BDEV_IO_TYPE_WRITE:
1446 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1447 [ # # ]: 0 : stat->num_write_ops++;
1448 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1449 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1450 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1451 : 0 : }
1452 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1453 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1454 : 0 : }
1455 : 0 : break;
1456 : 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
1457 [ # # # # ]: 0 : stat->bytes_unmapped += num_blocks * blocklen;
1458 [ # # ]: 0 : stat->num_unmap_ops++;
1459 [ # # # # ]: 0 : stat->unmap_latency_ticks += tsc_diff;
1460 [ # # # # : 0 : if (stat->max_unmap_latency_ticks < tsc_diff) {
# # ]
1461 [ # # # # ]: 0 : stat->max_unmap_latency_ticks = tsc_diff;
1462 : 0 : }
1463 [ # # # # : 0 : if (stat->min_unmap_latency_ticks > tsc_diff) {
# # ]
1464 [ # # # # ]: 0 : stat->min_unmap_latency_ticks = tsc_diff;
1465 : 0 : }
1466 : 0 : break;
1467 : 0 : case SPDK_BDEV_IO_TYPE_ZCOPY:
1468 : : /* Track the data in the start phase only */
1469 [ # # # # : 0 : if (!bdev_io->u.bdev.zcopy.start) {
# # # # #
# ]
1470 : 0 : break;
1471 : : }
1472 [ # # # # : 0 : if (bdev_io->u.bdev.zcopy.populate) {
# # # # #
# ]
1473 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1474 [ # # ]: 0 : stat->num_read_ops++;
1475 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1476 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1477 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1478 : 0 : }
1479 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1480 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1481 : 0 : }
1482 : 0 : } else {
1483 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1484 [ # # ]: 0 : stat->num_write_ops++;
1485 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1486 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1487 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1488 : 0 : }
1489 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1490 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1491 : 0 : }
1492 : : }
1493 : 0 : break;
1494 : 0 : case SPDK_BDEV_IO_TYPE_COPY:
1495 [ # # # # ]: 0 : stat->bytes_copied += num_blocks * blocklen;
1496 [ # # ]: 0 : stat->num_copy_ops++;
1497 [ # # # # ]: 0 : stat->copy_latency_ticks += tsc_diff;
1498 [ # # # # : 0 : if (stat->max_copy_latency_ticks < tsc_diff) {
# # ]
1499 [ # # # # ]: 0 : stat->max_copy_latency_ticks = tsc_diff;
1500 : 0 : }
1501 [ # # # # : 0 : if (stat->min_copy_latency_ticks > tsc_diff) {
# # ]
1502 [ # # # # ]: 0 : stat->min_copy_latency_ticks = tsc_diff;
1503 : 0 : }
1504 : 0 : break;
1505 : 0 : default:
1506 : 0 : break;
1507 : : }
1508 : 619877 : }
1509 : :
1510 : : static bool
1511 : 10173 : bdev_nvme_check_retry_io(struct nvme_bdev_io *bio,
1512 : : const struct spdk_nvme_cpl *cpl,
1513 : : struct nvme_bdev_channel *nbdev_ch,
1514 : : uint64_t *_delay_ms)
1515 : : {
1516 [ # # # # ]: 10173 : struct nvme_io_path *io_path = bio->io_path;
1517 [ # # # # : 10173 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
1518 : : const struct spdk_nvme_ctrlr_data *cdata;
1519 : :
1520 [ + + - + : 10177 : if (spdk_nvme_cpl_is_path_error(cpl) ||
# # # # #
# ]
1521 [ + + + + : 7649 : spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
# # # # #
# # # #
# ]
1522 [ + + ]: 3329 : !nvme_io_path_is_available(io_path) ||
1523 [ - + ]: 3333 : !nvme_ctrlr_is_available(nvme_ctrlr)) {
1524 : 6848 : bdev_nvme_clear_current_io_path(nbdev_ch);
1525 [ # # # # ]: 6848 : bio->io_path = NULL;
1526 [ + + + - : 6848 : if (spdk_nvme_cpl_is_ana_error(cpl)) {
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1527 [ + + ]: 2516 : if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) {
1528 [ # # # # : 20 : io_path->nvme_ns->ana_state_updating = true;
# # # # ]
1529 : 1 : }
1530 : 1 : }
1531 [ + + ]: 6836 : if (!any_io_path_may_become_available(nbdev_ch)) {
1532 : 512 : return false;
1533 : : }
1534 [ # # ]: 6324 : *_delay_ms = 0;
1535 : 3 : } else {
1536 [ # # # # ]: 3333 : bio->retry_count++;
1537 : :
1538 [ # # # # ]: 3333 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
1539 : :
1540 [ + + # # : 3333 : if (cpl->status.crd != 0) {
# # # # ]
1541 [ # # # # : 4 : *_delay_ms = cdata->crdt[cpl->status.crd] * 100;
# # # # #
# # # # #
# # # # ]
1542 : 1 : } else {
1543 [ # # ]: 3329 : *_delay_ms = 0;
1544 : : }
1545 : : }
1546 : :
1547 : 9657 : return true;
1548 : 7 : }
1549 : :
1550 : : static inline void
1551 : 27688122 : bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio,
1552 : : const struct spdk_nvme_cpl *cpl)
1553 : : {
1554 : 27688122 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1555 : : struct nvme_bdev_channel *nbdev_ch;
1556 : 6915812 : uint64_t delay_ms;
1557 : :
1558 [ + + + - : 27688122 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
+ - # # ]
1559 : :
1560 [ + + + + : 27688122 : if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
+ - - + +
- + - + -
+ - ]
1561 : 27677862 : bdev_nvme_update_io_path_stat(bio);
1562 : 27677862 : goto complete;
1563 : : }
1564 : :
1565 : : /* Update error counts before deciding if retry is needed.
1566 : : * Hence, error counts may be more than the number of I/O errors.
1567 : : */
1568 : 10260 : bdev_nvme_update_nvme_error_stat(bdev_io, cpl);
1569 : :
1570 [ + + + + : 10267 : if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) ||
+ + # # #
# # # # #
# # # # #
# # # #
# ]
1571 [ + + + + : 10185 : (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) {
# # # # #
# ]
1572 : 101 : goto complete;
1573 : : }
1574 : :
1575 : : /* At this point we don't know whether the sequence was successfully executed or not, so we
1576 : : * cannot retry the IO */
1577 [ - + # # : 10169 : if (bdev_io->u.bdev.accel_sequence != NULL) {
# # # # #
# ]
1578 : 0 : goto complete;
1579 : : }
1580 : :
1581 : 10169 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1582 : :
1583 [ + + ]: 10169 : if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) {
1584 : 9657 : bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
1585 : 9657 : return;
1586 : : }
1587 : :
1588 : 6916303 : complete:
1589 [ + - + - ]: 27678457 : bio->retry_count = 0;
1590 [ + - + - ]: 27678457 : bio->submit_tsc = 0;
1591 [ + - + - : 27678457 : bdev_io->u.bdev.accel_sequence = NULL;
+ - + - ]
1592 : 27678457 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
1593 : 620170 : }
1594 : :
1595 : : static inline void
1596 : 1674108 : bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc)
1597 : : {
1598 : 1674108 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1599 : : struct nvme_bdev_channel *nbdev_ch;
1600 : : enum spdk_bdev_io_status io_status;
1601 : :
1602 [ - + # # : 1674108 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
1603 : :
1604 [ + + + + ]: 1674108 : switch (rc) {
1605 : 1123938 : case 0:
1606 : 1123939 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1607 : 1123939 : break;
1608 : 164476 : case -ENOMEM:
1609 : 164476 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1610 : 164476 : break;
1611 : 385681 : case -ENXIO:
1612 [ + + + - : 385696 : if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) {
# # # # #
# # # ]
1613 : 385693 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1614 : :
1615 : 385693 : bdev_nvme_clear_current_io_path(nbdev_ch);
1616 [ # # # # ]: 385693 : bio->io_path = NULL;
1617 : :
1618 [ + + ]: 385693 : if (any_io_path_may_become_available(nbdev_ch)) {
1619 : 9444 : bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1620 : 9444 : return;
1621 : : }
1622 : 3 : }
1623 : :
1624 : : /* fallthrough */
1625 : : default:
1626 [ # # # # : 376249 : spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence);
# # # # ]
1627 [ # # # # : 376249 : bdev_io->u.bdev.accel_sequence = NULL;
# # # # ]
1628 : 376249 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1629 : 376249 : break;
1630 : : }
1631 : :
1632 [ # # # # ]: 1664664 : bio->retry_count = 0;
1633 [ # # # # ]: 1664664 : bio->submit_tsc = 0;
1634 : 1664664 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1635 : 13 : }
1636 : :
1637 : : static inline void
1638 : 16 : bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc)
1639 : : {
1640 : 16 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1641 : : enum spdk_bdev_io_status io_status;
1642 : :
1643 [ + + + + ]: 16 : switch (rc) {
1644 : 3 : case 0:
1645 : 4 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1646 : 4 : break;
1647 : 0 : case -ENOMEM:
1648 : 0 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1649 : 0 : break;
1650 : 10 : case -ENXIO:
1651 : : /* fallthrough */
1652 : : default:
1653 : 12 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1654 : 12 : break;
1655 : : }
1656 : :
1657 : 16 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1658 : 16 : }
1659 : :
1660 : : static void
1661 : 1541 : bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr,
1662 : : void *ctx, int status)
1663 : : {
1664 [ - + # # ]: 1541 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1665 : :
1666 [ + + # # : 1541 : assert(nvme_ctrlr->io_path_cache_clearing == true);
# # ]
1667 [ # # ]: 1541 : nvme_ctrlr->io_path_cache_clearing = false;
1668 : :
1669 [ + + ]: 1541 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1670 [ - + # # ]: 1541 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1671 : 1541 : return;
1672 : : }
1673 : :
1674 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1675 : :
1676 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
1677 : 3 : }
1678 : :
1679 : : static void
1680 : 10126 : _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair)
1681 : : {
1682 : : struct nvme_io_path *io_path;
1683 : :
1684 [ + + + - : 17667 : TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) {
+ - + + +
- + - +
- ]
1685 [ + + + - : 7541 : if (io_path->nbdev_ch == NULL) {
- + ]
1686 : 4809 : continue;
1687 : : }
1688 [ # # # # ]: 2732 : bdev_nvme_clear_current_io_path(io_path->nbdev_ch);
1689 : 167 : }
1690 : 10126 : }
1691 : :
1692 : : static void
1693 : 1492 : bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i,
1694 : : struct nvme_ctrlr *nvme_ctrlr,
1695 : : struct nvme_ctrlr_channel *ctrlr_ch,
1696 : : void *ctx)
1697 : : {
1698 [ + + # # : 1492 : assert(ctrlr_ch->qpair != NULL);
# # # # ]
1699 : :
1700 [ # # # # ]: 1492 : _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair);
1701 : :
1702 : 1492 : nvme_ctrlr_for_each_channel_continue(i, 0);
1703 : 1492 : }
1704 : :
1705 : : static void
1706 : 7737 : bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr)
1707 : : {
1708 [ - + # # ]: 7737 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1709 [ + + + + ]: 7737 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
1710 [ # # ]: 3 : nvme_ctrlr->io_path_cache_clearing) {
1711 [ - + # # ]: 6196 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1712 : 6196 : return;
1713 : : }
1714 : :
1715 [ # # ]: 1541 : nvme_ctrlr->io_path_cache_clearing = true;
1716 [ - + # # ]: 1541 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1717 : :
1718 : 1541 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
1719 : : bdev_nvme_clear_io_path_cache,
1720 : : NULL,
1721 : : bdev_nvme_clear_io_path_caches_done);
1722 : 15 : }
1723 : :
1724 : : static struct nvme_qpair *
1725 : 2697 : nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
1726 : : {
1727 : : struct nvme_qpair *nvme_qpair;
1728 : :
1729 [ + + + - : 2856 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
+ - - + #
# # # #
# ]
1730 [ + + + - : 2856 : if (nvme_qpair->qpair == qpair) {
- + ]
1731 : 2697 : break;
1732 : : }
1733 : 17 : }
1734 : :
1735 : 2697 : return nvme_qpair;
1736 : : }
1737 : :
1738 : : static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair);
1739 : :
1740 : : static void
1741 : 2697 : bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
1742 : : {
1743 : 2697 : struct nvme_poll_group *group = poll_group_ctx;
1744 : : struct nvme_qpair *nvme_qpair;
1745 : : struct nvme_ctrlr *nvme_ctrlr;
1746 : : struct nvme_ctrlr_channel *ctrlr_ch;
1747 : : int status;
1748 : :
1749 : 2697 : nvme_qpair = nvme_poll_group_get_qpair(group, qpair);
1750 [ + + ]: 2697 : if (nvme_qpair == NULL) {
1751 : 0 : return;
1752 : : }
1753 : :
1754 [ + + + - : 2697 : if (nvme_qpair->qpair != NULL) {
- + ]
1755 [ + - + - ]: 2697 : spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair);
1756 [ + - + - ]: 2697 : nvme_qpair->qpair = NULL;
1757 : 143 : }
1758 : :
1759 : 2697 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1760 : :
1761 [ + - + - ]: 2697 : nvme_ctrlr = nvme_qpair->ctrlr;
1762 [ + - + - ]: 2697 : ctrlr_ch = nvme_qpair->ctrlr_ch;
1763 : :
1764 [ + + ]: 2697 : if (ctrlr_ch != NULL) {
1765 [ + + # # : 385 : if (ctrlr_ch->reset_iter != NULL) {
# # ]
1766 : : /* We are in a full reset sequence. */
1767 [ - + # # : 324 : if (ctrlr_ch->connect_poller != NULL) {
# # ]
1768 : : /* qpair was failed to connect. Abort the reset sequence. */
1769 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1770 : : "qpair %p was failed to connect. abort the reset ctrlr sequence.\n",
1771 : : qpair);
1772 [ # # ]: 0 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
1773 : 0 : status = -1;
1774 : 0 : } else {
1775 : : /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */
1776 [ + + - + : 324 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1777 : : "qpair %p was disconnected and freed in a reset ctrlr sequence.\n",
1778 : : qpair);
1779 : 324 : status = 0;
1780 : : }
1781 [ # # # # ]: 324 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status);
1782 [ # # # # ]: 324 : ctrlr_ch->reset_iter = NULL;
1783 : 70 : } else {
1784 : : /* qpair was disconnected unexpectedly. Reset controller for recovery. */
1785 [ + + - + : 61 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. reset controller.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1786 : : qpair);
1787 : 61 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1788 : : }
1789 : 75 : } else {
1790 : : /* In this case, ctrlr_channel is already deleted. */
1791 [ + + + + : 2312 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. delete nvme_qpair.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1792 : : qpair);
1793 : 2312 : nvme_qpair_delete(nvme_qpair);
1794 : : }
1795 : 143 : }
1796 : :
1797 : : static void
1798 : 15 : bdev_nvme_check_io_qpairs(struct nvme_poll_group *group)
1799 : : {
1800 : : struct nvme_qpair *nvme_qpair;
1801 : :
1802 [ + + # # : 30 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
# # # # #
# # # #
# ]
1803 [ + + - + : 15 : if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) {
# # # # #
# # # ]
1804 : 1 : continue;
1805 : : }
1806 : :
1807 [ + - # # : 14 : if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
# # ]
1808 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1809 : 14 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1810 : 0 : }
1811 : 0 : }
1812 : 15 : }
1813 : :
1814 : : static int
1815 : 486178283 : bdev_nvme_poll(void *arg)
1816 : : {
1817 : 486178283 : struct nvme_poll_group *group = arg;
1818 : : int64_t num_completions;
1819 : :
1820 [ + + + + : 486178283 : if (group->collect_spin_stat && group->start_ticks == 0) {
+ - - + #
# # # #
# ]
1821 [ # # # # ]: 0 : group->start_ticks = spdk_get_ticks();
1822 : 0 : }
1823 : :
1824 [ + - + - ]: 486178283 : num_completions = spdk_nvme_poll_group_process_completions(group->group, 0,
1825 : : bdev_nvme_disconnected_qpair_cb);
1826 [ + + + + : 486178283 : if (group->collect_spin_stat) {
+ - + - ]
1827 [ # # ]: 0 : if (num_completions > 0) {
1828 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
1829 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
1830 [ # # # # ]: 0 : group->end_ticks = 0;
1831 : 0 : }
1832 [ # # # # ]: 0 : group->start_ticks = 0;
1833 : 0 : } else {
1834 [ # # # # ]: 0 : group->end_ticks = spdk_get_ticks();
1835 : : }
1836 : 0 : }
1837 : :
1838 [ + + ]: 486178283 : if (spdk_unlikely(num_completions < 0)) {
1839 : 15 : bdev_nvme_check_io_qpairs(group);
1840 : 0 : }
1841 : :
1842 : 486178283 : return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
1843 : : }
1844 : :
1845 : : static int bdev_nvme_poll_adminq(void *arg);
1846 : :
1847 : : static void
1848 : 1446 : bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us)
1849 : : {
1850 [ - + ]: 1446 : if (spdk_interrupt_mode_is_enabled()) {
1851 : 0 : return;
1852 : : }
1853 : :
1854 [ # # ]: 1446 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
1855 : :
1856 [ # # # # ]: 1446 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq,
1857 : : nvme_ctrlr, new_period_us);
1858 : 146 : }
1859 : :
1860 : : static int
1861 : 511696 : bdev_nvme_poll_adminq(void *arg)
1862 : : {
1863 : : int32_t rc;
1864 : 511696 : struct nvme_ctrlr *nvme_ctrlr = arg;
1865 : : nvme_ctrlr_disconnected_cb disconnected_cb;
1866 : :
1867 [ + + # # ]: 511696 : assert(nvme_ctrlr != NULL);
1868 : :
1869 [ + - + - ]: 511696 : rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr);
1870 [ + + ]: 511696 : if (rc < 0) {
1871 [ # # # # ]: 1525 : disconnected_cb = nvme_ctrlr->disconnected_cb;
1872 [ # # # # ]: 1525 : nvme_ctrlr->disconnected_cb = NULL;
1873 : :
1874 [ + + ]: 1525 : if (disconnected_cb != NULL) {
1875 : 796 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr,
1876 [ # # ]: 73 : g_opts.nvme_adminq_poll_period_us);
1877 [ # # # # ]: 723 : disconnected_cb(nvme_ctrlr);
1878 : 73 : } else {
1879 : 802 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1880 : : }
1881 [ + + + - : 510259 : } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) !=
+ - ]
1882 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1883 : 7619 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
1884 : 12 : }
1885 : :
1886 : 511696 : return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
1887 : : }
1888 : :
1889 : : static void
1890 : 1481 : nvme_bdev_free(void *io_device)
1891 : : {
1892 : 1481 : struct nvme_bdev *nvme_disk = io_device;
1893 : :
1894 [ + + + - ]: 1481 : pthread_mutex_destroy(&nvme_disk->mutex);
1895 [ + - + - : 1481 : free(nvme_disk->disk.name);
+ - ]
1896 [ + - + - ]: 1481 : free(nvme_disk->err_stat);
1897 : 1481 : free(nvme_disk);
1898 : 1481 : }
1899 : :
1900 : : static int
1901 : 1475 : bdev_nvme_destruct(void *ctx)
1902 : : {
1903 : 1475 : struct nvme_bdev *nvme_disk = ctx;
1904 : : struct nvme_ns *nvme_ns, *tmp_nvme_ns;
1905 : :
1906 : 394 : SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nvme_disk->nbdev_ctrlr->name, nvme_disk->nsid);
1907 : :
1908 [ + + + + : 2960 : TAILQ_FOREACH_SAFE(nvme_ns, &nvme_disk->nvme_ns_list, tailq, tmp_nvme_ns) {
+ - + + +
- + - + -
+ + ]
1909 [ + + + - : 1485 : pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1910 : :
1911 [ + - + - ]: 1485 : nvme_ns->bdev = NULL;
1912 : :
1913 [ + + + - : 1485 : assert(nvme_ns->id > 0);
+ - # # ]
1914 : :
1915 [ + + + - : 1485 : if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) {
+ - + - +
- ]
1916 [ - + # # : 515 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
# # # # ]
1917 : :
1918 [ # # # # ]: 515 : nvme_ctrlr_put_ref(nvme_ns->ctrlr);
1919 : 515 : nvme_ns_free(nvme_ns);
1920 : 3 : } else {
1921 [ + + + - : 970 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1922 : : }
1923 : 50 : }
1924 : :
1925 [ + + ]: 1475 : pthread_mutex_lock(&g_bdev_nvme_mutex);
1926 [ + + + - : 1475 : TAILQ_REMOVE(&nvme_disk->nbdev_ctrlr->bdevs, nvme_disk, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
1927 [ + + ]: 1475 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
1928 : :
1929 : 1475 : spdk_io_device_unregister(nvme_disk, nvme_bdev_free);
1930 : :
1931 : 1475 : return 0;
1932 : : }
1933 : :
1934 : : static int
1935 : 2701 : bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair)
1936 : : {
1937 : : struct nvme_ctrlr *nvme_ctrlr;
1938 : 1021 : struct spdk_nvme_io_qpair_opts opts;
1939 : : struct spdk_nvme_qpair *qpair;
1940 : : int rc;
1941 : :
1942 [ + - + - ]: 2701 : nvme_ctrlr = nvme_qpair->ctrlr;
1943 : :
1944 [ + - + - ]: 2701 : spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1945 [ + - ]: 2701 : opts.create_only = true;
1946 : : /* In interrupt mode qpairs must be created in sync mode, else it will never be connected.
1947 : : * delay_cmd_submit must be false as in interrupt mode requests cannot be submitted in
1948 : : * completion context.
1949 : : */
1950 [ + + ]: 2701 : if (!spdk_interrupt_mode_is_enabled()) {
1951 : 2699 : opts.async_mode = true;
1952 [ + + + - : 2699 : opts.delay_cmd_submit = g_opts.delay_cmd_submit;
+ - + - ]
1953 : 144 : }
1954 [ + + + - : 2701 : opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
- + # # +
- + - ]
1955 [ + - + - ]: 2701 : g_opts.io_queue_requests = opts.io_queue_requests;
1956 : :
1957 [ + - + - ]: 2701 : qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1958 [ + + ]: 2701 : if (qpair == NULL) {
1959 : 0 : return -1;
1960 : : }
1961 : :
1962 : 601 : SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name,
1963 : : spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread));
1964 : :
1965 [ + + + - : 2701 : assert(nvme_qpair->group != NULL);
+ - # # ]
1966 : :
1967 [ + - + - : 2701 : rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair);
+ - + - ]
1968 [ - + ]: 2701 : if (rc != 0) {
1969 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to begin polling on NVMe Channel.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1970 : 0 : goto err;
1971 : : }
1972 : :
1973 [ + - + - ]: 2701 : rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair);
1974 [ - + ]: 2701 : if (rc != 0) {
1975 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to connect I/O qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1976 : 0 : goto err;
1977 : : }
1978 : :
1979 [ + - + - ]: 2701 : nvme_qpair->qpair = qpair;
1980 : :
1981 [ + + + + ]: 2701 : if (!g_opts.disable_auto_failback) {
1982 : 2553 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1983 : 107 : }
1984 : :
1985 [ + + + + : 2701 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Connecting qpair %p:%u started.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1986 : : qpair, spdk_nvme_qpair_get_id(qpair));
1987 : :
1988 : 2701 : return 0;
1989 : :
1990 : 0 : err:
1991 : 0 : spdk_nvme_ctrlr_free_io_qpair(qpair);
1992 : :
1993 : 0 : return rc;
1994 : 144 : }
1995 : :
1996 : : static void bdev_nvme_reset_io_continue(void *cb_arg, int rc);
1997 : :
1998 : : static void
1999 : 723 : bdev_nvme_complete_pending_resets(struct nvme_ctrlr *nvme_ctrlr, bool success)
2000 : : {
2001 : 723 : int rc = 0;
2002 : : struct nvme_bdev_io *bio;
2003 : :
2004 [ + + # # ]: 723 : if (!success) {
2005 : 493 : rc = -1;
2006 : 33 : }
2007 : :
2008 [ + + # # : 771 : while (!TAILQ_EMPTY(&nvme_ctrlr->pending_resets)) {
# # # # ]
2009 [ # # # # : 48 : bio = TAILQ_FIRST(&nvme_ctrlr->pending_resets);
# # ]
2010 [ - + # # : 48 : TAILQ_REMOVE(&nvme_ctrlr->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2011 : :
2012 : 48 : bdev_nvme_reset_io_continue(bio, rc);
2013 : : }
2014 : 723 : }
2015 : :
2016 : : /* This function marks the current trid as failed by storing the current ticks
2017 : : * and then sets the next trid to the active trid within a controller if exists.
2018 : : *
2019 : : * The purpose of the boolean return value is to request the caller to disconnect
2020 : : * the current trid now to try connecting the next trid.
2021 : : */
2022 : : static bool
2023 : 953 : bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start)
2024 : : {
2025 : : struct nvme_path_id *path_id, *next_path;
2026 : : int rc __attribute__((unused));
2027 : :
2028 [ # # # # : 953 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
2029 [ + + # # ]: 953 : assert(path_id);
2030 [ + + # # : 953 : assert(path_id == nvme_ctrlr->active_path_id);
# # # # ]
2031 [ # # # # : 953 : next_path = TAILQ_NEXT(path_id, link);
# # ]
2032 : :
2033 : : /* Update the last failed time. It means the trid is failed if its last
2034 : : * failed time is non-zero.
2035 : : */
2036 [ # # # # ]: 953 : path_id->last_failed_tsc = spdk_get_ticks();
2037 : :
2038 [ + + ]: 953 : if (next_path == NULL) {
2039 : : /* There is no alternate trid within a controller. */
2040 : 893 : return false;
2041 : : }
2042 : :
2043 [ + + + + : 60 : if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2044 : : /* Connect is not retried in a controller reset sequence. Connecting
2045 : : * the next trid will be done by the next bdev_nvme_failover_ctrlr() call.
2046 : : */
2047 : 12 : return false;
2048 : : }
2049 : :
2050 [ + + # # : 48 : assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE);
# # # # #
# ]
2051 : :
2052 [ + - # # : 48 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Start failover from %s:%s to %s:%s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2053 : : path_id->trid.traddr, path_id->trid.trsvcid,
2054 : : next_path->trid.traddr, next_path->trid.trsvcid);
2055 : :
2056 [ # # # # ]: 48 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2057 [ # # # # ]: 48 : nvme_ctrlr->active_path_id = next_path;
2058 [ # # # # : 48 : rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid);
# # ]
2059 [ - + # # ]: 48 : assert(rc == 0);
2060 [ + - # # : 48 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2061 [ + + # # ]: 48 : if (!remove) {
2062 : : /** Shuffle the old trid to the end of the list and use the new one.
2063 : : * Allows for round robin through multiple connections.
2064 : : */
2065 [ # # # # : 36 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2066 : 6 : } else {
2067 : 12 : free(path_id);
2068 : : }
2069 : :
2070 [ + + + + : 48 : if (start || next_path->last_failed_tsc == 0) {
# # # # #
# ]
2071 : : /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed
2072 : : * or used yet. Try the next trid now.
2073 : : */
2074 : 44 : return true;
2075 : : }
2076 : :
2077 [ - + - + : 5 : if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() *
# # ]
2078 [ - + # # : 4 : nvme_ctrlr->opts.reconnect_delay_sec) {
# # ]
2079 : : /* Enough backoff passed since the next trid failed. Try the next trid now. */
2080 : 0 : return true;
2081 : : }
2082 : :
2083 : : /* The next trid will be tried after reconnect_delay_sec seconds. */
2084 : 4 : return false;
2085 : 62 : }
2086 : :
2087 : : static bool
2088 : 702313 : bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr)
2089 : : {
2090 : : int32_t elapsed;
2091 : :
2092 [ + + + + : 702313 : if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||
# # # # #
# ]
2093 [ + + # # : 224711 : nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {
# # ]
2094 : 484456 : return false;
2095 : : }
2096 : :
2097 [ - + # # : 217857 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2098 [ + + # # : 217857 : if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {
# # # # ]
2099 : 70 : return true;
2100 : : } else {
2101 : 217787 : return false;
2102 : : }
2103 : 301 : }
2104 : :
2105 : : static bool
2106 : 93 : bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr)
2107 : : {
2108 : : uint32_t elapsed;
2109 : :
2110 [ + + # # : 93 : if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {
# # # # ]
2111 : 62 : return false;
2112 : : }
2113 : :
2114 [ - + # # : 31 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2115 [ + + # # : 31 : if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {
# # # # ]
2116 : 15 : return true;
2117 : : } else {
2118 : 16 : return false;
2119 : : }
2120 : 12 : }
2121 : :
2122 : : static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success);
2123 : :
2124 : : static void
2125 : 727 : nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn)
2126 : : {
2127 : : int rc;
2128 : :
2129 [ + + + + : 727 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2130 : :
2131 [ # # # # ]: 727 : rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
2132 [ + + ]: 727 : if (rc != 0) {
2133 [ + - # # : 4 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "disconnecting ctrlr failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2134 : :
2135 : : /* Disconnect fails if ctrlr is already resetting or removed. In this case,
2136 : : * fail the reset sequence immediately.
2137 : : */
2138 : 4 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2139 : 4 : return;
2140 : : }
2141 : :
2142 : : /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
2143 : : * Set callback here to execute the specified operation after ctrlr is really disconnected.
2144 : : */
2145 [ + + # # : 723 : assert(nvme_ctrlr->disconnected_cb == NULL);
# # # # ]
2146 [ # # # # ]: 723 : nvme_ctrlr->disconnected_cb = cb_fn;
2147 : :
2148 : : /* During disconnection, reduce the period to poll adminq more often. */
2149 : 723 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0);
2150 : 74 : }
2151 : :
2152 : : enum bdev_nvme_op_after_reset {
2153 : : OP_NONE,
2154 : : OP_COMPLETE_PENDING_DESTRUCT,
2155 : : OP_DESTRUCT,
2156 : : OP_DELAYED_RECONNECT,
2157 : : OP_FAILOVER,
2158 : : };
2159 : :
2160 : : typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset;
2161 : :
2162 : : static _bdev_nvme_op_after_reset
2163 : 723 : bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success)
2164 : : {
2165 [ + + ]: 723 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
2166 : : /* Complete pending destruct after reset completes. */
2167 : 15 : return OP_COMPLETE_PENDING_DESTRUCT;
2168 [ + + # # ]: 708 : } else if (nvme_ctrlr->pending_failover) {
2169 [ # # ]: 12 : nvme_ctrlr->pending_failover = false;
2170 [ # # # # ]: 12 : nvme_ctrlr->reset_start_tsc = 0;
2171 : 12 : return OP_FAILOVER;
2172 [ + + + + : 696 : } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2173 [ # # # # ]: 572 : nvme_ctrlr->reset_start_tsc = 0;
2174 : 572 : return OP_NONE;
2175 [ + + ]: 124 : } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2176 : 31 : return OP_DESTRUCT;
2177 : : } else {
2178 [ + + ]: 93 : if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) {
2179 [ # # ]: 15 : nvme_ctrlr->fast_io_fail_timedout = true;
2180 : 2 : }
2181 : 93 : return OP_DELAYED_RECONNECT;
2182 : : }
2183 : 73 : }
2184 : :
2185 : : static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug);
2186 : : static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
2187 : :
2188 : : static int
2189 : 81 : bdev_nvme_reconnect_delay_timer_expired(void *ctx)
2190 : : {
2191 : 81 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2192 : :
2193 : 36 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name);
2194 [ - + # # ]: 81 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2195 : :
2196 [ # # ]: 81 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2197 : :
2198 [ + + # # ]: 81 : if (!nvme_ctrlr->reconnect_is_delayed) {
2199 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2200 : 0 : return SPDK_POLLER_BUSY;
2201 : : }
2202 : :
2203 [ # # ]: 81 : nvme_ctrlr->reconnect_is_delayed = false;
2204 : :
2205 [ - + # # ]: 81 : if (nvme_ctrlr->destruct) {
2206 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2207 : 0 : return SPDK_POLLER_BUSY;
2208 : : }
2209 : :
2210 [ - + # # : 81 : assert(nvme_ctrlr->resetting == false);
# # ]
2211 [ # # ]: 81 : nvme_ctrlr->resetting = true;
2212 : :
2213 [ - + # # ]: 81 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2214 : :
2215 [ # # # # ]: 81 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2216 : :
2217 : 81 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2218 : 81 : return SPDK_POLLER_BUSY;
2219 : 9 : }
2220 : :
2221 : : static void
2222 : 93 : bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr)
2223 : : {
2224 [ # # # # ]: 93 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2225 : :
2226 [ + + # # : 93 : assert(nvme_ctrlr->reconnect_is_delayed == false);
# # ]
2227 [ # # ]: 93 : nvme_ctrlr->reconnect_is_delayed = true;
2228 : :
2229 [ + + # # : 93 : assert(nvme_ctrlr->reconnect_delay_timer == NULL);
# # # # ]
2230 [ # # # # : 93 : nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,
# # # # #
# ]
2231 : : nvme_ctrlr,
2232 : : nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);
2233 : 93 : }
2234 : :
2235 : : static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr);
2236 : :
2237 : : static void
2238 : 723 : bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
2239 : : {
2240 [ # # # # ]: 723 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2241 [ # # # # ]: 723 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2242 : : enum bdev_nvme_op_after_reset op_after_reset;
2243 : :
2244 [ + + # # : 723 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2245 : :
2246 [ - + # # ]: 723 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2247 [ + + # # ]: 723 : if (!success) {
2248 : : /* Connecting the active trid failed. Set the next alternate trid to the
2249 : : * active trid if it exists.
2250 : : */
2251 [ + + ]: 501 : if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) {
2252 : : /* The next alternate trid exists and is ready to try. Try it now. */
2253 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2254 : :
2255 [ + + - + : 8 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Try the next alternate trid %s:%s now.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2256 : : nvme_ctrlr->active_path_id->trid.traddr,
2257 : : nvme_ctrlr->active_path_id->trid.trsvcid);
2258 : :
2259 : 8 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2260 : 8 : return;
2261 : : }
2262 : :
2263 : : /* We came here if there is no alternate trid or if the next trid exists but
2264 : : * is not ready to try. We will try the active trid after reconnect_delay_sec
2265 : : * seconds if it is non-zero or at the next reset call otherwise.
2266 : : */
2267 : 33 : } else {
2268 : : /* Connecting the active trid succeeded. Clear the last failed time because it
2269 : : * means the trid is failed if its last failed time is non-zero.
2270 : : */
2271 [ # # # # : 222 : nvme_ctrlr->active_path_id->last_failed_tsc = 0;
# # # # ]
2272 : : }
2273 : :
2274 [ + + + + : 715 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Clear pending resets.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2275 : :
2276 : : /* Make sure we clear any pending resets before returning. */
2277 [ # # ]: 715 : bdev_nvme_complete_pending_resets(nvme_ctrlr, success);
2278 : :
2279 [ + + # # ]: 715 : if (!success) {
2280 [ + - # # : 493 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Resetting controller failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2281 : 33 : } else {
2282 [ + + # # : 222 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Resetting controller successful.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2283 : : }
2284 : :
2285 [ # # ]: 715 : nvme_ctrlr->resetting = false;
2286 [ # # ]: 715 : nvme_ctrlr->dont_retry = false;
2287 [ # # ]: 715 : nvme_ctrlr->in_failover = false;
2288 : :
2289 [ # # # # ]: 715 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2290 [ # # # # ]: 715 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2291 : :
2292 [ # # ]: 715 : op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success);
2293 [ - + # # ]: 715 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2294 : :
2295 : : /* Delay callbacks when the next operation is a failover. */
2296 [ + + + + ]: 715 : if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) {
2297 [ + + # # : 118 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1);
# # ]
2298 : 19 : }
2299 : :
2300 [ + + + + : 715 : switch (op_after_reset) {
+ ]
2301 : 15 : case OP_COMPLETE_PENDING_DESTRUCT:
2302 : 15 : nvme_ctrlr_unregister(nvme_ctrlr);
2303 : 15 : break;
2304 : 29 : case OP_DESTRUCT:
2305 : 31 : bdev_nvme_delete_ctrlr(nvme_ctrlr, false);
2306 : 31 : remove_discovery_entry(nvme_ctrlr);
2307 : 31 : break;
2308 : 81 : case OP_DELAYED_RECONNECT:
2309 : 93 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer);
2310 : 93 : break;
2311 : 9 : case OP_FAILOVER:
2312 [ # # # # ]: 12 : nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn;
2313 [ # # # # ]: 12 : nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg;
2314 : 12 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
2315 : 12 : break;
2316 : 510 : default:
2317 : 564 : break;
2318 : : }
2319 : 73 : }
2320 : :
2321 : : static void
2322 : 0 : bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2323 : : {
2324 : 0 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2325 : 0 : }
2326 : :
2327 : : static void
2328 : 737 : bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i,
2329 : : struct nvme_ctrlr *nvme_ctrlr,
2330 : : struct nvme_ctrlr_channel *ctrlr_ch, void *ctx)
2331 : : {
2332 : : struct nvme_qpair *nvme_qpair;
2333 : : struct spdk_nvme_qpair *qpair;
2334 : :
2335 [ # # # # ]: 737 : nvme_qpair = ctrlr_ch->qpair;
2336 [ + + # # ]: 737 : assert(nvme_qpair != NULL);
2337 : :
2338 : 737 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2339 : :
2340 [ # # # # ]: 737 : qpair = nvme_qpair->qpair;
2341 [ + + ]: 737 : if (qpair != NULL) {
2342 [ + + - + : 324 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting qpair %p:%u.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2343 : : qpair, spdk_nvme_qpair_get_id(qpair));
2344 : :
2345 [ + + # # : 324 : if (nvme_qpair->ctrlr->dont_retry) {
# # # # ]
2346 : 252 : spdk_nvme_qpair_set_abort_dnr(qpair, true);
2347 : 54 : }
2348 : 324 : spdk_nvme_ctrlr_disconnect_io_qpair(qpair);
2349 : :
2350 : : /* The current full reset sequence will move to the next
2351 : : * ctrlr_channel after the qpair is actually disconnected.
2352 : : */
2353 [ - + # # : 324 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2354 [ # # # # ]: 324 : ctrlr_ch->reset_iter = i;
2355 : 70 : } else {
2356 : 413 : nvme_ctrlr_for_each_channel_continue(i, 0);
2357 : : }
2358 : 737 : }
2359 : :
2360 : : static void
2361 : 222 : bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2362 : : {
2363 [ + + ]: 222 : if (status == 0) {
2364 [ + + - + : 222 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were created after ctrlr reset.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2365 : :
2366 : 222 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true);
2367 : 38 : } else {
2368 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were failed to create after ctrlr reset.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2369 : :
2370 : : /* Delete the added qpairs and quiesce ctrlr to make the states clean. */
2371 : 0 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2372 : : bdev_nvme_reset_destroy_qpair,
2373 : : NULL,
2374 : : bdev_nvme_reset_create_qpairs_failed);
2375 : : }
2376 : 222 : }
2377 : :
2378 : : static int
2379 : 13063 : bdev_nvme_reset_check_qpair_connected(void *ctx)
2380 : : {
2381 : 13063 : struct nvme_ctrlr_channel *ctrlr_ch = ctx;
2382 [ # # # # ]: 13063 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2383 : : struct spdk_nvme_qpair *qpair;
2384 : :
2385 [ + + # # : 13063 : if (ctrlr_ch->reset_iter == NULL) {
# # ]
2386 : : /* qpair was already failed to connect and the reset sequence is being aborted. */
2387 [ # # # # : 0 : assert(ctrlr_ch->connect_poller == NULL);
# # # # ]
2388 [ # # # # : 0 : assert(nvme_qpair->qpair == NULL);
# # # # ]
2389 : :
2390 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2391 : : "qpair was already failed to connect. reset is being aborted.\n");
2392 : 0 : return SPDK_POLLER_BUSY;
2393 : : }
2394 : :
2395 [ # # # # ]: 13063 : qpair = nvme_qpair->qpair;
2396 [ + + # # ]: 13063 : assert(qpair != NULL);
2397 : :
2398 [ + + ]: 13063 : if (!spdk_nvme_qpair_is_connected(qpair)) {
2399 : 12759 : return SPDK_POLLER_BUSY;
2400 : : }
2401 : :
2402 [ + + - + : 304 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr, "qpair %p:%u was connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2403 : : qpair, spdk_nvme_qpair_get_id(qpair));
2404 : :
2405 [ # # ]: 304 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
2406 : :
2407 : : /* qpair was completed to connect. Move to the next ctrlr_channel */
2408 [ # # # # ]: 304 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
2409 [ # # # # ]: 304 : ctrlr_ch->reset_iter = NULL;
2410 : :
2411 [ + + + + ]: 304 : if (!g_opts.disable_auto_failback) {
2412 : 236 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2413 : 45 : }
2414 : :
2415 : 304 : return SPDK_POLLER_BUSY;
2416 : 71 : }
2417 : :
2418 : : static void
2419 : 304 : bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i,
2420 : : struct nvme_ctrlr *nvme_ctrlr,
2421 : : struct nvme_ctrlr_channel *ctrlr_ch,
2422 : : void *ctx)
2423 : : {
2424 [ # # # # ]: 304 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2425 : : struct spdk_nvme_qpair *qpair;
2426 : 304 : int rc = 0;
2427 : :
2428 [ + + # # : 304 : if (nvme_qpair->qpair == NULL) {
# # ]
2429 : 304 : rc = bdev_nvme_create_qpair(nvme_qpair);
2430 : 62 : }
2431 [ + + ]: 304 : if (rc == 0) {
2432 [ # # # # ]: 304 : ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected,
2433 : : ctrlr_ch, 0);
2434 : :
2435 [ # # # # ]: 304 : qpair = nvme_qpair->qpair;
2436 : :
2437 [ + + - + : 304 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start checking qpair %p:%u to be connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2438 : : qpair, spdk_nvme_qpair_get_id(qpair));
2439 : :
2440 : : /* The current full reset sequence will move to the next
2441 : : * ctrlr_channel after the qpair is actually connected.
2442 : : */
2443 [ - + # # : 304 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2444 [ # # # # ]: 304 : ctrlr_ch->reset_iter = i;
2445 : 62 : } else {
2446 : 0 : nvme_ctrlr_for_each_channel_continue(i, rc);
2447 : : }
2448 : 304 : }
2449 : :
2450 : : static void
2451 : 222 : nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr)
2452 : : {
2453 [ # # # # ]: 222 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
2454 : : struct nvme_ns *nvme_ns;
2455 : :
2456 [ + + ]: 245 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
2457 [ + + ]: 392 : nvme_ns != NULL;
2458 : 170 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
2459 [ + + # # : 170 : if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
2460 [ + + - + : 4 : SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id);
# # # # #
# ]
2461 : : /* NS can be added again. Just nullify nvme_ns->ns. */
2462 [ # # # # ]: 4 : nvme_ns->ns = NULL;
2463 : 1 : }
2464 : 23 : }
2465 : 222 : }
2466 : :
2467 : :
2468 : : static int
2469 : 702169 : bdev_nvme_reconnect_ctrlr_poll(void *arg)
2470 : : {
2471 : 702169 : struct nvme_ctrlr *nvme_ctrlr = arg;
2472 : : struct spdk_nvme_transport_id *trid;
2473 : 702169 : int rc = -ETIMEDOUT;
2474 : :
2475 [ + + ]: 702169 : if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2476 : : /* Mark the ctrlr as failed. The next call to
2477 : : * spdk_nvme_ctrlr_reconnect_poll_async() will then
2478 : : * do the necessary cleanup and return failure.
2479 : : */
2480 [ # # # # ]: 31 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2481 : 2 : }
2482 : :
2483 [ # # # # ]: 702169 : rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr);
2484 [ + + ]: 702169 : if (rc == -EAGAIN) {
2485 : 701450 : return SPDK_POLLER_BUSY;
2486 : : }
2487 : :
2488 [ # # ]: 719 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
2489 [ + + ]: 719 : if (rc == 0) {
2490 [ # # # # : 222 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
2491 : :
2492 [ + + # # : 222 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
# # ]
2493 [ + + - + : 184 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected to %s:%s. Create qpairs.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2494 : : trid->traddr, trid->trsvcid);
2495 : 36 : } else {
2496 [ + + - + : 38 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected. Create qpairs.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2497 : : }
2498 : :
2499 : 222 : nvme_ctrlr_check_namespaces(nvme_ctrlr);
2500 : :
2501 : : /* Recreate all of the I/O queue pairs */
2502 : 222 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2503 : : bdev_nvme_reset_create_qpair,
2504 : : NULL,
2505 : : bdev_nvme_reset_create_qpairs_done);
2506 : 38 : } else {
2507 [ + + + + : 497 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr could not be connected.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2508 : :
2509 : 497 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2510 : : }
2511 : 719 : return SPDK_POLLER_BUSY;
2512 : 282 : }
2513 : :
2514 : : static void
2515 : 719 : bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2516 : : {
2517 [ + + + + : 719 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start reconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2518 : :
2519 [ # # # # ]: 719 : spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr);
2520 : :
2521 : 103 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name);
2522 [ + + # # : 719 : assert(nvme_ctrlr->reset_detach_poller == NULL);
# # # # ]
2523 [ # # # # ]: 719 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll,
2524 : : nvme_ctrlr, 0);
2525 : 719 : }
2526 : :
2527 : : static void
2528 : 622 : bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2529 : : {
2530 : 67 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name);
2531 [ + + # # ]: 622 : assert(status == 0);
2532 : :
2533 [ + + + + : 622 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were deleted.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2534 : :
2535 [ + + # # : 622 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2536 : 38 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2537 : 2 : } else {
2538 : 584 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2539 : : }
2540 : 622 : }
2541 : :
2542 : : static void
2543 : 622 : bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2544 : : {
2545 [ + + + + : 622 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Delete qpairs for reset.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2546 : :
2547 : 622 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2548 : : bdev_nvme_reset_destroy_qpair,
2549 : : NULL,
2550 : : bdev_nvme_reset_destroy_qpair_done);
2551 : 622 : }
2552 : :
2553 : : static void
2554 : 12 : bdev_nvme_reconnect_ctrlr_now(void *ctx)
2555 : : {
2556 : 12 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2557 : :
2558 [ + + # # : 12 : assert(nvme_ctrlr->resetting == true);
# # ]
2559 [ + + # # : 12 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2560 : :
2561 [ # # ]: 12 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2562 : :
2563 [ # # # # ]: 12 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2564 : :
2565 : 12 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2566 : 12 : }
2567 : :
2568 : : static void
2569 : 622 : _bdev_nvme_reset_ctrlr(void *ctx)
2570 : : {
2571 : 622 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2572 : :
2573 [ + + # # : 622 : assert(nvme_ctrlr->resetting == true);
# # ]
2574 [ - + # # : 622 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2575 : :
2576 [ + + # # : 622 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2577 : 38 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs);
2578 : 2 : } else {
2579 : 584 : bdev_nvme_reset_destroy_qpairs(nvme_ctrlr);
2580 : : }
2581 : 622 : }
2582 : :
2583 : : static int
2584 : 250 : bdev_nvme_reset_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, spdk_msg_fn *msg_fn)
2585 : : {
2586 [ + + # # ]: 250 : if (nvme_ctrlr->destruct) {
2587 : 12 : return -ENXIO;
2588 : : }
2589 : :
2590 [ + + # # ]: 238 : if (nvme_ctrlr->resetting) {
2591 [ + - # # : 56 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2592 : 56 : return -EBUSY;
2593 : : }
2594 : :
2595 [ + + # # ]: 182 : if (nvme_ctrlr->disabled) {
2596 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset. Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2597 : 4 : return -EALREADY;
2598 : : }
2599 : :
2600 [ # # ]: 178 : nvme_ctrlr->resetting = true;
2601 [ # # ]: 178 : nvme_ctrlr->dont_retry = true;
2602 : :
2603 [ + + # # ]: 178 : if (nvme_ctrlr->reconnect_is_delayed) {
2604 [ + + - + : 4 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2605 [ # # ]: 4 : *msg_fn = bdev_nvme_reconnect_ctrlr_now;
2606 [ # # ]: 4 : nvme_ctrlr->reconnect_is_delayed = false;
2607 : 1 : } else {
2608 [ # # ]: 174 : *msg_fn = _bdev_nvme_reset_ctrlr;
2609 [ + + # # : 174 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
2610 : : }
2611 : :
2612 [ # # # # ]: 178 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2613 : :
2614 : 178 : return 0;
2615 : 52 : }
2616 : :
2617 : : static int
2618 : 106 : bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2619 : : {
2620 : 75 : spdk_msg_fn msg_fn;
2621 : : int rc;
2622 : :
2623 [ - + # # ]: 106 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2624 : 106 : rc = bdev_nvme_reset_ctrlr_unsafe(nvme_ctrlr, &msg_fn);
2625 [ - + # # ]: 106 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2626 : :
2627 [ + + ]: 106 : if (rc == 0) {
2628 [ # # # # ]: 86 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2629 : 20 : }
2630 : :
2631 : 106 : return rc;
2632 : : }
2633 : :
2634 : : static int
2635 : 12 : bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2636 : : {
2637 [ - + # # ]: 12 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2638 [ - + # # ]: 12 : if (nvme_ctrlr->destruct) {
2639 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2640 : 0 : return -ENXIO;
2641 : : }
2642 : :
2643 [ - + # # ]: 12 : if (nvme_ctrlr->resetting) {
2644 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2645 : 0 : return -EBUSY;
2646 : : }
2647 : :
2648 [ + + # # ]: 12 : if (!nvme_ctrlr->disabled) {
2649 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2650 : 4 : return -EALREADY;
2651 : : }
2652 : :
2653 [ # # ]: 8 : nvme_ctrlr->disabled = false;
2654 [ # # ]: 8 : nvme_ctrlr->resetting = true;
2655 : :
2656 [ # # # # ]: 8 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2657 : :
2658 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2659 : :
2660 [ # # # # ]: 8 : spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr);
2661 : 8 : return 0;
2662 : 3 : }
2663 : :
2664 : : static void
2665 : 8 : bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr)
2666 : : {
2667 [ # # # # ]: 8 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2668 [ # # # # ]: 8 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2669 : : enum bdev_nvme_op_after_reset op_after_disable;
2670 : :
2671 [ + + # # : 8 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2672 : :
2673 [ # # # # ]: 8 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2674 [ # # # # ]: 8 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2675 : :
2676 [ - + # # ]: 8 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2677 : :
2678 [ # # ]: 8 : nvme_ctrlr->resetting = false;
2679 [ # # ]: 8 : nvme_ctrlr->dont_retry = false;
2680 : :
2681 : 8 : op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true);
2682 : :
2683 [ # # ]: 8 : nvme_ctrlr->disabled = true;
2684 [ # # # # ]: 8 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2685 : :
2686 : : /* Make sure we clear any pending resets before returning. */
2687 : 8 : bdev_nvme_complete_pending_resets(nvme_ctrlr, true);
2688 : :
2689 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2690 : :
2691 [ + + ]: 8 : if (ctrlr_op_cb_fn) {
2692 [ # # # # ]: 0 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0);
2693 : 0 : }
2694 : :
2695 [ + + ]: 8 : switch (op_after_disable) {
2696 : 0 : case OP_COMPLETE_PENDING_DESTRUCT:
2697 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
2698 : 0 : break;
2699 : 6 : default:
2700 : 8 : break;
2701 : : }
2702 : 8 : }
2703 : :
2704 : : static void
2705 : 4 : bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2706 : : {
2707 [ + + # # ]: 4 : assert(status == 0);
2708 : :
2709 [ + + # # : 4 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2710 : 0 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2711 : 0 : } else {
2712 : 4 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete);
2713 : : }
2714 : 4 : }
2715 : :
2716 : : static void
2717 : 4 : bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2718 : : {
2719 : 4 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2720 : : bdev_nvme_reset_destroy_qpair,
2721 : : NULL,
2722 : : bdev_nvme_disable_destroy_qpairs_done);
2723 : 4 : }
2724 : :
2725 : : static void
2726 : 4 : _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx)
2727 : : {
2728 : 4 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2729 : :
2730 [ + + # # : 4 : assert(nvme_ctrlr->resetting == true);
# # ]
2731 [ + + # # : 4 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2732 : :
2733 [ # # ]: 4 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2734 : :
2735 : 4 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2736 : 4 : }
2737 : :
2738 : : static void
2739 : 4 : _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx)
2740 : : {
2741 : 4 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2742 : :
2743 [ + + # # : 4 : assert(nvme_ctrlr->resetting == true);
# # ]
2744 [ - + # # : 4 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2745 : :
2746 [ + + # # : 4 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2747 : 0 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs);
2748 : 0 : } else {
2749 : 4 : bdev_nvme_disable_destroy_qpairs(nvme_ctrlr);
2750 : : }
2751 : 4 : }
2752 : :
2753 : : static int
2754 : 20 : bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2755 : : {
2756 : : spdk_msg_fn msg_fn;
2757 : :
2758 [ - + # # ]: 20 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2759 [ + + # # ]: 20 : if (nvme_ctrlr->destruct) {
2760 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2761 : 4 : return -ENXIO;
2762 : : }
2763 : :
2764 [ + + # # ]: 16 : if (nvme_ctrlr->resetting) {
2765 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2766 : 4 : return -EBUSY;
2767 : : }
2768 : :
2769 [ + + # # ]: 12 : if (nvme_ctrlr->disabled) {
2770 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2771 : 4 : return -EALREADY;
2772 : : }
2773 : :
2774 [ # # ]: 8 : nvme_ctrlr->resetting = true;
2775 [ # # ]: 8 : nvme_ctrlr->dont_retry = true;
2776 : :
2777 [ + + # # ]: 8 : if (nvme_ctrlr->reconnect_is_delayed) {
2778 : 4 : msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr;
2779 [ # # ]: 4 : nvme_ctrlr->reconnect_is_delayed = false;
2780 : 1 : } else {
2781 : 4 : msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr;
2782 : : }
2783 : :
2784 [ # # # # ]: 8 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2785 : :
2786 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2787 : :
2788 [ # # # # ]: 8 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2789 : 8 : return 0;
2790 : 5 : }
2791 : :
2792 : : static int
2793 : 34 : nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2794 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2795 : : {
2796 : : int rc;
2797 : :
2798 [ + + - + ]: 34 : switch (op) {
2799 : 24 : case NVME_CTRLR_OP_RESET:
2800 : 30 : rc = bdev_nvme_reset_ctrlr(nvme_ctrlr);
2801 : 30 : break;
2802 : 0 : case NVME_CTRLR_OP_ENABLE:
2803 : 0 : rc = bdev_nvme_enable_ctrlr(nvme_ctrlr);
2804 : 0 : break;
2805 : 0 : case NVME_CTRLR_OP_DISABLE:
2806 : 0 : rc = bdev_nvme_disable_ctrlr(nvme_ctrlr);
2807 : 0 : break;
2808 : 3 : default:
2809 : 4 : rc = -EINVAL;
2810 : 4 : break;
2811 : : }
2812 : :
2813 [ + + ]: 34 : if (rc == 0) {
2814 [ + + # # : 22 : assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL);
# # # # ]
2815 [ - + # # : 22 : assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL);
# # # # ]
2816 [ # # # # ]: 22 : nvme_ctrlr->ctrlr_op_cb_fn = cb_fn;
2817 [ # # # # ]: 22 : nvme_ctrlr->ctrlr_op_cb_arg = cb_arg;
2818 : 4 : }
2819 : 34 : return rc;
2820 : : }
2821 : :
2822 : : struct nvme_ctrlr_op_rpc_ctx {
2823 : : struct nvme_ctrlr *nvme_ctrlr;
2824 : : struct spdk_thread *orig_thread;
2825 : : enum nvme_ctrlr_op op;
2826 : : int rc;
2827 : : bdev_nvme_ctrlr_op_cb cb_fn;
2828 : : void *cb_arg;
2829 : : };
2830 : :
2831 : : static void
2832 : 16 : _nvme_ctrlr_op_rpc_complete(void *_ctx)
2833 : : {
2834 : 16 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2835 : :
2836 [ + + # # ]: 16 : assert(ctx != NULL);
2837 [ + + # # : 16 : assert(ctx->cb_fn != NULL);
# # # # ]
2838 : :
2839 [ # # # # : 16 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2840 : :
2841 : 16 : free(ctx);
2842 : 16 : }
2843 : :
2844 : : static void
2845 : 16 : nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc)
2846 : : {
2847 : 16 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2848 : :
2849 [ # # # # ]: 16 : ctx->rc = rc;
2850 : :
2851 [ # # # # ]: 16 : spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx);
2852 : 16 : }
2853 : :
2854 : : void
2855 : 16 : nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2856 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2857 : : {
2858 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2859 : : int rc;
2860 : :
2861 [ + + # # ]: 16 : assert(cb_fn != NULL);
2862 : :
2863 : 16 : ctx = calloc(1, sizeof(*ctx));
2864 [ + + ]: 16 : if (ctx == NULL) {
2865 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2866 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2867 : 0 : return;
2868 : : }
2869 : :
2870 [ # # # # ]: 16 : ctx->orig_thread = spdk_get_thread();
2871 [ # # # # ]: 16 : ctx->cb_fn = cb_fn;
2872 [ # # # # ]: 16 : ctx->cb_arg = cb_arg;
2873 : :
2874 : 16 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx);
2875 [ + + ]: 16 : if (rc == 0) {
2876 : 4 : return;
2877 [ + + ]: 12 : } else if (rc == -EALREADY) {
2878 : 0 : rc = 0;
2879 : 0 : }
2880 : :
2881 : 12 : nvme_ctrlr_op_rpc_complete(ctx, rc);
2882 : 4 : }
2883 : :
2884 : : static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc);
2885 : :
2886 : : static void
2887 : 18 : _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx)
2888 : : {
2889 : 18 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2890 : : struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr;
2891 : : int rc;
2892 : :
2893 [ # # # # ]: 18 : prev_nvme_ctrlr = ctx->nvme_ctrlr;
2894 [ # # # # ]: 18 : ctx->nvme_ctrlr = NULL;
2895 : :
2896 [ - + # # : 18 : if (ctx->rc != 0) {
# # ]
2897 : 0 : goto complete;
2898 : : }
2899 : :
2900 [ # # # # : 18 : next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq);
# # ]
2901 [ + + ]: 18 : if (next_nvme_ctrlr == NULL) {
2902 : 14 : goto complete;
2903 : : }
2904 : :
2905 [ # # # # ]: 4 : rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2906 [ + + ]: 4 : if (rc == 0) {
2907 [ # # # # ]: 4 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2908 : 4 : return;
2909 [ # # ]: 0 : } else if (rc == -EALREADY) {
2910 [ # # # # ]: 0 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2911 : 0 : rc = 0;
2912 : 0 : }
2913 : :
2914 [ # # # # ]: 0 : ctx->rc = rc;
2915 : :
2916 : 12 : complete:
2917 [ # # # # : 14 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2918 : 14 : free(ctx);
2919 : 3 : }
2920 : :
2921 : : static void
2922 : 18 : nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc)
2923 : : {
2924 : 18 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2925 : :
2926 [ # # # # ]: 18 : ctx->rc = rc;
2927 : :
2928 [ # # # # ]: 18 : spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx);
2929 : 18 : }
2930 : :
2931 : : void
2932 : 14 : nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op,
2933 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2934 : : {
2935 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2936 : : struct nvme_ctrlr *nvme_ctrlr;
2937 : : int rc;
2938 : :
2939 [ + + # # ]: 14 : assert(cb_fn != NULL);
2940 : :
2941 : 14 : ctx = calloc(1, sizeof(*ctx));
2942 [ + + ]: 14 : if (ctx == NULL) {
2943 : 0 : SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
2944 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2945 : 0 : return;
2946 : : }
2947 : :
2948 [ # # # # ]: 14 : ctx->orig_thread = spdk_get_thread();
2949 [ # # # # ]: 14 : ctx->op = op;
2950 [ # # # # ]: 14 : ctx->cb_fn = cb_fn;
2951 [ # # # # ]: 14 : ctx->cb_arg = cb_arg;
2952 : :
2953 [ # # # # : 14 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # ]
2954 [ - + # # ]: 14 : assert(nvme_ctrlr != NULL);
2955 : :
2956 : 14 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2957 [ + + ]: 14 : if (rc == 0) {
2958 [ # # # # ]: 14 : ctx->nvme_ctrlr = nvme_ctrlr;
2959 : 14 : return;
2960 [ # # ]: 0 : } else if (rc == -EALREADY) {
2961 [ # # # # ]: 0 : ctx->nvme_ctrlr = nvme_ctrlr;
2962 : 0 : rc = 0;
2963 : 0 : }
2964 : :
2965 : 0 : nvme_bdev_ctrlr_op_rpc_continue(ctx, rc);
2966 : 2 : }
2967 : :
2968 : : static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
2969 : :
2970 : : static void
2971 : 104 : bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
2972 : : {
2973 : 104 : struct nvme_bdev_io *bio = ctx;
2974 : : enum spdk_bdev_io_status io_status;
2975 : :
2976 [ + + # # : 104 : if (bio->cpl.cdw0 == 0) {
# # # # ]
2977 : 88 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
2978 : 13 : } else {
2979 : 16 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
2980 : : }
2981 : :
2982 [ + + - + : 104 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p completed, status:%d\n", bio, io_status);
# # # # #
# # # ]
2983 : :
2984 : 104 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL);
2985 : 104 : }
2986 : :
2987 : : static void
2988 : 168 : bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i,
2989 : : struct nvme_bdev *nbdev,
2990 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
2991 : : {
2992 : 168 : bdev_nvme_abort_retry_ios(nbdev_ch);
2993 [ # # # # ]: 168 : nbdev_ch->resetting = false;
2994 : :
2995 : 168 : nvme_bdev_for_each_channel_continue(i, 0);
2996 : 168 : }
2997 : :
2998 : : static void
2999 : 104 : bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
3000 : : {
3001 : 104 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3002 [ # # # # : 104 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3003 : :
3004 : : /* Abort all queued I/Os for retry. */
3005 : 121 : nvme_bdev_for_each_channel(nbdev,
3006 : : bdev_nvme_unfreeze_bdev_channel,
3007 : 17 : bio,
3008 : : bdev_nvme_unfreeze_bdev_channel_done);
3009 : 104 : }
3010 : :
3011 : : static void
3012 : 144 : _bdev_nvme_reset_io_continue(void *ctx)
3013 : : {
3014 : 144 : struct nvme_bdev_io *bio = ctx;
3015 : : struct nvme_io_path *prev_io_path, *next_io_path;
3016 : : int rc;
3017 : :
3018 [ # # # # ]: 144 : prev_io_path = bio->io_path;
3019 [ # # # # ]: 144 : bio->io_path = NULL;
3020 : :
3021 [ # # # # : 144 : next_io_path = STAILQ_NEXT(prev_io_path, stailq);
# # ]
3022 [ + + ]: 144 : if (next_io_path == NULL) {
3023 : 104 : goto complete;
3024 : : }
3025 : :
3026 : 40 : rc = _bdev_nvme_reset_io(next_io_path, bio);
3027 [ + - ]: 40 : if (rc == 0) {
3028 : 40 : return;
3029 : : }
3030 : :
3031 : 0 : complete:
3032 : 104 : bdev_nvme_reset_io_complete(bio);
3033 : 27 : }
3034 : :
3035 : : static void
3036 : 144 : bdev_nvme_reset_io_continue(void *cb_arg, int rc)
3037 : : {
3038 : 144 : struct nvme_bdev_io *bio = cb_arg;
3039 : 144 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3040 [ # # # # : 144 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3041 : :
3042 [ + + - + : 144 : NVME_BDEV_INFOLOG(nbdev, "continue reset_io %p, rc:%d\n", bio, rc);
# # # # #
# # # ]
3043 : :
3044 : : /* Reset status is initialized as "failed". Set to "success" once we have at least one
3045 : : * successfully reset nvme_ctrlr.
3046 : : */
3047 [ + + ]: 144 : if (rc == 0) {
3048 [ # # # # : 104 : bio->cpl.cdw0 = 0;
# # ]
3049 : 17 : }
3050 : :
3051 : 144 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio);
3052 : 144 : }
3053 : :
3054 : : static int
3055 : 144 : _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio)
3056 : : {
3057 : 144 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3058 [ # # # # : 144 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3059 [ # # # # : 144 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
3060 : 97 : spdk_msg_fn msg_fn;
3061 : : int rc;
3062 : :
3063 [ + + # # : 144 : assert(bio->io_path == NULL);
# # # # ]
3064 [ # # # # ]: 144 : bio->io_path = io_path;
3065 : :
3066 [ - + # # ]: 144 : pthread_mutex_lock(&nvme_ctrlr->mutex);
3067 : 144 : rc = bdev_nvme_reset_ctrlr_unsafe(nvme_ctrlr, &msg_fn);
3068 [ + + ]: 144 : if (rc == -EBUSY) {
3069 : : /*
3070 : : * Reset call is queued only if it is from the app framework. This is on purpose so that
3071 : : * we don't interfere with the app framework reset strategy. i.e. we are deferring to the
3072 : : * upper level. If they are in the middle of a reset, we won't try to schedule another one.
3073 : : */
3074 [ # # # # : 48 : TAILQ_INSERT_TAIL(&nvme_ctrlr->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3075 : 12 : }
3076 [ - + # # ]: 144 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
3077 : :
3078 [ + + ]: 144 : if (rc == 0) {
3079 [ + + # # : 92 : assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL);
# # # # ]
3080 [ + + # # : 92 : assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL);
# # # # ]
3081 [ # # # # ]: 92 : nvme_ctrlr->ctrlr_op_cb_fn = bdev_nvme_reset_io_continue;
3082 [ # # # # ]: 92 : nvme_ctrlr->ctrlr_op_cb_arg = bio;
3083 : :
3084 [ # # # # ]: 92 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
3085 : :
3086 [ + + - + : 92 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started resetting ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3087 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3088 [ + + ]: 66 : } else if (rc == -EBUSY) {
3089 : 48 : rc = 0;
3090 : :
3091 [ + + - + : 48 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p was queued to ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3092 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3093 : 12 : } else {
3094 [ + + - + : 4 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p could not reset ctrlr [%s, %u], rc:%d\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3095 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr), rc);
3096 : : }
3097 : :
3098 : 144 : return rc;
3099 : : }
3100 : :
3101 : : static void
3102 : 104 : bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
3103 : : {
3104 : 104 : struct nvme_bdev_io *bio = ctx;
3105 : 104 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3106 : : struct nvme_bdev_channel *nbdev_ch;
3107 : : struct nvme_io_path *io_path;
3108 : : int rc;
3109 : :
3110 : 104 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
3111 : :
3112 : : /* Initialize with failed status. With multipath it is enough to have at least one successful
3113 : : * nvme_ctrlr reset. If there is none, reset status will remain failed.
3114 : : */
3115 [ # # # # : 104 : bio->cpl.cdw0 = 1;
# # ]
3116 : :
3117 : : /* Reset all nvme_ctrlrs of a bdev controller sequentially. */
3118 [ # # # # : 104 : io_path = STAILQ_FIRST(&nbdev_ch->io_path_list);
# # ]
3119 [ + + # # ]: 104 : assert(io_path != NULL);
3120 : :
3121 : 104 : rc = _bdev_nvme_reset_io(io_path, bio);
3122 [ + + ]: 104 : if (rc != 0) {
3123 : : /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */
3124 [ + + ]: 4 : rc = (rc == -EALREADY) ? 0 : rc;
3125 : :
3126 : 4 : bdev_nvme_reset_io_continue(bio, rc);
3127 : 1 : }
3128 : 104 : }
3129 : :
3130 : : static void
3131 : 160 : bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i,
3132 : : struct nvme_bdev *nbdev,
3133 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
3134 : : {
3135 [ # # # # ]: 160 : nbdev_ch->resetting = true;
3136 : :
3137 : 160 : nvme_bdev_for_each_channel_continue(i, 0);
3138 : 160 : }
3139 : :
3140 : : static void
3141 : 100 : bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio)
3142 : : {
3143 [ + + - + : 100 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started.\n", bio);
# # # # #
# # # ]
3144 : :
3145 : 116 : nvme_bdev_for_each_channel(nbdev,
3146 : : bdev_nvme_freeze_bdev_channel,
3147 : 16 : bio,
3148 : : bdev_nvme_freeze_bdev_channel_done);
3149 : 100 : }
3150 : :
3151 : : static int
3152 : 915 : bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove)
3153 : : {
3154 [ + + # # ]: 915 : if (nvme_ctrlr->destruct) {
3155 : : /* Don't bother resetting if the controller is in the process of being destructed. */
3156 : 409 : return -ENXIO;
3157 : : }
3158 : :
3159 [ + + # # ]: 506 : if (nvme_ctrlr->resetting) {
3160 [ + + # # ]: 54 : if (!nvme_ctrlr->in_failover) {
3161 [ + - # # : 12 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3162 : : "Reset is already in progress. Defer failover until reset completes.\n");
3163 : :
3164 : : /* Defer failover until reset completes. */
3165 [ # # ]: 12 : nvme_ctrlr->pending_failover = true;
3166 : 12 : return -EINPROGRESS;
3167 : : } else {
3168 [ + - # # : 42 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform failover, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3169 : 42 : return -EBUSY;
3170 : : }
3171 : : }
3172 : :
3173 [ # # ]: 452 : bdev_nvme_failover_trid(nvme_ctrlr, remove, true);
3174 : :
3175 [ + + # # ]: 452 : if (nvme_ctrlr->reconnect_is_delayed) {
3176 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3177 : :
3178 : : /* We rely on the next reconnect for the failover. */
3179 : 4 : return -EALREADY;
3180 : : }
3181 : :
3182 [ - + # # ]: 448 : if (nvme_ctrlr->disabled) {
3183 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3184 : :
3185 : : /* We rely on the enablement for the failover. */
3186 : 0 : return -EALREADY;
3187 : : }
3188 : :
3189 [ # # ]: 448 : nvme_ctrlr->resetting = true;
3190 [ # # ]: 448 : nvme_ctrlr->in_failover = true;
3191 : :
3192 [ + + # # : 448 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
3193 [ # # # # ]: 448 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
3194 : :
3195 : 448 : return 0;
3196 : 32 : }
3197 : :
3198 : : static int
3199 : 903 : bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
3200 : : {
3201 : : int rc;
3202 : :
3203 [ - + # # ]: 903 : pthread_mutex_lock(&nvme_ctrlr->mutex);
3204 : 903 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false);
3205 [ - + # # ]: 903 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
3206 : :
3207 [ + + ]: 903 : if (rc == 0) {
3208 [ # # # # ]: 440 : spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr);
3209 [ + + ]: 488 : } else if (rc == -EALREADY) {
3210 : 0 : rc = 0;
3211 : 0 : }
3212 : :
3213 : 903 : return rc;
3214 : : }
3215 : :
3216 : : static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3217 : : uint64_t num_blocks);
3218 : :
3219 : : static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3220 : : uint64_t num_blocks);
3221 : :
3222 : : static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks,
3223 : : uint64_t src_offset_blocks,
3224 : : uint64_t num_blocks);
3225 : :
3226 : : static void
3227 : 6580821 : bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
3228 : : bool success)
3229 : : {
3230 [ # # ]: 6580821 : struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3231 : : int ret;
3232 : :
3233 [ + + # # ]: 6580821 : if (!success) {
3234 : 0 : ret = -EINVAL;
3235 : 0 : goto exit;
3236 : : }
3237 : :
3238 [ + + # # : 6580821 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
3239 : 0 : ret = -ENXIO;
3240 : 0 : goto exit;
3241 : : }
3242 : :
3243 : 6854664 : ret = bdev_nvme_readv(bio,
3244 [ # # # # : 273843 : bdev_io->u.bdev.iovs,
# # # # ]
3245 [ # # # # : 273843 : bdev_io->u.bdev.iovcnt,
# # # # ]
3246 [ # # # # : 273843 : bdev_io->u.bdev.md_buf,
# # # # ]
3247 [ # # # # : 273843 : bdev_io->u.bdev.num_blocks,
# # # # ]
3248 [ # # # # : 273843 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3249 [ # # # # : 273843 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3250 [ # # # # : 273843 : bdev_io->u.bdev.memory_domain,
# # # # ]
3251 [ # # # # : 273843 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3252 [ # # # # : 273843 : bdev_io->u.bdev.accel_sequence);
# # # # ]
3253 : :
3254 : 6306978 : exit:
3255 [ + + ]: 6580821 : if (spdk_unlikely(ret != 0)) {
3256 : 82292 : bdev_nvme_io_complete(bio, ret);
3257 : 0 : }
3258 : 6580821 : }
3259 : :
3260 : : static inline void
3261 : 28983771 : _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
3262 : : {
3263 [ + - ]: 28983771 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3264 [ + - + - ]: 28983771 : struct spdk_bdev *bdev = bdev_io->bdev;
3265 : : struct nvme_bdev_io *nbdev_io_to_abort;
3266 : 28983771 : int rc = 0;
3267 : :
3268 [ + + + + : 28983771 : switch (bdev_io->type) {
+ + + + +
+ + + + +
- + + + -
- - - ]
3269 : 12924409 : case SPDK_BDEV_IO_TYPE_READ:
3270 [ + + + + : 13198099 : if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) {
+ - + - +
- + - + -
+ - + - +
- + - + -
- + ]
3271 : :
3272 : 6617689 : rc = bdev_nvme_readv(nbdev_io,
3273 [ - + - + : 129 : bdev_io->u.bdev.iovs,
- + - + ]
3274 [ - + - + : 129 : bdev_io->u.bdev.iovcnt,
- + - + ]
3275 [ - + - + : 129 : bdev_io->u.bdev.md_buf,
- + - + ]
3276 [ - + - + : 129 : bdev_io->u.bdev.num_blocks,
- + - + ]
3277 [ - + - + : 129 : bdev_io->u.bdev.offset_blocks,
- + - + ]
3278 [ - + - + : 129 : bdev_io->u.bdev.dif_check_flags,
- + - + ]
3279 [ - + - + : 129 : bdev_io->u.bdev.memory_domain,
- + - + ]
3280 [ - + - + : 129 : bdev_io->u.bdev.memory_domain_ctx,
- + - + ]
3281 [ - + - + : 129 : bdev_io->u.bdev.accel_sequence);
- + - + ]
3282 : 129 : } else {
3283 : 6854100 : spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb,
3284 [ # # # # : 6580539 : bdev_io->u.bdev.num_blocks * bdev->blocklen);
# # # # #
# # # ]
3285 : 6580539 : rc = 0;
3286 : : }
3287 : 13198099 : break;
3288 : 13375050 : case SPDK_BDEV_IO_TYPE_WRITE:
3289 : 13922626 : rc = bdev_nvme_writev(nbdev_io,
3290 [ # # # # : 273788 : bdev_io->u.bdev.iovs,
# # # # ]
3291 [ # # # # : 273788 : bdev_io->u.bdev.iovcnt,
# # # # ]
3292 [ # # # # : 273788 : bdev_io->u.bdev.md_buf,
# # # # ]
3293 [ # # # # : 273788 : bdev_io->u.bdev.num_blocks,
# # # # ]
3294 [ # # # # : 273788 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3295 [ # # # # : 273788 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3296 [ # # # # : 273788 : bdev_io->u.bdev.memory_domain,
# # # # ]
3297 [ # # # # : 273788 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3298 [ # # # # : 273788 : bdev_io->u.bdev.accel_sequence,
# # # # ]
3299 [ # # # # : 273788 : bdev_io->u.bdev.nvme_cdw12,
# # ]
3300 [ # # # # : 273788 : bdev_io->u.bdev.nvme_cdw13);
# # ]
3301 : 13648838 : break;
3302 : 53 : case SPDK_BDEV_IO_TYPE_COMPARE:
3303 : 59 : rc = bdev_nvme_comparev(nbdev_io,
3304 [ # # # # : 3 : bdev_io->u.bdev.iovs,
# # # # ]
3305 [ # # # # : 3 : bdev_io->u.bdev.iovcnt,
# # # # ]
3306 [ # # # # : 3 : bdev_io->u.bdev.md_buf,
# # # # ]
3307 [ # # # # : 3 : bdev_io->u.bdev.num_blocks,
# # # # ]
3308 [ # # # # : 3 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3309 [ # # # # : 3 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3310 : 56 : break;
3311 : 51 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3312 : 55 : rc = bdev_nvme_comparev_and_writev(nbdev_io,
3313 [ # # # # : 2 : bdev_io->u.bdev.iovs,
# # # # ]
3314 [ # # # # : 2 : bdev_io->u.bdev.iovcnt,
# # # # ]
3315 [ # # # # : 2 : bdev_io->u.bdev.fused_iovs,
# # # # ]
3316 [ # # # # : 2 : bdev_io->u.bdev.fused_iovcnt,
# # # # ]
3317 [ # # # # : 2 : bdev_io->u.bdev.md_buf,
# # # # ]
3318 [ # # # # : 2 : bdev_io->u.bdev.num_blocks,
# # # # ]
3319 [ # # # # : 2 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3320 [ # # # # : 2 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3321 : 53 : break;
3322 : 79815 : case SPDK_BDEV_IO_TYPE_UNMAP:
3323 : 79821 : rc = bdev_nvme_unmap(nbdev_io,
3324 [ # # # # : 3 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3325 [ # # # # : 3 : bdev_io->u.bdev.num_blocks);
# # # # ]
3326 : 79818 : break;
3327 : 621801 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3328 : 766809 : rc = bdev_nvme_write_zeroes(nbdev_io,
3329 [ # # # # : 72504 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3330 [ # # # # : 72504 : bdev_io->u.bdev.num_blocks);
# # # # ]
3331 : 694305 : break;
3332 : 84 : case SPDK_BDEV_IO_TYPE_RESET:
3333 [ # # # # ]: 100 : nbdev_io->io_path = NULL;
3334 [ # # # # ]: 100 : bdev_nvme_reset_io(bdev->ctxt, nbdev_io);
3335 : 100 : return;
3336 : :
3337 : 1123938 : case SPDK_BDEV_IO_TYPE_FLUSH:
3338 : 1123939 : bdev_nvme_io_complete(nbdev_io, 0);
3339 : 1123939 : return;
3340 : :
3341 : 231053 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3342 : 231053 : rc = bdev_nvme_zone_appendv(nbdev_io,
3343 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3344 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3345 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3346 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3347 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3348 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3349 : 231053 : break;
3350 : 1 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3351 : 1 : rc = bdev_nvme_get_zone_info(nbdev_io,
3352 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3353 [ # # # # : 0 : bdev_io->u.zone_mgmt.num_zones,
# # # # ]
3354 [ # # # # : 1 : bdev_io->u.zone_mgmt.buf);
# # # # ]
3355 : 1 : break;
3356 : 43 : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3357 : 43 : rc = bdev_nvme_zone_management(nbdev_io,
3358 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3359 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_action);
# # # # ]
3360 : 43 : break;
3361 : 136 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3362 [ # # # # ]: 142 : nbdev_io->io_path = NULL;
3363 : 148 : bdev_nvme_admin_passthru(nbdev_ch,
3364 : 6 : nbdev_io,
3365 [ # # # # : 6 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3366 [ # # # # : 6 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3367 [ # # # # : 6 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3368 : 142 : return;
3369 : :
3370 : 108 : case SPDK_BDEV_IO_TYPE_NVME_IO:
3371 : 114 : rc = bdev_nvme_io_passthru(nbdev_io,
3372 [ # # # # : 3 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3373 [ # # # # : 3 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3374 [ # # # # : 3 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3375 : 111 : break;
3376 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3377 : 0 : rc = bdev_nvme_io_passthru_md(nbdev_io,
3378 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3379 [ # # # # : 0 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3380 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3381 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3382 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3383 : 0 : break;
3384 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IOV_MD:
3385 : 0 : rc = bdev_nvme_iov_passthru_md(nbdev_io,
3386 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3387 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovs,
# # # # ]
3388 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovcnt,
# # # # ]
3389 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3390 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3391 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3392 : 0 : break;
3393 : 7170 : case SPDK_BDEV_IO_TYPE_ABORT:
3394 [ # # # # ]: 7176 : nbdev_io->io_path = NULL;
3395 [ # # # # : 7176 : nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx;
# # # # #
# ]
3396 : 7182 : bdev_nvme_abort(nbdev_ch,
3397 : 6 : nbdev_io,
3398 : 6 : nbdev_io_to_abort);
3399 : 7176 : return;
3400 : :
3401 : 36 : case SPDK_BDEV_IO_TYPE_COPY:
3402 : 38 : rc = bdev_nvme_copy(nbdev_io,
3403 [ # # # # : 1 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3404 [ # # # # : 1 : bdev_io->u.bdev.copy.src_offset_blocks,
# # # # #
# ]
3405 [ # # # # : 1 : bdev_io->u.bdev.num_blocks);
# # # # ]
3406 : 37 : break;
3407 : 0 : default:
3408 : 0 : rc = -EINVAL;
3409 : 0 : break;
3410 : : }
3411 : :
3412 [ + + ]: 27852414 : if (spdk_unlikely(rc != 0)) {
3413 : 82184 : bdev_nvme_io_complete(nbdev_io, rc);
3414 : 0 : }
3415 : 620023 : }
3416 : :
3417 : : static void
3418 : 29365090 : bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
3419 : : {
3420 : 29365090 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
3421 [ + - ]: 29365090 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3422 : :
3423 [ + + + - : 29365090 : if (spdk_likely(nbdev_io->submit_tsc == 0)) {
- + ]
3424 [ + - + - ]: 29293274 : nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io);
3425 : 618985 : } else {
3426 : : /* There are cases where submit_tsc != 0, i.e. retry I/O.
3427 : : * We need to update submit_tsc here.
3428 : : */
3429 [ # # # # ]: 71816 : nbdev_io->submit_tsc = spdk_get_ticks();
3430 : : }
3431 : :
3432 [ + + + + : 29365090 : spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io);
+ - + - +
- + - + -
+ - ]
3433 [ + - + - ]: 29365090 : nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch);
3434 [ + + + - : 29365090 : if (spdk_unlikely(!nbdev_io->io_path)) {
+ - ]
3435 [ + + # # : 385697 : if (!bdev_nvme_io_type_is_admin(bdev_io->type)) {
# # ]
3436 : 385693 : bdev_nvme_io_complete(nbdev_io, -ENXIO);
3437 : 385693 : return;
3438 : : }
3439 : :
3440 : : /* Admin commands do not use the optimal I/O path.
3441 : : * Simply fall through even if it is not found.
3442 : : */
3443 : 1 : }
3444 : :
3445 : 28979397 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
3446 : 618987 : }
3447 : :
3448 : : static bool
3449 : 3494258 : bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi)
3450 : : {
3451 [ + + + ]: 3494258 : switch (csi) {
3452 : 3349055 : case SPDK_NVME_CSI_NVM:
3453 : 3494229 : return true;
3454 : 29 : case SPDK_NVME_CSI_ZNS:
3455 : 29 : return true;
3456 : 0 : default:
3457 : 0 : return false;
3458 : : }
3459 : 145174 : }
3460 : :
3461 : : static bool
3462 : 3494258 : bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
3463 : : {
3464 : 3494258 : struct nvme_bdev *nbdev = ctx;
3465 : : struct nvme_ns *nvme_ns;
3466 : : struct spdk_nvme_ns *ns;
3467 : : struct spdk_nvme_ctrlr *ctrlr;
3468 : : const struct spdk_nvme_ctrlr_data *cdata;
3469 : :
3470 [ + - + - : 3494258 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
+ - ]
3471 [ + + # # ]: 3494258 : assert(nvme_ns != NULL);
3472 [ + - + - ]: 3494258 : ns = nvme_ns->ns;
3473 [ + + ]: 3494258 : if (ns == NULL) {
3474 : 0 : return false;
3475 : : }
3476 : :
3477 [ + + ]: 3494258 : if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) {
3478 [ # # # ]: 0 : switch (io_type) {
3479 : 0 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3480 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3481 : 0 : return true;
3482 : :
3483 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3484 : 0 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3485 : :
3486 : 0 : default:
3487 : 0 : return false;
3488 : : }
3489 : : }
3490 : :
3491 : 3494258 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
3492 : :
3493 [ + + + + : 3494258 : switch (io_type) {
+ + + + +
+ ]
3494 : 1306978 : case SPDK_BDEV_IO_TYPE_READ:
3495 : : case SPDK_BDEV_IO_TYPE_WRITE:
3496 : : case SPDK_BDEV_IO_TYPE_RESET:
3497 : : case SPDK_BDEV_IO_TYPE_FLUSH:
3498 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3499 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3500 : : case SPDK_BDEV_IO_TYPE_ABORT:
3501 : 1307025 : return true;
3502 : :
3503 : 3794 : case SPDK_BDEV_IO_TYPE_COMPARE:
3504 : 3800 : return spdk_nvme_ns_supports_compare(ns);
3505 : :
3506 : 1102 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3507 : 1104 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3508 : :
3509 : 12186 : case SPDK_BDEV_IO_TYPE_UNMAP:
3510 : 12192 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3511 [ # # # # ]: 12192 : return cdata->oncs.dsm;
3512 : :
3513 : 1988449 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3514 : 2133492 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3515 [ + - + - ]: 2133492 : return cdata->oncs.write_zeroes;
3516 : :
3517 : 3792 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3518 [ + + ]: 3798 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
3519 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) {
3520 : 236 : return true;
3521 : : }
3522 : 3562 : return false;
3523 : :
3524 : 7436 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3525 : : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3526 : 7444 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS;
3527 : :
3528 : 3719 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3529 [ + + ]: 3725 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
3530 [ + - ]: 2 : spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
3531 : :
3532 : 6722 : case SPDK_BDEV_IO_TYPE_COPY:
3533 : 6758 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3534 [ + - + - ]: 6758 : return cdata->oncs.copy;
3535 : :
3536 : 14906 : default:
3537 : 14922 : return false;
3538 : : }
3539 : 145174 : }
3540 : :
3541 : : static int
3542 : 2397 : nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch)
3543 : : {
3544 : : struct nvme_qpair *nvme_qpair;
3545 : : struct spdk_io_channel *pg_ch;
3546 : : int rc;
3547 : :
3548 : 2397 : nvme_qpair = calloc(1, sizeof(*nvme_qpair));
3549 [ + + ]: 2397 : if (!nvme_qpair) {
3550 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to alloc nvme_qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3551 : 0 : return -1;
3552 : : }
3553 : :
3554 [ + - + - : 2397 : TAILQ_INIT(&nvme_qpair->io_path_list);
+ - + - +
- + - + -
+ - ]
3555 : :
3556 [ + - + - ]: 2397 : nvme_qpair->ctrlr = nvme_ctrlr;
3557 [ + - + - ]: 2397 : nvme_qpair->ctrlr_ch = ctrlr_ch;
3558 : :
3559 : 2397 : pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs);
3560 [ + + ]: 2397 : if (!pg_ch) {
3561 : 0 : free(nvme_qpair);
3562 : 0 : return -1;
3563 : : }
3564 : :
3565 [ + - + - ]: 2397 : nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch);
3566 : :
3567 : : #ifdef SPDK_CONFIG_VTUNE
3568 : : nvme_qpair->group->collect_spin_stat = true;
3569 : : #else
3570 [ + - + - : 2397 : nvme_qpair->group->collect_spin_stat = false;
+ - + - ]
3571 : : #endif
3572 : :
3573 [ + + - + ]: 2397 : if (!nvme_ctrlr->disabled) {
3574 : : /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will
3575 : : * be created when it's enabled.
3576 : : */
3577 : 2397 : rc = bdev_nvme_create_qpair(nvme_qpair);
3578 [ + + ]: 2397 : if (rc != 0) {
3579 : : /* nvme_ctrlr can't create IO qpair if connection is down.
3580 : : * If reconnect_delay_sec is non-zero, creating IO qpair is retried
3581 : : * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero,
3582 : : * submitted IO will be queued until IO qpair is successfully created.
3583 : : *
3584 : : * Hence, if both are satisfied, ignore the failure.
3585 : : */
3586 [ # # # # : 0 : if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) {
# # # # #
# # # ]
3587 : 0 : spdk_put_io_channel(pg_ch);
3588 : 0 : free(nvme_qpair);
3589 : 0 : return rc;
3590 : : }
3591 : 0 : }
3592 : 82 : }
3593 : :
3594 [ + - + - : 2397 : TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
3595 : :
3596 [ + - + - ]: 2397 : ctrlr_ch->qpair = nvme_qpair;
3597 : :
3598 : 2397 : nvme_ctrlr_get_ref(nvme_ctrlr);
3599 : :
3600 : 2397 : return 0;
3601 : 82 : }
3602 : :
3603 : : static int
3604 : 2397 : bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3605 : : {
3606 : 2397 : struct nvme_ctrlr *nvme_ctrlr = io_device;
3607 : 2397 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3608 : :
3609 : 2397 : return nvme_qpair_create(nvme_ctrlr, ctrlr_ch);
3610 : : }
3611 : :
3612 : : static void
3613 : 2397 : nvme_qpair_delete(struct nvme_qpair *nvme_qpair)
3614 : : {
3615 : : struct nvme_io_path *io_path, *next;
3616 : :
3617 [ + + + - : 2397 : assert(nvme_qpair->group != NULL);
+ - # # ]
3618 : :
3619 [ + + + + : 4828 : TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) {
+ - + + +
- + - + -
+ + ]
3620 [ + + + - : 2431 : TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
3621 : 2431 : nvme_io_path_free(io_path);
3622 : 60 : }
3623 : :
3624 [ + + + - : 2397 : TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
3625 : :
3626 [ + - + - ]: 2397 : spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group));
3627 : :
3628 [ + - + - ]: 2397 : nvme_ctrlr_put_ref(nvme_qpair->ctrlr);
3629 : :
3630 : 2397 : free(nvme_qpair);
3631 : 2397 : }
3632 : :
3633 : : static void
3634 : 2397 : bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3635 : : {
3636 : 2397 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3637 : : struct nvme_qpair *nvme_qpair;
3638 : :
3639 [ + - + - ]: 2397 : nvme_qpair = ctrlr_ch->qpair;
3640 [ + + # # ]: 2397 : assert(nvme_qpair != NULL);
3641 : :
3642 : 2397 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
3643 : :
3644 [ + + + - : 2397 : if (nvme_qpair->qpair != NULL) {
+ - ]
3645 : : /* Always try to disconnect the qpair, even if a reset is in progress.
3646 : : * The qpair may have been created after the reset process started.
3647 : : */
3648 [ + - + - ]: 2312 : spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
3649 [ + + + - : 2312 : if (ctrlr_ch->reset_iter) {
- + ]
3650 : : /* Skip current ctrlr_channel in a full reset sequence because
3651 : : * it is being deleted now.
3652 : : */
3653 [ # # # # ]: 0 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
3654 : 0 : }
3655 : :
3656 : : /* We cannot release a reference to the poll group now.
3657 : : * The qpair may be disconnected asynchronously later.
3658 : : * We need to poll it until it is actually disconnected.
3659 : : * Just detach the qpair from the deleting ctrlr_channel.
3660 : : */
3661 [ + - + - ]: 2312 : nvme_qpair->ctrlr_ch = NULL;
3662 : 68 : } else {
3663 [ - + # # : 85 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
3664 : :
3665 : 85 : nvme_qpair_delete(nvme_qpair);
3666 : : }
3667 : 2397 : }
3668 : :
3669 : : static inline struct spdk_io_channel *
3670 : 758989 : bdev_nvme_get_accel_channel(struct nvme_poll_group *group)
3671 : : {
3672 [ + + # # : 758989 : if (spdk_unlikely(!group->accel_channel)) {
# # ]
3673 [ # # # # ]: 53 : group->accel_channel = spdk_accel_get_io_channel();
3674 [ - + # # : 53 : if (!group->accel_channel) {
# # ]
3675 : 0 : SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
3676 : : group);
3677 : 0 : return NULL;
3678 : : }
3679 : 0 : }
3680 : :
3681 [ # # # # ]: 758989 : return group->accel_channel;
3682 : 0 : }
3683 : :
3684 : : static void
3685 : 758989 : bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
3686 : : {
3687 : 758989 : spdk_accel_sequence_finish(seq, cb_fn, cb_arg);
3688 : 758989 : }
3689 : :
3690 : : static void
3691 : 0 : bdev_nvme_abort_sequence(void *seq)
3692 : : {
3693 : 0 : spdk_accel_sequence_abort(seq);
3694 : 0 : }
3695 : :
3696 : : static void
3697 : 374748 : bdev_nvme_reverse_sequence(void *seq)
3698 : : {
3699 : 374748 : spdk_accel_sequence_reverse(seq);
3700 : 374748 : }
3701 : :
3702 : : static int
3703 : 758989 : bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
3704 : : struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed,
3705 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3706 : : {
3707 : : struct spdk_io_channel *ch;
3708 : 758989 : struct nvme_poll_group *group = ctx;
3709 : :
3710 : 758989 : ch = bdev_nvme_get_accel_channel(group);
3711 [ - + ]: 758989 : if (spdk_unlikely(ch == NULL)) {
3712 : 0 : return -ENOMEM;
3713 : : }
3714 : :
3715 : 758989 : return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt,
3716 : 0 : domain, domain_ctx, seed, cb_fn, cb_arg);
3717 : 0 : }
3718 : :
3719 : : static int
3720 : 0 : bdev_nvme_append_copy(void *ctx, void **seq, struct iovec *dst_iovs, uint32_t dst_iovcnt,
3721 : : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
3722 : : struct iovec *src_iovs, uint32_t src_iovcnt,
3723 : : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
3724 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3725 : : {
3726 : : struct spdk_io_channel *ch;
3727 : 0 : struct nvme_poll_group *group = ctx;
3728 : :
3729 : 0 : ch = bdev_nvme_get_accel_channel(group);
3730 [ # # ]: 0 : if (spdk_unlikely(ch == NULL)) {
3731 : 0 : return -ENOMEM;
3732 : : }
3733 : :
3734 : 0 : return spdk_accel_append_copy((struct spdk_accel_sequence **)seq, ch,
3735 : 0 : dst_iovs, dst_iovcnt, dst_domain, dst_domain_ctx,
3736 : 0 : src_iovs, src_iovcnt, src_domain, src_domain_ctx,
3737 : 0 : cb_fn, cb_arg);
3738 : 0 : }
3739 : :
3740 : : static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
3741 : : .table_size = sizeof(struct spdk_nvme_accel_fn_table),
3742 : : .append_crc32c = bdev_nvme_append_crc32c,
3743 : : .append_copy = bdev_nvme_append_copy,
3744 : : .finish_sequence = bdev_nvme_finish_sequence,
3745 : : .reverse_sequence = bdev_nvme_reverse_sequence,
3746 : : .abort_sequence = bdev_nvme_abort_sequence,
3747 : : };
3748 : :
3749 : : static int
3750 : 125546 : bdev_nvme_interrupt_wrapper(void *ctx)
3751 : : {
3752 : : int num_events;
3753 : 125546 : struct nvme_poll_group *group = ctx;
3754 : :
3755 [ # # # # ]: 125546 : num_events = spdk_nvme_poll_group_wait(group->group, bdev_nvme_disconnected_qpair_cb);
3756 [ - + ]: 125546 : if (spdk_unlikely(num_events < 0)) {
3757 : 0 : bdev_nvme_check_io_qpairs(group);
3758 : 0 : }
3759 : :
3760 : 125546 : return num_events;
3761 : : }
3762 : :
3763 : : static int
3764 : 2235 : bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
3765 : : {
3766 : 2235 : struct nvme_poll_group *group = ctx_buf;
3767 : : uint64_t period;
3768 : : int fd;
3769 : :
3770 [ + - + - : 2235 : TAILQ_INIT(&group->qpair_list);
+ - + - +
- + - + -
+ - ]
3771 : :
3772 [ + - + - ]: 2235 : group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
3773 [ + + + - : 2235 : if (group->group == NULL) {
+ - ]
3774 : 0 : return -1;
3775 : : }
3776 : :
3777 [ + + + - ]: 2235 : period = spdk_interrupt_mode_is_enabled() ? 0 : g_opts.nvme_ioq_poll_period_us;
3778 [ + - + - ]: 2235 : group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, period);
3779 : :
3780 [ + + + - : 2235 : if (group->poller == NULL) {
+ - ]
3781 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3782 : 0 : return -1;
3783 : : }
3784 : :
3785 [ + + ]: 2235 : if (spdk_interrupt_mode_is_enabled()) {
3786 [ # # # # ]: 2 : spdk_poller_register_interrupt(group->poller, NULL, NULL);
3787 : :
3788 [ # # # # ]: 2 : fd = spdk_nvme_poll_group_get_fd(group->group);
3789 [ - + ]: 2 : if (fd < 0) {
3790 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3791 : 0 : return -1;
3792 : : }
3793 : :
3794 [ # # # # ]: 2 : group->intr = SPDK_INTERRUPT_REGISTER(fd, bdev_nvme_interrupt_wrapper, group);
3795 [ - + # # : 2 : if (!group->intr) {
# # ]
3796 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3797 : 0 : return -1;
3798 : : }
3799 : 0 : }
3800 : :
3801 : 2235 : return 0;
3802 : 67 : }
3803 : :
3804 : : static void
3805 : 2235 : bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf)
3806 : : {
3807 : 2235 : struct nvme_poll_group *group = ctx_buf;
3808 : :
3809 [ + + + - : 2235 : assert(TAILQ_EMPTY(&group->qpair_list));
+ - + - #
# ]
3810 : :
3811 [ + + + - : 2235 : if (group->accel_channel) {
+ - ]
3812 [ # # # # ]: 53 : spdk_put_io_channel(group->accel_channel);
3813 : 0 : }
3814 : :
3815 [ + + ]: 2235 : if (spdk_interrupt_mode_is_enabled()) {
3816 [ # # ]: 2 : spdk_interrupt_unregister(&group->intr);
3817 : 0 : }
3818 : :
3819 [ + - ]: 2235 : spdk_poller_unregister(&group->poller);
3820 [ + + + - : 2235 : if (spdk_nvme_poll_group_destroy(group->group)) {
+ - ]
3821 : 0 : SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n");
3822 [ # # ]: 0 : assert(false);
3823 : : }
3824 : 2235 : }
3825 : :
3826 : : static struct spdk_io_channel *
3827 : 2279 : bdev_nvme_get_io_channel(void *ctx)
3828 : : {
3829 : 2279 : struct nvme_bdev *nvme_bdev = ctx;
3830 : :
3831 : 2279 : return spdk_get_io_channel(nvme_bdev);
3832 : : }
3833 : :
3834 : : static void *
3835 : 0 : bdev_nvme_get_module_ctx(void *ctx)
3836 : : {
3837 : 0 : struct nvme_bdev *nvme_bdev = ctx;
3838 : : struct nvme_ns *nvme_ns;
3839 : :
3840 [ # # # # : 0 : if (!nvme_bdev || nvme_bdev->disk.module != &nvme_if) {
# # # # #
# ]
3841 : 0 : return NULL;
3842 : : }
3843 : :
3844 [ # # # # : 0 : nvme_ns = TAILQ_FIRST(&nvme_bdev->nvme_ns_list);
# # ]
3845 [ # # ]: 0 : if (!nvme_ns) {
3846 : 0 : return NULL;
3847 : : }
3848 : :
3849 [ # # # # ]: 0 : return nvme_ns->ns;
3850 : 0 : }
3851 : :
3852 : : static const char *
3853 : 0 : _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state)
3854 : : {
3855 [ # # # # : 0 : switch (ana_state) {
# # ]
3856 : 0 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
3857 : 0 : return "optimized";
3858 : 0 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
3859 : 0 : return "non_optimized";
3860 : 0 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
3861 : 0 : return "inaccessible";
3862 : 0 : case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
3863 : 0 : return "persistent_loss";
3864 : 0 : case SPDK_NVME_ANA_CHANGE_STATE:
3865 : 0 : return "change";
3866 : 0 : default:
3867 : 0 : return NULL;
3868 : : }
3869 : 0 : }
3870 : :
3871 : : static int
3872 : 10367 : bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
3873 : : {
3874 : 10367 : struct spdk_memory_domain **_domains = NULL;
3875 : 10367 : struct nvme_bdev *nbdev = ctx;
3876 : : struct nvme_ns *nvme_ns;
3877 : 10367 : int i = 0, _array_size = array_size;
3878 : 10367 : int rc = 0;
3879 : :
3880 [ + + + - : 20810 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
3881 [ + + + + ]: 10443 : if (domains && array_size >= i) {
3882 [ # # ]: 275 : _domains = &domains[i];
3883 : 11 : } else {
3884 : 10168 : _domains = NULL;
3885 : : }
3886 [ + - + - : 10443 : rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size);
+ - + - ]
3887 [ + + ]: 10443 : if (rc > 0) {
3888 [ # # ]: 4312 : i += rc;
3889 [ + + ]: 4312 : if (_array_size >= rc) {
3890 [ # # ]: 267 : _array_size -= rc;
3891 : 9 : } else {
3892 : 4045 : _array_size = 0;
3893 : : }
3894 [ + + ]: 6144 : } else if (rc < 0) {
3895 : 0 : return rc;
3896 : : }
3897 : 77 : }
3898 : :
3899 : 10367 : return i;
3900 : 71 : }
3901 : :
3902 : : static const char *
3903 : 745 : nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr)
3904 : : {
3905 [ - + # # ]: 745 : if (nvme_ctrlr->destruct) {
3906 : 0 : return "deleting";
3907 [ - + # # : 745 : } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
3908 : 0 : return "failed";
3909 [ + + # # ]: 745 : } else if (nvme_ctrlr->resetting) {
3910 : 12 : return "resetting";
3911 [ + + # # ]: 733 : } else if (nvme_ctrlr->reconnect_is_delayed > 0) {
3912 : 5 : return "reconnect_is_delayed";
3913 [ + + # # ]: 728 : } else if (nvme_ctrlr->disabled) {
3914 : 0 : return "disabled";
3915 : : } else {
3916 : 728 : return "enabled";
3917 : : }
3918 : 1 : }
3919 : :
3920 : : void
3921 : 745 : nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr)
3922 : 744 : {
3923 : : struct spdk_nvme_transport_id *trid;
3924 : : const struct spdk_nvme_ctrlr_opts *opts;
3925 : : const struct spdk_nvme_ctrlr_data *cdata;
3926 : : struct nvme_path_id *path_id;
3927 : : int32_t numa_id;
3928 : :
3929 : 745 : spdk_json_write_object_begin(w);
3930 : :
3931 : 745 : spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr));
3932 : :
3933 : : #ifdef SPDK_CONFIG_NVME_CUSE
3934 : 744 : size_t cuse_name_size = 128;
3935 [ - + ]: 744 : char cuse_name[cuse_name_size];
3936 : :
3937 [ # # # # ]: 744 : int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size);
3938 [ + + ]: 744 : if (rc == 0) {
3939 : 3 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
3940 : 0 : }
3941 : : #endif
3942 [ # # # # : 745 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
3943 : 745 : spdk_json_write_named_object_begin(w, "trid");
3944 : 745 : nvme_bdev_dump_trid_json(trid, w);
3945 : 745 : spdk_json_write_object_end(w);
3946 : :
3947 [ # # # # : 745 : path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link);
# # # # #
# ]
3948 [ + + ]: 745 : if (path_id != NULL) {
3949 : 12 : spdk_json_write_named_array_begin(w, "alternate_trids");
3950 : 0 : do {
3951 [ # # ]: 16 : trid = &path_id->trid;
3952 : 16 : spdk_json_write_object_begin(w);
3953 : 16 : nvme_bdev_dump_trid_json(trid, w);
3954 : 16 : spdk_json_write_object_end(w);
3955 : :
3956 [ # # # # : 16 : path_id = TAILQ_NEXT(path_id, link);
# # ]
3957 [ + + ]: 16 : } while (path_id != NULL);
3958 : 12 : spdk_json_write_array_end(w);
3959 : 0 : }
3960 : :
3961 [ # # # # ]: 745 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
3962 [ # # # # ]: 745 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
3963 : :
3964 [ # # # # ]: 745 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
3965 : 745 : spdk_json_write_named_object_begin(w, "host");
3966 [ # # ]: 745 : spdk_json_write_named_string(w, "nqn", opts->hostnqn);
3967 [ # # ]: 745 : spdk_json_write_named_string(w, "addr", opts->src_addr);
3968 [ # # ]: 745 : spdk_json_write_named_string(w, "svcid", opts->src_svcid);
3969 : 745 : spdk_json_write_object_end(w);
3970 : :
3971 [ # # # # ]: 745 : numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr);
3972 [ + + ]: 745 : if (numa_id != SPDK_ENV_NUMA_ID_ANY) {
3973 : 357 : spdk_json_write_named_uint32(w, "numa_id", numa_id);
3974 : 0 : }
3975 : 745 : spdk_json_write_object_end(w);
3976 : 745 : }
3977 : :
3978 : : static void
3979 : 1114 : nvme_namespace_info_json(struct spdk_json_write_ctx *w,
3980 : : struct nvme_ns *nvme_ns)
3981 : 1112 : {
3982 : : struct spdk_nvme_ns *ns;
3983 : : struct spdk_nvme_ctrlr *ctrlr;
3984 : : const struct spdk_nvme_ctrlr_data *cdata;
3985 : : const struct spdk_nvme_transport_id *trid;
3986 : : union spdk_nvme_vs_register vs;
3987 : : const struct spdk_nvme_ns_data *nsdata;
3988 : 889 : char buf[128];
3989 : :
3990 [ # # # # ]: 1114 : ns = nvme_ns->ns;
3991 [ + + ]: 1114 : if (ns == NULL) {
3992 : 0 : return;
3993 : : }
3994 : :
3995 : 1114 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
3996 : :
3997 : 1114 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3998 : 1114 : trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
3999 : 1114 : vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
4000 : :
4001 : 1114 : spdk_json_write_object_begin(w);
4002 : :
4003 [ + + # # : 1114 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
4004 [ # # ]: 886 : spdk_json_write_named_string(w, "pci_address", trid->traddr);
4005 : 2 : }
4006 : :
4007 : 1114 : spdk_json_write_named_object_begin(w, "trid");
4008 : :
4009 : 1114 : nvme_bdev_dump_trid_json(trid, w);
4010 : :
4011 : 1114 : spdk_json_write_object_end(w);
4012 : :
4013 : : #ifdef SPDK_CONFIG_NVME_CUSE
4014 : 1112 : size_t cuse_name_size = 128;
4015 [ - + ]: 1112 : char cuse_name[cuse_name_size];
4016 : :
4017 : 1112 : int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns),
4018 : : cuse_name, &cuse_name_size);
4019 [ + + ]: 1112 : if (rc == 0) {
4020 : 4 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
4021 : 0 : }
4022 : : #endif
4023 : :
4024 : 1114 : spdk_json_write_named_object_begin(w, "ctrlr_data");
4025 : :
4026 [ # # # # ]: 1114 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
4027 : :
4028 [ # # # # ]: 1114 : spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);
4029 : :
4030 [ - + ]: 1114 : snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn);
4031 : 1114 : spdk_str_trim(buf);
4032 : 1114 : spdk_json_write_named_string(w, "model_number", buf);
4033 : :
4034 [ - + ]: 1114 : snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn);
4035 : 1114 : spdk_str_trim(buf);
4036 : 1114 : spdk_json_write_named_string(w, "serial_number", buf);
4037 : :
4038 [ - + ]: 1114 : snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr);
4039 : 1114 : spdk_str_trim(buf);
4040 : 1114 : spdk_json_write_named_string(w, "firmware_revision", buf);
4041 : :
4042 [ + + # # : 1114 : if (cdata->subnqn[0] != '\0') {
# # # # #
# ]
4043 [ # # ]: 371 : spdk_json_write_named_string(w, "subnqn", cdata->subnqn);
4044 : 2 : }
4045 : :
4046 : 1114 : spdk_json_write_named_object_begin(w, "oacs");
4047 : :
4048 [ # # # # ]: 1114 : spdk_json_write_named_uint32(w, "security", cdata->oacs.security);
4049 [ # # # # ]: 1114 : spdk_json_write_named_uint32(w, "format", cdata->oacs.format);
4050 [ # # # # ]: 1114 : spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware);
4051 [ # # # # ]: 1114 : spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage);
4052 : :
4053 : 1114 : spdk_json_write_object_end(w);
4054 : :
4055 [ # # # # ]: 1114 : spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr);
4056 [ # # # # ]: 1114 : spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting);
4057 : :
4058 : 1114 : spdk_json_write_object_end(w);
4059 : :
4060 : 1114 : spdk_json_write_named_object_begin(w, "vs");
4061 : :
4062 : 1114 : spdk_json_write_name(w, "nvme_version");
4063 [ + + ]: 1114 : if (vs.bits.ter) {
4064 : 0 : spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter);
4065 : 0 : } else {
4066 : 1114 : spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr);
4067 : : }
4068 : :
4069 : 1114 : spdk_json_write_object_end(w);
4070 : :
4071 : 1114 : nsdata = spdk_nvme_ns_get_data(ns);
4072 : :
4073 : 1114 : spdk_json_write_named_object_begin(w, "ns_data");
4074 : :
4075 : 1114 : spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns));
4076 : :
4077 [ + + # # : 1114 : if (cdata->cmic.ana_reporting) {
# # ]
4078 : 0 : spdk_json_write_named_string(w, "ana_state",
4079 [ # # # # ]: 0 : _nvme_ana_state_str(nvme_ns->ana_state));
4080 : 0 : }
4081 : :
4082 [ # # # # ]: 1114 : spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share);
4083 : :
4084 : 1114 : spdk_json_write_object_end(w);
4085 : :
4086 [ + + # # : 1114 : if (cdata->oacs.security) {
# # ]
4087 : 21 : spdk_json_write_named_object_begin(w, "security");
4088 : :
4089 [ - + # # : 21 : spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal);
# # # # #
# ]
4090 : :
4091 : 21 : spdk_json_write_object_end(w);
4092 : 0 : }
4093 : :
4094 : 1114 : spdk_json_write_object_end(w);
4095 : 2 : }
4096 : :
4097 : : static const char *
4098 : 1104 : nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev)
4099 : : {
4100 [ + - + # : 1104 : switch (nbdev->mp_policy) {
# # # ]
4101 : 1102 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
4102 : 1104 : return "active_passive";
4103 : 0 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
4104 : 0 : return "active_active";
4105 : 0 : default:
4106 [ # # ]: 0 : assert(false);
4107 : : return "invalid";
4108 : : }
4109 : 2 : }
4110 : :
4111 : : static const char *
4112 : 0 : nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev)
4113 : : {
4114 [ # # # # : 0 : switch (nbdev->mp_selector) {
# # # ]
4115 : 0 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
4116 : 0 : return "round_robin";
4117 : 0 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
4118 : 0 : return "queue_depth";
4119 : 0 : default:
4120 [ # # ]: 0 : assert(false);
4121 : : return "invalid";
4122 : : }
4123 : 0 : }
4124 : :
4125 : : static int
4126 : 1104 : bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
4127 : : {
4128 : 1104 : struct nvme_bdev *nvme_bdev = ctx;
4129 : : struct nvme_ns *nvme_ns;
4130 : :
4131 [ - + # # ]: 1104 : pthread_mutex_lock(&nvme_bdev->mutex);
4132 : 1104 : spdk_json_write_named_array_begin(w, "nvme");
4133 [ + + # # : 2218 : TAILQ_FOREACH(nvme_ns, &nvme_bdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
4134 : 1114 : nvme_namespace_info_json(w, nvme_ns);
4135 : 2 : }
4136 : 1104 : spdk_json_write_array_end(w);
4137 : 1104 : spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nvme_bdev));
4138 [ + + # # : 1104 : if (nvme_bdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
4139 : 0 : spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nvme_bdev));
4140 [ # # # # : 0 : if (nvme_bdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
4141 [ # # # # ]: 0 : spdk_json_write_named_uint32(w, "rr_min_io", nvme_bdev->rr_min_io);
4142 : 0 : }
4143 : 0 : }
4144 [ - + # # ]: 1104 : pthread_mutex_unlock(&nvme_bdev->mutex);
4145 : :
4146 : 1104 : return 0;
4147 : : }
4148 : :
4149 : : static void
4150 : 169 : bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
4151 : : {
4152 : : /* No config per bdev needed */
4153 : 169 : }
4154 : :
4155 : : static uint64_t
4156 : 0 : bdev_nvme_get_spin_time(struct spdk_io_channel *ch)
4157 : : {
4158 : 0 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
4159 : : struct nvme_io_path *io_path;
4160 : : struct nvme_poll_group *group;
4161 : 0 : uint64_t spin_time = 0;
4162 : :
4163 [ # # # # : 0 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
4164 [ # # # # : 0 : group = io_path->qpair->group;
# # # # ]
4165 : :
4166 [ # # # # : 0 : if (!group || !group->collect_spin_stat) {
# # # # #
# ]
4167 : 0 : continue;
4168 : : }
4169 : :
4170 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
4171 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
4172 [ # # # # ]: 0 : group->end_ticks = 0;
4173 : 0 : }
4174 : :
4175 [ # # # # ]: 0 : spin_time += group->spin_ticks;
4176 [ # # # # ]: 0 : group->start_ticks = 0;
4177 [ # # # # ]: 0 : group->spin_ticks = 0;
4178 : 0 : }
4179 : :
4180 [ # # ]: 0 : return (spin_time * 1000000ULL) / spdk_get_ticks_hz();
4181 : : }
4182 : :
4183 : : static void
4184 : 0 : bdev_nvme_reset_device_stat(void *ctx)
4185 : : {
4186 : 0 : struct nvme_bdev *nbdev = ctx;
4187 : :
4188 [ # # # # : 0 : if (nbdev->err_stat != NULL) {
# # ]
4189 [ # # # # : 0 : memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat));
# # ]
4190 : 0 : }
4191 : 0 : }
4192 : :
4193 : : /* JSON string should be lowercases and underscore delimited string. */
4194 : : static void
4195 : 24 : bdev_nvme_format_nvme_status(char *dst, const char *src)
4196 : : {
4197 : 0 : char tmp[256];
4198 : :
4199 : 24 : spdk_strcpy_replace(dst, 256, src, " - ", "_");
4200 : 24 : spdk_strcpy_replace(tmp, 256, dst, "-", "_");
4201 : 24 : spdk_strcpy_replace(dst, 256, tmp, " ", "_");
4202 : 24 : spdk_strlwr(dst);
4203 : 24 : }
4204 : :
4205 : : static void
4206 : 31 : bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w)
4207 : : {
4208 : 31 : struct nvme_bdev *nbdev = ctx;
4209 : 31 : struct spdk_nvme_status status = {};
4210 : : uint16_t sct, sc;
4211 : 0 : char status_json[256];
4212 : : const char *status_str;
4213 : :
4214 [ + + # # : 31 : if (nbdev->err_stat == NULL) {
# # ]
4215 : 19 : return;
4216 : : }
4217 : :
4218 : 12 : spdk_json_write_named_object_begin(w, "nvme_error");
4219 : :
4220 : 12 : spdk_json_write_named_object_begin(w, "status_type");
4221 [ + + ]: 108 : for (sct = 0; sct < 8; sct++) {
4222 [ + + # # : 96 : if (nbdev->err_stat->status_type[sct] == 0) {
# # # # #
# # # #
# ]
4223 : 84 : continue;
4224 : : }
4225 : 12 : status.sct = sct;
4226 : :
4227 : 12 : status_str = spdk_nvme_cpl_get_status_type_string(&status);
4228 [ - + # # ]: 12 : assert(status_str != NULL);
4229 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4230 : :
4231 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]);
# # # # #
# # # ]
4232 : 0 : }
4233 : 12 : spdk_json_write_object_end(w);
4234 : :
4235 : 12 : spdk_json_write_named_object_begin(w, "status_code");
4236 [ + + ]: 60 : for (sct = 0; sct < 4; sct++) {
4237 : 48 : status.sct = sct;
4238 [ + + ]: 12336 : for (sc = 0; sc < 256; sc++) {
4239 [ + + # # : 12288 : if (nbdev->err_stat->status[sct][sc] == 0) {
# # # # #
# # # # #
# # ]
4240 : 12276 : continue;
4241 : : }
4242 : 12 : status.sc = sc;
4243 : :
4244 : 12 : status_str = spdk_nvme_cpl_get_status_string(&status);
4245 [ - + # # ]: 12 : assert(status_str != NULL);
4246 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4247 : :
4248 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]);
# # # # #
# # # #
# ]
4249 : 0 : }
4250 : 0 : }
4251 : 12 : spdk_json_write_object_end(w);
4252 : :
4253 : 12 : spdk_json_write_object_end(w);
4254 : 0 : }
4255 : :
4256 : : static bool
4257 : 161469 : bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
4258 : : {
4259 : 161469 : struct nvme_bdev *nbdev = ctx;
4260 : : struct nvme_ns *nvme_ns;
4261 : : struct spdk_nvme_ctrlr *ctrlr;
4262 : :
4263 [ + + + + : 161469 : if (!g_opts.allow_accel_sequence) {
- + ]
4264 : 161385 : return false;
4265 : : }
4266 : :
4267 [ + + ]: 84 : switch (type) {
4268 : 8 : case SPDK_BDEV_IO_TYPE_WRITE:
4269 : : case SPDK_BDEV_IO_TYPE_READ:
4270 : 8 : break;
4271 : 76 : default:
4272 : 76 : return false;
4273 : : }
4274 : :
4275 [ # # # # : 8 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
4276 [ - + # # ]: 8 : assert(nvme_ns != NULL);
4277 : :
4278 [ # # # # : 8 : ctrlr = nvme_ns->ctrlr->ctrlr;
# # # # ]
4279 [ - + # # ]: 8 : assert(ctrlr != NULL);
4280 : :
4281 : 8 : return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED;
4282 : 1281 : }
4283 : :
4284 : : static const struct spdk_bdev_fn_table nvmelib_fn_table = {
4285 : : .destruct = bdev_nvme_destruct,
4286 : : .submit_request = bdev_nvme_submit_request,
4287 : : .io_type_supported = bdev_nvme_io_type_supported,
4288 : : .get_io_channel = bdev_nvme_get_io_channel,
4289 : : .dump_info_json = bdev_nvme_dump_info_json,
4290 : : .write_config_json = bdev_nvme_write_config_json,
4291 : : .get_spin_time = bdev_nvme_get_spin_time,
4292 : : .get_module_ctx = bdev_nvme_get_module_ctx,
4293 : : .get_memory_domains = bdev_nvme_get_memory_domains,
4294 : : .accel_sequence_supported = bdev_nvme_accel_sequence_supported,
4295 : : .reset_device_stat = bdev_nvme_reset_device_stat,
4296 : : .dump_device_stat_json = bdev_nvme_dump_device_stat_json,
4297 : : };
4298 : :
4299 : : typedef int (*bdev_nvme_parse_ana_log_page_cb)(
4300 : : const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
4301 : :
4302 : : static int
4303 : 596 : bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
4304 : : bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
4305 : : {
4306 : : struct spdk_nvme_ana_group_descriptor *copied_desc;
4307 : : uint8_t *orig_desc;
4308 : : uint32_t i, desc_size, copy_len;
4309 : 596 : int rc = 0;
4310 : :
4311 [ + + # # : 596 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
4312 : 0 : return -EINVAL;
4313 : : }
4314 : :
4315 [ # # # # ]: 596 : copied_desc = nvme_ctrlr->copied_ana_desc;
4316 : :
4317 [ # # # # : 596 : orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
# # ]
4318 [ # # # # ]: 596 : copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
4319 : :
4320 [ + + # # : 820 : for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) {
# # # # #
# ]
4321 [ - + - + ]: 696 : memcpy(copied_desc, orig_desc, copy_len);
4322 : :
4323 [ # # # # ]: 696 : rc = cb_fn(copied_desc, cb_arg);
4324 [ + + ]: 696 : if (rc != 0) {
4325 : 472 : break;
4326 : : }
4327 : :
4328 : 224 : desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
4329 [ # # # # ]: 224 : copied_desc->num_of_nsid * sizeof(uint32_t);
4330 [ # # ]: 224 : orig_desc += desc_size;
4331 : 224 : copy_len -= desc_size;
4332 : 30 : }
4333 : :
4334 : 596 : return rc;
4335 : 42 : }
4336 : :
4337 : : static int
4338 : 22 : nvme_ns_ana_transition_timedout(void *ctx)
4339 : : {
4340 : 22 : struct nvme_ns *nvme_ns = ctx;
4341 : :
4342 [ # # ]: 22 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4343 [ # # # # ]: 22 : nvme_ns->ana_transition_timedout = true;
4344 : :
4345 : 22 : return SPDK_POLLER_BUSY;
4346 : : }
4347 : :
4348 : : static void
4349 : 612 : _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns,
4350 : : const struct spdk_nvme_ana_group_descriptor *desc)
4351 : : {
4352 : : const struct spdk_nvme_ctrlr_data *cdata;
4353 : :
4354 [ # # # # : 612 : nvme_ns->ana_group_id = desc->ana_group_id;
# # # # ]
4355 [ # # # # : 612 : nvme_ns->ana_state = desc->ana_state;
# # ]
4356 [ # # # # ]: 612 : nvme_ns->ana_state_updating = false;
4357 : :
4358 [ + + + # : 612 : switch (nvme_ns->ana_state) {
# # # ]
4359 : 513 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
4360 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
4361 [ # # # # ]: 552 : nvme_ns->ana_transition_timedout = false;
4362 [ # # ]: 552 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4363 : 552 : break;
4364 : :
4365 : 50 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
4366 : : case SPDK_NVME_ANA_CHANGE_STATE:
4367 [ + + # # : 56 : if (nvme_ns->anatt_timer != NULL) {
# # ]
4368 : 18 : break;
4369 : : }
4370 : :
4371 [ # # # # : 38 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
4372 [ # # # # : 38 : nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout,
# # # # ]
4373 : : nvme_ns,
4374 : : cdata->anatt * SPDK_SEC_TO_USEC);
4375 : 38 : break;
4376 : 3 : default:
4377 : 4 : break;
4378 : : }
4379 : 612 : }
4380 : :
4381 : : static int
4382 : 564 : nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg)
4383 : : {
4384 : 564 : struct nvme_ns *nvme_ns = cb_arg;
4385 : : uint32_t i;
4386 : :
4387 [ + + # # : 564 : assert(nvme_ns->ns != NULL);
# # # # ]
4388 : :
4389 [ + + # # : 652 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
4390 [ + + # # : 560 : if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) {
# # # # #
# # # ]
4391 : 88 : continue;
4392 : : }
4393 : :
4394 : 472 : _nvme_ns_set_ana_state(nvme_ns, desc);
4395 : 472 : return 1;
4396 : : }
4397 : :
4398 : 92 : return 0;
4399 : 60 : }
4400 : :
4401 : : static int
4402 : 20 : nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid)
4403 : : {
4404 : 20 : int rc = 0;
4405 : 15 : struct spdk_uuid new_uuid, namespace_uuid;
4406 : 20 : char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'};
4407 : : /* This namespace UUID was generated using uuid_generate() method. */
4408 : 20 : const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"};
4409 : : int size;
4410 : :
4411 [ + + - + : 20 : assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN);
# # ]
4412 : :
4413 : 20 : spdk_uuid_set_null(&new_uuid);
4414 : 20 : spdk_uuid_set_null(&namespace_uuid);
4415 : :
4416 : 20 : size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid);
4417 [ + - - + ]: 20 : if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) {
4418 : 0 : return -EINVAL;
4419 : : }
4420 : :
4421 : 20 : spdk_uuid_parse(&namespace_uuid, namespace_str);
4422 : :
4423 : 20 : rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size);
4424 [ + + ]: 20 : if (rc == 0) {
4425 [ # # # # ]: 20 : memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid));
4426 : 5 : }
4427 : :
4428 : 20 : return rc;
4429 : 5 : }
4430 : :
4431 : : static int
4432 : 1481 : nvme_disk_create(struct spdk_bdev *disk, const char *base_name,
4433 : : struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns,
4434 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx)
4435 : : {
4436 : : const struct spdk_uuid *uuid;
4437 : : const uint8_t *nguid;
4438 : : const struct spdk_nvme_ctrlr_data *cdata;
4439 : : const struct spdk_nvme_ns_data *nsdata;
4440 : : const struct spdk_nvme_ctrlr_opts *opts;
4441 : : enum spdk_nvme_csi csi;
4442 : : uint32_t atomic_bs, phys_bs, bs;
4443 : 1481 : char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'};
4444 : : int rc;
4445 : :
4446 : 1481 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4447 : 1481 : csi = spdk_nvme_ns_get_csi(ns);
4448 : 1481 : opts = spdk_nvme_ctrlr_get_opts(ctrlr);
4449 : :
4450 [ + + - ]: 1481 : switch (csi) {
4451 : 1429 : case SPDK_NVME_CSI_NVM:
4452 [ + - + - ]: 1479 : disk->product_name = "NVMe disk";
4453 : 1479 : break;
4454 : 2 : case SPDK_NVME_CSI_ZNS:
4455 [ # # # # ]: 2 : disk->product_name = "NVMe ZNS disk";
4456 [ # # # # ]: 2 : disk->zoned = true;
4457 [ # # # # ]: 2 : disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
4458 [ # # # # : 2 : disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) /
# # # # ]
4459 : 2 : spdk_nvme_ns_get_extended_sector_size(ns);
4460 [ # # # # ]: 2 : disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns);
4461 [ # # # # ]: 2 : disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns);
4462 : 2 : break;
4463 : 0 : default:
4464 [ # # # # : 0 : if (bdev_opts->allow_unrecognized_csi) {
# # # # ]
4465 [ # # # # ]: 0 : disk->product_name = "NVMe Passthrough disk";
4466 : 0 : break;
4467 : : }
4468 : 0 : SPDK_ERRLOG("unsupported CSI: %u\n", csi);
4469 : 0 : return -ENOTSUP;
4470 : : }
4471 : :
4472 : 1481 : nguid = spdk_nvme_ns_get_nguid(ns);
4473 [ + + ]: 1481 : if (!nguid) {
4474 : 1099 : uuid = spdk_nvme_ns_get_uuid(ns);
4475 [ + + ]: 1099 : if (uuid) {
4476 [ # # ]: 360 : disk->uuid = *uuid;
4477 [ + + + + : 751 : } else if (g_opts.generate_uuids) {
+ - ]
4478 [ # # ]: 0 : spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0');
4479 [ # # ]: 0 : rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid);
4480 [ # # ]: 0 : if (rc < 0) {
4481 [ # # ]: 0 : SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc));
4482 : 0 : return rc;
4483 : : }
4484 : 0 : }
4485 : 50 : } else {
4486 [ # # # # : 382 : memcpy(&disk->uuid, nguid, sizeof(disk->uuid));
# # ]
4487 : : }
4488 : :
4489 [ + - + - ]: 1481 : disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns));
4490 [ + + + - : 1481 : if (!disk->name) {
- + ]
4491 : 0 : return -ENOMEM;
4492 : : }
4493 : :
4494 [ + - + - ]: 1481 : disk->write_cache = 0;
4495 [ + + + - : 1481 : if (cdata->vwc.present) {
+ - ]
4496 : : /* Enable if the Volatile Write Cache exists */
4497 [ # # # # ]: 1235 : disk->write_cache = 1;
4498 : 10 : }
4499 [ + + + - : 1481 : if (cdata->oncs.write_zeroes) {
+ - ]
4500 [ # # # # ]: 1243 : disk->max_write_zeroes = UINT16_MAX + 1;
4501 : 10 : }
4502 [ + - + - ]: 1481 : disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns);
4503 [ + - + - ]: 1481 : disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns);
4504 [ + - + - ]: 1481 : disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr);
4505 [ + - + - : 1481 : disk->ctratt.raw = cdata->ctratt.raw;
+ - + - +
- + - ]
4506 [ + - + - ]: 1481 : disk->nsid = spdk_nvme_ns_get_id(ns);
4507 : : /* NVMe driver will split one request into multiple requests
4508 : : * based on MDTS and stripe boundary, the bdev layer will use
4509 : : * max_segment_size and max_num_segments to split one big IO
4510 : : * into multiple requests, then small request can't run out
4511 : : * of NVMe internal requests data structure.
4512 : : */
4513 [ + - + + : 1481 : if (opts && opts->io_queue_requests) {
+ - - + ]
4514 [ + - + - : 1325 : disk->max_num_segments = opts->io_queue_requests / 2;
+ - + - +
- ]
4515 : 11 : }
4516 [ + + ]: 1481 : if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
4517 : : /* The nvme driver will try to split I/O that have too many
4518 : : * SGEs, but it doesn't work if that last SGE doesn't end on
4519 : : * an aggregate total that is block aligned. The bdev layer has
4520 : : * a more robust splitting framework, so use that instead for
4521 : : * this case. (See issue #3269.)
4522 : : */
4523 : 1235 : uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr);
4524 : :
4525 [ + + # # : 1235 : if (disk->max_num_segments == 0) {
# # ]
4526 [ # # # # ]: 0 : disk->max_num_segments = max_sges;
4527 : 0 : } else {
4528 [ - + # # : 1235 : disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges);
# # # # #
# # # #
# ]
4529 : : }
4530 : 10 : }
4531 [ + - + - ]: 1481 : disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns);
4532 : :
4533 : 1481 : nsdata = spdk_nvme_ns_get_data(ns);
4534 : 1481 : bs = spdk_nvme_ns_get_sector_size(ns);
4535 : 1481 : atomic_bs = bs;
4536 : 1481 : phys_bs = bs;
4537 [ + + + - : 1481 : if (nsdata->nabo == 0) {
- + ]
4538 [ + + + + : 1481 : if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) {
- + # # #
# # # ]
4539 [ # # # # : 8 : atomic_bs = bs * (1 + nsdata->nawupf);
# # ]
4540 : 0 : } else {
4541 [ + - + - : 1473 : atomic_bs = bs * (1 + cdata->awupf);
+ - ]
4542 : : }
4543 : 50 : }
4544 [ + + + - : 1481 : if (nsdata->nsfeat.optperf) {
+ - ]
4545 [ # # # # : 1243 : phys_bs = bs * (1 + nsdata->npwg);
# # ]
4546 : 10 : }
4547 [ - + + - : 1481 : disk->phys_blocklen = spdk_min(phys_bs, atomic_bs);
+ - ]
4548 : :
4549 [ + - + - ]: 1481 : disk->md_len = spdk_nvme_ns_get_md_size(ns);
4550 [ + + + - : 1481 : if (disk->md_len != 0) {
- + ]
4551 [ # # # # : 47 : disk->md_interleave = nsdata->flbas.extended;
# # # # ]
4552 [ # # # # ]: 47 : disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns);
4553 [ - + # # : 47 : if (disk->dif_type != SPDK_DIF_DISABLE) {
# # ]
4554 [ # # # # : 0 : disk->dif_is_head_of_md = nsdata->dps.md_start;
# # # # ]
4555 [ # # # # : 0 : disk->dif_check_flags = bdev_opts->prchk_flags;
# # # # ]
4556 [ # # # # ]: 0 : disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns);
4557 : 0 : }
4558 : 0 : }
4559 : :
4560 [ + + ]: 1481 : if (!(spdk_nvme_ctrlr_get_flags(ctrlr) &
4561 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) {
4562 [ + - + - ]: 1107 : disk->acwu = 0;
4563 [ + - # # : 424 : } else if (nsdata->nsfeat.ns_atomic_write_unit) {
# # ]
4564 [ # # # # : 374 : disk->acwu = nsdata->nacwu + 1; /* 0-based */
# # # # #
# ]
4565 : 0 : } else {
4566 [ # # # # : 0 : disk->acwu = cdata->acwu + 1; /* 0-based */
# # # # #
# ]
4567 : : }
4568 : :
4569 [ + + + - : 1481 : if (cdata->oncs.copy) {
+ - ]
4570 : : /* For now bdev interface allows only single segment copy */
4571 [ # # # # : 923 : disk->max_copy = nsdata->mssrl;
# # # # ]
4572 : 10 : }
4573 : :
4574 [ + - + - ]: 1481 : disk->ctxt = ctx;
4575 [ + - + - ]: 1481 : disk->fn_table = &nvmelib_fn_table;
4576 [ + - + - ]: 1481 : disk->module = &nvme_if;
4577 : :
4578 [ + - + - ]: 1481 : disk->numa.id_valid = 1;
4579 [ + - + - ]: 1481 : disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr);
4580 : :
4581 : 1481 : return 0;
4582 : 50 : }
4583 : :
4584 : : static struct nvme_bdev *
4585 : 1481 : nvme_bdev_alloc(void)
4586 : : {
4587 : : struct nvme_bdev *bdev;
4588 : : int rc;
4589 : :
4590 : 1481 : bdev = calloc(1, sizeof(*bdev));
4591 [ + + ]: 1481 : if (!bdev) {
4592 : 0 : SPDK_ERRLOG("bdev calloc() failed\n");
4593 : 0 : return NULL;
4594 : : }
4595 : :
4596 [ + + + + : 1481 : if (g_opts.nvme_error_stat) {
+ - ]
4597 [ # # # # ]: 12 : bdev->err_stat = calloc(1, sizeof(struct nvme_error_stat));
4598 [ - + # # : 12 : if (!bdev->err_stat) {
# # ]
4599 : 0 : SPDK_ERRLOG("err_stat calloc() failed\n");
4600 : 0 : free(bdev);
4601 : 0 : return NULL;
4602 : : }
4603 : 0 : }
4604 : :
4605 [ + + + - ]: 1481 : rc = pthread_mutex_init(&bdev->mutex, NULL);
4606 [ - + ]: 1481 : if (rc != 0) {
4607 [ # # # # ]: 0 : free(bdev->err_stat);
4608 : 0 : free(bdev);
4609 : 0 : return NULL;
4610 : : }
4611 : :
4612 [ - + - + ]: 1481 : bdev->ref = 1;
4613 [ - + - + ]: 1481 : bdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE;
4614 [ - + - + ]: 1481 : bdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN;
4615 [ - + - + ]: 1481 : bdev->rr_min_io = UINT32_MAX;
4616 [ - + - + : 1481 : TAILQ_INIT(&bdev->nvme_ns_list);
- + - + -
+ - + - +
- + ]
4617 : :
4618 : 1481 : return bdev;
4619 : 50 : }
4620 : :
4621 : : static int
4622 : 1481 : nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4623 : : {
4624 : : struct nvme_bdev *bdev;
4625 [ + - + - ]: 1481 : struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr;
4626 : : int rc;
4627 : :
4628 : 1481 : bdev = nvme_bdev_alloc();
4629 [ + + ]: 1481 : if (bdev == NULL) {
4630 : 0 : SPDK_ERRLOG("Failed to allocate NVMe bdev\n");
4631 : 0 : return -ENOMEM;
4632 : : }
4633 : :
4634 [ + - + - : 1481 : bdev->opal = nvme_ctrlr->opal_dev != NULL;
+ - + - ]
4635 : :
4636 [ + - + - : 1531 : rc = nvme_disk_create(&bdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr,
+ - + - +
- ]
4637 [ + - + - : 50 : nvme_ns->ns, &nvme_ctrlr->opts, bdev);
+ - ]
4638 [ - + ]: 1481 : if (rc != 0) {
4639 : 0 : SPDK_ERRLOG("Failed to create NVMe disk\n");
4640 : 0 : nvme_bdev_free(bdev);
4641 : 0 : return rc;
4642 : : }
4643 : :
4644 : 1531 : spdk_io_device_register(bdev,
4645 : : bdev_nvme_create_bdev_channel_cb,
4646 : : bdev_nvme_destroy_bdev_channel_cb,
4647 : : sizeof(struct nvme_bdev_channel),
4648 [ + - + - : 1481 : bdev->disk.name);
+ - ]
4649 : :
4650 [ + - + - ]: 1481 : nvme_ns->bdev = bdev;
4651 [ + - + - : 1481 : bdev->nsid = nvme_ns->id;
+ - + - ]
4652 [ + - + - : 1481 : TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4653 : :
4654 [ + - + - ]: 1481 : bdev->nbdev_ctrlr = nbdev_ctrlr;
4655 [ + - + - : 1481 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, bdev, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4656 : :
4657 [ + - ]: 1481 : rc = spdk_bdev_register(&bdev->disk);
4658 [ + + ]: 1481 : if (rc != 0) {
4659 : 6 : SPDK_ERRLOG("spdk_bdev_register() failed\n");
4660 : 6 : spdk_io_device_unregister(bdev, NULL);
4661 [ # # # # ]: 6 : nvme_ns->bdev = NULL;
4662 [ + + # # : 6 : TAILQ_REMOVE(&nbdev_ctrlr->bdevs, bdev, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4663 : 6 : nvme_bdev_free(bdev);
4664 : 6 : return rc;
4665 : : }
4666 : :
4667 : 1475 : return 0;
4668 : 50 : }
4669 : :
4670 : : static bool
4671 : 126 : bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
4672 : : {
4673 : : const struct spdk_nvme_ns_data *nsdata1, *nsdata2;
4674 : : const struct spdk_uuid *uuid1, *uuid2;
4675 : :
4676 : 126 : nsdata1 = spdk_nvme_ns_get_data(ns1);
4677 : 126 : nsdata2 = spdk_nvme_ns_get_data(ns2);
4678 : 126 : uuid1 = spdk_nvme_ns_get_uuid(ns1);
4679 : 126 : uuid2 = spdk_nvme_ns_get_uuid(ns2);
4680 : :
4681 [ + + - + : 196 : return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 &&
# # # # #
# ]
4682 [ + + + + : 122 : nsdata1->eui64 == nsdata2->eui64 &&
# # # # #
# ]
4683 [ + + + + ]: 118 : ((uuid1 == NULL && uuid2 == NULL) ||
4684 [ + + + - : 308 : (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) &&
+ + + + ]
4685 : 106 : spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2);
4686 : : }
4687 : :
4688 : : static bool
4689 : 56 : hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4690 : : struct spdk_nvme_ctrlr_opts *opts)
4691 : : {
4692 : : struct nvme_probe_skip_entry *entry;
4693 : :
4694 [ - + # # : 56 : TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) {
# # # # ]
4695 [ # # # # ]: 0 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
4696 : 0 : return false;
4697 : : }
4698 : 0 : }
4699 : :
4700 [ # # # # : 56 : opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst;
# # ]
4701 [ # # # # : 56 : opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight;
# # ]
4702 [ # # # # : 56 : opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight;
# # ]
4703 [ # # # # : 56 : opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight;
# # ]
4704 [ # # # # ]: 56 : opts->disable_read_ana_log_page = true;
4705 : :
4706 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr);
# # # # ]
4707 : :
4708 : 56 : return true;
4709 : 0 : }
4710 : :
4711 : : static void
4712 : 0 : nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
4713 : : {
4714 : 0 : struct nvme_ctrlr *nvme_ctrlr = ctx;
4715 : :
4716 [ # # # # : 0 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
4717 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Abort failed. Resetting controller. sc is %u, sct is %u.\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4718 : : cpl->status.sc, cpl->status.sct);
4719 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4720 [ # # # # : 0 : } else if (cpl->cdw0 & 0x1) {
# # ]
4721 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Specified command could not be aborted.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4722 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4723 : 0 : }
4724 : 0 : }
4725 : :
4726 : : static void
4727 : 0 : timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
4728 : : struct spdk_nvme_qpair *qpair, uint16_t cid)
4729 : : {
4730 : 0 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
4731 : : union spdk_nvme_csts_register csts;
4732 : : int rc;
4733 : :
4734 [ # # # # : 0 : assert(nvme_ctrlr->ctrlr == ctrlr);
# # # # ]
4735 : :
4736 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4737 : : ctrlr, qpair, cid);
4738 : :
4739 : : /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O
4740 : : * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we
4741 : : * would submit another fabrics cmd on the admin queue to read CSTS and check for its
4742 : : * completion recursively.
4743 : : */
4744 [ # # # # : 0 : if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) {
# # # # #
# # # #
# ]
4745 : 0 : csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr);
4746 [ # # ]: 0 : if (csts.bits.cfs) {
4747 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Controller Fatal Status, reset required\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4748 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4749 : 0 : return;
4750 : : }
4751 : 0 : }
4752 : :
4753 [ # # # # : 0 : switch (g_opts.action_on_timeout) {
# # ]
4754 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
4755 [ # # ]: 0 : if (qpair) {
4756 : : /* Don't send abort to ctrlr when ctrlr is not available. */
4757 [ # # # # ]: 0 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4758 [ # # ]: 0 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
4759 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4760 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Quit abort. Ctrlr is not available.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4761 : 0 : return;
4762 : : }
4763 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4764 : :
4765 : 0 : rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
4766 : 0 : nvme_abort_cpl, nvme_ctrlr);
4767 [ # # ]: 0 : if (rc == 0) {
4768 : 0 : return;
4769 : : }
4770 : :
4771 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to send abort. Resetting, rc is %d.\n", rc);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4772 : 0 : }
4773 : :
4774 : : /* FALLTHROUGH */
4775 : : case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
4776 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4777 : 0 : break;
4778 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
4779 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "No action for nvme controller timeout.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4780 : 0 : break;
4781 : 0 : default:
4782 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "An invalid timeout action value is found.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4783 : 0 : break;
4784 : : }
4785 : 0 : }
4786 : :
4787 : : static struct nvme_ns *
4788 : 1551 : nvme_ns_alloc(void)
4789 : : {
4790 : : struct nvme_ns *nvme_ns;
4791 : :
4792 : 1551 : nvme_ns = calloc(1, sizeof(struct nvme_ns));
4793 [ + + ]: 1551 : if (nvme_ns == NULL) {
4794 : 0 : return NULL;
4795 : : }
4796 : :
4797 [ + + + + ]: 1551 : if (g_opts.io_path_stat) {
4798 [ # # # # ]: 0 : nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
4799 [ # # # # : 0 : if (nvme_ns->stat == NULL) {
# # ]
4800 : 0 : free(nvme_ns);
4801 : 0 : return NULL;
4802 : : }
4803 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
4804 : 0 : }
4805 : :
4806 : 1551 : return nvme_ns;
4807 : 63 : }
4808 : :
4809 : : static void
4810 : 1551 : nvme_ns_free(struct nvme_ns *nvme_ns)
4811 : : {
4812 [ + - + - ]: 1551 : free(nvme_ns->stat);
4813 : 1551 : free(nvme_ns);
4814 : 1551 : }
4815 : :
4816 : : static void
4817 : 1551 : nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc)
4818 : : {
4819 [ + - + - ]: 1551 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4820 [ + - + - ]: 1551 : struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx;
4821 : :
4822 [ + + ]: 1551 : if (rc == 0) {
4823 [ - + - + ]: 1541 : nvme_ns->probe_ctx = NULL;
4824 : 1541 : nvme_ctrlr_get_ref(nvme_ctrlr);
4825 : 61 : } else {
4826 [ # # ]: 10 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4827 : 10 : nvme_ns_free(nvme_ns);
4828 : : }
4829 : :
4830 [ + + ]: 1551 : if (ctx) {
4831 [ + - ]: 1486 : ctx->populates_in_progress--;
4832 [ + + + - : 1486 : if (ctx->populates_in_progress == 0) {
+ - ]
4833 : 61 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
4834 : 12 : }
4835 : 62 : }
4836 : 1551 : }
4837 : :
4838 : : static void
4839 : 8 : bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i,
4840 : : struct nvme_bdev *nbdev,
4841 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4842 : : {
4843 : 8 : struct nvme_ns *nvme_ns = ctx;
4844 : : int rc;
4845 : :
4846 : 8 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
4847 [ - + ]: 8 : if (rc != 0) {
4848 : 0 : SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n");
4849 : 0 : }
4850 : :
4851 : 8 : nvme_bdev_for_each_channel_continue(i, rc);
4852 : 8 : }
4853 : :
4854 : : static void
4855 : 8 : bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i,
4856 : : struct nvme_bdev *nbdev,
4857 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4858 : : {
4859 : 8 : struct nvme_ns *nvme_ns = ctx;
4860 : : struct nvme_io_path *io_path;
4861 : :
4862 : 8 : io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns);
4863 [ + - ]: 8 : if (io_path != NULL) {
4864 : 8 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
4865 : 2 : }
4866 : :
4867 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
4868 : 8 : }
4869 : :
4870 : : static void
4871 : 0 : bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status)
4872 : : {
4873 : 0 : struct nvme_ns *nvme_ns = ctx;
4874 : :
4875 : 0 : nvme_ctrlr_populate_namespace_done(nvme_ns, -1);
4876 : 0 : }
4877 : :
4878 : : static void
4879 : 66 : bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
4880 : : {
4881 : 66 : struct nvme_ns *nvme_ns = ctx;
4882 : :
4883 [ + - ]: 66 : if (status == 0) {
4884 : 66 : nvme_ctrlr_populate_namespace_done(nvme_ns, 0);
4885 : 12 : } else {
4886 : : /* Delete the added io_paths and fail populating the namespace. */
4887 : 0 : nvme_bdev_for_each_channel(nbdev,
4888 : : bdev_nvme_delete_io_path,
4889 : 0 : nvme_ns,
4890 : : bdev_nvme_add_io_path_failed);
4891 : : }
4892 : 66 : }
4893 : :
4894 : : static int
4895 : 70 : nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns)
4896 : : {
4897 : : struct nvme_ns *tmp_ns;
4898 : : const struct spdk_nvme_ns_data *nsdata;
4899 : :
4900 [ # # # # ]: 70 : nsdata = spdk_nvme_ns_get_data(nvme_ns->ns);
4901 [ - + # # : 70 : if (!nsdata->nmic.can_share) {
# # ]
4902 : 0 : SPDK_ERRLOG("Namespace cannot be shared.\n");
4903 : 0 : return -EINVAL;
4904 : : }
4905 : :
4906 [ - + # # ]: 70 : pthread_mutex_lock(&bdev->mutex);
4907 : :
4908 [ # # # # : 70 : tmp_ns = TAILQ_FIRST(&bdev->nvme_ns_list);
# # ]
4909 [ - + # # ]: 70 : assert(tmp_ns != NULL);
4910 : :
4911 [ + - + + : 70 : if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) {
# # # # #
# # # # #
# # ]
4912 [ - + # # ]: 4 : pthread_mutex_unlock(&bdev->mutex);
4913 : 4 : SPDK_ERRLOG("Namespaces are not identical.\n");
4914 : 4 : return -EINVAL;
4915 : : }
4916 : :
4917 [ # # # # ]: 66 : bdev->ref++;
4918 [ # # # # : 66 : TAILQ_INSERT_TAIL(&bdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4919 [ # # # # ]: 66 : nvme_ns->bdev = bdev;
4920 : :
4921 [ - + # # ]: 66 : pthread_mutex_unlock(&bdev->mutex);
4922 : :
4923 : : /* Add nvme_io_path to nvme_bdev_channels dynamically. */
4924 : 78 : nvme_bdev_for_each_channel(bdev,
4925 : : bdev_nvme_add_io_path,
4926 : 12 : nvme_ns,
4927 : : bdev_nvme_add_io_path_done);
4928 : :
4929 : 66 : return 0;
4930 : 13 : }
4931 : :
4932 : : static void
4933 : 1551 : nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4934 : : {
4935 : : struct spdk_nvme_ns *ns;
4936 : : struct nvme_bdev *bdev;
4937 : 1551 : int rc = 0;
4938 : :
4939 [ + - + - : 1551 : ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id);
+ - + - ]
4940 [ - + ]: 1551 : if (!ns) {
4941 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "Invalid NS %d\n", nvme_ns->id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4942 : 0 : rc = -EINVAL;
4943 : 0 : goto done;
4944 : : }
4945 : :
4946 [ + - + - ]: 1551 : nvme_ns->ns = ns;
4947 [ + - + - ]: 1551 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
4948 : :
4949 [ + + + - : 1551 : if (nvme_ctrlr->ana_log_page != NULL) {
+ - ]
4950 : 476 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns);
4951 : 38 : }
4952 : :
4953 [ + - + - : 1551 : bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id);
+ - + - ]
4954 [ + + ]: 1602 : if (bdev == NULL) {
4955 : 1481 : rc = nvme_bdev_create(nvme_ctrlr, nvme_ns);
4956 : 50 : } else {
4957 : 70 : rc = nvme_bdev_add_ns(bdev, nvme_ns);
4958 [ + + ]: 70 : if (rc == 0) {
4959 : 66 : return;
4960 : : }
4961 : : }
4962 : 3 : done:
4963 : 1485 : nvme_ctrlr_populate_namespace_done(nvme_ns, rc);
4964 : 63 : }
4965 : :
4966 : : static void
4967 : 1541 : nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns)
4968 : : {
4969 [ + - + - ]: 1541 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4970 : :
4971 [ + + # # ]: 1541 : assert(nvme_ctrlr != NULL);
4972 : :
4973 [ + + + - ]: 1541 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4974 : :
4975 [ + - ]: 1541 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4976 : :
4977 [ + + + - : 1541 : if (nvme_ns->bdev != NULL) {
- + ]
4978 [ - + # # ]: 515 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4979 : 515 : return;
4980 : : }
4981 : :
4982 : 1026 : nvme_ns_free(nvme_ns);
4983 [ + + + - ]: 1026 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4984 : :
4985 : 1026 : nvme_ctrlr_put_ref(nvme_ctrlr);
4986 : 61 : }
4987 : :
4988 : : static void
4989 : 56 : bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
4990 : : {
4991 : 56 : struct nvme_ns *nvme_ns = ctx;
4992 : :
4993 : 56 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
4994 : 56 : }
4995 : :
4996 : : static void
4997 : 1541 : nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4998 : : {
4999 : : struct nvme_bdev *bdev;
5000 : :
5001 [ + - ]: 1541 : spdk_poller_unregister(&nvme_ns->anatt_timer);
5002 : :
5003 [ + - + - ]: 1541 : bdev = nvme_ns->bdev;
5004 [ + + ]: 1541 : if (bdev != NULL) {
5005 [ - + # # ]: 711 : pthread_mutex_lock(&bdev->mutex);
5006 : :
5007 [ + + # # : 711 : assert(bdev->ref > 0);
# # # # ]
5008 [ # # # # ]: 711 : bdev->ref--;
5009 [ + + # # : 711 : if (bdev->ref == 0) {
# # ]
5010 [ - + # # ]: 655 : pthread_mutex_unlock(&bdev->mutex);
5011 : :
5012 [ # # ]: 655 : spdk_bdev_unregister(&bdev->disk, NULL, NULL);
5013 : 38 : } else {
5014 : : /* spdk_bdev_unregister() is not called until the last nvme_ns is
5015 : : * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list
5016 : : * and clear nvme_ns->bdev here.
5017 : : */
5018 [ + + # # : 56 : TAILQ_REMOVE(&bdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5019 [ # # # # ]: 56 : nvme_ns->bdev = NULL;
5020 : :
5021 [ - + # # ]: 56 : pthread_mutex_unlock(&bdev->mutex);
5022 : :
5023 : : /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that,
5024 : : * we call depopulate_namespace_done() to avoid use-after-free.
5025 : : */
5026 : 67 : nvme_bdev_for_each_channel(bdev,
5027 : : bdev_nvme_delete_io_path,
5028 : 11 : nvme_ns,
5029 : : bdev_nvme_delete_io_path_done);
5030 : 56 : return;
5031 : : }
5032 : 38 : }
5033 : :
5034 : 1485 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
5035 : 61 : }
5036 : :
5037 : : static void
5038 : 1798 : nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
5039 : : struct nvme_async_probe_ctx *ctx)
5040 : : {
5041 [ + - + - ]: 1798 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5042 : : struct nvme_ns *nvme_ns, *next;
5043 : : struct spdk_nvme_ns *ns;
5044 : : struct nvme_bdev *bdev;
5045 : : uint32_t nsid;
5046 : : int rc;
5047 : : uint64_t num_sectors;
5048 : :
5049 [ + + ]: 1798 : if (ctx) {
5050 : : /* Initialize this count to 1 to handle the populate functions
5051 : : * calling nvme_ctrlr_populate_namespace_done() immediately.
5052 : : */
5053 [ + - + - ]: 1652 : ctx->populates_in_progress = 1;
5054 : 58 : }
5055 : :
5056 : : /* First loop over our existing namespaces and see if they have been
5057 : : * removed. */
5058 : 1798 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5059 [ + + ]: 1832 : while (nvme_ns != NULL) {
5060 : 34 : next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
5061 : :
5062 [ + + # # : 34 : if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
5063 : : /* NS is still there or added again. Its attributes may have changed. */
5064 [ # # # # ]: 21 : ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id);
5065 [ + + # # : 21 : if (nvme_ns->ns != ns) {
# # ]
5066 [ + + # # : 4 : assert(nvme_ns->ns == NULL);
# # # # ]
5067 [ # # # # ]: 4 : nvme_ns->ns = ns;
5068 [ + + - + : 4 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "NSID %u was added\n", nvme_ns->id);
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5069 : 1 : }
5070 : :
5071 : 21 : num_sectors = spdk_nvme_ns_get_num_sectors(ns);
5072 [ # # # # ]: 21 : bdev = nvme_ns->bdev;
5073 [ + + # # ]: 21 : assert(bdev != NULL);
5074 [ + + # # : 21 : if (bdev->disk.blockcnt != num_sectors) {
# # # # ]
5075 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5076 : : "NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n",
5077 : : nvme_ns->id,
5078 : : bdev->disk.name,
5079 : : bdev->disk.blockcnt,
5080 : : num_sectors);
5081 [ # # ]: 4 : rc = spdk_bdev_notify_blockcnt_change(&bdev->disk, num_sectors);
5082 [ + + ]: 4 : if (rc != 0) {
5083 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5084 : : "Could not change num blocks for nvme bdev: name %s, errno: %d.\n",
5085 : : bdev->disk.name, rc);
5086 : 0 : }
5087 : 1 : }
5088 : 3 : } else {
5089 : : /* Namespace was removed */
5090 : 13 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5091 : : }
5092 : :
5093 : 34 : nvme_ns = next;
5094 : : }
5095 : :
5096 : : /* Loop through all of the namespaces at the nvme level and see if any of them are new */
5097 : 1798 : nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5098 [ + + ]: 3370 : while (nsid != 0) {
5099 : 1572 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5100 : :
5101 [ + + ]: 1572 : if (nvme_ns == NULL) {
5102 : : /* Found a new one */
5103 : 1551 : nvme_ns = nvme_ns_alloc();
5104 [ + + ]: 1551 : if (nvme_ns == NULL) {
5105 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate namespace\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5106 : : /* This just fails to attach the namespace. It may work on a future attempt. */
5107 : 0 : continue;
5108 : : }
5109 : :
5110 [ + - + - ]: 1551 : nvme_ns->id = nsid;
5111 [ + - + - ]: 1551 : nvme_ns->ctrlr = nvme_ctrlr;
5112 : :
5113 [ + - + - ]: 1551 : nvme_ns->bdev = NULL;
5114 : :
5115 [ + + ]: 1551 : if (ctx) {
5116 [ + - ]: 1486 : ctx->populates_in_progress++;
5117 : 62 : }
5118 [ + - + - ]: 1551 : nvme_ns->probe_ctx = ctx;
5119 : :
5120 [ + - ]: 1551 : RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
5121 : :
5122 : 1551 : nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns);
5123 : 63 : }
5124 : :
5125 : 1572 : nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid);
5126 : : }
5127 : :
5128 [ + + ]: 1798 : if (ctx) {
5129 : : /* Decrement this count now that the loop is over to account
5130 : : * for the one we started with. If the count is then 0, we
5131 : : * know any populate_namespace functions completed immediately,
5132 : : * so we'll kick the callback here.
5133 : : */
5134 [ + - ]: 1652 : ctx->populates_in_progress--;
5135 [ + + + - : 1652 : if (ctx->populates_in_progress == 0) {
- + ]
5136 : 1591 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
5137 : 46 : }
5138 : 58 : }
5139 : :
5140 : 1798 : }
5141 : :
5142 : : static void
5143 : 1768 : nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr)
5144 : : {
5145 : : struct nvme_ns *nvme_ns, *tmp;
5146 : :
5147 [ + + + + : 3296 : RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) {
+ + ]
5148 : 1528 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5149 : 60 : }
5150 : 1768 : }
5151 : :
5152 : : static uint32_t
5153 : 3084 : nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr)
5154 : : {
5155 [ # # # # ]: 3084 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5156 : : const struct spdk_nvme_ctrlr_data *cdata;
5157 : 3084 : uint32_t nsid, ns_count = 0;
5158 : :
5159 : 3084 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5160 : :
5161 [ + + ]: 3129 : for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5162 [ + + ]: 6200 : nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
5163 : 3116 : ns_count++;
5164 : 45 : }
5165 : :
5166 [ # # # # ]: 3121 : return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5167 : 3084 : sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count *
5168 : : sizeof(uint32_t);
5169 : : }
5170 : :
5171 : : static int
5172 : 132 : nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
5173 : : void *cb_arg)
5174 : : {
5175 : 132 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
5176 : : struct nvme_ns *nvme_ns;
5177 : : uint32_t i, nsid;
5178 : :
5179 [ + + # # : 260 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
5180 [ # # # # : 128 : nsid = desc->nsid[i];
# # ]
5181 [ + + ]: 128 : if (nsid == 0) {
5182 : 0 : continue;
5183 : : }
5184 : :
5185 : 128 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5186 : :
5187 [ + + ]: 128 : if (nvme_ns == NULL) {
5188 : : /* Target told us that an inactive namespace had an ANA change */
5189 : 4 : continue;
5190 : : }
5191 : :
5192 : 124 : _nvme_ns_set_ana_state(nvme_ns, desc);
5193 : 5 : }
5194 : :
5195 : 132 : return 0;
5196 : : }
5197 : :
5198 : : static void
5199 : 2 : bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5200 : : {
5201 : : struct nvme_ns *nvme_ns;
5202 : :
5203 [ # # # # ]: 2 : spdk_free(nvme_ctrlr->ana_log_page);
5204 [ # # # # ]: 2 : nvme_ctrlr->ana_log_page = NULL;
5205 : :
5206 [ # # ]: 2 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5207 [ + + ]: 4 : nvme_ns != NULL;
5208 : 2 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
5209 [ # # # # ]: 2 : nvme_ns->ana_state_updating = false;
5210 [ # # # # ]: 2 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
5211 : 0 : }
5212 : 2 : }
5213 : :
5214 : : static void
5215 : 118 : nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl)
5216 : : {
5217 : 118 : struct nvme_ctrlr *nvme_ctrlr = ctx;
5218 : :
5219 [ + - + + : 118 : if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) {
+ + # # #
# # # # #
# # # # ]
5220 : 119 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states,
5221 : 3 : nvme_ctrlr);
5222 : 3 : } else {
5223 : 2 : bdev_nvme_disable_read_ana_log_page(nvme_ctrlr);
5224 : : }
5225 : :
5226 [ - + # # ]: 118 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5227 : :
5228 [ + + # # : 118 : assert(nvme_ctrlr->ana_log_page_updating == true);
# # ]
5229 [ # # ]: 118 : nvme_ctrlr->ana_log_page_updating = false;
5230 : :
5231 [ - + ]: 118 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
5232 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5233 : :
5234 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
5235 : 0 : } else {
5236 [ - + # # ]: 118 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5237 : :
5238 : 118 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
5239 : : }
5240 : 118 : }
5241 : :
5242 : : static int
5243 : 2636 : nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5244 : : {
5245 : : uint32_t ana_log_page_size;
5246 : : int rc;
5247 : :
5248 [ + + # # : 2636 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5249 : 0 : return -EINVAL;
5250 : : }
5251 : :
5252 : 2636 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5253 : :
5254 [ - + # # : 2636 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5255 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5256 : : "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
5257 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5258 : 0 : return -EINVAL;
5259 : : }
5260 : :
5261 [ - + # # ]: 2636 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5262 [ + + + + ]: 2636 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
5263 [ # # ]: 5 : nvme_ctrlr->ana_log_page_updating) {
5264 [ - + # # ]: 2518 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5265 : 2518 : return -EBUSY;
5266 : : }
5267 : :
5268 [ # # ]: 118 : nvme_ctrlr->ana_log_page_updating = true;
5269 [ - + # # ]: 118 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5270 : :
5271 [ # # # # ]: 121 : rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr,
5272 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5273 : : SPDK_NVME_GLOBAL_NS_TAG,
5274 [ # # # # ]: 118 : nvme_ctrlr->ana_log_page,
5275 : 3 : ana_log_page_size, 0,
5276 : : nvme_ctrlr_read_ana_log_page_done,
5277 : 3 : nvme_ctrlr);
5278 [ - + ]: 118 : if (rc != 0) {
5279 : 0 : nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL);
5280 : 0 : }
5281 : :
5282 : 118 : return rc;
5283 : 6 : }
5284 : :
5285 : : static void
5286 : 0 : dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
5287 : : {
5288 : 0 : }
5289 : :
5290 : : struct bdev_nvme_set_preferred_path_ctx {
5291 : : struct spdk_bdev_desc *desc;
5292 : : struct nvme_ns *nvme_ns;
5293 : : bdev_nvme_set_preferred_path_cb cb_fn;
5294 : : void *cb_arg;
5295 : : };
5296 : :
5297 : : static void
5298 : 12 : bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5299 : : {
5300 : 12 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5301 : :
5302 [ + + # # ]: 12 : assert(ctx != NULL);
5303 [ + + # # : 12 : assert(ctx->desc != NULL);
# # # # ]
5304 [ + + # # : 12 : assert(ctx->cb_fn != NULL);
# # # # ]
5305 : :
5306 [ # # # # ]: 12 : spdk_bdev_close(ctx->desc);
5307 : :
5308 [ # # # # : 12 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5309 : :
5310 : 12 : free(ctx);
5311 : 12 : }
5312 : :
5313 : : static void
5314 : 8 : _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i,
5315 : : struct nvme_bdev *nbdev,
5316 : : struct nvme_bdev_channel *nbdev_ch, void *_ctx)
5317 : : {
5318 : 8 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5319 : : struct nvme_io_path *io_path, *prev;
5320 : :
5321 : 8 : prev = NULL;
5322 [ + + # # : 12 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
5323 [ + + # # : 12 : if (io_path->nvme_ns == ctx->nvme_ns) {
# # # # #
# ]
5324 : 8 : break;
5325 : : }
5326 : 4 : prev = io_path;
5327 : 1 : }
5328 : :
5329 [ + - ]: 8 : if (io_path != NULL) {
5330 [ + + ]: 8 : if (prev != NULL) {
5331 [ - + # # : 4 : STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5332 [ - + # # : 4 : STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5333 : 1 : }
5334 : :
5335 : : /* We can set io_path to nbdev_ch->current_io_path directly here.
5336 : : * However, it needs to be conditional. To simplify the code,
5337 : : * just clear nbdev_ch->current_io_path and let find_io_path()
5338 : : * fill it.
5339 : : *
5340 : : * Automatic failback may be disabled. Hence even if the io_path is
5341 : : * already at the head, clear nbdev_ch->current_io_path.
5342 : : */
5343 : 8 : bdev_nvme_clear_current_io_path(nbdev_ch);
5344 : 2 : }
5345 : :
5346 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
5347 : 8 : }
5348 : :
5349 : : static struct nvme_ns *
5350 : 12 : bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid)
5351 : : {
5352 : : struct nvme_ns *nvme_ns, *prev;
5353 : : const struct spdk_nvme_ctrlr_data *cdata;
5354 : :
5355 : 12 : prev = NULL;
5356 [ + + # # : 24 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
5357 [ # # # # : 24 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
5358 : :
5359 [ + + # # : 24 : if (cdata->cntlid == cntlid) {
# # ]
5360 : 12 : break;
5361 : : }
5362 : 12 : prev = nvme_ns;
5363 : 3 : }
5364 : :
5365 [ + - + + ]: 12 : if (nvme_ns != NULL && prev != NULL) {
5366 [ + + # # : 8 : TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5367 [ + + # # : 8 : TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5368 : 2 : }
5369 : :
5370 : 12 : return nvme_ns;
5371 : : }
5372 : :
5373 : : /* This function supports only multipath mode. There is only a single I/O path
5374 : : * for each NVMe-oF controller. Hence, just move the matched I/O path to the
5375 : : * head of the I/O path list for each NVMe bdev channel.
5376 : : *
5377 : : * NVMe bdev channel may be acquired after completing this function. move the
5378 : : * matched namespace to the head of the namespace list for the NVMe bdev too.
5379 : : */
5380 : : void
5381 : 12 : bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
5382 : : bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg)
5383 : : {
5384 : : struct bdev_nvme_set_preferred_path_ctx *ctx;
5385 : : struct spdk_bdev *bdev;
5386 : : struct nvme_bdev *nbdev;
5387 : 12 : int rc = 0;
5388 : :
5389 [ + + # # ]: 12 : assert(cb_fn != NULL);
5390 : :
5391 : 12 : ctx = calloc(1, sizeof(*ctx));
5392 [ + + ]: 12 : if (ctx == NULL) {
5393 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5394 : 0 : rc = -ENOMEM;
5395 : 0 : goto err_alloc;
5396 : : }
5397 : :
5398 [ # # # # ]: 12 : ctx->cb_fn = cb_fn;
5399 [ # # # # ]: 12 : ctx->cb_arg = cb_arg;
5400 : :
5401 [ # # ]: 12 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5402 [ - + ]: 12 : if (rc != 0) {
5403 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5404 : 0 : goto err_open;
5405 : : }
5406 : :
5407 [ # # # # ]: 12 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5408 : :
5409 [ - + # # : 12 : if (bdev->module != &nvme_if) {
# # ]
5410 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5411 : 0 : rc = -ENODEV;
5412 : 0 : goto err_bdev;
5413 : : }
5414 : :
5415 : 12 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5416 : :
5417 [ - + # # ]: 12 : pthread_mutex_lock(&nbdev->mutex);
5418 : :
5419 [ # # # # ]: 12 : ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid);
5420 [ + + # # : 12 : if (ctx->nvme_ns == NULL) {
# # ]
5421 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
5422 : :
5423 : 0 : SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid);
5424 : 0 : rc = -ENODEV;
5425 : 0 : goto err_bdev;
5426 : : }
5427 : :
5428 [ - + # # ]: 12 : pthread_mutex_unlock(&nbdev->mutex);
5429 : :
5430 : 15 : nvme_bdev_for_each_channel(nbdev,
5431 : : _bdev_nvme_set_preferred_path,
5432 : 3 : ctx,
5433 : : bdev_nvme_set_preferred_path_done);
5434 : 12 : return;
5435 : :
5436 : 0 : err_bdev:
5437 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5438 : 0 : err_open:
5439 : 0 : free(ctx);
5440 : 0 : err_alloc:
5441 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5442 : 3 : }
5443 : :
5444 : : struct bdev_nvme_set_multipath_policy_ctx {
5445 : : struct spdk_bdev_desc *desc;
5446 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn;
5447 : : void *cb_arg;
5448 : : };
5449 : :
5450 : : static void
5451 : 16 : bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5452 : : {
5453 : 16 : struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx;
5454 : :
5455 [ + + # # ]: 16 : assert(ctx != NULL);
5456 [ + + # # : 16 : assert(ctx->desc != NULL);
# # # # ]
5457 [ + + # # : 16 : assert(ctx->cb_fn != NULL);
# # # # ]
5458 : :
5459 [ # # # # ]: 16 : spdk_bdev_close(ctx->desc);
5460 : :
5461 [ # # # # : 16 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5462 : :
5463 : 16 : free(ctx);
5464 : 16 : }
5465 : :
5466 : : static void
5467 : 8 : _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i,
5468 : : struct nvme_bdev *nbdev,
5469 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
5470 : : {
5471 [ # # # # : 8 : nbdev_ch->mp_policy = nbdev->mp_policy;
# # # # ]
5472 [ # # # # : 8 : nbdev_ch->mp_selector = nbdev->mp_selector;
# # # # ]
5473 [ # # # # : 8 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
# # # # ]
5474 : 8 : bdev_nvme_clear_current_io_path(nbdev_ch);
5475 : :
5476 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
5477 : 8 : }
5478 : :
5479 : : void
5480 : 16 : spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy,
5481 : : enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io,
5482 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg)
5483 : : {
5484 : : struct bdev_nvme_set_multipath_policy_ctx *ctx;
5485 : : struct spdk_bdev *bdev;
5486 : : struct nvme_bdev *nbdev;
5487 : : int rc;
5488 : :
5489 [ + + # # ]: 16 : assert(cb_fn != NULL);
5490 : :
5491 [ + + + ]: 16 : switch (policy) {
5492 : 3 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
5493 : 4 : break;
5494 [ + + - ]: 10 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
5495 [ + + + ]: 6 : switch (selector) {
5496 : 7 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
5497 [ + + ]: 8 : if (rr_min_io == UINT32_MAX) {
5498 : 4 : rr_min_io = 1;
5499 [ - + ]: 4 : } else if (rr_min_io == 0) {
5500 : 0 : rc = -EINVAL;
5501 : 0 : goto exit;
5502 : : }
5503 : 8 : break;
5504 : 3 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
5505 : 4 : break;
5506 : 0 : default:
5507 : 0 : rc = -EINVAL;
5508 : 0 : goto exit;
5509 : : }
5510 : 12 : break;
5511 : 0 : default:
5512 : 0 : rc = -EINVAL;
5513 : 0 : goto exit;
5514 : : }
5515 : :
5516 : 16 : ctx = calloc(1, sizeof(*ctx));
5517 [ + + ]: 16 : if (ctx == NULL) {
5518 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5519 : 0 : rc = -ENOMEM;
5520 : 0 : goto exit;
5521 : : }
5522 : :
5523 [ # # # # ]: 16 : ctx->cb_fn = cb_fn;
5524 [ # # # # ]: 16 : ctx->cb_arg = cb_arg;
5525 : :
5526 [ # # ]: 16 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5527 [ - + ]: 16 : if (rc != 0) {
5528 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5529 : 0 : rc = -ENODEV;
5530 : 0 : goto err_open;
5531 : : }
5532 : :
5533 [ # # # # ]: 16 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5534 [ - + # # : 16 : if (bdev->module != &nvme_if) {
# # ]
5535 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5536 : 0 : rc = -ENODEV;
5537 : 0 : goto err_module;
5538 : : }
5539 : 16 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5540 : :
5541 [ - + # # ]: 16 : pthread_mutex_lock(&nbdev->mutex);
5542 [ # # # # ]: 16 : nbdev->mp_policy = policy;
5543 [ # # # # ]: 16 : nbdev->mp_selector = selector;
5544 [ # # # # ]: 16 : nbdev->rr_min_io = rr_min_io;
5545 [ - + # # ]: 16 : pthread_mutex_unlock(&nbdev->mutex);
5546 : :
5547 : 19 : nvme_bdev_for_each_channel(nbdev,
5548 : : _bdev_nvme_set_multipath_policy,
5549 : 3 : ctx,
5550 : : bdev_nvme_set_multipath_policy_done);
5551 : 16 : return;
5552 : :
5553 : 0 : err_module:
5554 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5555 : 0 : err_open:
5556 : 0 : free(ctx);
5557 : 0 : exit:
5558 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5559 : 3 : }
5560 : :
5561 : : static void
5562 : 138 : aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
5563 : : {
5564 : 138 : struct nvme_ctrlr *nvme_ctrlr = arg;
5565 : : union spdk_nvme_async_event_completion event;
5566 : :
5567 [ + - - + : 138 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5568 : 0 : SPDK_WARNLOG("AER request execute failed\n");
5569 : 0 : return;
5570 : : }
5571 : :
5572 [ # # # # ]: 138 : event.raw = cpl->cdw0;
5573 [ + - + + ]: 138 : if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5574 [ + + ]: 138 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
5575 : 34 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL);
5576 [ + - - + ]: 106 : } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5577 [ + - ]: 104 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
5578 : 104 : nvme_ctrlr_read_ana_log_page(nvme_ctrlr);
5579 : 1 : }
5580 : 3 : }
5581 : :
5582 : : static void
5583 : 1756 : free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx)
5584 : : {
5585 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.tls_psk);
+ - ]
5586 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.dhchap_key);
+ - ]
5587 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key);
+ - ]
5588 : 1756 : free(ctx);
5589 : 1756 : }
5590 : :
5591 : : static void
5592 : 1747 : populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc)
5593 : : {
5594 [ + + + - : 1747 : if (ctx->cb_fn) {
+ - ]
5595 [ + - + - : 1747 : ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc);
- + + - +
- + - + -
+ - ]
5596 : 64 : }
5597 : :
5598 [ + - + - ]: 1747 : ctx->namespaces_populated = true;
5599 [ + + + + : 1747 : if (ctx->probe_done) {
+ - + - ]
5600 : : /* The probe was already completed, so we need to free the context
5601 : : * here. This can happen for cases like OCSSD, where we need to
5602 : : * send additional commands to the SSD after attach.
5603 : : */
5604 : 514 : free_nvme_async_probe_ctx(ctx);
5605 : 32 : }
5606 : 1747 : }
5607 : :
5608 : : static int
5609 : 36865 : bdev_nvme_remove_poller(void *ctx)
5610 : : {
5611 : 15288 : struct spdk_nvme_transport_id trid_pcie;
5612 : :
5613 [ + + ]: 36865 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
5614 : 869 : spdk_poller_unregister(&g_hotplug_poller);
5615 : 869 : return SPDK_POLLER_IDLE;
5616 : : }
5617 : :
5618 [ + + ]: 35996 : memset(&trid_pcie, 0, sizeof(trid_pcie));
5619 : 35996 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
5620 : :
5621 [ + + ]: 35996 : if (spdk_nvme_scan_attached(&trid_pcie)) {
5622 [ # # # # ]: 0 : SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n");
5623 : 0 : }
5624 : :
5625 : 35996 : return SPDK_POLLER_BUSY;
5626 : 271 : }
5627 : :
5628 : : static void
5629 : 1764 : nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr,
5630 : : struct nvme_async_probe_ctx *ctx)
5631 : : {
5632 [ + - + - : 1764 : struct spdk_nvme_transport_id *trid = &nvme_ctrlr->active_path_id->trid;
+ - ]
5633 : :
5634 [ + + + - : 1764 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
- + ]
5635 [ + + + + : 1216 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created to %s:%s\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5636 : : trid->traddr, trid->trsvcid);
5637 : 61 : } else {
5638 [ + + + + : 548 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5639 : : }
5640 : :
5641 : 1836 : spdk_io_device_register(nvme_ctrlr,
5642 : : bdev_nvme_create_ctrlr_channel_cb,
5643 : : bdev_nvme_destroy_ctrlr_channel_cb,
5644 : : sizeof(struct nvme_ctrlr_channel),
5645 [ + - + - : 1764 : nvme_ctrlr->nbdev_ctrlr->name);
+ - + - ]
5646 : :
5647 : 1764 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx);
5648 : :
5649 [ + + ]: 1764 : if (g_hotplug_poller == NULL) {
5650 : 1261 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL,
5651 : : NVME_HOTPLUG_POLL_PERIOD_DEFAULT);
5652 : 13 : }
5653 : 1764 : }
5654 : :
5655 : : static void
5656 : 448 : nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl)
5657 : : {
5658 : 448 : struct nvme_ctrlr *nvme_ctrlr = _ctx;
5659 [ # # # # ]: 448 : struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx;
5660 : :
5661 [ # # # # ]: 448 : nvme_ctrlr->probe_ctx = NULL;
5662 : :
5663 [ + - + + : 448 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5664 : 0 : nvme_ctrlr_delete(nvme_ctrlr);
5665 : :
5666 [ # # ]: 0 : if (ctx != NULL) {
5667 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
5668 : 0 : populate_namespaces_cb(ctx, -1);
5669 : 0 : }
5670 : 0 : return;
5671 : : }
5672 : :
5673 : 448 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
5674 : 31 : }
5675 : :
5676 : : static int
5677 : 448 : nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
5678 : : struct nvme_async_probe_ctx *ctx)
5679 : : {
5680 [ # # # # ]: 448 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5681 : : const struct spdk_nvme_ctrlr_data *cdata;
5682 : : uint32_t ana_log_page_size;
5683 : :
5684 : 448 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5685 : :
5686 : : /* Set buffer size enough to include maximum number of allowed namespaces. */
5687 [ # # # # ]: 479 : ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5688 [ # # # # ]: 448 : sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan *
5689 : : sizeof(uint32_t);
5690 : :
5691 [ # # # # ]: 448 : nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL,
5692 : : SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA);
5693 [ + + # # : 448 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5694 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate ANA log page buffer\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5695 : 0 : return -ENXIO;
5696 : : }
5697 : :
5698 : : /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned.
5699 : : * Hence copy each descriptor to a temporary area when parsing it.
5700 : : *
5701 : : * Allocate a buffer whose size is as large as ANA log page buffer because
5702 : : * we do not know the size of a descriptor until actually reading it.
5703 : : */
5704 [ # # # # ]: 448 : nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size);
5705 [ + + # # : 448 : if (nvme_ctrlr->copied_ana_desc == NULL) {
# # ]
5706 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate a buffer to parse ANA descriptor\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5707 : 0 : return -ENOMEM;
5708 : : }
5709 : :
5710 [ # # # # ]: 448 : nvme_ctrlr->max_ana_log_page_size = ana_log_page_size;
5711 : :
5712 [ # # # # ]: 448 : nvme_ctrlr->probe_ctx = ctx;
5713 : :
5714 : : /* Then, set the read size only to include the current active namespaces. */
5715 : 448 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5716 : :
5717 [ - + # # : 448 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5718 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5719 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5720 : 0 : return -EINVAL;
5721 : : }
5722 : :
5723 : 479 : return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
5724 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5725 : : SPDK_NVME_GLOBAL_NS_TAG,
5726 [ # # # # ]: 448 : nvme_ctrlr->ana_log_page,
5727 : 31 : ana_log_page_size, 0,
5728 : : nvme_ctrlr_init_ana_log_page_done,
5729 : 31 : nvme_ctrlr);
5730 : 31 : }
5731 : :
5732 : : /* hostnqn and subnqn were already verified before attaching a controller.
5733 : : * Hence check only the multipath capability and cntlid here.
5734 : : */
5735 : : static bool
5736 : 77 : bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr)
5737 : : {
5738 : : struct nvme_ctrlr *tmp;
5739 : : const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata;
5740 : :
5741 : 77 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5742 : :
5743 [ - + # # : 77 : if (!cdata->cmic.multi_ctrlr) {
# # ]
5744 [ # # # # ]: 0 : SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
5745 : 0 : return false;
5746 : : }
5747 : :
5748 [ + + # # : 158 : TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5749 [ # # # # ]: 85 : tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr);
5750 : :
5751 [ + + # # : 85 : if (!tmp_cdata->cmic.multi_ctrlr) {
# # ]
5752 [ # # # # : 0 : NVME_CTRLR_ERRLOG(tmp, "Ctrlr%u does not support multipath.\n", cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5753 : 0 : return false;
5754 : : }
5755 [ + + # # : 85 : if (cdata->cntlid == tmp_cdata->cntlid) {
# # # # #
# ]
5756 [ + - # # : 4 : NVME_CTRLR_ERRLOG(tmp, "cntlid %u are duplicated.\n", tmp_cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5757 : 4 : return false;
5758 : : }
5759 : 17 : }
5760 : :
5761 : 73 : return true;
5762 : 16 : }
5763 : :
5764 : :
5765 : : static int
5766 : 1768 : nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr)
5767 : : {
5768 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
5769 [ + - + - ]: 1768 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5770 : : struct nvme_ctrlr *nctrlr;
5771 : 1768 : int rc = 0;
5772 : :
5773 [ + + ]: 1768 : pthread_mutex_lock(&g_bdev_nvme_mutex);
5774 : :
5775 : 1768 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
5776 [ + + ]: 1768 : if (nbdev_ctrlr != NULL) {
5777 [ + + ]: 77 : if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) {
5778 : 4 : rc = -EINVAL;
5779 : 4 : goto exit;
5780 : : }
5781 [ + + # # : 154 : TAILQ_FOREACH(nctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5782 [ + + - + : 81 : if (nctrlr->opts.multipath != nvme_ctrlr->opts.multipath) {
- + # # #
# # # # #
# # # # ]
5783 : : /* All controllers with the same name must be configured the same
5784 : : * way, either for multipath or failover. If the configuration doesn't
5785 : : * match - report error.
5786 : : */
5787 : 0 : rc = -EINVAL;
5788 : 0 : goto exit;
5789 : : }
5790 : 17 : }
5791 : 15 : } else {
5792 : 1691 : nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr));
5793 [ + + ]: 1691 : if (nbdev_ctrlr == NULL) {
5794 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5795 : 0 : rc = -ENOMEM;
5796 : 0 : goto exit;
5797 : : }
5798 [ + + + - : 1691 : nbdev_ctrlr->name = strdup(name);
+ - ]
5799 [ + + + - : 1691 : if (nbdev_ctrlr->name == NULL) {
+ - ]
5800 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate name of nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5801 : 0 : free(nbdev_ctrlr);
5802 : 0 : goto exit;
5803 : : }
5804 [ + - + - : 1691 : TAILQ_INIT(&nbdev_ctrlr->ctrlrs);
+ - + - +
- + - + -
+ - ]
5805 [ + - + - : 1691 : TAILQ_INIT(&nbdev_ctrlr->bdevs);
+ - + - +
- + - + -
+ - ]
5806 [ + - + - : 1691 : TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
5807 : : }
5808 [ + - + - ]: 1764 : nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr;
5809 [ + - + - : 1764 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5810 : 1695 : exit:
5811 [ + + ]: 1768 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
5812 : 1768 : return rc;
5813 : : }
5814 : :
5815 : : static int
5816 : 1768 : nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
5817 : : const char *name,
5818 : : const struct spdk_nvme_transport_id *trid,
5819 : : struct nvme_async_probe_ctx *ctx)
5820 : : {
5821 : : struct nvme_ctrlr *nvme_ctrlr;
5822 : : struct nvme_path_id *path_id;
5823 : : const struct spdk_nvme_ctrlr_data *cdata;
5824 : 1768 : struct spdk_event_handler_opts opts = {
5825 : : .opts_size = SPDK_SIZEOF(&opts, fd_type),
5826 : : };
5827 : : uint64_t period;
5828 : : int fd, rc;
5829 : :
5830 : 1768 : nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
5831 [ + + ]: 1768 : if (nvme_ctrlr == NULL) {
5832 : 0 : SPDK_ERRLOG("Failed to allocate device struct\n");
5833 : 0 : return -ENOMEM;
5834 : : }
5835 : :
5836 [ + + + - ]: 1768 : rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL);
5837 [ - + ]: 1768 : if (rc != 0) {
5838 : 0 : free(nvme_ctrlr);
5839 : 0 : return rc;
5840 : : }
5841 : :
5842 [ + - + - : 1768 : TAILQ_INIT(&nvme_ctrlr->trids);
+ - + - +
- + - + -
+ - ]
5843 [ + - + - : 1768 : TAILQ_INIT(&nvme_ctrlr->pending_resets);
+ - + - +
- + - + -
+ - ]
5844 [ + - + - : 1768 : RB_INIT(&nvme_ctrlr->namespaces);
+ - ]
5845 : :
5846 : : /* Get another reference to the key, so the first one can be released from probe_ctx */
5847 [ + + ]: 1768 : if (ctx != NULL) {
5848 [ + + + - : 1656 : if (ctx->drv_opts.tls_psk != NULL) {
+ - + - ]
5849 [ # # # # ]: 42 : nvme_ctrlr->psk = spdk_keyring_get_key(
5850 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.tls_psk));
# # ]
5851 [ - + # # : 42 : if (nvme_ctrlr->psk == NULL) {
# # ]
5852 : : /* Could only happen if the key was removed in the meantime */
5853 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5854 : : spdk_key_get_name(ctx->drv_opts.tls_psk));
5855 : 0 : rc = -ENOKEY;
5856 : 0 : goto err;
5857 : : }
5858 : 0 : }
5859 : :
5860 [ + + + - : 1656 : if (ctx->drv_opts.dhchap_key != NULL) {
+ - + - ]
5861 [ # # # # ]: 628 : nvme_ctrlr->dhchap_key = spdk_keyring_get_key(
5862 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_key));
# # ]
5863 [ - + # # : 628 : if (nvme_ctrlr->dhchap_key == NULL) {
# # ]
5864 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5865 : : spdk_key_get_name(ctx->drv_opts.dhchap_key));
5866 : 0 : rc = -ENOKEY;
5867 : 0 : goto err;
5868 : : }
5869 : 0 : }
5870 : :
5871 [ + + + - : 1656 : if (ctx->drv_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
5872 [ # # # # ]: 484 : nvme_ctrlr->dhchap_ctrlr_key =
5873 : 484 : spdk_keyring_get_key(
5874 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
# # ]
5875 [ - + # # : 484 : if (nvme_ctrlr->dhchap_ctrlr_key == NULL) {
# # ]
5876 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5877 : : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
5878 : 0 : rc = -ENOKEY;
5879 : 0 : goto err;
5880 : : }
5881 : 0 : }
5882 : 59 : }
5883 : :
5884 : : /* Check if we manage to enable interrupts on the controller. */
5885 [ + + + - : 1768 : if (spdk_interrupt_mode_is_enabled() && ctx != NULL && !ctx->drv_opts.enable_interrupts) {
- + - + #
# # # #
# ]
5886 : 0 : SPDK_ERRLOG("Failed to enable interrupts on the controller\n");
5887 : 0 : rc = -ENOTSUP;
5888 : 0 : goto err;
5889 : : }
5890 : :
5891 : 1768 : path_id = calloc(1, sizeof(*path_id));
5892 [ + + ]: 1768 : if (path_id == NULL) {
5893 : 0 : SPDK_ERRLOG("Failed to allocate trid entry pointer\n");
5894 : 0 : rc = -ENOMEM;
5895 : 0 : goto err;
5896 : : }
5897 : :
5898 [ + - ]: 1768 : path_id->trid = *trid;
5899 [ + + ]: 1768 : if (ctx != NULL) {
5900 [ + + + + : 1656 : memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr));
+ - + - +
- + - ]
5901 [ + + + + : 1656 : memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid));
+ - + - +
- + - ]
5902 : 59 : }
5903 [ + - + - ]: 1768 : nvme_ctrlr->active_path_id = path_id;
5904 [ + + + - : 1768 : TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link);
+ - + - +
- + - - +
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5905 : :
5906 [ + - + - ]: 1768 : nvme_ctrlr->thread = spdk_get_thread();
5907 [ + - + - ]: 1768 : nvme_ctrlr->ctrlr = ctrlr;
5908 [ + - + - ]: 1768 : nvme_ctrlr->ref = 1;
5909 : :
5910 [ - + ]: 1768 : if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) {
5911 : 0 : SPDK_ERRLOG("OCSSDs are not supported");
5912 : 0 : rc = -ENOTSUP;
5913 : 0 : goto err;
5914 : : }
5915 : :
5916 [ + + ]: 1768 : if (ctx != NULL) {
5917 [ + + + + : 1656 : memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));
+ - + - ]
5918 : 59 : } else {
5919 [ # # ]: 112 : spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
5920 : : }
5921 : :
5922 [ + + + - ]: 1768 : period = spdk_interrupt_mode_is_enabled() ? 0 : g_opts.nvme_adminq_poll_period_us;
5923 : :
5924 [ + - + - ]: 1768 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
5925 : : period);
5926 : :
5927 [ + + ]: 1768 : if (spdk_interrupt_mode_is_enabled()) {
5928 [ # # # # ]: 1 : spdk_poller_register_interrupt(nvme_ctrlr->adminq_timer_poller, NULL, NULL);
5929 : :
5930 [ # # # # ]: 1 : fd = spdk_nvme_ctrlr_get_admin_qp_fd(nvme_ctrlr->ctrlr, &opts);
5931 [ - + ]: 1 : if (fd < 0) {
5932 : 0 : rc = fd;
5933 : 0 : goto err;
5934 : : }
5935 : :
5936 [ # # # # ]: 1 : nvme_ctrlr->intr = SPDK_INTERRUPT_REGISTER_EXT(fd, bdev_nvme_poll_adminq,
5937 : : nvme_ctrlr, &opts);
5938 [ - + # # : 1 : if (!nvme_ctrlr->intr) {
# # ]
5939 : 0 : rc = -EINVAL;
5940 : 0 : goto err;
5941 : : }
5942 : 0 : }
5943 : :
5944 [ + + + - ]: 1768 : if (g_opts.timeout_us > 0) {
5945 : : /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
5946 : : /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */
5947 [ # # # # ]: 0 : uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ?
5948 [ # # # # ]: 0 : g_opts.timeout_us : g_opts.timeout_admin_us;
5949 [ # # ]: 0 : spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
5950 : 0 : adm_timeout_us, timeout_cb, nvme_ctrlr);
5951 : 0 : }
5952 : :
5953 : 1768 : spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr);
5954 : 1768 : spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr);
5955 : :
5956 [ + + ]: 1768 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
5957 : : SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
5958 [ # # # # ]: 36 : nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr);
5959 : 0 : }
5960 : :
5961 : 1768 : rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr);
5962 [ + + ]: 1768 : if (rc != 0) {
5963 : 4 : goto err;
5964 : : }
5965 : :
5966 : 1764 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5967 : :
5968 [ + + - + : 1764 : if (cdata->cmic.ana_reporting) {
- + ]
5969 : 448 : rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx);
5970 [ + - ]: 448 : if (rc == 0) {
5971 : 448 : return 0;
5972 : : }
5973 : 0 : } else {
5974 : 1316 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
5975 : 1316 : return 0;
5976 : : }
5977 : :
5978 : 3 : err:
5979 : 4 : nvme_ctrlr_delete(nvme_ctrlr);
5980 : 4 : return rc;
5981 : 73 : }
5982 : :
5983 : : void
5984 : 1707 : spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts)
5985 : : {
5986 [ + - + - ]: 1707 : opts->prchk_flags = 0;
5987 [ + - + - : 1707 : opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec;
+ - ]
5988 [ + - + - : 1707 : opts->reconnect_delay_sec = g_opts.reconnect_delay_sec;
+ - ]
5989 [ + - + - : 1707 : opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec;
+ - ]
5990 [ + - + - ]: 1707 : opts->multipath = true;
5991 : 1707 : }
5992 : :
5993 : : static void
5994 : 56 : attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
5995 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts)
5996 : : {
5997 : : char *name;
5998 : :
5999 [ # # ]: 56 : name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++);
6000 [ - + ]: 56 : if (!name) {
6001 : 0 : SPDK_ERRLOG("Failed to assign name to NVMe device\n");
6002 : 0 : return;
6003 : : }
6004 : :
6005 [ + - ]: 56 : if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) {
6006 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name);
# # # # ]
6007 : 0 : } else {
6008 [ # # ]: 0 : SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name);
6009 : : }
6010 : :
6011 : 56 : free(name);
6012 : 0 : }
6013 : :
6014 : : static void
6015 : 1764 : _nvme_ctrlr_destruct(void *ctx)
6016 : : {
6017 : 1764 : struct nvme_ctrlr *nvme_ctrlr = ctx;
6018 : :
6019 : 1764 : nvme_ctrlr_depopulate_namespaces(nvme_ctrlr);
6020 : 1764 : nvme_ctrlr_put_ref(nvme_ctrlr);
6021 : 1764 : }
6022 : :
6023 : : static int
6024 : 1027 : bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
6025 : : {
6026 : : struct nvme_probe_skip_entry *entry;
6027 : :
6028 : : /* The controller's destruction was already started */
6029 [ - + # # ]: 1027 : if (nvme_ctrlr->destruct) {
6030 : 0 : return -EALREADY;
6031 : : }
6032 : :
6033 [ + + + + : 1027 : if (!hotplug &&
# # ]
6034 [ + + # # : 979 : nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # # # #
# ]
6035 : 50 : entry = calloc(1, sizeof(*entry));
6036 [ - + ]: 50 : if (!entry) {
6037 : 0 : return -ENOMEM;
6038 : : }
6039 [ # # # # : 50 : entry->trid = nvme_ctrlr->active_path_id->trid;
# # # # ]
6040 [ # # # # : 50 : TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
6041 : 3 : }
6042 : :
6043 [ # # ]: 1027 : nvme_ctrlr->destruct = true;
6044 : 1027 : return 0;
6045 : 61 : }
6046 : :
6047 : : static int
6048 : 79 : bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
6049 : : {
6050 : : int rc;
6051 : :
6052 [ - + # # ]: 79 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6053 [ # # ]: 79 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug);
6054 [ - + # # ]: 79 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6055 : :
6056 [ + + ]: 79 : if (rc == 0) {
6057 : 79 : _nvme_ctrlr_destruct(nvme_ctrlr);
6058 [ # # ]: 2 : } else if (rc == -EALREADY) {
6059 : 0 : rc = 0;
6060 : 0 : }
6061 : :
6062 : 79 : return rc;
6063 : : }
6064 : :
6065 : : static void
6066 : 48 : remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
6067 : : {
6068 : 48 : struct nvme_ctrlr *nvme_ctrlr = cb_ctx;
6069 : :
6070 : 48 : bdev_nvme_delete_ctrlr(nvme_ctrlr, true);
6071 : 48 : }
6072 : :
6073 : : static int
6074 : 19973 : bdev_nvme_hotplug_probe(void *arg)
6075 : : {
6076 [ - + ]: 19973 : if (g_hotplug_probe_ctx == NULL) {
6077 : 0 : spdk_poller_unregister(&g_hotplug_probe_poller);
6078 : 0 : return SPDK_POLLER_IDLE;
6079 : : }
6080 : :
6081 [ + + ]: 19973 : if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) {
6082 : 3880 : g_hotplug_probe_ctx = NULL;
6083 : 3880 : spdk_poller_unregister(&g_hotplug_probe_poller);
6084 : 0 : }
6085 : :
6086 : 19973 : return SPDK_POLLER_BUSY;
6087 : 0 : }
6088 : :
6089 : : static int
6090 : 4027 : bdev_nvme_hotplug(void *arg)
6091 : : {
6092 : 1959 : struct spdk_nvme_transport_id trid_pcie;
6093 : :
6094 [ + + ]: 4027 : if (g_hotplug_probe_ctx) {
6095 : 147 : return SPDK_POLLER_BUSY;
6096 : : }
6097 : :
6098 [ - + ]: 3880 : memset(&trid_pcie, 0, sizeof(trid_pcie));
6099 : 3880 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
6100 : :
6101 : 3880 : g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL,
6102 : : hotplug_probe_cb, attach_cb, NULL);
6103 : :
6104 [ + - ]: 3880 : if (g_hotplug_probe_ctx) {
6105 [ - + # # ]: 3880 : assert(g_hotplug_probe_poller == NULL);
6106 : 3880 : g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000);
6107 : 0 : }
6108 : :
6109 : 3880 : return SPDK_POLLER_BUSY;
6110 : 0 : }
6111 : :
6112 : : void
6113 : 1032 : spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts, size_t opts_size)
6114 : : {
6115 [ + + ]: 1032 : if (!opts) {
6116 : 0 : SPDK_ERRLOG("opts should not be NULL\n");
6117 : 0 : return;
6118 : : }
6119 : :
6120 [ + + ]: 1032 : if (!opts_size) {
6121 : 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
6122 : 0 : return;
6123 : : }
6124 : :
6125 [ + - + - ]: 1032 : opts->opts_size = opts_size;
6126 : :
6127 : : #define SET_FIELD(field, defval) \
6128 : : opts->field = SPDK_GET_FIELD(&g_opts, field, defval, opts_size); \
6129 : :
6130 [ + + + - : 1032 : SET_FIELD(action_on_timeout, 0);
+ - + - ]
6131 [ + + + - : 1032 : SET_FIELD(keep_alive_timeout_ms, 0);
+ - + - ]
6132 [ + + + - : 1032 : SET_FIELD(timeout_us, 0);
+ - + - ]
6133 [ + + + - : 1032 : SET_FIELD(timeout_admin_us, 0);
+ - + - ]
6134 [ + + + - : 1032 : SET_FIELD(transport_retry_count, 0);
+ - + - ]
6135 [ + + + - : 1032 : SET_FIELD(arbitration_burst, 0);
+ - + - ]
6136 [ + + + - : 1032 : SET_FIELD(low_priority_weight, 0);
+ - + - ]
6137 [ + + + - : 1032 : SET_FIELD(medium_priority_weight, 0);
+ - + - ]
6138 [ + + + - : 1032 : SET_FIELD(high_priority_weight, 0);
+ - + - ]
6139 [ + + + - : 1032 : SET_FIELD(io_queue_requests, 0);
+ - + - ]
6140 [ + + + - : 1032 : SET_FIELD(nvme_adminq_poll_period_us, 0);
+ - + - ]
6141 [ + + + - : 1032 : SET_FIELD(nvme_ioq_poll_period_us, 0);
+ - + - ]
6142 [ + + + + : 1032 : SET_FIELD(delay_cmd_submit, 0);
+ - + - +
- ]
6143 [ + + + - : 1032 : SET_FIELD(bdev_retry_count, 0);
+ - + - ]
6144 [ + + + - : 1032 : SET_FIELD(ctrlr_loss_timeout_sec, 0);
+ - + - ]
6145 [ + + + - : 1032 : SET_FIELD(reconnect_delay_sec, 0);
+ - + - ]
6146 [ + + + - : 1032 : SET_FIELD(fast_io_fail_timeout_sec, 0);
+ - + - ]
6147 [ + + + - : 1032 : SET_FIELD(transport_ack_timeout, 0);
+ - + - ]
6148 [ + + + + : 1032 : SET_FIELD(disable_auto_failback, false);
+ + + - ]
6149 [ + + + + : 1032 : SET_FIELD(generate_uuids, false);
+ + + - +
- ]
6150 [ + + + - : 1032 : SET_FIELD(transport_tos, 0);
+ - ]
6151 [ + + + + : 1032 : SET_FIELD(nvme_error_stat, false);
+ + + - +
- ]
6152 [ + + + + : 1032 : SET_FIELD(io_path_stat, false);
+ + + - ]
6153 [ + + + + : 1032 : SET_FIELD(allow_accel_sequence, false);
+ + + - +
- ]
6154 [ + + + - : 1032 : SET_FIELD(rdma_srq_size, 0);
+ - + - ]
6155 [ + + + - : 1032 : SET_FIELD(rdma_max_cq_size, 0);
+ - + - ]
6156 [ + + + - : 1032 : SET_FIELD(rdma_cm_event_timeout_ms, 0);
+ - + - ]
6157 [ + + + - : 1032 : SET_FIELD(dhchap_digests, 0);
+ - + - ]
6158 [ + - + - : 1032 : SET_FIELD(dhchap_dhgroups, 0);
+ - + - ]
6159 : :
6160 : : #undef SET_FIELD
6161 : :
6162 : : /* Do not remove this statement, you should always update this statement when you adding a new field,
6163 : : * and do not forget to add the SET_FIELD statement for your added field. */
6164 : : SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_nvme_opts) == 120, "Incorrect size");
6165 : 43 : }
6166 : :
6167 : : static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6168 : : uint32_t reconnect_delay_sec,
6169 : : uint32_t fast_io_fail_timeout_sec);
6170 : :
6171 : : static int
6172 : 1032 : bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts)
6173 : : {
6174 [ + + + + : 1032 : if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) {
+ - + - +
- + - ]
6175 : : /* Can't set timeout_admin_us without also setting timeout_us */
6176 : 0 : SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n");
6177 : 0 : return -EINVAL;
6178 : : }
6179 : :
6180 [ + + + - : 1032 : if (opts->bdev_retry_count < -1) {
- + ]
6181 : 0 : SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n");
6182 : 0 : return -EINVAL;
6183 : : }
6184 : :
6185 [ + + + - : 1075 : if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec,
+ - ]
6186 [ + - + - ]: 1032 : opts->reconnect_delay_sec,
6187 [ + - + - ]: 1032 : opts->fast_io_fail_timeout_sec)) {
6188 : 0 : return -EINVAL;
6189 : : }
6190 : :
6191 : 1032 : return 0;
6192 : 43 : }
6193 : :
6194 : : int
6195 : 1032 : spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
6196 : : {
6197 [ + + ]: 1032 : if (!opts) {
6198 : 0 : SPDK_ERRLOG("opts cannot be NULL\n");
6199 : 0 : return -1;
6200 : : }
6201 : :
6202 [ + + + - : 1032 : if (!opts->opts_size) {
- + ]
6203 : 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
6204 : 0 : return -1;
6205 : : }
6206 : :
6207 : : int ret;
6208 : :
6209 : 1032 : ret = bdev_nvme_validate_opts(opts);
6210 [ - + ]: 1032 : if (ret) {
6211 : 0 : SPDK_WARNLOG("Failed to set nvme opts.\n");
6212 : 0 : return ret;
6213 : : }
6214 : :
6215 [ + + ]: 1032 : if (g_bdev_nvme_init_thread != NULL) {
6216 [ - + ]: 653 : if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
6217 : 0 : return -EPERM;
6218 : : }
6219 : 1 : }
6220 : :
6221 [ + - + + : 1075 : if (opts->rdma_srq_size != 0 ||
+ - - + ]
6222 [ + - + - : 1032 : opts->rdma_max_cq_size != 0 ||
+ - ]
6223 [ - + - + ]: 1032 : opts->rdma_cm_event_timeout_ms != 0) {
6224 : 0 : struct spdk_nvme_transport_opts drv_opts;
6225 : :
6226 : 0 : spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts));
6227 [ # # # # : 0 : if (opts->rdma_srq_size != 0) {
# # ]
6228 [ # # # # ]: 0 : drv_opts.rdma_srq_size = opts->rdma_srq_size;
6229 : 0 : }
6230 [ # # # # : 0 : if (opts->rdma_max_cq_size != 0) {
# # ]
6231 [ # # # # : 0 : drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size;
# # ]
6232 : 0 : }
6233 [ # # # # : 0 : if (opts->rdma_cm_event_timeout_ms != 0) {
# # ]
6234 [ # # # # : 0 : drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms;
# # ]
6235 : 0 : }
6236 : :
6237 : 0 : ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts));
6238 [ # # ]: 0 : if (ret) {
6239 : 0 : SPDK_ERRLOG("Failed to set NVMe transport opts.\n");
6240 : 0 : return ret;
6241 : : }
6242 : 0 : }
6243 : :
6244 : : #define SET_FIELD(field, defval) \
6245 : : g_opts.field = SPDK_GET_FIELD(opts, field, defval, opts->opts_size); \
6246 : :
6247 [ + - + - : 1032 : SET_FIELD(action_on_timeout, 0);
+ - + - +
- + - ]
6248 [ + - + - : 1032 : SET_FIELD(keep_alive_timeout_ms, 0);
+ - + - +
- + - ]
6249 [ + - + - : 1032 : SET_FIELD(timeout_us, 0);
+ - + - +
- + - ]
6250 [ + - + - : 1032 : SET_FIELD(timeout_admin_us, 0);
+ - + - +
- + - ]
6251 [ + - + - : 1032 : SET_FIELD(transport_retry_count, 0);
+ - + - +
- + - ]
6252 [ + - + - : 1032 : SET_FIELD(arbitration_burst, 0);
+ - + - +
- + - ]
6253 [ + - + - : 1032 : SET_FIELD(low_priority_weight, 0);
+ - + - +
- + - ]
6254 [ + - + - : 1032 : SET_FIELD(medium_priority_weight, 0);
+ - + - +
- + - ]
6255 [ + - + - : 1032 : SET_FIELD(high_priority_weight, 0);
+ - + - +
- + - ]
6256 [ + - + - : 1032 : SET_FIELD(io_queue_requests, 0);
+ - + - +
- + - ]
6257 [ + - + - : 1032 : SET_FIELD(nvme_adminq_poll_period_us, 0);
+ - + - +
- + - ]
6258 [ + - + - : 1032 : SET_FIELD(nvme_ioq_poll_period_us, 0);
+ - + - +
- + - ]
6259 [ + - + + : 1032 : SET_FIELD(delay_cmd_submit, 0);
+ - + - +
- + - +
- ]
6260 [ + - + - : 1032 : SET_FIELD(bdev_retry_count, 0);
+ - + - +
- + - ]
6261 [ + - + - : 1032 : SET_FIELD(ctrlr_loss_timeout_sec, 0);
+ - + - +
- + - ]
6262 [ + - + - : 1032 : SET_FIELD(reconnect_delay_sec, 0);
+ - + - +
- + - ]
6263 [ + - + - : 1032 : SET_FIELD(fast_io_fail_timeout_sec, 0);
+ - + - +
- + - ]
6264 [ + - + - : 1032 : SET_FIELD(transport_ack_timeout, 0);
+ - + - +
- + - ]
6265 [ + - + + : 1032 : SET_FIELD(disable_auto_failback, false);
+ + + - +
- + - ]
6266 [ + - + + : 1032 : SET_FIELD(generate_uuids, false);
+ + + - +
- + - +
- ]
6267 [ + - + - : 1032 : SET_FIELD(transport_tos, 0);
+ - + - +
- ]
6268 [ + - + + : 1032 : SET_FIELD(nvme_error_stat, false);
+ + + - +
- + - +
- ]
6269 [ + - + + : 1032 : SET_FIELD(io_path_stat, false);
+ + + - +
- + - ]
6270 [ + - + + : 1032 : SET_FIELD(allow_accel_sequence, false);
+ + + - +
- + - +
- ]
6271 [ + - + - : 1032 : SET_FIELD(rdma_srq_size, 0);
+ - + - +
- + - ]
6272 [ + - + - : 1032 : SET_FIELD(rdma_max_cq_size, 0);
+ - + - +
- + - ]
6273 [ + - + - : 1032 : SET_FIELD(rdma_cm_event_timeout_ms, 0);
+ - + - +
- + - ]
6274 [ + - + - : 1032 : SET_FIELD(dhchap_digests, 0);
+ - + - +
- + - ]
6275 [ + - + - : 1032 : SET_FIELD(dhchap_dhgroups, 0);
+ - + - +
- + - ]
6276 : :
6277 [ + - + - ]: 1032 : g_opts.opts_size = opts->opts_size;
6278 : :
6279 : : #undef SET_FIELD
6280 : :
6281 : 1032 : return 0;
6282 : 43 : }
6283 : :
6284 : : struct set_nvme_hotplug_ctx {
6285 : : uint64_t period_us;
6286 : : bool enabled;
6287 : : spdk_msg_fn fn;
6288 : : void *fn_ctx;
6289 : : };
6290 : :
6291 : : static void
6292 : 311 : set_nvme_hotplug_period_cb(void *_ctx)
6293 : : {
6294 : 311 : struct set_nvme_hotplug_ctx *ctx = _ctx;
6295 : :
6296 : 311 : spdk_poller_unregister(&g_hotplug_poller);
6297 [ + + + + : 311 : if (ctx->enabled) {
+ - - + ]
6298 [ # # # # ]: 12 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us);
6299 : 0 : } else {
6300 : 299 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL,
6301 : : NVME_HOTPLUG_POLL_PERIOD_DEFAULT);
6302 : : }
6303 : :
6304 [ + - + - ]: 311 : g_nvme_hotplug_poll_period_us = ctx->period_us;
6305 [ + + + - : 311 : g_nvme_hotplug_enabled = ctx->enabled;
+ - ]
6306 [ + + + - : 311 : if (ctx->fn) {
+ - ]
6307 [ + - + - : 311 : ctx->fn(ctx->fn_ctx);
- + + - +
- + - ]
6308 : 41 : }
6309 : :
6310 : 311 : free(ctx);
6311 : 311 : }
6312 : :
6313 : : int
6314 : 311 : bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx)
6315 : : {
6316 : : struct set_nvme_hotplug_ctx *ctx;
6317 : :
6318 [ + + - + : 311 : if (enabled == true && !spdk_process_is_primary()) {
# # ]
6319 : 0 : return -EPERM;
6320 : : }
6321 : :
6322 : 311 : ctx = calloc(1, sizeof(*ctx));
6323 [ + + ]: 311 : if (ctx == NULL) {
6324 : 0 : return -ENOMEM;
6325 : : }
6326 : :
6327 [ + + ]: 311 : period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
6328 [ + - - + : 311 : ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
- + ]
6329 [ - + - + : 311 : ctx->enabled = enabled;
- + ]
6330 [ - + - + ]: 311 : ctx->fn = cb;
6331 [ - + - + ]: 311 : ctx->fn_ctx = cb_ctx;
6332 : :
6333 : 311 : spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
6334 : 311 : return 0;
6335 : 41 : }
6336 : :
6337 : : static void
6338 : 1652 : nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
6339 : : struct nvme_async_probe_ctx *ctx)
6340 : : {
6341 : : struct nvme_ns *nvme_ns;
6342 : : struct nvme_bdev *nvme_bdev;
6343 : : size_t j;
6344 : :
6345 [ + + # # ]: 1652 : assert(nvme_ctrlr != NULL);
6346 : :
6347 [ + + + - : 1652 : if (ctx->names == NULL) {
+ - ]
6348 [ # # # # ]: 37 : ctx->reported_bdevs = 0;
6349 : 37 : populate_namespaces_cb(ctx, 0);
6350 : 37 : return;
6351 : : }
6352 : :
6353 : : /*
6354 : : * Report the new bdevs that were created in this call.
6355 : : * There can be more than one bdev per NVMe controller.
6356 : : */
6357 : 1615 : j = 0;
6358 : 1615 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6359 [ + + ]: 3039 : while (nvme_ns != NULL) {
6360 [ + - + - ]: 1424 : nvme_bdev = nvme_ns->bdev;
6361 [ + - + - : 1424 : if (j < ctx->max_bdevs) {
+ - ]
6362 [ + - + - : 1424 : ctx->names[j] = nvme_bdev->disk.name;
+ - + - +
- + - +
- ]
6363 : 1424 : j++;
6364 : 60 : } else {
6365 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6366 : : "Maximum number of namespaces supported per NVMe controller is %du. "
6367 : : "Unable to return all names of created bdevs\n",
6368 : : ctx->max_bdevs);
6369 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
6370 : 0 : populate_namespaces_cb(ctx, -ERANGE);
6371 : 0 : return;
6372 : : }
6373 : :
6374 : 1424 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6375 : : }
6376 : :
6377 [ + - + - ]: 1615 : ctx->reported_bdevs = j;
6378 : 1615 : populate_namespaces_cb(ctx, 0);
6379 : 58 : }
6380 : :
6381 : : static int
6382 : 52 : bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6383 : : struct spdk_nvme_ctrlr *new_ctrlr,
6384 : : struct spdk_nvme_transport_id *trid)
6385 : : {
6386 : : struct nvme_path_id *tmp_trid;
6387 : :
6388 [ - + # # : 52 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
6389 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "PCIe failover is not supported.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6390 : 0 : return -ENOTSUP;
6391 : : }
6392 : :
6393 : : /* Currently we only support failover to the same transport type. */
6394 [ - + # # : 52 : if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) {
# # # # #
# # # # #
# # ]
6395 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6396 : : "Failover from trtype: %s to a different trtype: %s is not supported currently\n",
6397 : : spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype),
6398 : : spdk_nvme_transport_id_trtype_str(trid->trtype));
6399 : 0 : return -EINVAL;
6400 : : }
6401 : :
6402 : :
6403 : : /* Currently we only support failover to the same NQN. */
6404 [ - + - + : 52 : if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
- + # # #
# # # # #
# # ]
6405 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6406 : : "Failover from subnqn: %s to a different subnqn: %s is not supported currently\n",
6407 : : nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn);
6408 : 0 : return -EINVAL;
6409 : : }
6410 : :
6411 : : /* Skip all the other checks if we've already registered this path. */
6412 [ + + # # : 124 : TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # #
# ]
6413 [ + + # # ]: 72 : if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) {
6414 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "This path (traddr: %s subnqn: %s) is already registered\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6415 : : trid->traddr, trid->subnqn);
6416 : 0 : return -EALREADY;
6417 : : }
6418 : 12 : }
6419 : :
6420 : 52 : return 0;
6421 : 9 : }
6422 : :
6423 : : static int
6424 : 52 : bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr,
6425 : : struct spdk_nvme_ctrlr *new_ctrlr)
6426 : : {
6427 : : struct nvme_ns *nvme_ns;
6428 : : struct spdk_nvme_ns *new_ns;
6429 : :
6430 : 52 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6431 [ + + ]: 68 : while (nvme_ns != NULL) {
6432 [ # # # # ]: 16 : new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id);
6433 [ - + # # ]: 16 : assert(new_ns != NULL);
6434 : :
6435 [ - + # # : 16 : if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) {
# # ]
6436 : 0 : return -EINVAL;
6437 : : }
6438 : :
6439 : 16 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6440 : : }
6441 : :
6442 : 52 : return 0;
6443 : 9 : }
6444 : :
6445 : : static int
6446 : 52 : _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6447 : : struct spdk_nvme_transport_id *trid)
6448 : : {
6449 : : struct nvme_path_id *active_id, *new_trid, *tmp_trid;
6450 : :
6451 : 52 : new_trid = calloc(1, sizeof(*new_trid));
6452 [ + + ]: 52 : if (new_trid == NULL) {
6453 : 0 : return -ENOMEM;
6454 : : }
6455 [ # # ]: 52 : new_trid->trid = *trid;
6456 : :
6457 [ # # # # ]: 52 : active_id = nvme_ctrlr->active_path_id;
6458 [ + + # # ]: 52 : assert(active_id != NULL);
6459 [ - + # # : 52 : assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
6460 : :
6461 : : /* Skip the active trid not to replace it until it is failed. */
6462 [ # # # # : 52 : tmp_trid = TAILQ_NEXT(active_id, link);
# # ]
6463 [ + + ]: 52 : if (tmp_trid == NULL) {
6464 : 32 : goto add_tail;
6465 : : }
6466 : :
6467 : : /* It means the trid is faled if its last failed time is non-zero.
6468 : : * Insert the new alternate trid before any failed trid.
6469 : : */
6470 [ - + + + : 32 : TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # # #
# # ]
6471 [ + + # # : 20 : if (tmp_trid->last_failed_tsc != 0) {
# # ]
6472 [ # # # # : 8 : TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6473 : 8 : return 0;
6474 : : }
6475 : 4 : }
6476 : :
6477 : 10 : add_tail:
6478 [ # # # # : 44 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6479 : 44 : return 0;
6480 : 9 : }
6481 : :
6482 : : /* This is the case that a secondary path is added to an existing
6483 : : * nvme_ctrlr for failover. After checking if it can access the same
6484 : : * namespaces as the primary path, it is disconnected until failover occurs.
6485 : : */
6486 : : static int
6487 : 52 : bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6488 : : struct spdk_nvme_ctrlr *new_ctrlr,
6489 : : struct spdk_nvme_transport_id *trid)
6490 : : {
6491 : : int rc;
6492 : :
6493 [ + + # # ]: 52 : assert(nvme_ctrlr != NULL);
6494 : :
6495 [ - + # # ]: 52 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6496 : :
6497 : 52 : rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid);
6498 [ - + ]: 52 : if (rc != 0) {
6499 : 0 : goto exit;
6500 : : }
6501 : :
6502 : 52 : rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr);
6503 [ - + ]: 52 : if (rc != 0) {
6504 : 0 : goto exit;
6505 : : }
6506 : :
6507 : 52 : rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid);
6508 : :
6509 : 43 : exit:
6510 [ - + # # ]: 52 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6511 : :
6512 : 52 : spdk_nvme_detach(new_ctrlr);
6513 : :
6514 : 52 : return rc;
6515 : : }
6516 : :
6517 : : static void
6518 : 1656 : connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6519 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
6520 : : {
6521 : 1656 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6522 : : struct nvme_async_probe_ctx *ctx;
6523 : : int rc;
6524 : :
6525 : 1656 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6526 [ + - + - ]: 1656 : ctx->ctrlr_attached = true;
6527 : :
6528 [ + - + - : 1656 : rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx);
+ - ]
6529 [ + + ]: 1656 : if (rc != 0) {
6530 [ # # # # ]: 4 : ctx->reported_bdevs = 0;
6531 : 4 : populate_namespaces_cb(ctx, rc);
6532 : 1 : }
6533 : 1656 : }
6534 : :
6535 : :
6536 : : static void
6537 : 32 : connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6538 : : struct spdk_nvme_ctrlr *ctrlr,
6539 : : const struct spdk_nvme_ctrlr_opts *opts)
6540 : : {
6541 : 32 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6542 : : struct nvme_ctrlr *nvme_ctrlr;
6543 : : struct nvme_async_probe_ctx *ctx;
6544 : : int rc;
6545 : :
6546 : 32 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6547 [ # # # # ]: 32 : ctx->ctrlr_attached = true;
6548 : :
6549 [ # # # # ]: 32 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6550 [ + - ]: 32 : if (nvme_ctrlr) {
6551 [ # # ]: 32 : rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid);
6552 : 4 : } else {
6553 : 0 : rc = -ENODEV;
6554 : : }
6555 : :
6556 [ # # # # ]: 32 : ctx->reported_bdevs = 0;
6557 : 32 : populate_namespaces_cb(ctx, rc);
6558 : 32 : }
6559 : :
6560 : : static int
6561 : 270994 : bdev_nvme_async_poll(void *arg)
6562 : : {
6563 : 270994 : struct nvme_async_probe_ctx *ctx = arg;
6564 : : int rc;
6565 : :
6566 [ + - + - ]: 270994 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
6567 [ + + ]: 270994 : if (spdk_unlikely(rc != -EAGAIN)) {
6568 [ + - + - ]: 1747 : ctx->probe_done = true;
6569 [ + - ]: 1747 : spdk_poller_unregister(&ctx->poller);
6570 [ + + + + : 1747 : if (!ctx->ctrlr_attached) {
+ - + - ]
6571 : : /* The probe is done, but no controller was attached.
6572 : : * That means we had a failure, so report -EIO back to
6573 : : * the caller (usually the RPC). populate_namespaces_cb()
6574 : : * will take care of freeing the nvme_async_probe_ctx.
6575 : : */
6576 [ # # # # ]: 59 : ctx->reported_bdevs = 0;
6577 : 59 : populate_namespaces_cb(ctx, -EIO);
6578 [ + + + + : 1689 : } else if (ctx->namespaces_populated) {
+ - - + ]
6579 : : /* The namespaces for the attached controller were all
6580 : : * populated and the response was already sent to the
6581 : : * caller (usually the RPC). So free the context here.
6582 : : */
6583 : 1233 : free_nvme_async_probe_ctx(ctx);
6584 : 32 : }
6585 : 64 : }
6586 : :
6587 : 270994 : return SPDK_POLLER_BUSY;
6588 : : }
6589 : :
6590 : : static bool
6591 : 2864 : bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6592 : : uint32_t reconnect_delay_sec,
6593 : : uint32_t fast_io_fail_timeout_sec)
6594 : : {
6595 [ + + ]: 2864 : if (ctrlr_loss_timeout_sec < -1) {
6596 : 4 : SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n");
6597 : 4 : return false;
6598 [ + + ]: 2860 : } else if (ctrlr_loss_timeout_sec == -1) {
6599 [ + + ]: 68 : if (reconnect_delay_sec == 0) {
6600 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6601 : 4 : return false;
6602 [ + + + + ]: 64 : } else if (fast_io_fail_timeout_sec != 0 &&
6603 : 3 : fast_io_fail_timeout_sec < reconnect_delay_sec) {
6604 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n");
6605 : 4 : return false;
6606 : : }
6607 [ + + ]: 2804 : } else if (ctrlr_loss_timeout_sec != 0) {
6608 [ + + ]: 72 : if (reconnect_delay_sec == 0) {
6609 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6610 : 4 : return false;
6611 [ + + ]: 68 : } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6612 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n");
6613 : 4 : return false;
6614 [ + + ]: 64 : } else if (fast_io_fail_timeout_sec != 0) {
6615 [ + + ]: 32 : if (fast_io_fail_timeout_sec < reconnect_delay_sec) {
6616 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n");
6617 : 4 : return false;
6618 [ + + ]: 28 : } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6619 : 4 : SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n");
6620 : 4 : return false;
6621 : : }
6622 : 4 : }
6623 [ + + + + ]: 2727 : } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) {
6624 : 8 : SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n");
6625 : 8 : return false;
6626 : : }
6627 : :
6628 : 2828 : return true;
6629 : 126 : }
6630 : :
6631 : : int
6632 : 1756 : spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
6633 : : const char *base_name,
6634 : : const char **names,
6635 : : uint32_t count,
6636 : : spdk_bdev_nvme_create_cb cb_fn,
6637 : : void *cb_ctx,
6638 : : struct spdk_nvme_ctrlr_opts *drv_opts,
6639 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts)
6640 : : {
6641 : : struct nvme_probe_skip_entry *entry, *tmp;
6642 : : struct nvme_async_probe_ctx *ctx;
6643 : : spdk_nvme_attach_cb attach_cb;
6644 : : struct nvme_ctrlr *nvme_ctrlr;
6645 : : int len;
6646 : :
6647 : : /* TODO expand this check to include both the host and target TRIDs.
6648 : : * Only if both are the same should we fail.
6649 : : */
6650 [ + + - + ]: 1756 : if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) {
6651 [ # # # # ]: 0 : SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) "
6652 : : "already exists.\n", trid->traddr, drv_opts->hostnqn);
6653 : 0 : return -EEXIST;
6654 : : }
6655 : :
6656 [ + + ]: 1756 : len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX);
6657 : :
6658 [ + - - + ]: 1756 : if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) {
6659 : 0 : SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1);
6660 : 0 : return -EINVAL;
6661 : : }
6662 : :
6663 [ + - + - ]: 1757 : if (bdev_opts != NULL &&
6664 [ + + + - ]: 1820 : !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec,
6665 [ + - + - ]: 64 : bdev_opts->reconnect_delay_sec,
6666 [ + - + - ]: 64 : bdev_opts->fast_io_fail_timeout_sec)) {
6667 : 0 : return -EINVAL;
6668 : : }
6669 : :
6670 : 1756 : ctx = calloc(1, sizeof(*ctx));
6671 [ + + ]: 1756 : if (!ctx) {
6672 : 0 : return -ENOMEM;
6673 : : }
6674 [ + - + - ]: 1756 : ctx->base_name = base_name;
6675 [ + - + - ]: 1756 : ctx->names = names;
6676 [ + - + - ]: 1756 : ctx->max_bdevs = count;
6677 [ + - + - ]: 1756 : ctx->cb_fn = cb_fn;
6678 [ + - + - ]: 1756 : ctx->cb_ctx = cb_ctx;
6679 [ + - ]: 1756 : ctx->trid = *trid;
6680 : :
6681 [ + + ]: 1756 : if (bdev_opts) {
6682 [ + + + + : 1756 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
+ - ]
6683 : 64 : } else {
6684 [ # # ]: 0 : spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts);
6685 : : }
6686 : :
6687 [ + + + - : 1756 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
- + ]
6688 [ + + + - : 492 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
# # # # +
- ]
6689 [ + - # # ]: 2 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
6690 [ - + # # : 2 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6691 : 2 : free(entry);
6692 : 2 : break;
6693 : : }
6694 : 0 : }
6695 : 11 : }
6696 : :
6697 [ + + + + : 1756 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
+ - ]
6698 [ + - + - : 1756 : ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count;
+ - + - ]
6699 [ + - + - : 1756 : ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout;
+ - + - ]
6700 [ + - + - : 1756 : ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms;
+ - + - ]
6701 [ + - + - : 1756 : ctx->drv_opts.disable_read_ana_log_page = true;
+ - ]
6702 [ + - + - : 1756 : ctx->drv_opts.transport_tos = g_opts.transport_tos;
+ - ]
6703 : :
6704 [ + + ]: 1756 : if (spdk_interrupt_mode_is_enabled()) {
6705 [ + - # # : 1 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
6706 [ # # # # : 1 : ctx->drv_opts.enable_interrupts = true;
# # ]
6707 : 0 : } else {
6708 : 0 : SPDK_ERRLOG("Interrupt mode is only supported with PCIe transport\n");
6709 : 0 : free_nvme_async_probe_ctx(ctx);
6710 : 0 : return -ENOTSUP;
6711 : : }
6712 : 0 : }
6713 : :
6714 [ + + + - : 1756 : if (ctx->bdev_opts.psk != NULL) {
+ - + - ]
6715 [ # # # # : 66 : ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk);
# # # # #
# # # ]
6716 [ + + # # : 66 : if (ctx->drv_opts.tls_psk == NULL) {
# # # # ]
6717 [ # # # # : 6 : SPDK_ERRLOG("Could not load PSK: %s\n", ctx->bdev_opts.psk);
# # ]
6718 : 6 : free_nvme_async_probe_ctx(ctx);
6719 : 6 : return -ENOKEY;
6720 : : }
6721 : 0 : }
6722 : :
6723 [ + + + - : 1750 : if (ctx->bdev_opts.dhchap_key != NULL) {
+ - + - ]
6724 [ # # # # : 664 : ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key);
# # # # #
# # # ]
6725 [ - + # # : 664 : if (ctx->drv_opts.dhchap_key == NULL) {
# # # # ]
6726 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n",
# # ]
6727 : : ctx->bdev_opts.dhchap_key);
6728 : 0 : free_nvme_async_probe_ctx(ctx);
6729 : 0 : return -ENOKEY;
6730 : : }
6731 : :
6732 [ # # # # : 664 : ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests;
# # # # ]
6733 [ # # # # : 664 : ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups;
# # # # ]
6734 : 0 : }
6735 [ + + + - : 1750 : if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
6736 [ # # # # : 500 : ctx->drv_opts.dhchap_ctrlr_key =
# # ]
6737 [ # # # # : 500 : spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key);
# # ]
6738 [ - + # # : 500 : if (ctx->drv_opts.dhchap_ctrlr_key == NULL) {
# # # # ]
6739 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n",
# # ]
6740 : : ctx->bdev_opts.dhchap_ctrlr_key);
6741 : 0 : free_nvme_async_probe_ctx(ctx);
6742 : 0 : return -ENOKEY;
6743 : : }
6744 : 0 : }
6745 : :
6746 [ + + + + : 1750 : if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || ctx->bdev_opts.multipath) {
+ + # # #
# # # ]
6747 : 1718 : attach_cb = connect_attach_cb;
6748 : 60 : } else {
6749 : 32 : attach_cb = connect_set_failover_cb;
6750 : : }
6751 : :
6752 [ + - + - ]: 1750 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6753 [ + + + + : 1750 : if (nvme_ctrlr && nvme_ctrlr->opts.multipath != ctx->bdev_opts.multipath) {
- + - + #
# # # # #
# # # # #
# ]
6754 : : /* All controllers with the same name must be configured the same
6755 : : * way, either for multipath or failover. If the configuration doesn't
6756 : : * match - report error.
6757 : : */
6758 : 0 : free_nvme_async_probe_ctx(ctx);
6759 : 0 : return -EINVAL;
6760 : : }
6761 : :
6762 [ + - + - : 1750 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb);
+ - ]
6763 [ + + + - : 1750 : if (ctx->probe_ctx == NULL) {
+ - ]
6764 [ # # ]: 3 : SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr);
6765 : 3 : free_nvme_async_probe_ctx(ctx);
6766 : 3 : return -ENODEV;
6767 : : }
6768 [ + - + - ]: 1747 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000);
6769 : :
6770 : 1747 : return 0;
6771 : 64 : }
6772 : :
6773 : : struct bdev_nvme_delete_ctx {
6774 : : char *name;
6775 : : struct nvme_path_id path_id;
6776 : : bdev_nvme_delete_done_fn delete_done;
6777 : : void *delete_done_ctx;
6778 : : uint64_t timeout_ticks;
6779 : : struct spdk_poller *poller;
6780 : : };
6781 : :
6782 : : static void
6783 : 706 : free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx)
6784 : : {
6785 [ + + ]: 706 : if (ctx != NULL) {
6786 [ # # # # ]: 702 : free(ctx->name);
6787 : 702 : free(ctx);
6788 : 4 : }
6789 : 706 : }
6790 : :
6791 : : static bool
6792 : 42905 : nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id)
6793 : : {
6794 [ + + # # : 42905 : if (path_id->trid.trtype != 0) {
# # # # ]
6795 [ - + # # : 226 : if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
# # # # ]
6796 [ # # # # : 0 : if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) {
# # # # #
# # # #
# ]
6797 : 0 : return false;
6798 : : }
6799 : 0 : } else {
6800 [ - + # # : 226 : if (path_id->trid.trtype != p->trid.trtype) {
# # # # #
# # # #
# ]
6801 : 0 : return false;
6802 : : }
6803 : : }
6804 : 21 : }
6805 : :
6806 [ + + # # : 42905 : if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) {
# # ]
6807 [ + + - + : 226 : if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) {
+ + # # #
# # # #
# ]
6808 : 44 : return false;
6809 : : }
6810 : 10 : }
6811 : :
6812 [ + + # # : 42861 : if (path_id->trid.adrfam != 0) {
# # # # ]
6813 [ - + # # : 142 : if (path_id->trid.adrfam != p->trid.adrfam) {
# # # # #
# # # #
# ]
6814 : 0 : return false;
6815 : : }
6816 : 0 : }
6817 : :
6818 [ + + # # : 42861 : if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) {
# # ]
6819 [ - + - + : 182 : if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) {
+ + # # #
# # # #
# ]
6820 : 49 : return false;
6821 : : }
6822 : 10 : }
6823 : :
6824 [ + + # # : 42812 : if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) {
# # ]
6825 [ - + - + : 133 : if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) {
- + # # #
# # # #
# ]
6826 : 0 : return false;
6827 : : }
6828 : 10 : }
6829 : :
6830 [ + + # # : 42812 : if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) {
# # ]
6831 [ # # # # : 0 : if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) {
# # # # #
# # # #
# ]
6832 : 0 : return false;
6833 : : }
6834 : 0 : }
6835 : :
6836 [ + + # # : 42812 : if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) {
# # ]
6837 [ # # # # : 0 : if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) {
# # # # #
# # # #
# ]
6838 : 0 : return false;
6839 : : }
6840 : 0 : }
6841 : :
6842 : 42812 : return true;
6843 : 94 : }
6844 : :
6845 : : static bool
6846 : 42526 : nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id)
6847 : : {
6848 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
6849 : : struct nvme_ctrlr *ctrlr;
6850 : : struct nvme_path_id *p;
6851 : :
6852 [ - + ]: 42526 : pthread_mutex_lock(&g_bdev_nvme_mutex);
6853 : 42526 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
6854 [ + + ]: 42526 : if (!nbdev_ctrlr) {
6855 [ - + ]: 692 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6856 : 692 : return false;
6857 : : }
6858 : :
6859 [ + + # # : 41860 : TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
6860 [ - + # # ]: 41850 : pthread_mutex_lock(&ctrlr->mutex);
6861 [ + + # # : 41880 : TAILQ_FOREACH(p, &ctrlr->trids, link) {
# # # # #
# # # #
# ]
6862 [ + + ]: 41854 : if (nvme_path_id_compare(p, path_id)) {
6863 [ - + # # ]: 41824 : pthread_mutex_unlock(&ctrlr->mutex);
6864 [ - + ]: 41824 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6865 : 41824 : return true;
6866 : : }
6867 : 0 : }
6868 [ - + # # ]: 26 : pthread_mutex_unlock(&ctrlr->mutex);
6869 : 0 : }
6870 [ - + ]: 10 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6871 : :
6872 : 10 : return false;
6873 : 20 : }
6874 : :
6875 : : static int
6876 : 42526 : bdev_nvme_delete_complete_poll(void *arg)
6877 : : {
6878 : 42526 : struct bdev_nvme_delete_ctx *ctx = arg;
6879 : 42526 : int rc = 0;
6880 : :
6881 [ + + # # : 42526 : if (nvme_path_id_exists(ctx->name, &ctx->path_id)) {
# # # # ]
6882 [ + - # # : 41824 : if (ctx->timeout_ticks > spdk_get_ticks()) {
# # ]
6883 : 41824 : return SPDK_POLLER_BUSY;
6884 : : }
6885 : :
6886 [ # # # # ]: 0 : SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name);
6887 : 0 : rc = -ETIMEDOUT;
6888 : 0 : }
6889 : :
6890 [ # # ]: 702 : spdk_poller_unregister(&ctx->poller);
6891 : :
6892 [ # # # # : 702 : ctx->delete_done(ctx->delete_done_ctx, rc);
# # # # #
# # # ]
6893 : 702 : free_bdev_nvme_delete_ctx(ctx);
6894 : :
6895 : 702 : return SPDK_POLLER_BUSY;
6896 : 20 : }
6897 : :
6898 : : static int
6899 : 999 : _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)
6900 : : {
6901 : : struct nvme_path_id *p, *t;
6902 : : spdk_msg_fn msg_fn;
6903 : 999 : int rc = -ENXIO;
6904 : :
6905 [ - + # # ]: 999 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6906 : :
6907 [ + + - + : 1051 : TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6908 [ + + # # : 1051 : if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) {
# # # # ]
6909 : 999 : break;
6910 : : }
6911 : :
6912 [ + + ]: 52 : if (!nvme_path_id_compare(p, path_id)) {
6913 : 20 : continue;
6914 : : }
6915 : :
6916 : : /* We are not using the specified path. */
6917 [ + + # # : 32 : TAILQ_REMOVE(&nvme_ctrlr->trids, p, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6918 : 32 : free(p);
6919 : 32 : rc = 0;
6920 : 7 : }
6921 : :
6922 [ + - + + ]: 999 : if (p == NULL || !nvme_path_id_compare(p, path_id)) {
6923 [ - + # # ]: 43 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6924 : 43 : return rc;
6925 : : }
6926 : :
6927 : : /* If we made it here, then this path is a match! Now we need to remove it. */
6928 : :
6929 : : /* This is the active path in use right now. The active path is always the first in the list. */
6930 [ + + # # : 956 : assert(p == nvme_ctrlr->active_path_id);
# # # # ]
6931 : :
6932 [ + + # # : 956 : if (!TAILQ_NEXT(p, link)) {
# # # # ]
6933 : : /* The current path is the only path. */
6934 : 948 : msg_fn = _nvme_ctrlr_destruct;
6935 : 948 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false);
6936 : 59 : } else {
6937 : : /* There is an alternative path. */
6938 : 8 : msg_fn = _bdev_nvme_reset_ctrlr;
6939 : 8 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true);
6940 : : }
6941 : :
6942 [ - + # # ]: 956 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6943 : :
6944 [ + + ]: 956 : if (rc == 0) {
6945 [ # # # # ]: 956 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
6946 [ # # ]: 60 : } else if (rc == -EALREADY) {
6947 : 0 : rc = 0;
6948 : 0 : }
6949 : :
6950 : 956 : return rc;
6951 : 68 : }
6952 : :
6953 : : int
6954 : 932 : bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,
6955 : : bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx)
6956 : : {
6957 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
6958 : : struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr;
6959 : 932 : struct bdev_nvme_delete_ctx *ctx = NULL;
6960 : 932 : int rc = -ENXIO, _rc;
6961 : :
6962 [ + - - + ]: 932 : if (name == NULL || path_id == NULL) {
6963 : 0 : rc = -EINVAL;
6964 : 0 : goto exit;
6965 : : }
6966 : :
6967 [ - + ]: 932 : pthread_mutex_lock(&g_bdev_nvme_mutex);
6968 : :
6969 : 932 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
6970 [ + + ]: 932 : if (nbdev_ctrlr == NULL) {
6971 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6972 : :
6973 : 0 : SPDK_ERRLOG("Failed to find NVMe bdev controller\n");
6974 : 0 : rc = -ENODEV;
6975 : 0 : goto exit;
6976 : : }
6977 : :
6978 [ + + + + : 1931 : TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {
# # # # #
# # # # #
# # ]
6979 : 999 : _rc = _bdev_nvme_delete(nvme_ctrlr, path_id);
6980 [ + + + + ]: 999 : if (_rc < 0 && _rc != -ENXIO) {
6981 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6982 : 0 : rc = _rc;
6983 : 0 : goto exit;
6984 [ + + ]: 999 : } else if (_rc == 0) {
6985 : : /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr
6986 : : * was deleted successfully. To remember the successful deletion,
6987 : : * overwrite rc only if _rc is zero.
6988 : : */
6989 : 968 : rc = 0;
6990 : 62 : }
6991 : 68 : }
6992 : :
6993 [ - + ]: 932 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6994 : :
6995 [ + + + + ]: 932 : if (rc != 0 || delete_done == NULL) {
6996 : 230 : goto exit;
6997 : : }
6998 : :
6999 : 702 : ctx = calloc(1, sizeof(*ctx));
7000 [ + + ]: 702 : if (ctx == NULL) {
7001 : 0 : SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n");
7002 : 0 : rc = -ENOMEM;
7003 : 0 : goto exit;
7004 : : }
7005 : :
7006 [ - + # # : 702 : ctx->name = strdup(name);
# # ]
7007 [ + + # # : 702 : if (ctx->name == NULL) {
# # ]
7008 : 0 : SPDK_ERRLOG("Failed to copy controller name for deletion\n");
7009 : 0 : rc = -ENOMEM;
7010 : 0 : goto exit;
7011 : : }
7012 : :
7013 [ # # # # ]: 702 : ctx->delete_done = delete_done;
7014 [ # # # # ]: 702 : ctx->delete_done_ctx = delete_done_ctx;
7015 [ # # ]: 702 : ctx->path_id = *path_id;
7016 [ # # # # ]: 702 : ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz();
7017 [ # # # # ]: 702 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000);
7018 [ + + # # : 702 : if (ctx->poller == NULL) {
# # ]
7019 : 0 : SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n");
7020 : 0 : rc = -ENOMEM;
7021 : 0 : goto exit;
7022 : : }
7023 : :
7024 : 698 : exit:
7025 [ + + ]: 932 : if (rc != 0) {
7026 : 4 : free_bdev_nvme_delete_ctx(ctx);
7027 : 1 : }
7028 : :
7029 : 932 : return rc;
7030 : : }
7031 : :
7032 : : #define DISCOVERY_INFOLOG(ctx, format, ...) \
7033 : : SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
7034 : :
7035 : : #define DISCOVERY_ERRLOG(ctx, format, ...) \
7036 : : SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
7037 : :
7038 : : struct discovery_entry_ctx {
7039 : : char name[128];
7040 : : struct spdk_nvme_transport_id trid;
7041 : : struct spdk_nvme_ctrlr_opts drv_opts;
7042 : : struct spdk_nvmf_discovery_log_page_entry entry;
7043 : : TAILQ_ENTRY(discovery_entry_ctx) tailq;
7044 : : struct discovery_ctx *ctx;
7045 : : };
7046 : :
7047 : : struct discovery_ctx {
7048 : : char *name;
7049 : : spdk_bdev_nvme_start_discovery_fn start_cb_fn;
7050 : : spdk_bdev_nvme_stop_discovery_fn stop_cb_fn;
7051 : : void *cb_ctx;
7052 : : struct spdk_nvme_probe_ctx *probe_ctx;
7053 : : struct spdk_nvme_detach_ctx *detach_ctx;
7054 : : struct spdk_nvme_ctrlr *ctrlr;
7055 : : struct spdk_nvme_transport_id trid;
7056 : : struct discovery_entry_ctx *entry_ctx_in_use;
7057 : : struct spdk_poller *poller;
7058 : : struct spdk_nvme_ctrlr_opts drv_opts;
7059 : : struct spdk_bdev_nvme_ctrlr_opts bdev_opts;
7060 : : struct spdk_nvmf_discovery_log_page *log_page;
7061 : : TAILQ_ENTRY(discovery_ctx) tailq;
7062 : : TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs;
7063 : : TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs;
7064 : : int rc;
7065 : : bool wait_for_attach;
7066 : : uint64_t timeout_ticks;
7067 : : /* Denotes that the discovery service is being started. We're waiting
7068 : : * for the initial connection to the discovery controller to be
7069 : : * established and attach discovered NVM ctrlrs.
7070 : : */
7071 : : bool initializing;
7072 : : /* Denotes if a discovery is currently in progress for this context.
7073 : : * That includes connecting to newly discovered subsystems. Used to
7074 : : * ensure we do not start a new discovery until an existing one is
7075 : : * complete.
7076 : : */
7077 : : bool in_progress;
7078 : :
7079 : : /* Denotes if another discovery is needed after the one in progress
7080 : : * completes. Set when we receive an AER completion while a discovery
7081 : : * is already in progress.
7082 : : */
7083 : : bool pending;
7084 : :
7085 : : /* Signal to the discovery context poller that it should stop the
7086 : : * discovery service, including detaching from the current discovery
7087 : : * controller.
7088 : : */
7089 : : bool stop;
7090 : :
7091 : : struct spdk_thread *calling_thread;
7092 : : uint32_t index;
7093 : : uint32_t attach_in_progress;
7094 : : char *hostnqn;
7095 : :
7096 : : /* Denotes if the discovery service was started by the mdns discovery.
7097 : : */
7098 : : bool from_mdns_discovery_service;
7099 : : };
7100 : :
7101 : : TAILQ_HEAD(discovery_ctxs, discovery_ctx);
7102 : : static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs);
7103 : :
7104 : : static void get_discovery_log_page(struct discovery_ctx *ctx);
7105 : :
7106 : : static void
7107 : 34 : free_discovery_ctx(struct discovery_ctx *ctx)
7108 : : {
7109 [ # # # # ]: 34 : free(ctx->log_page);
7110 [ # # # # ]: 34 : free(ctx->hostnqn);
7111 [ # # # # ]: 34 : free(ctx->name);
7112 : 34 : free(ctx);
7113 : 34 : }
7114 : :
7115 : : static void
7116 : 49 : discovery_complete(struct discovery_ctx *ctx)
7117 : : {
7118 [ # # # # ]: 49 : ctx->initializing = false;
7119 [ # # # # ]: 49 : ctx->in_progress = false;
7120 [ - + + + : 49 : if (ctx->pending) {
# # # # ]
7121 [ # # # # ]: 5 : ctx->pending = false;
7122 : 5 : get_discovery_log_page(ctx);
7123 : 0 : }
7124 : 49 : }
7125 : :
7126 : : static void
7127 : 161 : build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid,
7128 : : struct spdk_nvmf_discovery_log_page_entry *entry)
7129 : : {
7130 : : char *space;
7131 : :
7132 [ # # # # : 161 : trid->trtype = entry->trtype;
# # # # ]
7133 [ # # # # : 161 : trid->adrfam = entry->adrfam;
# # # # ]
7134 [ - + - + : 161 : memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr));
# # # # ]
7135 [ # # # # : 161 : memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid));
# # # # ]
7136 : : /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and
7137 : : * before call to this function trid->subnqn is zeroed out, we need
7138 : : * to copy sizeof(trid->subnqn) minus one byte to make sure the last character
7139 : : * remains 0. Then we can shorten the string (replace ' ' with 0) if required
7140 : : */
7141 [ - + - + : 161 : memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1);
# # # # ]
7142 : :
7143 : : /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated.
7144 : : * But the log page entries typically pad them with spaces, not zeroes.
7145 : : * So add a NULL terminator to each of these fields at the appropriate
7146 : : * location.
7147 : : */
7148 [ - + # # ]: 161 : space = strchr(trid->traddr, ' ');
7149 [ + - ]: 161 : if (space) {
7150 [ # # ]: 161 : *space = 0;
7151 : 0 : }
7152 [ - + # # ]: 161 : space = strchr(trid->trsvcid, ' ');
7153 [ + - ]: 161 : if (space) {
7154 [ # # ]: 161 : *space = 0;
7155 : 0 : }
7156 [ - + # # ]: 161 : space = strchr(trid->subnqn, ' ');
7157 [ - + ]: 161 : if (space) {
7158 [ # # ]: 0 : *space = 0;
7159 : 0 : }
7160 : 161 : }
7161 : :
7162 : : static void
7163 : 34 : _stop_discovery(void *_ctx)
7164 : : {
7165 : 34 : struct discovery_ctx *ctx = _ctx;
7166 : :
7167 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
7168 : 0 : spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx);
7169 : 0 : return;
7170 : : }
7171 : :
7172 [ # # # # ]: 34 : ctx->stop = true;
7173 : :
7174 [ + + # # : 63 : while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {
# # # # ]
7175 : : struct discovery_entry_ctx *entry_ctx;
7176 : 29 : struct nvme_path_id path = {};
7177 : :
7178 [ # # # # : 29 : entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);
# # ]
7179 [ # # ]: 29 : path.trid = entry_ctx->trid;
7180 [ # # ]: 29 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
7181 [ - + # # : 29 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7182 : 29 : free(entry_ctx);
7183 : : }
7184 : :
7185 [ + + # # : 77 : while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) {
# # # # ]
7186 : : struct discovery_entry_ctx *entry_ctx;
7187 : :
7188 [ # # # # : 43 : entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # ]
7189 [ + + # # : 43 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7190 : 43 : free(entry_ctx);
7191 : : }
7192 : :
7193 [ # # # # ]: 34 : free(ctx->entry_ctx_in_use);
7194 [ # # # # ]: 34 : ctx->entry_ctx_in_use = NULL;
7195 : 0 : }
7196 : :
7197 : : static void
7198 : 34 : stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7199 : : {
7200 [ # # # # ]: 34 : ctx->stop_cb_fn = cb_fn;
7201 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7202 : :
7203 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
7204 [ # # # # : 0 : DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n",
# # # # #
# # # # #
# # # # ]
7205 : : ctx->attach_in_progress);
7206 : 0 : }
7207 : :
7208 : 34 : _stop_discovery(ctx);
7209 : 34 : }
7210 : :
7211 : : static void
7212 : 31 : remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr)
7213 : : {
7214 : : struct discovery_ctx *d_ctx;
7215 : : struct nvme_path_id *path_id;
7216 : 31 : struct spdk_nvme_transport_id trid = {};
7217 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7218 : :
7219 [ # # # # : 31 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
7220 : :
7221 [ + + # # : 34 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7222 [ + + # # : 6 : TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7223 [ # # ]: 3 : build_trid_from_log_page_entry(&trid, &entry_ctx->entry);
7224 [ - + # # ]: 3 : if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) {
7225 : 0 : continue;
7226 : : }
7227 : :
7228 [ - + # # : 3 : TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7229 : 3 : free(entry_ctx);
7230 [ - + + - : 3 : DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n",
# # # # #
# # # #
# ]
7231 : : trid.subnqn, trid.traddr, trid.trsvcid);
7232 : :
7233 : : /* Fail discovery ctrlr to force reattach attempt */
7234 [ # # # # ]: 3 : spdk_nvme_ctrlr_fail(d_ctx->ctrlr);
7235 : 0 : }
7236 : 0 : }
7237 : 31 : }
7238 : :
7239 : : static void
7240 : 49 : discovery_remove_controllers(struct discovery_ctx *ctx)
7241 : : {
7242 [ # # # # ]: 49 : struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page;
7243 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7244 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7245 : 49 : struct spdk_nvme_transport_id old_trid = {};
7246 : : uint64_t numrec, i;
7247 : : bool found;
7248 : :
7249 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7250 [ + + # # : 109 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7251 : 60 : found = false;
7252 [ # # ]: 60 : old_entry = &entry_ctx->entry;
7253 : 60 : build_trid_from_log_page_entry(&old_trid, old_entry);
7254 [ + + ]: 143 : for (i = 0; i < numrec; i++) {
7255 [ # # # # ]: 138 : new_entry = &log_page->entries[i];
7256 [ - + - + : 138 : if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) {
+ + ]
7257 [ - + + + : 55 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n",
# # # # #
# # # #
# ]
7258 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7259 : 55 : found = true;
7260 : 55 : break;
7261 : : }
7262 : 0 : }
7263 [ + + # # ]: 60 : if (!found) {
7264 : 5 : struct nvme_path_id path = {};
7265 : :
7266 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",
# # # # #
# # # #
# ]
7267 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7268 : :
7269 [ # # ]: 5 : path.trid = entry_ctx->trid;
7270 [ # # ]: 5 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
7271 [ + - # # : 5 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7272 : 5 : free(entry_ctx);
7273 : 0 : }
7274 : 0 : }
7275 : 49 : free(log_page);
7276 [ # # # # ]: 49 : ctx->log_page = NULL;
7277 : 49 : discovery_complete(ctx);
7278 : 49 : }
7279 : :
7280 : : static void
7281 : 42 : complete_discovery_start(struct discovery_ctx *ctx, int status)
7282 : : {
7283 [ # # # # ]: 42 : ctx->timeout_ticks = 0;
7284 [ # # # # ]: 42 : ctx->rc = status;
7285 [ + + # # : 42 : if (ctx->start_cb_fn) {
# # ]
7286 [ # # # # : 27 : ctx->start_cb_fn(ctx->cb_ctx, status);
# # # # #
# # # ]
7287 [ # # # # ]: 27 : ctx->start_cb_fn = NULL;
7288 [ # # # # ]: 27 : ctx->cb_ctx = NULL;
7289 : 0 : }
7290 : 42 : }
7291 : :
7292 : : static void
7293 : 37 : discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc)
7294 : : {
7295 : 37 : struct discovery_entry_ctx *entry_ctx = cb_ctx;
7296 [ # # # # ]: 37 : struct discovery_ctx *ctx = entry_ctx->ctx;
7297 : :
7298 [ - + + + : 37 : DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name);
# # # # #
# # # # #
# # ]
7299 [ # # ]: 37 : ctx->attach_in_progress--;
7300 [ + - # # : 37 : if (ctx->attach_in_progress == 0) {
# # ]
7301 [ # # # # ]: 37 : complete_discovery_start(ctx, ctx->rc);
7302 [ - + + + : 37 : if (ctx->initializing && ctx->rc != 0) {
- + # # #
# # # #
# ]
7303 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc);
# # # # #
# # # ]
7304 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7305 : 0 : } else {
7306 : 37 : discovery_remove_controllers(ctx);
7307 : : }
7308 : 0 : }
7309 : 37 : }
7310 : :
7311 : : static struct discovery_entry_ctx *
7312 : 95 : create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid)
7313 : : {
7314 : : struct discovery_entry_ctx *new_ctx;
7315 : :
7316 : 95 : new_ctx = calloc(1, sizeof(*new_ctx));
7317 [ - + ]: 95 : if (new_ctx == NULL) {
7318 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7319 : 0 : return NULL;
7320 : : }
7321 : :
7322 [ # # # # ]: 95 : new_ctx->ctx = ctx;
7323 [ - + - + : 95 : memcpy(&new_ctx->trid, trid, sizeof(*trid));
# # ]
7324 [ # # ]: 95 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7325 [ - + # # : 95 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7326 : 95 : return new_ctx;
7327 : 0 : }
7328 : :
7329 : : static void
7330 : 49 : discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl,
7331 : : struct spdk_nvmf_discovery_log_page *log_page)
7332 : : {
7333 : 49 : struct discovery_ctx *ctx = cb_arg;
7334 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7335 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7336 : : uint64_t numrec, i;
7337 : : bool found;
7338 : :
7339 [ + - + - : 49 : if (rc || spdk_nvme_cpl_is_error(cpl)) {
- + # # #
# # # # #
# # # # ]
7340 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7341 : 0 : return;
7342 : : }
7343 : :
7344 [ # # # # ]: 49 : ctx->log_page = log_page;
7345 [ - + # # : 49 : assert(ctx->attach_in_progress == 0);
# # # # ]
7346 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7347 [ + + # # : 71 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7348 [ + + # # : 22 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7349 : 22 : free(entry_ctx);
7350 : 0 : }
7351 [ + + ]: 165 : for (i = 0; i < numrec; i++) {
7352 : 116 : found = false;
7353 [ # # # # ]: 116 : new_entry = &log_page->entries[i];
7354 [ + + # # : 116 : if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT ||
# # # # ]
7355 [ - + # # ]: 55 : new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
7356 : : struct discovery_entry_ctx *new_ctx;
7357 : 61 : struct spdk_nvme_transport_id trid = {};
7358 : :
7359 : 61 : build_trid_from_log_page_entry(&trid, new_entry);
7360 : 61 : new_ctx = create_discovery_entry_ctx(ctx, &trid);
7361 [ - + ]: 61 : if (new_ctx == NULL) {
7362 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7363 : 0 : break;
7364 : : }
7365 : :
7366 [ # # # # : 61 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7367 : 61 : continue;
7368 : : }
7369 [ + + # # : 67 : TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7370 [ # # ]: 30 : old_entry = &entry_ctx->entry;
7371 [ - + - + : 30 : if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) {
+ + ]
7372 : 18 : found = true;
7373 : 18 : break;
7374 : : }
7375 : 0 : }
7376 [ + + # # ]: 55 : if (!found) {
7377 : 37 : struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx;
7378 : : struct discovery_ctx *d_ctx;
7379 : :
7380 [ + + # # : 77 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7381 [ + + # # : 51 : TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7382 [ - + - + : 11 : if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn,
+ + # # #
# # # ]
7383 : : sizeof(new_entry->subnqn))) {
7384 : 5 : break;
7385 : : }
7386 : 0 : }
7387 [ + + ]: 45 : if (subnqn_ctx) {
7388 : 5 : break;
7389 : : }
7390 : 0 : }
7391 : :
7392 : 37 : new_ctx = calloc(1, sizeof(*new_ctx));
7393 [ - + ]: 37 : if (new_ctx == NULL) {
7394 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7395 : 0 : break;
7396 : : }
7397 : :
7398 [ # # # # ]: 37 : new_ctx->ctx = ctx;
7399 [ - + - + : 37 : memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry));
# # ]
7400 [ # # ]: 37 : build_trid_from_log_page_entry(&new_ctx->trid, new_entry);
7401 [ + + ]: 37 : if (subnqn_ctx) {
7402 [ # # # # ]: 5 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name);
7403 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7404 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7405 : : new_ctx->name);
7406 : 0 : } else {
7407 [ # # # # : 32 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++);
# # # # ]
7408 [ - + + + : 32 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7409 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7410 : : new_ctx->name);
7411 : : }
7412 [ # # ]: 37 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7413 [ # # # # : 37 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7414 [ # # # # ]: 37 : rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,
7415 : 0 : discovery_attach_controller_done, new_ctx,
7416 [ # # # # ]: 0 : &new_ctx->drv_opts, &ctx->bdev_opts);
7417 [ + - ]: 37 : if (rc == 0) {
7418 [ # # # # : 37 : TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7419 [ # # ]: 37 : ctx->attach_in_progress++;
7420 : 0 : } else {
7421 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc));
# # # # #
# ]
7422 : : }
7423 : 0 : }
7424 : 0 : }
7425 : :
7426 [ + + # # : 49 : if (ctx->attach_in_progress == 0) {
# # ]
7427 : 12 : discovery_remove_controllers(ctx);
7428 : 0 : }
7429 : 0 : }
7430 : :
7431 : : static void
7432 : 49 : get_discovery_log_page(struct discovery_ctx *ctx)
7433 : : {
7434 : : int rc;
7435 : :
7436 [ - + - + : 49 : assert(ctx->in_progress == false);
# # # # #
# ]
7437 [ # # # # ]: 49 : ctx->in_progress = true;
7438 [ # # # # ]: 49 : rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx);
7439 [ - + ]: 49 : if (rc != 0) {
7440 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7441 : 0 : }
7442 [ - + + + : 49 : DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n");
# # # # #
# # # #
# ]
7443 : 49 : }
7444 : :
7445 : : static void
7446 : 16 : discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
7447 : : {
7448 : 16 : struct discovery_ctx *ctx = arg;
7449 [ # # # # : 16 : uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
# # ]
7450 : :
7451 [ + - - + : 16 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7452 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "aer failed\n");
# # # # ]
7453 : 0 : return;
7454 : : }
7455 : :
7456 [ - + ]: 16 : if (log_page_id != SPDK_NVME_LOG_DISCOVERY) {
7457 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id);
# # # # ]
7458 : 0 : return;
7459 : : }
7460 : :
7461 [ - + + - : 16 : DISCOVERY_INFOLOG(ctx, "got aer\n");
# # # # #
# # # #
# ]
7462 [ - + + + : 16 : if (ctx->in_progress) {
# # # # ]
7463 [ # # # # ]: 5 : ctx->pending = true;
7464 : 5 : return;
7465 : : }
7466 : :
7467 : 11 : get_discovery_log_page(ctx);
7468 : 0 : }
7469 : :
7470 : : static void
7471 : 33 : discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
7472 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
7473 : : {
7474 : 33 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
7475 : : struct discovery_ctx *ctx;
7476 : :
7477 : 33 : ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts);
7478 : :
7479 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n");
# # # # #
# # # #
# ]
7480 [ # # # # ]: 33 : ctx->probe_ctx = NULL;
7481 [ # # # # ]: 33 : ctx->ctrlr = ctrlr;
7482 : :
7483 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7484 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n",
# # # # #
# # # ]
7485 : : ctx->rc);
7486 : 0 : return;
7487 : : }
7488 : :
7489 [ # # # # ]: 33 : spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx);
7490 : 0 : }
7491 : :
7492 : : static int
7493 : 106551 : discovery_poller(void *arg)
7494 : : {
7495 : 106551 : struct discovery_ctx *ctx = arg;
7496 : : struct spdk_nvme_transport_id *trid;
7497 : : int rc;
7498 : :
7499 [ + + # # : 106551 : if (ctx->detach_ctx) {
# # ]
7500 [ # # # # ]: 243 : rc = spdk_nvme_detach_poll_async(ctx->detach_ctx);
7501 [ + + ]: 243 : if (rc != -EAGAIN) {
7502 [ # # # # ]: 33 : ctx->detach_ctx = NULL;
7503 [ # # # # ]: 33 : ctx->ctrlr = NULL;
7504 : 0 : }
7505 [ - + + + : 106308 : } else if (ctx->stop) {
# # # # ]
7506 [ + + # # : 60 : if (ctx->ctrlr != NULL) {
# # ]
7507 [ # # # # : 30 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7508 [ + - ]: 30 : if (rc == 0) {
7509 : 30 : return SPDK_POLLER_BUSY;
7510 : : }
7511 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7512 : 0 : }
7513 [ # # ]: 30 : spdk_poller_unregister(&ctx->poller);
7514 [ + + # # : 30 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7515 [ - + # # : 30 : assert(ctx->start_cb_fn == NULL);
# # # # ]
7516 [ + + # # : 30 : if (ctx->stop_cb_fn != NULL) {
# # ]
7517 [ # # # # : 25 : ctx->stop_cb_fn(ctx->cb_ctx);
# # # # #
# # # ]
7518 : 0 : }
7519 : 30 : free_discovery_ctx(ctx);
7520 [ + + + + : 106248 : } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) {
# # # # #
# # # ]
7521 [ + + + + : 50 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7522 [ # # # # : 4 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7523 [ - + - + : 4 : assert(ctx->initializing);
# # # # #
# ]
7524 [ # # ]: 4 : spdk_poller_unregister(&ctx->poller);
7525 [ - + # # : 4 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7526 : 4 : complete_discovery_start(ctx, -ETIMEDOUT);
7527 : 4 : stop_discovery(ctx, NULL, NULL);
7528 : 4 : free_discovery_ctx(ctx);
7529 : 4 : return SPDK_POLLER_BUSY;
7530 : : }
7531 : :
7532 [ - + # # : 46 : assert(ctx->entry_ctx_in_use == NULL);
# # # # ]
7533 [ # # # # : 46 : ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # # # #
# ]
7534 [ + + # # : 46 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7535 [ # # # # : 46 : trid = &ctx->entry_ctx_in_use->trid;
# # ]
7536 : :
7537 : : /* All controllers must be configured explicitely either for multipath or failover.
7538 : : * While discovery use multipath mode, we need to set this in bdev options as well.
7539 : : */
7540 [ # # # # : 46 : ctx->bdev_opts.multipath = true;
# # ]
7541 : :
7542 [ # # # # : 46 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb);
# # ]
7543 [ + + # # : 46 : if (ctx->probe_ctx) {
# # ]
7544 [ # # ]: 33 : spdk_poller_unregister(&ctx->poller);
7545 [ # # # # ]: 33 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000);
7546 : 0 : } else {
7547 [ # # # # : 13 : DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n");
# # # # ]
7548 [ # # # # : 13 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7549 [ # # # # ]: 13 : ctx->entry_ctx_in_use = NULL;
7550 : : }
7551 [ + + # # : 106198 : } else if (ctx->probe_ctx) {
# # ]
7552 [ + + - + : 33 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7553 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7554 : 0 : complete_discovery_start(ctx, -ETIMEDOUT);
7555 : 0 : return SPDK_POLLER_BUSY;
7556 : : }
7557 : :
7558 [ # # # # ]: 33 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
7559 [ + - ]: 33 : if (rc != -EAGAIN) {
7560 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7561 [ # # # # : 0 : assert(ctx->initializing);
# # # # #
# ]
7562 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7563 : 0 : } else {
7564 [ - + # # ]: 33 : assert(rc == 0);
7565 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n");
# # # # #
# # # #
# ]
7566 [ # # # # ]: 33 : ctx->rc = rc;
7567 : 33 : get_discovery_log_page(ctx);
7568 : : }
7569 : 0 : }
7570 : 0 : } else {
7571 [ + + + + : 106165 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7572 [ # # # # : 1 : DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n");
# # # # ]
7573 : 1 : complete_discovery_start(ctx, -ETIMEDOUT);
7574 : : /* We need to wait until all NVM ctrlrs are attached before we stop the
7575 : : * discovery service to make sure we don't detach a ctrlr that is still
7576 : : * being attached.
7577 : : */
7578 [ + - # # : 1 : if (ctx->attach_in_progress == 0) {
# # ]
7579 [ # # # # ]: 1 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7580 : 1 : return SPDK_POLLER_BUSY;
7581 : : }
7582 : 0 : }
7583 : :
7584 [ # # # # ]: 106164 : rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr);
7585 [ + + ]: 106164 : if (rc < 0) {
7586 [ # # ]: 3 : spdk_poller_unregister(&ctx->poller);
7587 [ # # # # ]: 3 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7588 [ # # # # : 3 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7589 [ # # # # ]: 3 : ctx->entry_ctx_in_use = NULL;
7590 : :
7591 [ # # # # : 3 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7592 [ - + ]: 3 : if (rc != 0) {
7593 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7594 [ # # # # ]: 0 : ctx->ctrlr = NULL;
7595 : 0 : }
7596 : 0 : }
7597 : : }
7598 : :
7599 : 106516 : return SPDK_POLLER_BUSY;
7600 : 0 : }
7601 : :
7602 : : static void
7603 : 34 : start_discovery_poller(void *arg)
7604 : : {
7605 : 34 : struct discovery_ctx *ctx = arg;
7606 : :
7607 [ # # # # : 34 : TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
7608 [ # # # # ]: 34 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7609 : 34 : }
7610 : :
7611 : : int
7612 : 40 : bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid,
7613 : : const char *base_name,
7614 : : struct spdk_nvme_ctrlr_opts *drv_opts,
7615 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts,
7616 : : uint64_t attach_timeout,
7617 : : bool from_mdns,
7618 : : spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx)
7619 : : {
7620 : : struct discovery_ctx *ctx;
7621 : : struct discovery_entry_ctx *discovery_entry_ctx;
7622 : :
7623 [ # # ]: 40 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
7624 [ + + # # : 50 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7625 [ - + - + : 16 : if (strcmp(ctx->name, base_name) == 0) {
+ + # # #
# ]
7626 : 3 : return -EEXIST;
7627 : : }
7628 : :
7629 [ + + # # : 13 : if (ctx->entry_ctx_in_use != NULL) {
# # ]
7630 [ + + # # : 11 : if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) {
# # # # ]
7631 : 3 : return -EEXIST;
7632 : : }
7633 : 0 : }
7634 : :
7635 [ + + # # : 20 : TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7636 [ - + # # ]: 10 : if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) {
7637 : 0 : return -EEXIST;
7638 : : }
7639 : 0 : }
7640 : 0 : }
7641 : :
7642 : 34 : ctx = calloc(1, sizeof(*ctx));
7643 [ - + ]: 34 : if (ctx == NULL) {
7644 : 0 : return -ENOMEM;
7645 : : }
7646 : :
7647 [ - + # # : 34 : ctx->name = strdup(base_name);
# # ]
7648 [ - + # # : 34 : if (ctx->name == NULL) {
# # ]
7649 : 0 : free_discovery_ctx(ctx);
7650 : 0 : return -ENOMEM;
7651 : : }
7652 [ - + - + : 34 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
# # ]
7653 [ - + - + : 34 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
# # ]
7654 [ # # # # : 34 : ctx->from_mdns_discovery_service = from_mdns;
# # ]
7655 [ # # # # : 34 : ctx->bdev_opts.from_discovery_service = true;
# # ]
7656 [ # # # # ]: 34 : ctx->calling_thread = spdk_get_thread();
7657 [ # # # # ]: 34 : ctx->start_cb_fn = cb_fn;
7658 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7659 [ # # # # ]: 34 : ctx->initializing = true;
7660 [ + + # # : 34 : if (ctx->start_cb_fn) {
# # ]
7661 : : /* We can use this when dumping json to denote if this RPC parameter
7662 : : * was specified or not.
7663 : : */
7664 [ # # # # ]: 27 : ctx->wait_for_attach = true;
7665 : 0 : }
7666 [ + + ]: 34 : if (attach_timeout != 0) {
7667 [ # # # # ]: 42 : ctx->timeout_ticks = spdk_get_ticks() + attach_timeout *
7668 [ # # # # ]: 21 : spdk_get_ticks_hz() / 1000ull;
7669 : 0 : }
7670 [ # # # # : 34 : TAILQ_INIT(&ctx->nvm_entry_ctxs);
# # # # #
# # # # #
# # ]
7671 [ # # # # : 34 : TAILQ_INIT(&ctx->discovery_entry_ctxs);
# # # # #
# # # # #
# # ]
7672 [ - + - + : 34 : memcpy(&ctx->trid, trid, sizeof(*trid));
# # ]
7673 : : /* Even if user did not specify hostnqn, we can still strdup("\0"); */
7674 [ - + # # : 34 : ctx->hostnqn = strdup(ctx->drv_opts.hostnqn);
# # # # #
# ]
7675 [ - + # # : 34 : if (ctx->hostnqn == NULL) {
# # ]
7676 : 0 : free_discovery_ctx(ctx);
7677 : 0 : return -ENOMEM;
7678 : : }
7679 : 34 : discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid);
7680 [ - + ]: 34 : if (discovery_entry_ctx == NULL) {
7681 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7682 : 0 : free_discovery_ctx(ctx);
7683 : 0 : return -ENOMEM;
7684 : : }
7685 : :
7686 [ # # # # : 34 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7687 : 34 : spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx);
7688 : 34 : return 0;
7689 : 0 : }
7690 : :
7691 : : int
7692 : 23 : bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7693 : : {
7694 : : struct discovery_ctx *ctx;
7695 : :
7696 [ + - # # : 27 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7697 [ - + - + : 27 : if (strcmp(name, ctx->name) == 0) {
+ + # # #
# ]
7698 [ - + - + : 23 : if (ctx->stop) {
# # # # ]
7699 : 0 : return -EALREADY;
7700 : : }
7701 : : /* If we're still starting the discovery service and ->rc is non-zero, we're
7702 : : * going to stop it as soon as we can
7703 : : */
7704 [ - + - + : 23 : if (ctx->initializing && ctx->rc != 0) {
- - # # #
# # # #
# ]
7705 : 0 : return -EALREADY;
7706 : : }
7707 : 23 : stop_discovery(ctx, cb_fn, cb_ctx);
7708 : 23 : return 0;
7709 : : }
7710 : 0 : }
7711 : :
7712 : 0 : return -ENOENT;
7713 : 0 : }
7714 : :
7715 : : static int
7716 : 1964 : bdev_nvme_library_init(void)
7717 : : {
7718 : 1964 : g_bdev_nvme_init_thread = spdk_get_thread();
7719 : :
7720 : 1964 : spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb,
7721 : : bdev_nvme_destroy_poll_group_cb,
7722 : : sizeof(struct nvme_poll_group), "nvme_poll_groups");
7723 : :
7724 : 1964 : return 0;
7725 : : }
7726 : :
7727 : : static void
7728 : 1964 : bdev_nvme_fini_destruct_ctrlrs(void)
7729 : : {
7730 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
7731 : : struct nvme_ctrlr *nvme_ctrlr;
7732 : :
7733 [ + + ]: 1964 : pthread_mutex_lock(&g_bdev_nvme_mutex);
7734 [ + + + - : 2683 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ - + - ]
7735 [ + + + - : 1444 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
+ - + + +
- + - +
- ]
7736 [ + + + - ]: 725 : pthread_mutex_lock(&nvme_ctrlr->mutex);
7737 [ + + - + ]: 725 : if (nvme_ctrlr->destruct) {
7738 : : /* This controller's destruction was already started
7739 : : * before the application started shutting down
7740 : : */
7741 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7742 : 0 : continue;
7743 : : }
7744 [ - + ]: 725 : nvme_ctrlr->destruct = true;
7745 [ - + - + ]: 725 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7746 : :
7747 [ - + - + ]: 732 : spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct,
7748 : 8 : nvme_ctrlr);
7749 : 8 : }
7750 : 8 : }
7751 : :
7752 : 1964 : g_bdev_nvme_module_finish = true;
7753 [ + + ]: 1964 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
7754 [ - + ]: 1428 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7755 : 1428 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
7756 : 1428 : spdk_bdev_module_fini_done();
7757 : 1428 : return;
7758 : : }
7759 : :
7760 [ + + ]: 536 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7761 : 93 : }
7762 : :
7763 : : static void
7764 : 6 : check_discovery_fini(void *arg)
7765 : : {
7766 [ + - ]: 6 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7767 : 6 : bdev_nvme_fini_destruct_ctrlrs();
7768 : 0 : }
7769 : 6 : }
7770 : :
7771 : : static void
7772 : 1964 : bdev_nvme_library_fini(void)
7773 : : {
7774 : : struct nvme_probe_skip_entry *entry, *entry_tmp;
7775 : : struct discovery_ctx *ctx;
7776 : :
7777 : 1964 : spdk_poller_unregister(&g_hotplug_poller);
7778 : 1964 : free(g_hotplug_probe_ctx);
7779 : 1964 : g_hotplug_probe_ctx = NULL;
7780 : :
7781 [ + + + + : 2012 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) {
# # # # -
+ ]
7782 [ + + # # : 48 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7783 : 48 : free(entry);
7784 : 3 : }
7785 : :
7786 [ + + # # ]: 1964 : assert(spdk_get_thread() == g_bdev_nvme_init_thread);
7787 [ + + ]: 1964 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7788 : 1958 : bdev_nvme_fini_destruct_ctrlrs();
7789 : 93 : } else {
7790 [ + + # # : 12 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7791 : 6 : stop_discovery(ctx, check_discovery_fini, NULL);
7792 : 0 : }
7793 : : }
7794 : 1964 : }
7795 : :
7796 : : static void
7797 : 0 : bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio)
7798 : : {
7799 : 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7800 [ # # # # ]: 0 : struct spdk_bdev *bdev = bdev_io->bdev;
7801 : 0 : struct spdk_dif_ctx dif_ctx;
7802 : 0 : struct spdk_dif_error err_blk = {};
7803 : : int rc;
7804 : 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
7805 : :
7806 : 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
7807 [ # # # # : 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
# # ]
7808 : 0 : rc = spdk_dif_ctx_init(&dif_ctx,
7809 [ # # # # : 0 : bdev->blocklen, bdev->md_len, bdev->md_interleave,
# # # # #
# # # #
# ]
7810 [ # # # # : 0 : bdev->dif_is_head_of_md, bdev->dif_type,
# # # # #
# ]
7811 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
7812 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts);
# # # # ]
7813 [ # # ]: 0 : if (rc != 0) {
7814 : 0 : SPDK_ERRLOG("Initialization of DIF context failed\n");
7815 : 0 : return;
7816 : : }
7817 : :
7818 [ # # # # : 0 : if (bdev->md_interleave) {
# # # # ]
7819 [ # # # # : 0 : rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7820 [ # # # # : 0 : bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7821 : 0 : } else {
7822 : 0 : struct iovec md_iov = {
7823 [ # # # # : 0 : .iov_base = bdev_io->u.bdev.md_buf,
# # # # ]
7824 [ # # # # : 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
# # # # #
# # # ]
7825 : : };
7826 : :
7827 [ # # # # : 0 : rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7828 [ # # # # : 0 : &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7829 : : }
7830 : :
7831 [ # # ]: 0 : if (rc != 0) {
7832 [ # # ]: 0 : SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
7833 : : err_blk.err_type, err_blk.err_offset);
7834 : 0 : } else {
7835 : 0 : SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n");
7836 : : }
7837 : 0 : }
7838 : :
7839 : : static void
7840 : 0 : bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7841 : : {
7842 : 0 : struct nvme_bdev_io *bio = ref;
7843 : :
7844 [ # # # # : 0 : if (spdk_nvme_cpl_is_success(cpl)) {
# # # # #
# # # # #
# # ]
7845 : : /* Run PI verification for read data buffer. */
7846 : 0 : bdev_nvme_verify_pi_error(bio);
7847 : 0 : }
7848 : :
7849 : : /* Return original completion status */
7850 [ # # ]: 0 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
7851 : 0 : }
7852 : :
7853 : : static void
7854 : 13116074 : bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7855 : : {
7856 : 13116074 : struct nvme_bdev_io *bio = ref;
7857 : 13116074 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7858 : : int ret;
7859 : :
7860 [ + + + - : 13116074 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
+ - + - -
- - - # #
# # # # #
# # # # #
# # # # #
# + - ]
7861 [ # # # # : 0 : SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7862 : : cpl->status.sct, cpl->status.sc);
7863 : :
7864 : : /* Save completion status to use after verifying PI error. */
7865 [ # # ]: 0 : bio->cpl = *cpl;
7866 : :
7867 [ # # # # : 0 : if (spdk_likely(nvme_io_path_is_available(bio->io_path))) {
# # ]
7868 : : /* Read without PI checking to verify PI error. */
7869 : 0 : ret = bdev_nvme_no_pi_readv(bio,
7870 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
7871 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
7872 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
7873 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
7874 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks);
# # # # ]
7875 [ # # ]: 0 : if (ret == 0) {
7876 : 0 : return;
7877 : : }
7878 : 0 : }
7879 : 0 : }
7880 : :
7881 : 13116074 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7882 : 274031 : }
7883 : :
7884 : : static void
7885 : 13566967 : bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7886 : : {
7887 : 13566967 : struct nvme_bdev_io *bio = ref;
7888 : :
7889 [ + + - - : 13566967 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
- - + - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # ]
7890 [ # # # # : 0 : SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7891 : : cpl->status.sct, cpl->status.sc);
7892 : : /* Run PI verification for write data buffer if PI error is detected. */
7893 : 0 : bdev_nvme_verify_pi_error(bio);
7894 : 0 : }
7895 : :
7896 : 13566967 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7897 : 13566967 : }
7898 : :
7899 : : static void
7900 : 231053 : bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7901 : : {
7902 : 231053 : struct nvme_bdev_io *bio = ref;
7903 : 231053 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7904 : :
7905 : : /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks.
7906 : : * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error().
7907 : : */
7908 [ # # # # : 231053 : bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0;
# # # # #
# # # ]
7909 : :
7910 [ - + - - : 231053 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7911 [ # # # # : 0 : SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7912 : : cpl->status.sct, cpl->status.sc);
7913 : : /* Run PI verification for zone append data buffer if PI error is detected. */
7914 : 0 : bdev_nvme_verify_pi_error(bio);
7915 : 0 : }
7916 : :
7917 : 231053 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7918 : 231053 : }
7919 : :
7920 : : static void
7921 : 56 : bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7922 : : {
7923 : 56 : struct nvme_bdev_io *bio = ref;
7924 : :
7925 [ + + + - : 56 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7926 [ # # # # : 0 : SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7927 : : cpl->status.sct, cpl->status.sc);
7928 : : /* Run PI verification for compare data buffer if PI error is detected. */
7929 : 0 : bdev_nvme_verify_pi_error(bio);
7930 : 0 : }
7931 : :
7932 : 56 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7933 : 56 : }
7934 : :
7935 : : static void
7936 : 106 : bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7937 : : {
7938 : 106 : struct nvme_bdev_io *bio = ref;
7939 : :
7940 : : /* Compare operation completion */
7941 [ + + + + : 106 : if (!bio->first_fused_completed) {
# # # # ]
7942 : : /* Save compare result for write callback */
7943 [ # # ]: 53 : bio->cpl = *cpl;
7944 [ # # # # ]: 53 : bio->first_fused_completed = true;
7945 : 53 : return;
7946 : : }
7947 : :
7948 : : /* Write operation completion */
7949 [ + + - + : 53 : if (spdk_nvme_cpl_is_error(&bio->cpl)) {
# # # # #
# # # # #
# # # # #
# ]
7950 : : /* If bio->cpl is already an error, it means the compare operation failed. In that case,
7951 : : * complete the IO with the compare operation's status.
7952 : : */
7953 [ + + + - : 40 : if (!spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7954 : 4 : SPDK_ERRLOG("Unexpected write success after compare failure.\n");
7955 : 1 : }
7956 : :
7957 [ # # ]: 40 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
7958 : 1 : } else {
7959 : 13 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7960 : : }
7961 : 4 : }
7962 : :
7963 : : static void
7964 : 774271 : bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl)
7965 : : {
7966 : 774271 : struct nvme_bdev_io *bio = ref;
7967 : :
7968 : 774271 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7969 : 774271 : }
7970 : :
7971 : : static int
7972 : 40 : fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc)
7973 : : {
7974 [ + - # # ]: 40 : switch (desc->zt) {
7975 : 40 : case SPDK_NVME_ZONE_TYPE_SEQWR:
7976 [ # # # # ]: 40 : info->type = SPDK_BDEV_ZONE_TYPE_SEQWR;
7977 : 40 : break;
7978 : 0 : default:
7979 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt);
7980 : 0 : return -EIO;
7981 : : }
7982 : :
7983 [ + - - - : 40 : switch (desc->zs) {
- - - - #
# ]
7984 : 40 : case SPDK_NVME_ZONE_STATE_EMPTY:
7985 [ # # # # ]: 40 : info->state = SPDK_BDEV_ZONE_STATE_EMPTY;
7986 : 40 : break;
7987 : 0 : case SPDK_NVME_ZONE_STATE_IOPEN:
7988 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN;
7989 : 0 : break;
7990 : 0 : case SPDK_NVME_ZONE_STATE_EOPEN:
7991 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN;
7992 : 0 : break;
7993 : 0 : case SPDK_NVME_ZONE_STATE_CLOSED:
7994 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_CLOSED;
7995 : 0 : break;
7996 : 0 : case SPDK_NVME_ZONE_STATE_RONLY:
7997 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY;
7998 : 0 : break;
7999 : 0 : case SPDK_NVME_ZONE_STATE_FULL:
8000 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_FULL;
8001 : 0 : break;
8002 : 0 : case SPDK_NVME_ZONE_STATE_OFFLINE:
8003 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_OFFLINE;
8004 : 0 : break;
8005 : 0 : default:
8006 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs);
8007 : 0 : return -EIO;
8008 : : }
8009 : :
8010 [ # # # # : 40 : info->zone_id = desc->zslba;
# # # # ]
8011 [ # # # # : 40 : info->write_pointer = desc->wp;
# # # # ]
8012 [ # # # # : 40 : info->capacity = desc->zcap;
# # # # ]
8013 : :
8014 : 40 : return 0;
8015 : 0 : }
8016 : :
8017 : : static void
8018 : 1 : bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl)
8019 : : {
8020 : 1 : struct nvme_bdev_io *bio = ref;
8021 : 1 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8022 [ # # # # : 1 : uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id;
# # # # ]
8023 [ # # # # : 1 : uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones;
# # # # ]
8024 [ # # # # : 1 : struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf;
# # # # ]
8025 : : uint64_t max_zones_per_buf, i;
8026 : : uint32_t zone_report_bufsize;
8027 : : struct spdk_nvme_ns *ns;
8028 : : struct spdk_nvme_qpair *qpair;
8029 : : int ret;
8030 : :
8031 [ + - - + : 1 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
8032 : 0 : goto out_complete_io_nvme_cpl;
8033 : : }
8034 : :
8035 [ - + # # : 1 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
8036 : 0 : ret = -ENXIO;
8037 : 0 : goto out_complete_io_ret;
8038 : : }
8039 : :
8040 [ # # # # : 1 : ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8041 [ # # # # : 1 : qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8042 : :
8043 : 1 : zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
8044 [ # # ]: 1 : max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) /
8045 : : sizeof(bio->zone_report_buf->descs[0]);
8046 : :
8047 [ - + # # : 1 : if (bio->zone_report_buf->nr_zones > max_zones_per_buf) {
# # # # #
# ]
8048 : 0 : ret = -EINVAL;
8049 : 0 : goto out_complete_io_ret;
8050 : : }
8051 : :
8052 [ - + # # : 1 : if (!bio->zone_report_buf->nr_zones) {
# # # # #
# ]
8053 : 0 : ret = -EINVAL;
8054 : 0 : goto out_complete_io_ret;
8055 : : }
8056 : :
8057 [ + + + - : 41 : for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) {
# # # # #
# # # # #
# # ]
8058 [ # # # # : 40 : ret = fill_zone_from_report(&info[bio->handled_zones],
# # ]
8059 [ # # # # : 40 : &bio->zone_report_buf->descs[i]);
# # # # ]
8060 [ - + ]: 40 : if (ret) {
8061 : 0 : goto out_complete_io_ret;
8062 : : }
8063 [ # # ]: 40 : bio->handled_zones++;
8064 : 0 : }
8065 : :
8066 [ - + # # : 1 : if (bio->handled_zones < zones_to_copy) {
# # ]
8067 : 0 : uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
8068 [ # # # # ]: 0 : uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones);
8069 : :
8070 [ # # # # : 0 : memset(bio->zone_report_buf, 0, zone_report_bufsize);
# # ]
8071 : 0 : ret = spdk_nvme_zns_report_zones(ns, qpair,
8072 [ # # # # ]: 0 : bio->zone_report_buf, zone_report_bufsize,
8073 : 0 : slba, SPDK_NVME_ZRA_LIST_ALL, true,
8074 : 0 : bdev_nvme_get_zone_info_done, bio);
8075 [ # # ]: 0 : if (!ret) {
8076 : 0 : return;
8077 : : } else {
8078 : 0 : goto out_complete_io_ret;
8079 : : }
8080 : : }
8081 : :
8082 : 1 : out_complete_io_nvme_cpl:
8083 [ # # # # ]: 1 : free(bio->zone_report_buf);
8084 [ # # # # ]: 1 : bio->zone_report_buf = NULL;
8085 : 1 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8086 : 1 : return;
8087 : :
8088 : 0 : out_complete_io_ret:
8089 [ # # # # ]: 0 : free(bio->zone_report_buf);
8090 [ # # # # ]: 0 : bio->zone_report_buf = NULL;
8091 : 0 : bdev_nvme_io_complete(bio, ret);
8092 : 0 : }
8093 : :
8094 : : static void
8095 : 43 : bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl)
8096 : : {
8097 : 43 : struct nvme_bdev_io *bio = ref;
8098 : :
8099 : 43 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8100 : 43 : }
8101 : :
8102 : : static void
8103 : 138 : bdev_nvme_admin_passthru_complete_nvme_status(void *ctx)
8104 : : {
8105 : 138 : struct nvme_bdev_io *bio = ctx;
8106 : 138 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8107 [ # # ]: 138 : const struct spdk_nvme_cpl *cpl = &bio->cpl;
8108 : :
8109 [ + + # # : 138 : assert(bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
8110 : :
8111 : 138 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
8112 : 138 : }
8113 : :
8114 : : static void
8115 : 7164 : bdev_nvme_abort_complete(void *ctx)
8116 : : {
8117 : 7164 : struct nvme_bdev_io *bio = ctx;
8118 : 7164 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8119 : :
8120 [ + - + - : 7164 : if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) {
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # ]
8121 : 12 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL);
8122 : 3 : } else {
8123 : 7152 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL);
8124 : : }
8125 : 7164 : }
8126 : :
8127 : : static void
8128 : 7164 : bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl)
8129 : : {
8130 : 7164 : struct nvme_bdev_io *bio = ref;
8131 : 7164 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8132 : :
8133 [ # # ]: 7164 : bio->cpl = *cpl;
8134 : 7164 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio);
8135 : 7164 : }
8136 : :
8137 : : static void
8138 : 138 : bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl)
8139 : : {
8140 : 138 : struct nvme_bdev_io *bio = ref;
8141 : 138 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8142 : :
8143 [ # # ]: 138 : bio->cpl = *cpl;
8144 : 143 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io),
8145 : 5 : bdev_nvme_admin_passthru_complete_nvme_status, bio);
8146 : 138 : }
8147 : :
8148 : : static void
8149 : 2011979 : bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset)
8150 : : {
8151 : 2011979 : struct nvme_bdev_io *bio = ref;
8152 : : struct iovec *iov;
8153 : :
8154 [ # # # # ]: 2011979 : bio->iov_offset = sgl_offset;
8155 [ + + # # : 5978393 : for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) {
# # # # #
# # # # #
# # # # ]
8156 [ # # # # : 5978393 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
8157 [ + + # # : 5978393 : if (bio->iov_offset < iov->iov_len) {
# # # # #
# ]
8158 : 2011979 : break;
8159 : : }
8160 : :
8161 [ # # # # : 3966414 : bio->iov_offset -= iov->iov_len;
# # # # ]
8162 : 74 : }
8163 : 2011979 : }
8164 : :
8165 : : static int
8166 : 3667292 : bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length)
8167 : : {
8168 : 3667292 : struct nvme_bdev_io *bio = ref;
8169 : : struct iovec *iov;
8170 : :
8171 [ + + # # : 3667292 : assert(bio->iovpos < bio->iovcnt);
# # # # #
# # # ]
8172 : :
8173 [ # # # # : 3667292 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
8174 : :
8175 [ # # # # : 3667292 : *address = iov->iov_base;
# # ]
8176 [ # # # # : 3667292 : *length = iov->iov_len;
# # ]
8177 : :
8178 [ + + # # : 3667292 : if (bio->iov_offset) {
# # ]
8179 [ - + # # : 360248 : assert(bio->iov_offset <= iov->iov_len);
# # # # #
# # # ]
8180 [ # # # # : 360248 : *address += bio->iov_offset;
# # ]
8181 [ # # # # : 360248 : *length -= bio->iov_offset;
# # ]
8182 : 0 : }
8183 : :
8184 [ # # # # : 3667292 : bio->iov_offset += *length;
# # ]
8185 [ + + # # : 3667292 : if (bio->iov_offset == iov->iov_len) {
# # # # #
# ]
8186 [ # # # # ]: 3667292 : bio->iovpos++;
8187 [ # # # # ]: 3667292 : bio->iov_offset = 0;
8188 : 162 : }
8189 : :
8190 : 3667292 : return 0;
8191 : : }
8192 : :
8193 : : static void
8194 : 90 : bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset)
8195 : : {
8196 : 90 : struct nvme_bdev_io *bio = ref;
8197 : : struct iovec *iov;
8198 : :
8199 [ # # # # ]: 90 : bio->fused_iov_offset = sgl_offset;
8200 [ + - # # : 90 : for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) {
# # # # #
# # # # #
# # # # ]
8201 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8202 [ + - # # : 90 : if (bio->fused_iov_offset < iov->iov_len) {
# # # # #
# ]
8203 : 90 : break;
8204 : : }
8205 : :
8206 [ # # # # : 0 : bio->fused_iov_offset -= iov->iov_len;
# # # # ]
8207 : 0 : }
8208 : 90 : }
8209 : :
8210 : : static int
8211 : 90 : bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length)
8212 : : {
8213 : 90 : struct nvme_bdev_io *bio = ref;
8214 : : struct iovec *iov;
8215 : :
8216 [ - + # # : 90 : assert(bio->fused_iovpos < bio->fused_iovcnt);
# # # # #
# # # ]
8217 : :
8218 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8219 : :
8220 [ # # # # : 90 : *address = iov->iov_base;
# # ]
8221 [ # # # # : 90 : *length = iov->iov_len;
# # ]
8222 : :
8223 [ - + # # : 90 : if (bio->fused_iov_offset) {
# # ]
8224 [ # # # # : 0 : assert(bio->fused_iov_offset <= iov->iov_len);
# # # # #
# # # ]
8225 [ # # # # : 0 : *address += bio->fused_iov_offset;
# # ]
8226 [ # # # # : 0 : *length -= bio->fused_iov_offset;
# # ]
8227 : 0 : }
8228 : :
8229 [ # # # # : 90 : bio->fused_iov_offset += *length;
# # ]
8230 [ + - # # : 90 : if (bio->fused_iov_offset == iov->iov_len) {
# # # # #
# ]
8231 [ # # # # ]: 90 : bio->fused_iovpos++;
8232 [ # # # # ]: 90 : bio->fused_iov_offset = 0;
8233 : 0 : }
8234 : :
8235 : 90 : return 0;
8236 : : }
8237 : :
8238 : : static int
8239 : 0 : bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8240 : : void *md, uint64_t lba_count, uint64_t lba)
8241 : : {
8242 : : int rc;
8243 : :
8244 [ # # # # : 0 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n",
# # ]
8245 : : lba_count, lba);
8246 : :
8247 [ # # # # ]: 0 : bio->iovs = iov;
8248 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8249 [ # # # # ]: 0 : bio->iovpos = 0;
8250 [ # # # # ]: 0 : bio->iov_offset = 0;
8251 : :
8252 [ # # # # : 0 : rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8253 [ # # # # : 0 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8254 : 0 : lba, lba_count,
8255 : 0 : bdev_nvme_no_pi_readv_done, bio, 0,
8256 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8257 : 0 : md, 0, 0);
8258 : :
8259 [ # # # # ]: 0 : if (rc != 0 && rc != -ENOMEM) {
8260 : 0 : SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc);
8261 : 0 : }
8262 : 0 : return rc;
8263 : : }
8264 : :
8265 : : static int
8266 : 13198401 : bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8267 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8268 : : struct spdk_memory_domain *domain, void *domain_ctx,
8269 : : struct spdk_accel_sequence *seq)
8270 : : {
8271 [ + - + - : 13198401 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
+ - + - +
- + - ]
8272 [ + - + - : 13198401 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
+ - + - +
- + - ]
8273 : : int rc;
8274 : :
8275 [ + + + + : 13198401 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n",
+ - ]
8276 : : lba_count, lba);
8277 : :
8278 [ + - + - ]: 13198401 : bio->iovs = iov;
8279 [ + - + - ]: 13198401 : bio->iovcnt = iovcnt;
8280 [ + - + - ]: 13198401 : bio->iovpos = 0;
8281 [ + - + - ]: 13198401 : bio->iov_offset = 0;
8282 : :
8283 [ + + + + ]: 13198401 : if (domain != NULL || seq != NULL) {
8284 [ # # # # : 361515 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8285 [ # # # # : 361515 : bio->ext_opts.memory_domain = domain;
# # ]
8286 [ # # # # : 361515 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8287 [ # # # # : 361515 : bio->ext_opts.io_flags = flags;
# # ]
8288 [ # # # # : 361515 : bio->ext_opts.metadata = md;
# # ]
8289 [ # # # # : 361515 : bio->ext_opts.accel_sequence = seq;
# # ]
8290 : :
8291 [ + - ]: 361515 : if (iovcnt == 1) {
8292 [ # # # # : 361512 : rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done,
# # ]
8293 [ # # ]: 1 : bio, &bio->ext_opts);
8294 : 1 : } else {
8295 : 0 : rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count,
8296 : 0 : bdev_nvme_readv_done, bio,
8297 : : bdev_nvme_queued_reset_sgl,
8298 : : bdev_nvme_queued_next_sge,
8299 [ # # ]: 0 : &bio->ext_opts);
8300 : : }
8301 [ + + ]: 12836891 : } else if (iovcnt == 1) {
8302 [ - + - + : 12890744 : rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base,
- + ]
8303 : 273986 : md, lba, lba_count, bdev_nvme_readv_done,
8304 : 273986 : bio, flags, 0, 0);
8305 : 273986 : } else {
8306 : 220134 : rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count,
8307 : 5 : bdev_nvme_readv_done, bio, flags,
8308 : : bdev_nvme_queued_reset_sgl,
8309 : 5 : bdev_nvme_queued_next_sge, md, 0, 0);
8310 : : }
8311 : :
8312 [ + + + + ]: 13198401 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8313 : 0 : SPDK_ERRLOG("readv failed: rc = %d\n", rc);
8314 : 0 : }
8315 : 13198401 : return rc;
8316 : : }
8317 : :
8318 : : static int
8319 : 13649008 : bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8320 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8321 : : struct spdk_memory_domain *domain, void *domain_ctx,
8322 : : struct spdk_accel_sequence *seq,
8323 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13)
8324 : : {
8325 [ # # # # : 13649008 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8326 [ # # # # : 13649008 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8327 : : int rc;
8328 : :
8329 [ + + - + : 13649008 : SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8330 : : lba_count, lba);
8331 : :
8332 [ # # # # ]: 13649008 : bio->iovs = iov;
8333 [ # # # # ]: 13649008 : bio->iovcnt = iovcnt;
8334 [ # # # # ]: 13649008 : bio->iovpos = 0;
8335 [ # # # # ]: 13649008 : bio->iov_offset = 0;
8336 : :
8337 [ + + + + ]: 13649008 : if (domain != NULL || seq != NULL) {
8338 [ # # # # : 198937 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8339 [ # # # # : 198937 : bio->ext_opts.memory_domain = domain;
# # ]
8340 [ # # # # : 198937 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8341 [ - + # # : 198937 : bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype);
# # # # #
# ]
8342 [ # # # # : 198937 : bio->ext_opts.cdw13 = cdw13.raw;
# # ]
8343 [ # # # # : 198937 : bio->ext_opts.metadata = md;
# # ]
8344 [ # # # # : 198937 : bio->ext_opts.accel_sequence = seq;
# # ]
8345 : :
8346 [ + - ]: 198937 : if (iovcnt == 1) {
8347 [ # # # # : 198911 : rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done,
# # ]
8348 [ # # ]: 0 : bio, &bio->ext_opts);
8349 : 0 : } else {
8350 : 0 : rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count,
8351 : 0 : bdev_nvme_writev_done, bio,
8352 : : bdev_nvme_queued_reset_sgl,
8353 : : bdev_nvme_queued_next_sge,
8354 [ # # ]: 0 : &bio->ext_opts);
8355 : : }
8356 [ + + ]: 13450097 : } else if (iovcnt == 1) {
8357 [ # # # # : 13615469 : rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base,
# # ]
8358 : 273956 : md, lba, lba_count, bdev_nvme_writev_done,
8359 : 273956 : bio, flags, 0, 0);
8360 : 273956 : } else {
8361 : 108586 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8362 : 2 : bdev_nvme_writev_done, bio, flags,
8363 : : bdev_nvme_queued_reset_sgl,
8364 : 2 : bdev_nvme_queued_next_sge, md, 0, 0);
8365 : : }
8366 : :
8367 [ + + - + ]: 13649008 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8368 : 0 : SPDK_ERRLOG("writev failed: rc = %d\n", rc);
8369 : 0 : }
8370 : 13649008 : return rc;
8371 : : }
8372 : :
8373 : : static int
8374 : 231053 : bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8375 : : void *md, uint64_t lba_count, uint64_t zslba,
8376 : : uint32_t flags)
8377 : : {
8378 [ # # # # : 231053 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8379 [ # # # # : 231053 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8380 : : int rc;
8381 : :
8382 [ - + # # : 231053 : SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n",
# # ]
8383 : : lba_count, zslba);
8384 : :
8385 [ # # # # ]: 231053 : bio->iovs = iov;
8386 [ # # # # ]: 231053 : bio->iovcnt = iovcnt;
8387 [ # # # # ]: 231053 : bio->iovpos = 0;
8388 [ # # # # ]: 231053 : bio->iov_offset = 0;
8389 : :
8390 [ + - ]: 231053 : if (iovcnt == 1) {
8391 [ # # # # : 231053 : rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba,
# # ]
8392 : 0 : lba_count,
8393 : 0 : bdev_nvme_zone_appendv_done, bio,
8394 : 0 : flags,
8395 : : 0, 0);
8396 : 0 : } else {
8397 : 0 : rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count,
8398 : 0 : bdev_nvme_zone_appendv_done, bio, flags,
8399 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8400 : 0 : md, 0, 0);
8401 : : }
8402 : :
8403 [ - + - - ]: 231053 : if (rc != 0 && rc != -ENOMEM) {
8404 : 0 : SPDK_ERRLOG("zone append failed: rc = %d\n", rc);
8405 : 0 : }
8406 : 231053 : return rc;
8407 : : }
8408 : :
8409 : : static int
8410 : 56 : bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8411 : : void *md, uint64_t lba_count, uint64_t lba,
8412 : : uint32_t flags)
8413 : : {
8414 : : int rc;
8415 : :
8416 [ + + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8417 : : lba_count, lba);
8418 : :
8419 [ # # # # ]: 56 : bio->iovs = iov;
8420 [ # # # # ]: 56 : bio->iovcnt = iovcnt;
8421 [ # # # # ]: 56 : bio->iovpos = 0;
8422 [ # # # # ]: 56 : bio->iov_offset = 0;
8423 : :
8424 [ # # # # : 59 : rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8425 [ # # # # : 56 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8426 : 3 : lba, lba_count,
8427 : 3 : bdev_nvme_comparev_done, bio, flags,
8428 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8429 : 3 : md, 0, 0);
8430 : :
8431 [ - + - - ]: 56 : if (rc != 0 && rc != -ENOMEM) {
8432 : 0 : SPDK_ERRLOG("comparev failed: rc = %d\n", rc);
8433 : 0 : }
8434 : 56 : return rc;
8435 : : }
8436 : :
8437 : : static int
8438 : 53 : bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt,
8439 : : struct iovec *write_iov, int write_iovcnt,
8440 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags)
8441 : : {
8442 [ # # # # : 53 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8443 [ # # # # : 53 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8444 : 53 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8445 : : int rc;
8446 : :
8447 [ + + - + : 53 : SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8448 : : lba_count, lba);
8449 : :
8450 [ # # # # ]: 53 : bio->iovs = cmp_iov;
8451 [ # # # # ]: 53 : bio->iovcnt = cmp_iovcnt;
8452 [ # # # # ]: 53 : bio->iovpos = 0;
8453 [ # # # # ]: 53 : bio->iov_offset = 0;
8454 [ # # # # ]: 53 : bio->fused_iovs = write_iov;
8455 [ # # # # ]: 53 : bio->fused_iovcnt = write_iovcnt;
8456 [ # # # # ]: 53 : bio->fused_iovpos = 0;
8457 [ # # # # ]: 53 : bio->fused_iov_offset = 0;
8458 : :
8459 [ + + # # : 53 : if (bdev_io->num_retries == 0) {
# # ]
8460 [ # # # # ]: 53 : bio->first_fused_submitted = false;
8461 [ # # # # ]: 53 : bio->first_fused_completed = false;
8462 : 2 : }
8463 : :
8464 [ + + + - : 53 : if (!bio->first_fused_submitted) {
# # # # ]
8465 [ # # # # ]: 53 : flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8466 [ - + # # ]: 53 : memset(&bio->cpl, 0, sizeof(bio->cpl));
8467 : :
8468 : 55 : rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count,
8469 : 2 : bdev_nvme_comparev_and_writev_done, bio, flags,
8470 : 2 : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0);
8471 [ + + ]: 53 : if (rc == 0) {
8472 [ # # # # ]: 53 : bio->first_fused_submitted = true;
8473 [ # # # # ]: 53 : flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8474 : 2 : } else {
8475 [ # # ]: 0 : if (rc != -ENOMEM) {
8476 : 0 : SPDK_ERRLOG("compare failed: rc = %d\n", rc);
8477 : 0 : }
8478 : 0 : return rc;
8479 : : }
8480 : 2 : }
8481 : :
8482 [ # # # # ]: 53 : flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND;
8483 : :
8484 : 55 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8485 : 2 : bdev_nvme_comparev_and_writev_done, bio, flags,
8486 : 2 : bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0);
8487 [ - + - - ]: 53 : if (rc != 0 && rc != -ENOMEM) {
8488 : 0 : SPDK_ERRLOG("write failed: rc = %d\n", rc);
8489 : 0 : rc = 0;
8490 : 0 : }
8491 : :
8492 : 53 : return rc;
8493 : 2 : }
8494 : :
8495 : : static int
8496 : 79818 : bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8497 : : {
8498 : 4950 : struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES];
8499 : : struct spdk_nvme_dsm_range *range;
8500 : : uint64_t offset, remaining;
8501 : : uint64_t num_ranges_u64;
8502 : : uint16_t num_ranges;
8503 : : int rc;
8504 : :
8505 [ # # ]: 79818 : num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) /
8506 : : SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8507 [ - + ]: 79818 : if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) {
8508 : 0 : SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks);
8509 : 0 : return -EINVAL;
8510 : : }
8511 : 79818 : num_ranges = (uint16_t)num_ranges_u64;
8512 : :
8513 : 79818 : offset = offset_blocks;
8514 : 79818 : remaining = num_blocks;
8515 [ # # # # ]: 79818 : range = &dsm_ranges[0];
8516 : :
8517 : : /* Fill max-size ranges until the remaining blocks fit into one range */
8518 [ + + ]: 79822 : while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
8519 [ # # # # : 4 : range->attributes.raw = 0;
# # ]
8520 [ # # # # ]: 4 : range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8521 [ # # # # ]: 4 : range->starting_lba = offset;
8522 : :
8523 : 4 : offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8524 : 4 : remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8525 [ # # ]: 4 : range++;
8526 : : }
8527 : :
8528 : : /* Final range describes the remaining blocks */
8529 [ # # # # : 79818 : range->attributes.raw = 0;
# # ]
8530 [ # # # # ]: 79818 : range->length = remaining;
8531 [ # # # # ]: 79818 : range->starting_lba = offset;
8532 : :
8533 [ # # # # : 79821 : rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8534 [ # # # # : 79818 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8535 : : SPDK_NVME_DSM_ATTR_DEALLOCATE,
8536 : 3 : dsm_ranges, num_ranges,
8537 : 3 : bdev_nvme_queued_done, bio);
8538 : :
8539 : 79818 : return rc;
8540 : 3 : }
8541 : :
8542 : : static int
8543 : 694305 : bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8544 : : {
8545 [ - + ]: 694305 : if (num_blocks > UINT16_MAX + 1) {
8546 : 0 : SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n");
8547 : 0 : return -EINVAL;
8548 : : }
8549 : :
8550 [ # # # # : 766809 : return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8551 [ # # # # : 694305 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8552 : 72504 : offset_blocks, num_blocks,
8553 : 72504 : bdev_nvme_queued_done, bio,
8554 : : 0);
8555 : 72504 : }
8556 : :
8557 : : static int
8558 : 1 : bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones,
8559 : : struct spdk_bdev_zone_info *info)
8560 : : {
8561 [ # # # # : 1 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8562 [ # # # # : 1 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8563 : 1 : uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
8564 : 1 : uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
8565 : 1 : uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns);
8566 : :
8567 [ - + # # ]: 1 : if (zone_id % zone_size != 0) {
8568 : 0 : return -EINVAL;
8569 : : }
8570 : :
8571 [ + - - + ]: 1 : if (num_zones > total_zones || !num_zones) {
8572 : 0 : return -EINVAL;
8573 : : }
8574 : :
8575 [ - + # # : 1 : assert(!bio->zone_report_buf);
# # # # ]
8576 [ # # # # ]: 1 : bio->zone_report_buf = calloc(1, zone_report_bufsize);
8577 [ - + # # : 1 : if (!bio->zone_report_buf) {
# # ]
8578 : 0 : return -ENOMEM;
8579 : : }
8580 : :
8581 [ # # # # ]: 1 : bio->handled_zones = 0;
8582 : :
8583 [ # # # # ]: 1 : return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize,
8584 : 0 : zone_id, SPDK_NVME_ZRA_LIST_ALL, true,
8585 : 0 : bdev_nvme_get_zone_info_done, bio);
8586 : 0 : }
8587 : :
8588 : : static int
8589 : 43 : bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
8590 : : enum spdk_bdev_zone_action action)
8591 : : {
8592 [ # # # # : 43 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8593 [ # # # # : 43 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8594 : :
8595 [ - - - + : 43 : switch (action) {
- - ]
8596 : 0 : case SPDK_BDEV_ZONE_CLOSE:
8597 : 0 : return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false,
8598 : 0 : bdev_nvme_zone_management_done, bio);
8599 : 0 : case SPDK_BDEV_ZONE_FINISH:
8600 : 0 : return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false,
8601 : 0 : bdev_nvme_zone_management_done, bio);
8602 : 0 : case SPDK_BDEV_ZONE_OPEN:
8603 : 0 : return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false,
8604 : 0 : bdev_nvme_zone_management_done, bio);
8605 : 43 : case SPDK_BDEV_ZONE_RESET:
8606 : 43 : return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false,
8607 : 0 : bdev_nvme_zone_management_done, bio);
8608 : 0 : case SPDK_BDEV_ZONE_OFFLINE:
8609 : 0 : return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false,
8610 : 0 : bdev_nvme_zone_management_done, bio);
8611 : 0 : default:
8612 : 0 : return -EINVAL;
8613 : : }
8614 : 0 : }
8615 : :
8616 : : static void
8617 : 142 : bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8618 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes)
8619 : : {
8620 : : struct nvme_io_path *io_path;
8621 : : struct nvme_ctrlr *nvme_ctrlr;
8622 : : uint32_t max_xfer_size;
8623 : 142 : int rc = -ENXIO;
8624 : :
8625 : : /* Choose the first ctrlr which is not failed. */
8626 [ + + # # : 154 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8627 [ # # # # : 150 : nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
8628 : :
8629 : : /* We should skip any unavailable nvme_ctrlr rather than checking
8630 : : * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO.
8631 : : */
8632 [ + + ]: 150 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
8633 : 12 : continue;
8634 : : }
8635 : :
8636 [ # # # # ]: 138 : max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr);
8637 : :
8638 [ + + ]: 138 : if (nbytes > max_xfer_size) {
8639 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8640 : 0 : rc = -EINVAL;
8641 : 0 : goto err;
8642 : : }
8643 : :
8644 [ # # # # ]: 143 : rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes,
8645 : 5 : bdev_nvme_admin_passthru_done, bio);
8646 [ + + ]: 138 : if (rc == 0) {
8647 : 138 : return;
8648 : : }
8649 : 1 : }
8650 : :
8651 : 3 : err:
8652 : 4 : bdev_nvme_admin_complete(bio, rc);
8653 : 6 : }
8654 : :
8655 : : static int
8656 : 111 : bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8657 : : void *buf, size_t nbytes)
8658 : : {
8659 [ # # # # : 111 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8660 [ # # # # : 111 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8661 : 111 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8662 : 111 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8663 : :
8664 [ - + ]: 111 : if (nbytes > max_xfer_size) {
8665 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8666 : 0 : return -EINVAL;
8667 : : }
8668 : :
8669 : : /*
8670 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8671 : : * so fill it out automatically.
8672 : : */
8673 [ # # # # ]: 111 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8674 : :
8675 : 114 : return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf,
8676 : 3 : (uint32_t)nbytes, bdev_nvme_queued_done, bio);
8677 : 3 : }
8678 : :
8679 : : static int
8680 : 0 : bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8681 : : void *buf, size_t nbytes, void *md_buf, size_t md_len)
8682 : : {
8683 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8684 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8685 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8686 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8687 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8688 : :
8689 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8690 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8691 : 0 : return -EINVAL;
8692 : : }
8693 : :
8694 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8695 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8696 : 0 : return -EINVAL;
8697 : : }
8698 : :
8699 : : /*
8700 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8701 : : * so fill it out automatically.
8702 : : */
8703 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8704 : :
8705 : 0 : return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf,
8706 : 0 : (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio);
8707 : 0 : }
8708 : :
8709 : : static int
8710 : 0 : bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio,
8711 : : struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt,
8712 : : size_t nbytes, void *md_buf, size_t md_len)
8713 : : {
8714 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8715 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8716 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8717 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8718 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8719 : :
8720 [ # # # # ]: 0 : bio->iovs = iov;
8721 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8722 [ # # # # ]: 0 : bio->iovpos = 0;
8723 [ # # # # ]: 0 : bio->iov_offset = 0;
8724 : :
8725 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8726 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8727 : 0 : return -EINVAL;
8728 : : }
8729 : :
8730 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8731 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8732 : 0 : return -EINVAL;
8733 : : }
8734 : :
8735 : : /*
8736 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands
8737 : : * require a nsid, so fill it out automatically.
8738 : : */
8739 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8740 : :
8741 : 0 : return spdk_nvme_ctrlr_cmd_iov_raw_with_md(
8742 : 0 : ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio,
8743 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
8744 : 0 : }
8745 : :
8746 : : static void
8747 : 7176 : bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8748 : : struct nvme_bdev_io *bio_to_abort)
8749 : : {
8750 : : struct nvme_io_path *io_path;
8751 : 7176 : int rc = 0;
8752 : :
8753 : 7176 : rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort);
8754 [ + + ]: 7176 : if (rc == 0) {
8755 : 4 : bdev_nvme_admin_complete(bio, 0);
8756 : 4 : return;
8757 : : }
8758 : :
8759 [ # # # # ]: 7172 : io_path = bio_to_abort->io_path;
8760 [ + + ]: 7172 : if (io_path != NULL) {
8761 [ # # # # : 7167 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8762 [ # # # # : 7164 : io_path->qpair->qpair,
# # # # ]
8763 : 3 : bio_to_abort,
8764 : 3 : bdev_nvme_abort_done, bio);
8765 : 3 : } else {
8766 [ + + # # : 12 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8767 [ # # # # : 10 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8768 : : NULL,
8769 : 2 : bio_to_abort,
8770 : 2 : bdev_nvme_abort_done, bio);
8771 : :
8772 [ + + ]: 8 : if (rc != -ENOENT) {
8773 : 4 : break;
8774 : : }
8775 : 1 : }
8776 : : }
8777 : :
8778 [ + + ]: 7172 : if (rc != 0) {
8779 : : /* If no command was found or there was any error, complete the abort
8780 : : * request with failure.
8781 : : */
8782 : 8 : bdev_nvme_admin_complete(bio, rc);
8783 : 2 : }
8784 : 6 : }
8785 : :
8786 : : static int
8787 : 37 : bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks,
8788 : : uint64_t num_blocks)
8789 : : {
8790 : 39 : struct spdk_nvme_scc_source_range range = {
8791 : 1 : .slba = src_offset_blocks,
8792 : 37 : .nlb = num_blocks - 1
8793 : : };
8794 : :
8795 [ # # # # : 46 : return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8796 [ # # # # : 37 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8797 : 1 : &range, 1, dst_offset_blocks,
8798 : 1 : bdev_nvme_queued_done, bio);
8799 : : }
8800 : :
8801 : : static void
8802 : 197 : bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w)
8803 : : {
8804 : : const char *action;
8805 : : uint32_t i;
8806 : :
8807 [ + + - + ]: 197 : if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
8808 : 0 : action = "reset";
8809 [ + + - + ]: 197 : } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
8810 : 7 : action = "abort";
8811 : 1 : } else {
8812 : 190 : action = "none";
8813 : : }
8814 : :
8815 : 197 : spdk_json_write_object_begin(w);
8816 : :
8817 : 197 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_options");
8818 : :
8819 : 197 : spdk_json_write_named_object_begin(w, "params");
8820 : 197 : spdk_json_write_named_string(w, "action_on_timeout", action);
8821 [ + - ]: 197 : spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
8822 [ + - ]: 197 : spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us);
8823 [ + - ]: 197 : spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms);
8824 [ + - ]: 197 : spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst);
8825 [ + - ]: 197 : spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight);
8826 [ + - ]: 197 : spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight);
8827 [ + - ]: 197 : spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight);
8828 [ + - ]: 197 : spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
8829 [ + - ]: 197 : spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
8830 [ + - ]: 197 : spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
8831 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit);
8832 [ + - ]: 197 : spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count);
8833 [ + - ]: 197 : spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count);
8834 [ + - ]: 197 : spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout);
8835 [ + - ]: 197 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec);
8836 [ + - ]: 197 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec);
8837 [ + - ]: 197 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec);
8838 [ + + ]: 197 : spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback);
8839 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids);
8840 : 197 : spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos);
8841 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat);
8842 [ + - ]: 197 : spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size);
8843 [ + + ]: 197 : spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat);
8844 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence);
8845 [ + - ]: 197 : spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size);
8846 [ + - ]: 197 : spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms);
8847 : 197 : spdk_json_write_named_array_begin(w, "dhchap_digests");
8848 [ + + ]: 6501 : for (i = 0; i < 32; ++i) {
8849 [ + + + + : 6304 : if (g_opts.dhchap_digests & SPDK_BIT(i)) {
+ + ]
8850 : 591 : spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i));
8851 : 33 : }
8852 : 352 : }
8853 : 197 : spdk_json_write_array_end(w);
8854 : 197 : spdk_json_write_named_array_begin(w, "dhchap_dhgroups");
8855 [ + + ]: 6501 : for (i = 0; i < 32; ++i) {
8856 [ + + + + : 6304 : if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) {
+ + ]
8857 : 1182 : spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i));
8858 : 66 : }
8859 : 352 : }
8860 : :
8861 : 197 : spdk_json_write_array_end(w);
8862 : 197 : spdk_json_write_object_end(w);
8863 : :
8864 : 197 : spdk_json_write_object_end(w);
8865 : 197 : }
8866 : :
8867 : : static void
8868 : 0 : bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx)
8869 : : {
8870 : 0 : struct spdk_nvme_transport_id trid;
8871 : :
8872 : 0 : spdk_json_write_object_begin(w);
8873 : :
8874 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery");
8875 : :
8876 : 0 : spdk_json_write_named_object_begin(w, "params");
8877 [ # # # # ]: 0 : spdk_json_write_named_string(w, "name", ctx->name);
8878 [ # # # # ]: 0 : spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn);
8879 : :
8880 [ # # ]: 0 : trid = ctx->trid;
8881 [ # # ]: 0 : memset(trid.subnqn, 0, sizeof(trid.subnqn));
8882 : 0 : nvme_bdev_dump_trid_json(&trid, w);
8883 : :
8884 [ # # # # : 0 : spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach);
# # ]
8885 [ # # # # : 0 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec);
# # ]
8886 [ # # # # : 0 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec);
# # ]
8887 : 0 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8888 [ # # # # : 0 : ctx->bdev_opts.fast_io_fail_timeout_sec);
# # ]
8889 : 0 : spdk_json_write_object_end(w);
8890 : :
8891 : 0 : spdk_json_write_object_end(w);
8892 : 0 : }
8893 : :
8894 : : #ifdef SPDK_CONFIG_NVME_CUSE
8895 : : static void
8896 : 116 : nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w,
8897 : : struct nvme_ctrlr *nvme_ctrlr)
8898 : 116 : {
8899 : 116 : size_t cuse_name_size = 128;
8900 [ - + ]: 116 : char cuse_name[cuse_name_size];
8901 : :
8902 [ + - # # : 116 : if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr,
# # # # ]
8903 : 0 : cuse_name, &cuse_name_size) != 0) {
8904 : 116 : return;
8905 : : }
8906 : :
8907 : 0 : spdk_json_write_object_begin(w);
8908 : :
8909 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register");
8910 : :
8911 : 0 : spdk_json_write_named_object_begin(w, "params");
8912 [ # # # # : 0 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8913 : 0 : spdk_json_write_object_end(w);
8914 : :
8915 : 0 : spdk_json_write_object_end(w);
8916 [ # # ]: 0 : }
8917 : : #endif
8918 : :
8919 : : static void
8920 : 121 : nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,
8921 : : struct nvme_ctrlr *nvme_ctrlr,
8922 : : struct nvme_path_id *path_id)
8923 : : {
8924 : : struct spdk_nvme_transport_id *trid;
8925 : : const struct spdk_nvme_ctrlr_opts *opts;
8926 : :
8927 [ - + - + : 121 : if (nvme_ctrlr->opts.from_discovery_service) {
# # # # #
# ]
8928 : : /* Do not emit an RPC for this - it will be implicitly
8929 : : * covered by a separate bdev_nvme_start_discovery or
8930 : : * bdev_nvme_start_mdns_discovery RPC.
8931 : : */
8932 : 0 : return;
8933 : : }
8934 : :
8935 [ # # ]: 121 : trid = &path_id->trid;
8936 : :
8937 : 121 : spdk_json_write_object_begin(w);
8938 : :
8939 : 121 : spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller");
8940 : :
8941 : 121 : spdk_json_write_named_object_begin(w, "params");
8942 [ # # # # : 121 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8943 : 121 : nvme_bdev_dump_trid_json(trid, w);
8944 : 126 : spdk_json_write_named_bool(w, "prchk_reftag",
8945 [ # # # # : 121 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);
# # # # ]
8946 : 126 : spdk_json_write_named_bool(w, "prchk_guard",
8947 [ # # # # : 121 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);
# # # # ]
8948 [ # # # # : 121 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);
# # ]
8949 [ # # # # : 121 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);
# # ]
8950 : 126 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8951 [ # # # # : 5 : nvme_ctrlr->opts.fast_io_fail_timeout_sec);
# # ]
8952 [ + + # # : 121 : if (nvme_ctrlr->psk != NULL) {
# # ]
8953 [ # # # # ]: 9 : spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk));
8954 : 0 : }
8955 [ + + # # : 121 : if (nvme_ctrlr->dhchap_key != NULL) {
# # ]
8956 : 0 : spdk_json_write_named_string(w, "dhchap_key",
8957 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_key));
8958 : 0 : }
8959 [ + + # # : 121 : if (nvme_ctrlr->dhchap_ctrlr_key != NULL) {
# # ]
8960 : 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
8961 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_ctrlr_key));
8962 : 0 : }
8963 [ # # # # ]: 121 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
8964 [ # # ]: 121 : spdk_json_write_named_string(w, "hostnqn", opts->hostnqn);
8965 [ - + # # : 121 : spdk_json_write_named_bool(w, "hdgst", opts->header_digest);
# # ]
8966 [ - + # # : 121 : spdk_json_write_named_bool(w, "ddgst", opts->data_digest);
# # ]
8967 [ + + # # : 121 : if (opts->src_addr[0] != '\0') {
# # # # #
# ]
8968 [ # # ]: 0 : spdk_json_write_named_string(w, "hostaddr", opts->src_addr);
8969 : 0 : }
8970 [ + + # # : 121 : if (opts->src_svcid[0] != '\0') {
# # # # #
# ]
8971 [ # # ]: 0 : spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid);
8972 : 0 : }
8973 : :
8974 [ + + + - : 121 : if (nvme_ctrlr->opts.multipath) {
# # # # #
# ]
8975 : 121 : spdk_json_write_named_string(w, "multipath", "multipath");
8976 : 5 : }
8977 : 121 : spdk_json_write_object_end(w);
8978 : :
8979 : 121 : spdk_json_write_object_end(w);
8980 : 5 : }
8981 : :
8982 : : static void
8983 : 197 : bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w)
8984 : : {
8985 : 197 : spdk_json_write_object_begin(w);
8986 : 197 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug");
8987 : :
8988 : 197 : spdk_json_write_named_object_begin(w, "params");
8989 : 197 : spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
8990 [ + + ]: 197 : spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
8991 : 197 : spdk_json_write_object_end(w);
8992 : :
8993 : 197 : spdk_json_write_object_end(w);
8994 : 197 : }
8995 : :
8996 : : static int
8997 : 197 : bdev_nvme_config_json(struct spdk_json_write_ctx *w)
8998 : : {
8999 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
9000 : : struct nvme_ctrlr *nvme_ctrlr;
9001 : : struct discovery_ctx *ctx;
9002 : : struct nvme_path_id *path_id;
9003 : :
9004 : 197 : bdev_nvme_opts_config_json(w);
9005 : :
9006 [ + + ]: 197 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9007 : :
9008 [ + + # # : 318 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
9009 [ + + # # : 242 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
9010 [ # # # # ]: 121 : path_id = nvme_ctrlr->active_path_id;
9011 [ + + # # : 121 : assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
9012 : 121 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
9013 : :
9014 [ # # # # : 121 : path_id = TAILQ_NEXT(path_id, link);
# # ]
9015 [ - + ]: 121 : while (path_id != NULL) {
9016 : 0 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
9017 [ # # # # : 0 : path_id = TAILQ_NEXT(path_id, link);
# # ]
9018 : : }
9019 : :
9020 : : #ifdef SPDK_CONFIG_NVME_CUSE
9021 : 116 : nvme_ctrlr_cuse_config_json(w, nvme_ctrlr);
9022 : : #endif
9023 : 5 : }
9024 : 5 : }
9025 : :
9026 [ - + # # : 197 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
9027 [ # # # # : 0 : if (!ctx->from_mdns_discovery_service) {
# # # # ]
9028 : 0 : bdev_nvme_discovery_config_json(w, ctx);
9029 : 0 : }
9030 : 0 : }
9031 : :
9032 : 197 : bdev_nvme_mdns_discovery_config_json(w);
9033 : :
9034 : : /* Dump as last parameter to give all NVMe bdevs chance to be constructed
9035 : : * before enabling hotplug poller.
9036 : : */
9037 : 197 : bdev_nvme_hotplug_config_json(w);
9038 : :
9039 [ + + ]: 197 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9040 : 197 : return 0;
9041 : : }
9042 : :
9043 : : struct spdk_nvme_ctrlr *
9044 : 11 : bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
9045 : : {
9046 : : struct nvme_bdev *nbdev;
9047 : : struct nvme_ns *nvme_ns;
9048 : :
9049 [ + - - + : 11 : if (!bdev || bdev->module != &nvme_if) {
# # # # ]
9050 : 0 : return NULL;
9051 : : }
9052 : :
9053 : 11 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
9054 [ # # # # : 11 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
9055 [ + + # # ]: 11 : assert(nvme_ns != NULL);
9056 : :
9057 [ # # # # : 11 : return nvme_ns->ctrlr->ctrlr;
# # # # ]
9058 : 2 : }
9059 : :
9060 : : static bool
9061 : 528 : nvme_io_path_is_current(struct nvme_io_path *io_path)
9062 : : {
9063 : : const struct nvme_bdev_channel *nbdev_ch;
9064 : : bool current;
9065 : :
9066 [ + + ]: 528 : if (!nvme_io_path_is_available(io_path)) {
9067 : 136 : return false;
9068 : : }
9069 : :
9070 [ # # # # ]: 392 : nbdev_ch = io_path->nbdev_ch;
9071 [ + + ]: 392 : if (nbdev_ch == NULL) {
9072 : 4 : current = false;
9073 [ + + # # : 389 : } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
9074 : 180 : struct nvme_io_path *optimized_io_path = NULL;
9075 : :
9076 [ + + # # : 384 : STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
9077 [ + + # # : 308 : if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) {
# # # # #
# ]
9078 : 104 : break;
9079 : : }
9080 : 3 : }
9081 : :
9082 : : /* A non-optimized path is only current if there are no optimized paths. */
9083 [ + + + + : 180 : current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) ||
# # # # #
# ]
9084 : 2 : (optimized_io_path == NULL);
9085 : 3 : } else {
9086 [ + + # # : 208 : if (nbdev_ch->current_io_path) {
# # ]
9087 [ # # # # ]: 124 : current = (io_path == nbdev_ch->current_io_path);
9088 : 1 : } else {
9089 : : struct nvme_io_path *first_path;
9090 : :
9091 : : /* We arrived here as there are no optimized paths for active-passive
9092 : : * mode. Check if this io_path is the first one available on the list.
9093 : : */
9094 : 84 : current = false;
9095 [ + + # # : 84 : STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
9096 [ + - ]: 84 : if (nvme_io_path_is_available(first_path)) {
9097 : 84 : current = (io_path == first_path);
9098 : 84 : break;
9099 : : }
9100 : 0 : }
9101 : : }
9102 : : }
9103 : :
9104 [ # # ]: 392 : return current;
9105 : 12 : }
9106 : :
9107 : : static struct nvme_ctrlr *
9108 : 40 : bdev_nvme_next_ctrlr_unsafe(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct nvme_ctrlr *prev)
9109 : : {
9110 : : struct nvme_ctrlr *next;
9111 : :
9112 : : /* Must be called under g_bdev_nvme_mutex */
9113 [ + + # # : 40 : next = prev != NULL ? TAILQ_NEXT(prev, tailq) : TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # # # #
# # # #
# ]
9114 [ + + ]: 40 : while (next != NULL) {
9115 : : /* ref can be 0 when the ctrlr was released, but hasn't been detached yet */
9116 [ - + # # ]: 28 : pthread_mutex_lock(&next->mutex);
9117 [ + - # # : 28 : if (next->ref > 0) {
# # ]
9118 [ # # # # ]: 28 : next->ref++;
9119 [ - + # # ]: 28 : pthread_mutex_unlock(&next->mutex);
9120 : 28 : return next;
9121 : : }
9122 : :
9123 [ # # # # ]: 0 : pthread_mutex_unlock(&next->mutex);
9124 [ # # # # : 0 : next = TAILQ_NEXT(next, tailq);
# # ]
9125 : : }
9126 : :
9127 : 12 : return NULL;
9128 : 0 : }
9129 : :
9130 : : struct bdev_nvme_set_keys_ctx {
9131 : : struct nvme_ctrlr *nctrlr;
9132 : : struct spdk_key *dhchap_key;
9133 : : struct spdk_key *dhchap_ctrlr_key;
9134 : : struct spdk_thread *thread;
9135 : : bdev_nvme_set_keys_cb cb_fn;
9136 : : void *cb_ctx;
9137 : : int status;
9138 : : };
9139 : :
9140 : : static void
9141 : 28 : bdev_nvme_free_set_keys_ctx(struct bdev_nvme_set_keys_ctx *ctx)
9142 : : {
9143 [ - + ]: 28 : if (ctx == NULL) {
9144 : 0 : return;
9145 : : }
9146 : :
9147 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_key);
9148 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_ctrlr_key);
9149 : 28 : free(ctx);
9150 : 0 : }
9151 : :
9152 : : static void
9153 : 28 : _bdev_nvme_set_keys_done(void *_ctx)
9154 : : {
9155 : 28 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
9156 : :
9157 [ # # # # : 28 : ctx->cb_fn(ctx->cb_ctx, ctx->status);
# # # # #
# # # # #
# # ]
9158 : :
9159 [ + + # # : 28 : if (ctx->nctrlr != NULL) {
# # ]
9160 [ # # # # ]: 16 : nvme_ctrlr_put_ref(ctx->nctrlr);
9161 : 0 : }
9162 : 28 : bdev_nvme_free_set_keys_ctx(ctx);
9163 : 28 : }
9164 : :
9165 : : static void
9166 : 28 : bdev_nvme_set_keys_done(struct bdev_nvme_set_keys_ctx *ctx, int status)
9167 : : {
9168 [ # # # # ]: 28 : ctx->status = status;
9169 [ # # # # ]: 28 : spdk_thread_exec_msg(ctx->thread, _bdev_nvme_set_keys_done, ctx);
9170 : 28 : }
9171 : :
9172 : : static void bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx);
9173 : :
9174 : : static void
9175 : 12 : bdev_nvme_authenticate_ctrlr_continue(struct bdev_nvme_set_keys_ctx *ctx)
9176 : : {
9177 : : struct nvme_ctrlr *next;
9178 : :
9179 [ - + ]: 12 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9180 [ # # # # ]: 12 : next = bdev_nvme_next_ctrlr_unsafe(NULL, ctx->nctrlr);
9181 [ - + ]: 12 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9182 : :
9183 [ # # # # ]: 12 : nvme_ctrlr_put_ref(ctx->nctrlr);
9184 [ # # # # ]: 12 : ctx->nctrlr = next;
9185 : :
9186 [ + - ]: 12 : if (next == NULL) {
9187 : 12 : bdev_nvme_set_keys_done(ctx, 0);
9188 : 0 : } else {
9189 : 0 : bdev_nvme_authenticate_ctrlr(ctx);
9190 : : }
9191 : 12 : }
9192 : :
9193 : : static void
9194 : 8 : bdev_nvme_authenticate_qpairs_done(struct spdk_io_channel_iter *i, int status)
9195 : : {
9196 : 8 : struct bdev_nvme_set_keys_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
9197 : :
9198 [ - + ]: 8 : if (status != 0) {
9199 : 0 : bdev_nvme_set_keys_done(ctx, status);
9200 : 0 : return;
9201 : : }
9202 : 8 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9203 : 0 : }
9204 : :
9205 : : static void
9206 : 0 : bdev_nvme_authenticate_qpair_done(void *ctx, int status)
9207 : : {
9208 : 0 : spdk_for_each_channel_continue(ctx, status);
9209 : 0 : }
9210 : :
9211 : : static void
9212 : 0 : bdev_nvme_authenticate_qpair(struct spdk_io_channel_iter *i)
9213 : : {
9214 : 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
9215 : 0 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
9216 [ # # # # ]: 0 : struct nvme_qpair *qpair = ctrlr_ch->qpair;
9217 : : int rc;
9218 : :
9219 [ # # ]: 0 : if (!nvme_qpair_is_connected(qpair)) {
9220 : 0 : spdk_for_each_channel_continue(i, 0);
9221 : 0 : return;
9222 : : }
9223 : :
9224 [ # # # # ]: 0 : rc = spdk_nvme_qpair_authenticate(qpair->qpair, bdev_nvme_authenticate_qpair_done, i);
9225 [ # # ]: 0 : if (rc != 0) {
9226 : 0 : spdk_for_each_channel_continue(i, rc);
9227 : 0 : }
9228 : 0 : }
9229 : :
9230 : : static void
9231 : 20 : bdev_nvme_authenticate_ctrlr_done(void *_ctx, int status)
9232 : : {
9233 : 20 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
9234 : :
9235 [ + + ]: 20 : if (status != 0) {
9236 : 12 : bdev_nvme_set_keys_done(ctx, status);
9237 : 12 : return;
9238 : : }
9239 : :
9240 [ # # # # ]: 8 : spdk_for_each_channel(ctx->nctrlr, bdev_nvme_authenticate_qpair, ctx,
9241 : : bdev_nvme_authenticate_qpairs_done);
9242 : 0 : }
9243 : :
9244 : : static void
9245 : 28 : bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx)
9246 : : {
9247 : 28 : struct spdk_nvme_ctrlr_key_opts opts = {};
9248 [ # # # # ]: 28 : struct nvme_ctrlr *nctrlr = ctx->nctrlr;
9249 : : int rc;
9250 : :
9251 : 28 : opts.size = SPDK_SIZEOF(&opts, dhchap_ctrlr_key);
9252 [ # # # # : 28 : opts.dhchap_key = ctx->dhchap_key;
# # ]
9253 [ # # # # : 28 : opts.dhchap_ctrlr_key = ctx->dhchap_ctrlr_key;
# # ]
9254 [ # # # # ]: 28 : rc = spdk_nvme_ctrlr_set_keys(nctrlr->ctrlr, &opts);
9255 [ - + ]: 28 : if (rc != 0) {
9256 : 0 : bdev_nvme_set_keys_done(ctx, rc);
9257 : 0 : return;
9258 : : }
9259 : :
9260 [ + + # # : 28 : if (ctx->dhchap_key != NULL) {
# # ]
9261 [ # # # # ]: 24 : rc = spdk_nvme_ctrlr_authenticate(nctrlr->ctrlr,
9262 : 0 : bdev_nvme_authenticate_ctrlr_done, ctx);
9263 [ + + ]: 24 : if (rc != 0) {
9264 : 4 : bdev_nvme_set_keys_done(ctx, rc);
9265 : 0 : }
9266 : 0 : } else {
9267 : 4 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9268 : : }
9269 : 0 : }
9270 : :
9271 : : int
9272 : 28 : bdev_nvme_set_keys(const char *name, const char *dhchap_key, const char *dhchap_ctrlr_key,
9273 : : bdev_nvme_set_keys_cb cb_fn, void *cb_ctx)
9274 : : {
9275 : : struct bdev_nvme_set_keys_ctx *ctx;
9276 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
9277 : : struct nvme_ctrlr *nctrlr;
9278 : :
9279 : 28 : ctx = calloc(1, sizeof(*ctx));
9280 [ - + ]: 28 : if (ctx == NULL) {
9281 : 0 : return -ENOMEM;
9282 : : }
9283 : :
9284 [ + + ]: 28 : if (dhchap_key != NULL) {
9285 [ # # # # ]: 24 : ctx->dhchap_key = spdk_keyring_get_key(dhchap_key);
9286 [ - + # # : 24 : if (ctx->dhchap_key == NULL) {
# # ]
9287 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_key, name);
9288 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9289 : 0 : return -ENOKEY;
9290 : : }
9291 : 0 : }
9292 [ + + ]: 28 : if (dhchap_ctrlr_key != NULL) {
9293 [ # # # # ]: 24 : ctx->dhchap_ctrlr_key = spdk_keyring_get_key(dhchap_ctrlr_key);
9294 [ - + # # : 24 : if (ctx->dhchap_ctrlr_key == NULL) {
# # ]
9295 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_ctrlr_key, name);
9296 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9297 : 0 : return -ENOKEY;
9298 : : }
9299 : 0 : }
9300 : :
9301 [ - + ]: 28 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9302 : 28 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
9303 [ - + ]: 28 : if (nbdev_ctrlr == NULL) {
9304 : 0 : SPDK_ERRLOG("Could not find bdev_ctrlr %s\n", name);
9305 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9306 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9307 : 0 : return -ENODEV;
9308 : : }
9309 : 28 : nctrlr = bdev_nvme_next_ctrlr_unsafe(nbdev_ctrlr, NULL);
9310 [ - + ]: 28 : if (nctrlr == NULL) {
9311 : 0 : SPDK_ERRLOG("Could not find any nvme_ctrlrs on bdev_ctrlr %s\n", name);
9312 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9313 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9314 : 0 : return -ENODEV;
9315 : : }
9316 [ - + ]: 28 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9317 : :
9318 [ # # # # ]: 28 : ctx->nctrlr = nctrlr;
9319 [ # # # # ]: 28 : ctx->cb_fn = cb_fn;
9320 [ # # # # ]: 28 : ctx->cb_ctx = cb_ctx;
9321 [ # # # # ]: 28 : ctx->thread = spdk_get_thread();
9322 : :
9323 : 28 : bdev_nvme_authenticate_ctrlr(ctx);
9324 : :
9325 : 28 : return 0;
9326 : 0 : }
9327 : :
9328 : : void
9329 : 480 : nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
9330 : : {
9331 [ # # # # ]: 480 : struct nvme_ns *nvme_ns = io_path->nvme_ns;
9332 [ # # # # : 480 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
9333 : : const struct spdk_nvme_ctrlr_data *cdata;
9334 : : const struct spdk_nvme_transport_id *trid;
9335 : : const char *adrfam_str;
9336 : :
9337 : 480 : spdk_json_write_object_begin(w);
9338 : :
9339 [ # # # # : 480 : spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
# # # # #
# ]
9340 : :
9341 [ # # # # ]: 480 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
9342 [ # # # # ]: 480 : trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr);
9343 : :
9344 [ # # # # ]: 480 : spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
9345 : 480 : spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path));
9346 [ # # # # ]: 480 : spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair));
9347 : 480 : spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
9348 : :
9349 : 480 : spdk_json_write_named_object_begin(w, "transport");
9350 [ # # ]: 480 : spdk_json_write_named_string(w, "trtype", trid->trstring);
9351 [ # # ]: 480 : spdk_json_write_named_string(w, "traddr", trid->traddr);
9352 [ + - # # : 480 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
9353 [ # # ]: 480 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
9354 : 0 : }
9355 [ # # # # ]: 480 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
9356 [ + - ]: 480 : if (adrfam_str) {
9357 : 480 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
9358 : 0 : }
9359 : 480 : spdk_json_write_object_end(w);
9360 : :
9361 : 480 : spdk_json_write_object_end(w);
9362 : 480 : }
9363 : :
9364 : : void
9365 : 77 : bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w)
9366 : : {
9367 : : struct discovery_ctx *ctx;
9368 : : struct discovery_entry_ctx *entry_ctx;
9369 : :
9370 : 77 : spdk_json_write_array_begin(w);
9371 [ + + # # : 150 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
9372 : 73 : spdk_json_write_object_begin(w);
9373 [ # # # # ]: 73 : spdk_json_write_named_string(w, "name", ctx->name);
9374 : :
9375 : 73 : spdk_json_write_named_object_begin(w, "trid");
9376 [ # # ]: 73 : nvme_bdev_dump_trid_json(&ctx->trid, w);
9377 : 73 : spdk_json_write_object_end(w);
9378 : :
9379 : 73 : spdk_json_write_named_array_begin(w, "referrals");
9380 [ + + # # : 171 : TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
9381 : 98 : spdk_json_write_object_begin(w);
9382 : 98 : spdk_json_write_named_object_begin(w, "trid");
9383 [ # # ]: 98 : nvme_bdev_dump_trid_json(&entry_ctx->trid, w);
9384 : 98 : spdk_json_write_object_end(w);
9385 : 98 : spdk_json_write_object_end(w);
9386 : 0 : }
9387 : 73 : spdk_json_write_array_end(w);
9388 : :
9389 : 73 : spdk_json_write_object_end(w);
9390 : 0 : }
9391 : 77 : spdk_json_write_array_end(w);
9392 : 77 : }
9393 : :
9394 : 2149 : SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)
9395 : :
9396 : : static void
9397 : 1973 : bdev_nvme_trace(void)
9398 : : {
9399 : 1973 : struct spdk_trace_tpoint_opts opts[] = {
9400 : : {
9401 : : "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START,
9402 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1,
9403 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9404 : : },
9405 : : {
9406 : : "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE,
9407 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0,
9408 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9409 : : }
9410 : : };
9411 : :
9412 : :
9413 : 1973 : spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N');
9414 : 1973 : spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
9415 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9416 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9417 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9418 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9419 : 1973 : }
9420 : 2149 : SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME)
|