Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2017 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #include "spdk/stdinc.h"
8 : :
9 : : #include "nvmf_internal.h"
10 : :
11 : : #include "spdk/bdev.h"
12 : : #include "spdk/endian.h"
13 : : #include "spdk/thread.h"
14 : : #include "spdk/likely.h"
15 : : #include "spdk/nvme.h"
16 : : #include "spdk/nvmf_cmd.h"
17 : : #include "spdk/nvmf_spec.h"
18 : : #include "spdk/trace.h"
19 : : #include "spdk/scsi_spec.h"
20 : : #include "spdk/string.h"
21 : : #include "spdk/util.h"
22 : :
23 : : #include "spdk/log.h"
24 : :
25 : : static bool
26 : 5646 : nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem,
27 : : enum spdk_bdev_io_type io_type)
28 : : {
29 : : struct spdk_nvmf_ns *ns;
30 : :
31 [ + + ]: 8993 : for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
32 : 3347 : ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
33 [ + + + - : 3721 : if (ns->bdev == NULL) {
+ - ]
34 : 0 : continue;
35 : : }
36 : :
37 [ + + + - : 3721 : if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) {
+ - ]
38 [ - + - + : 374 : SPDK_DEBUGLOG(nvmf,
# # # # #
# # # # #
# # ]
39 : : "Subsystem %s namespace %u (%s) does not support io_type %d\n",
40 : : spdk_nvmf_subsystem_get_nqn(subsystem),
41 : : ns->opts.nsid, spdk_bdev_get_name(ns->bdev), (int)io_type);
42 : 374 : return false;
43 : : }
44 : 1524 : }
45 : :
46 [ + + + + : 5272 : SPDK_DEBUGLOG(nvmf, "All devices in Subsystem %s support io_type %d\n",
+ - ]
47 : : spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type);
48 : 5272 : return true;
49 : 1524 : }
50 : :
51 : : bool
52 : 2823 : nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr)
53 : : {
54 [ + - + - ]: 2823 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP);
55 : : }
56 : :
57 : : bool
58 : 2823 : nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr)
59 : : {
60 [ + - + - ]: 2823 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES);
61 : : }
62 : :
63 : : bool
64 : 0 : nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr)
65 : : {
66 [ # # # # ]: 0 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_COPY);
67 : : }
68 : :
69 : : static void
70 : 21933410 : nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success,
71 : : void *cb_arg)
72 : : {
73 : 21933410 : struct spdk_nvmf_request *req = cb_arg;
74 [ + - + - : 21933410 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
+ - ]
75 : 21933410 : int sc = 0, sct = 0;
76 : 21933410 : uint32_t cdw0 = 0;
77 : :
78 [ + + + - : 21933410 : if (spdk_unlikely(req->first_fused)) {
+ - - + ]
79 [ # # # # ]: 3117 : struct spdk_nvmf_request *first_req = req->first_fused_req;
80 [ # # # # : 3117 : struct spdk_nvme_cpl *first_response = &first_req->rsp->nvme_cpl;
# # ]
81 : 3117 : int first_sc = 0, first_sct = 0;
82 : :
83 : : /* get status for both operations */
84 : 3117 : spdk_bdev_io_get_nvme_fused_status(bdev_io, &cdw0, &first_sct, &first_sc, &sct, &sc);
85 [ # # # # ]: 3117 : first_response->cdw0 = cdw0;
86 [ # # # # : 3117 : first_response->status.sc = first_sc;
# # ]
87 [ # # # # : 3117 : first_response->status.sct = first_sct;
# # ]
88 : :
89 : : /* first request should be completed */
90 : 3117 : spdk_nvmf_request_complete(first_req);
91 [ # # # # ]: 3117 : req->first_fused_req = NULL;
92 [ # # # # : 3117 : req->first_fused = false;
# # ]
93 : 0 : } else {
94 : 21930293 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
95 : : }
96 : :
97 [ + - + - ]: 21933410 : response->cdw0 = cdw0;
98 [ + - + - : 21933410 : response->status.sc = sc;
+ - ]
99 [ + - + - : 21933410 : response->status.sct = sct;
+ - ]
100 : :
101 : 21933410 : spdk_nvmf_request_complete(req);
102 : 21933410 : spdk_bdev_free_io(bdev_io);
103 : 21933410 : }
104 : :
105 : : static void
106 : 102 : nvmf_bdev_ctrlr_complete_admin_cmd(struct spdk_bdev_io *bdev_io, bool success,
107 : : void *cb_arg)
108 : : {
109 : 102 : struct spdk_nvmf_request *req = cb_arg;
110 : :
111 [ + + # # : 102 : if (req->cmd_cb_fn) {
# # ]
112 [ # # # # : 94 : req->cmd_cb_fn(req);
# # # # ]
113 : 0 : }
114 : :
115 [ # # ]: 102 : nvmf_bdev_ctrlr_complete_cmd(bdev_io, success, req);
116 : 102 : }
117 : :
118 : : void
119 : 2495 : nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata,
120 : : bool dif_insert_or_strip)
121 : : {
122 [ + - + - ]: 2495 : struct spdk_bdev *bdev = ns->bdev;
123 [ + - + - ]: 2495 : struct spdk_bdev_desc *desc = ns->desc;
124 : : uint64_t num_blocks;
125 : : uint32_t phys_blocklen;
126 : : uint32_t max_copy;
127 : :
128 : 2495 : num_blocks = spdk_bdev_get_num_blocks(bdev);
129 : :
130 [ + - + - ]: 2495 : nsdata->nsze = num_blocks;
131 [ + - + - ]: 2495 : nsdata->ncap = num_blocks;
132 [ + - + - ]: 2495 : nsdata->nuse = num_blocks;
133 [ + - + - ]: 2495 : nsdata->nlbaf = 0;
134 [ + - + - ]: 2495 : nsdata->flbas.format = 0;
135 [ + - + - ]: 2495 : nsdata->flbas.msb_format = 0;
136 [ + - + - : 2495 : nsdata->nacwu = spdk_bdev_get_acwu(bdev) - 1; /* nacwu is 0-based */
+ - ]
137 [ + + - + ]: 2495 : if (!dif_insert_or_strip) {
138 [ + - + - : 2461 : nsdata->lbaf[0].ms = spdk_bdev_desc_get_md_size(desc);
+ - + - ]
139 [ + - + - : 2461 : nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_desc_get_block_size(desc));
+ - + - ]
140 [ + + + - : 2461 : if (nsdata->lbaf[0].ms != 0) {
+ - + - +
- ]
141 [ # # # # ]: 4 : nsdata->flbas.extended = 1;
142 [ # # # # ]: 4 : nsdata->mc.extended = 1;
143 [ # # # # ]: 4 : nsdata->mc.pointer = 0;
144 [ # # # # ]: 4 : nsdata->dps.md_start = spdk_bdev_desc_is_dif_head_of_md(desc);
145 : :
146 [ + + - - ]: 4 : switch (spdk_bdev_get_dif_type(bdev)) {
147 : 3 : case SPDK_DIF_TYPE1:
148 [ # # # # ]: 4 : nsdata->dpc.pit1 = 1;
149 [ # # # # ]: 4 : nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_TYPE1;
150 : 4 : break;
151 : 0 : case SPDK_DIF_TYPE2:
152 [ # # # # ]: 0 : nsdata->dpc.pit2 = 1;
153 [ # # # # ]: 0 : nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_TYPE2;
154 : 0 : break;
155 : 0 : case SPDK_DIF_TYPE3:
156 [ # # # # ]: 0 : nsdata->dpc.pit3 = 1;
157 [ # # # # ]: 0 : nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_TYPE3;
158 : 0 : break;
159 : 0 : default:
160 [ # # # # : 0 : SPDK_DEBUGLOG(nvmf, "Protection Disabled\n");
# # ]
161 [ # # # # ]: 0 : nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE;
162 : 0 : break;
163 : : }
164 : 1 : }
165 : 763 : } else {
166 [ # # # # : 34 : nsdata->lbaf[0].ms = 0;
# # # # ]
167 [ # # # # : 34 : nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_data_block_size(bdev));
# # # # ]
168 : : }
169 : :
170 : 2495 : phys_blocklen = spdk_bdev_get_physical_block_size(bdev);
171 [ + + # # ]: 2495 : assert(phys_blocklen > 0);
172 : : /* Linux driver uses min(nawupf, npwg) to set physical_block_size */
173 [ + - + - ]: 2495 : nsdata->nsfeat.optperf = 1;
174 [ + - + - ]: 2495 : nsdata->nsfeat.ns_atomic_write_unit = 1;
175 [ + + + - : 2495 : nsdata->npwg = (phys_blocklen >> nsdata->lbaf[0].lbads) - 1;
+ - + - +
- + - +
- ]
176 [ + - + - : 2495 : nsdata->nawupf = nsdata->npwg;
+ - + - ]
177 [ + - + - : 2495 : nsdata->npwa = nsdata->npwg;
+ - + - ]
178 [ + - + - : 2495 : nsdata->npdg = nsdata->npwg;
+ - + - ]
179 [ + - + - : 2495 : nsdata->npda = nsdata->npwg;
+ - + - ]
180 : :
181 [ + + ]: 2495 : if (spdk_bdev_get_write_unit_size(bdev) == 1) {
182 [ + - + - ]: 2495 : nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev);
183 : 764 : }
184 [ + - + - ]: 2495 : nsdata->nmic.can_share = 1;
185 [ + + ]: 2495 : if (nvmf_ns_is_ptpl_capable(ns)) {
186 [ # # # # : 8 : nsdata->nsrescap.rescap.persist = 1;
# # ]
187 : 2 : }
188 [ + - + - : 2495 : nsdata->nsrescap.rescap.write_exclusive = 1;
+ - ]
189 [ + - + - : 2495 : nsdata->nsrescap.rescap.exclusive_access = 1;
+ - ]
190 [ + - + - : 2495 : nsdata->nsrescap.rescap.write_exclusive_reg_only = 1;
+ - ]
191 [ + - + - : 2495 : nsdata->nsrescap.rescap.exclusive_access_reg_only = 1;
+ - ]
192 [ + - + - : 2495 : nsdata->nsrescap.rescap.write_exclusive_all_reg = 1;
+ - ]
193 [ + - + - : 2495 : nsdata->nsrescap.rescap.exclusive_access_all_reg = 1;
+ - ]
194 [ + - + - : 2495 : nsdata->nsrescap.rescap.ignore_existing_key = 1;
+ - ]
195 : :
196 : : SPDK_STATIC_ASSERT(sizeof(nsdata->nguid) == sizeof(ns->opts.nguid), "size mismatch");
197 [ + - + - : 2495 : memcpy(nsdata->nguid, ns->opts.nguid, sizeof(nsdata->nguid));
+ - + - +
- ]
198 : :
199 : : SPDK_STATIC_ASSERT(sizeof(nsdata->eui64) == sizeof(ns->opts.eui64), "size mismatch");
200 [ + - + - : 2495 : memcpy(&nsdata->eui64, ns->opts.eui64, sizeof(nsdata->eui64));
+ - + - +
- ]
201 : :
202 : : /* For now we support just one source range for copy command */
203 [ + - + - ]: 2495 : nsdata->msrc = 0;
204 : :
205 : 2495 : max_copy = spdk_bdev_get_max_copy(bdev);
206 [ + + - + ]: 2495 : if (max_copy == 0 || max_copy > UINT16_MAX) {
207 : : /* Zero means copy size is unlimited */
208 [ + - + - ]: 1691 : nsdata->mcl = UINT16_MAX;
209 [ + - + - ]: 1691 : nsdata->mssrl = UINT16_MAX;
210 : 764 : } else {
211 [ # # # # ]: 804 : nsdata->mcl = max_copy;
212 [ # # # # ]: 804 : nsdata->mssrl = max_copy;
213 : : }
214 : 2495 : }
215 : :
216 : : void
217 : 0 : nvmf_bdev_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ns *ns,
218 : : struct spdk_nvme_nvm_ns_data *nsdata_nvm)
219 : : {
220 [ # # # # ]: 0 : struct spdk_bdev_desc *desc = ns->desc;
221 : : uint8_t _16bpists;
222 : : uint32_t sts, pif;
223 : :
224 [ # # ]: 0 : if (spdk_bdev_desc_get_dif_type(desc) == SPDK_DIF_DISABLE) {
225 : 0 : return;
226 : : }
227 : :
228 : 0 : pif = spdk_bdev_desc_get_dif_pi_format(desc);
229 : :
230 : : /*
231 : : * 16BPISTS shall be 1 for 32/64b Guard PI.
232 : : * STCRS shall be 1 if 16BPISTS is 1.
233 : : * 16 is the minimum value of STS for 32b Guard PI.
234 : : */
235 [ # # # # ]: 0 : switch (pif) {
236 : 0 : case SPDK_DIF_PI_FORMAT_16:
237 : 0 : _16bpists = 0;
238 : 0 : sts = 0;
239 : 0 : break;
240 : 0 : case SPDK_DIF_PI_FORMAT_32:
241 : 0 : _16bpists = 1;
242 : 0 : sts = 16;
243 : 0 : break;
244 : 0 : case SPDK_DIF_PI_FORMAT_64:
245 : 0 : _16bpists = 1;
246 : 0 : sts = 0;
247 : 0 : break;
248 : 0 : default:
249 : 0 : SPDK_WARNLOG("PI format %u is not supported\n", pif);
250 : 0 : return;
251 : : }
252 : :
253 : : /* For 16b Guard PI, Storage Tag is not available because we set STS to 0.
254 : : * In this case, we do not have to set 16BPISTM to 1. For simplicity,
255 : : * set 16BPISTM to 0 and set LBSTM to all zeroes.
256 : : *
257 : : * We will revisit here when we find any OS uses Storage Tag.
258 : : */
259 [ # # # # ]: 0 : nsdata_nvm->lbstm = 0;
260 [ # # # # ]: 0 : nsdata_nvm->pic._16bpistm = 0;
261 : :
262 [ # # # # ]: 0 : nsdata_nvm->pic._16bpists = _16bpists;
263 [ # # # # ]: 0 : nsdata_nvm->pic.stcrs = 0;
264 [ # # # # : 0 : nsdata_nvm->elbaf[0].sts = sts;
# # # # ]
265 [ # # # # : 0 : nsdata_nvm->elbaf[0].pif = pif;
# # # # ]
266 : 0 : }
267 : :
268 : : static void
269 : 23140986 : nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba,
270 : : uint64_t *num_blocks)
271 : : {
272 : : /* SLBA: CDW10 and CDW11 */
273 [ + - + - : 23140986 : *start_lba = from_le64(&cmd->cdw10);
+ - ]
274 : :
275 : : /* NLB: CDW12 bits 15:00, 0's based */
276 [ + - + - : 23140986 : *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1;
+ - ]
277 : 23140986 : }
278 : :
279 : : static void
280 : 22042479 : nvmf_bdev_ctrlr_get_rw_ext_params(const struct spdk_nvme_cmd *cmd,
281 : : struct spdk_bdev_ext_io_opts *opts)
282 : : {
283 : : /* Get CDW12 values */
284 [ + - + - : 22042479 : opts->nvme_cdw12.raw = from_le32(&cmd->cdw12);
+ - + - +
- ]
285 : :
286 : : /* Get CDW13 values */
287 [ + - + - : 22042479 : opts->nvme_cdw13.raw = from_le32(&cmd->cdw13);
+ - + - +
- ]
288 : :
289 [ + - + - : 22042479 : opts->dif_check_flags_exclude_mask = (~opts->nvme_cdw12.raw) & SPDK_NVME_IO_FLAGS_PRCHK_MASK;
+ - + - +
- ]
290 : 22042479 : }
291 : :
292 : : static bool
293 : 23137889 : nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba,
294 : : uint64_t io_num_blocks)
295 : : {
296 [ + + + + ]: 23137889 : if (io_start_lba + io_num_blocks > bdev_num_blocks ||
297 [ + + ]: 23133457 : io_start_lba + io_num_blocks < io_start_lba) {
298 : 4440 : return false;
299 : : }
300 : :
301 : 23133449 : return true;
302 : 139019 : }
303 : :
304 : : static void
305 : 120706 : nvmf_ctrlr_process_io_cmd_resubmit(void *arg)
306 : : {
307 : 120706 : struct spdk_nvmf_request *req = arg;
308 : : int rc;
309 : :
310 : 120706 : rc = nvmf_ctrlr_process_io_cmd(req);
311 [ - + ]: 120706 : if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) {
312 : 0 : spdk_nvmf_request_complete(req);
313 : 0 : }
314 : 120706 : }
315 : :
316 : : static void
317 : 0 : nvmf_ctrlr_process_admin_cmd_resubmit(void *arg)
318 : : {
319 : 0 : struct spdk_nvmf_request *req = arg;
320 : : int rc;
321 : :
322 : 0 : rc = nvmf_ctrlr_process_admin_cmd(req);
323 [ # # ]: 0 : if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) {
324 : 0 : spdk_nvmf_request_complete(req);
325 : 0 : }
326 : 0 : }
327 : :
328 : : static void
329 : 182086 : nvmf_bdev_ctrl_queue_io(struct spdk_nvmf_request *req, struct spdk_bdev *bdev,
330 : : struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn, void *cb_arg)
331 : : {
332 : : int rc;
333 : :
334 [ # # # # : 182086 : req->bdev_io_wait.bdev = bdev;
# # ]
335 [ # # # # : 182086 : req->bdev_io_wait.cb_fn = cb_fn;
# # ]
336 [ # # # # : 182086 : req->bdev_io_wait.cb_arg = cb_arg;
# # ]
337 : :
338 [ # # ]: 182086 : rc = spdk_bdev_queue_io_wait(bdev, ch, &req->bdev_io_wait);
339 [ - + ]: 182086 : if (rc != 0) {
340 [ # # ]: 0 : assert(false);
341 : : }
342 [ # # # # : 182086 : req->qpair->group->stat.pending_bdev_io++;
# # # # #
# # # ]
343 : 182086 : }
344 : :
345 : : bool
346 : 0 : nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev)
347 : : {
348 : 0 : return spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY);
349 : : }
350 : :
351 : : int
352 : 12439476 : nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
353 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
354 : : {
355 : 49364655 : struct spdk_bdev_ext_io_opts opts = {
356 : : .size = SPDK_SIZEOF(&opts, accel_sequence),
357 [ + - + - ]: 12439476 : .memory_domain = req->memory_domain,
358 [ + - + - ]: 12439476 : .memory_domain_ctx = req->memory_domain_ctx,
359 [ + - + - ]: 12439476 : .accel_sequence = req->accel_sequence,
360 : : };
361 : 12439476 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
362 : 12439476 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
363 [ + - + - : 12439476 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
+ - ]
364 [ + - + - : 12439476 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
+ - ]
365 : 131083 : uint64_t start_lba;
366 : 131083 : uint64_t num_blocks;
367 : : int rc;
368 : :
369 : 12439476 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
370 : 12439476 : nvmf_bdev_ctrlr_get_rw_ext_params(cmd, &opts);
371 : :
372 [ - + ]: 12439476 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
373 : 0 : SPDK_ERRLOG("end of media\n");
374 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
375 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
376 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
377 : : }
378 : :
379 [ + + + - : 12439476 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
- + ]
380 [ # # # # ]: 0 : SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
381 : : num_blocks, block_size, req->length);
382 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
383 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
384 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
385 : : }
386 : :
387 [ + + # # ]: 12439476 : assert(!spdk_nvmf_request_using_zcopy(req));
388 : :
389 [ + - + - : 12439477 : rc = spdk_bdev_readv_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
+ - ]
390 : 69497 : nvmf_bdev_ctrlr_complete_cmd, req, &opts);
391 [ + + ]: 12439476 : if (spdk_unlikely(rc)) {
392 [ + - ]: 57343 : if (rc == -ENOMEM) {
393 : 57343 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
394 : 57343 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
395 : : }
396 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
397 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
398 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
399 : : }
400 : :
401 : 12382133 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
402 : 69497 : }
403 : :
404 : : int
405 : 9602999 : nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
406 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
407 : : {
408 : 38094233 : struct spdk_bdev_ext_io_opts opts = {
409 : : .size = SPDK_SIZEOF(&opts, nvme_cdw13),
410 [ + - + - ]: 9602999 : .memory_domain = req->memory_domain,
411 [ + - + - ]: 9602999 : .memory_domain_ctx = req->memory_domain_ctx,
412 [ + - + - ]: 9602999 : .accel_sequence = req->accel_sequence,
413 : : };
414 : 9602999 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
415 : 9602999 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
416 [ + - + - : 9602999 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
+ - ]
417 [ + - + - : 9602999 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
+ - ]
418 : 105921 : uint64_t start_lba;
419 : 105921 : uint64_t num_blocks;
420 : : int rc;
421 : :
422 : 9602999 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
423 : 9602999 : nvmf_bdev_ctrlr_get_rw_ext_params(cmd, &opts);
424 : :
425 [ - + ]: 9602999 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
426 : 0 : SPDK_ERRLOG("end of media\n");
427 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
428 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
429 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
430 : : }
431 : :
432 [ + + + - : 9602999 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
- + ]
433 [ # # # # ]: 0 : SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
434 : : num_blocks, block_size, req->length);
435 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
436 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
437 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
438 : : }
439 : :
440 [ + + # # ]: 9602999 : assert(!spdk_nvmf_request_using_zcopy(req));
441 : :
442 [ + - + - : 9603000 : rc = spdk_bdev_writev_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
+ - ]
443 : 69497 : nvmf_bdev_ctrlr_complete_cmd, req, &opts);
444 [ + + ]: 9602999 : if (spdk_unlikely(rc)) {
445 [ + - ]: 63363 : if (rc == -ENOMEM) {
446 : 63363 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
447 : 63363 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
448 : : }
449 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
450 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
451 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
452 : : }
453 : :
454 : 9539636 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
455 : 69497 : }
456 : :
457 : : int
458 : 16 : nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
459 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
460 : : {
461 : 16 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
462 : 16 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
463 [ # # # # : 16 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
464 [ # # # # : 16 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
465 : 12 : uint64_t start_lba;
466 : 12 : uint64_t num_blocks;
467 : : int rc;
468 : :
469 : 16 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
470 : :
471 [ + + ]: 16 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
472 : 4 : SPDK_ERRLOG("end of media\n");
473 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
474 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
475 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
476 : : }
477 : :
478 [ + + # # : 12 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
# # ]
479 [ # # # # ]: 4 : SPDK_ERRLOG("Compare NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
480 : : num_blocks, block_size, req->length);
481 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
482 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
483 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
484 : : }
485 : :
486 [ # # # # : 10 : rc = spdk_bdev_comparev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
# # ]
487 : 2 : nvmf_bdev_ctrlr_complete_cmd, req);
488 [ + + ]: 8 : if (spdk_unlikely(rc)) {
489 [ - + ]: 4 : if (rc == -ENOMEM) {
490 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
491 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
492 : : }
493 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
494 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
495 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
496 : : }
497 : :
498 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
499 : 4 : }
500 : :
501 : : int
502 : 3133 : nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
503 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req)
504 : : {
505 : 3133 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
506 : 3133 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
507 [ # # # # : 3133 : struct spdk_nvme_cmd *cmp_cmd = &cmp_req->cmd->nvme_cmd;
# # ]
508 [ # # # # : 3133 : struct spdk_nvme_cmd *write_cmd = &write_req->cmd->nvme_cmd;
# # ]
509 [ # # # # : 3133 : struct spdk_nvme_cpl *rsp = &write_req->rsp->nvme_cpl;
# # ]
510 : 12 : uint64_t write_start_lba, cmp_start_lba;
511 : 12 : uint64_t write_num_blocks, cmp_num_blocks;
512 : : int rc;
513 : :
514 : 3133 : nvmf_bdev_ctrlr_get_rw_params(cmp_cmd, &cmp_start_lba, &cmp_num_blocks);
515 : 3133 : nvmf_bdev_ctrlr_get_rw_params(write_cmd, &write_start_lba, &write_num_blocks);
516 : :
517 [ + + + + ]: 3133 : if (spdk_unlikely(write_start_lba != cmp_start_lba || write_num_blocks != cmp_num_blocks)) {
518 : 4 : SPDK_ERRLOG("Fused command start lba / num blocks mismatch\n");
519 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
520 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
521 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
522 : : }
523 : :
524 [ + + ]: 3129 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, write_start_lba,
525 : : write_num_blocks))) {
526 : 4 : SPDK_ERRLOG("end of media\n");
527 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
528 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
529 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
530 : : }
531 : :
532 [ + + # # : 3125 : if (spdk_unlikely(write_num_blocks * block_size > write_req->length)) {
# # ]
533 [ # # # # ]: 4 : SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
534 : : write_num_blocks, block_size, write_req->length);
535 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
536 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
537 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
538 : : }
539 : :
540 [ # # # # : 3122 : rc = spdk_bdev_comparev_and_writev_blocks(desc, ch, cmp_req->iov, cmp_req->iovcnt, write_req->iov,
# # # # ]
541 [ # # # # ]: 3121 : write_req->iovcnt, write_start_lba, write_num_blocks, nvmf_bdev_ctrlr_complete_cmd, write_req);
542 [ - + ]: 3121 : if (spdk_unlikely(rc)) {
543 [ # # ]: 0 : if (rc == -ENOMEM) {
544 : 0 : nvmf_bdev_ctrl_queue_io(cmp_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, cmp_req);
545 : 0 : nvmf_bdev_ctrl_queue_io(write_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, write_req);
546 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
547 : : }
548 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
549 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
550 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
551 : : }
552 : :
553 : 3121 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
554 : 4 : }
555 : :
556 : : int
557 : 7928 : nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
558 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
559 : : {
560 : 7928 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
561 [ # # # # : 7928 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
562 [ # # # # : 7928 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
563 [ # # # # : 7928 : uint64_t max_write_zeroes_size = req->qpair->ctrlr->subsys->max_write_zeroes_size_kib;
# # # # #
# # # # #
# # ]
564 : 12 : uint64_t start_lba;
565 : 12 : uint64_t num_blocks;
566 : : int rc;
567 : :
568 : 7928 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
569 [ + + + + : 7928 : if (spdk_unlikely(max_write_zeroes_size > 0 &&
+ - # # ]
570 : : num_blocks > (max_write_zeroes_size << 10) / spdk_bdev_desc_get_block_size(desc))) {
571 : 4 : SPDK_ERRLOG("invalid write zeroes size, should not exceed %" PRIu64 "Kib\n", max_write_zeroes_size);
572 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
573 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
574 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
575 : : }
576 : :
577 [ + + ]: 7924 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
578 : 4404 : SPDK_ERRLOG("end of media\n");
579 [ # # # # : 4404 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
580 [ # # # # : 4404 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
581 : 4404 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
582 : : }
583 : :
584 [ - + # # : 3520 : if (spdk_unlikely(cmd->cdw12_bits.write_zeroes.deac)) {
# # # # #
# ]
585 : 0 : SPDK_ERRLOG("Write Zeroes Deallocate is not supported\n");
586 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
587 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
588 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
589 : : }
590 : :
591 : 3522 : rc = spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks,
592 : 2 : nvmf_bdev_ctrlr_complete_cmd, req);
593 [ + + ]: 3520 : if (spdk_unlikely(rc)) {
594 [ - + ]: 4 : if (rc == -ENOMEM) {
595 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
596 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
597 : : }
598 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
599 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
600 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
601 : : }
602 : :
603 : 3516 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
604 : 4 : }
605 : :
606 : : int
607 : 4913 : nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
608 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
609 : : {
610 [ # # # # : 4913 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
611 : : int rc;
612 : :
613 : : /* As for NVMeoF controller, SPDK always set volatile write
614 : : * cache bit to 1, return success for those block devices
615 : : * which can't support FLUSH command.
616 : : */
617 [ + + ]: 4913 : if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) {
618 [ # # # # : 4 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
619 [ # # # # : 4 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
620 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
621 : : }
622 : :
623 : 4911 : rc = spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev),
624 : 2 : nvmf_bdev_ctrlr_complete_cmd, req);
625 [ + + ]: 4909 : if (spdk_unlikely(rc)) {
626 [ - + ]: 4 : if (rc == -ENOMEM) {
627 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
628 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
629 : : }
630 [ # # # # : 4 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
631 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
632 : : }
633 : 4905 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
634 : 3 : }
635 : :
636 : : struct nvmf_bdev_ctrlr_unmap {
637 : : struct spdk_nvmf_request *req;
638 : : uint32_t count;
639 : : struct spdk_bdev_desc *desc;
640 : : struct spdk_bdev *bdev;
641 : : struct spdk_io_channel *ch;
642 : : uint32_t range_index;
643 : : };
644 : :
645 : : static void
646 : 61402 : nvmf_bdev_ctrlr_unmap_cpl(struct spdk_bdev_io *bdev_io, bool success,
647 : : void *cb_arg)
648 : : {
649 : 61402 : struct nvmf_bdev_ctrlr_unmap *unmap_ctx = cb_arg;
650 [ # # # # ]: 61402 : struct spdk_nvmf_request *req = unmap_ctx->req;
651 [ # # # # : 61402 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
652 : 0 : int sc, sct;
653 : 0 : uint32_t cdw0;
654 : :
655 [ # # ]: 61402 : unmap_ctx->count--;
656 : :
657 [ + - # # : 61402 : if (response->status.sct == SPDK_NVME_SCT_GENERIC &&
# # # # #
# ]
658 [ + - # # : 61402 : response->status.sc == SPDK_NVME_SC_SUCCESS) {
# # ]
659 : 61402 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
660 [ # # # # ]: 61402 : response->cdw0 = cdw0;
661 [ # # # # : 61402 : response->status.sc = sc;
# # ]
662 [ # # # # : 61402 : response->status.sct = sct;
# # ]
663 : 0 : }
664 : :
665 [ + - # # : 61402 : if (unmap_ctx->count == 0) {
# # ]
666 : 61402 : spdk_nvmf_request_complete(req);
667 : 61402 : free(unmap_ctx);
668 : 0 : }
669 : 61402 : spdk_bdev_free_io(bdev_io);
670 : 61402 : }
671 : :
672 : : static int nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
673 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
674 : : struct nvmf_bdev_ctrlr_unmap *unmap_ctx);
675 : : static void
676 : 61372 : nvmf_bdev_ctrlr_unmap_resubmit(void *arg)
677 : : {
678 : 61372 : struct nvmf_bdev_ctrlr_unmap *unmap_ctx = arg;
679 [ # # # # ]: 61372 : struct spdk_nvmf_request *req = unmap_ctx->req;
680 [ # # # # ]: 61372 : struct spdk_bdev_desc *desc = unmap_ctx->desc;
681 [ # # # # ]: 61372 : struct spdk_bdev *bdev = unmap_ctx->bdev;
682 [ # # # # ]: 61372 : struct spdk_io_channel *ch = unmap_ctx->ch;
683 : :
684 : 61372 : nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, unmap_ctx);
685 : 61372 : }
686 : :
687 : : static int
688 : 122774 : nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
689 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
690 : : struct nvmf_bdev_ctrlr_unmap *unmap_ctx)
691 : : {
692 : : uint16_t nr, i;
693 [ # # # # : 122774 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
694 [ # # # # : 122774 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
695 [ # # # # : 122774 : uint64_t max_discard_size = req->qpair->ctrlr->subsys->max_discard_size_kib;
# # # # #
# # # # #
# # ]
696 : 122774 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
697 : 0 : struct spdk_iov_xfer ix;
698 : : uint64_t lba;
699 : : uint32_t lba_count;
700 : : int rc;
701 : :
702 [ # # # # : 122774 : nr = cmd->cdw10_bits.dsm.nr + 1;
# # # # #
# ]
703 [ - + # # : 122774 : if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) {
# # ]
704 : 0 : SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n");
705 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
706 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
707 : : }
708 : :
709 [ + + ]: 122774 : if (unmap_ctx == NULL) {
710 : 61402 : unmap_ctx = calloc(1, sizeof(*unmap_ctx));
711 [ - + ]: 61402 : if (!unmap_ctx) {
712 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
713 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
714 : : }
715 : :
716 [ # # # # ]: 61402 : unmap_ctx->req = req;
717 [ # # # # ]: 61402 : unmap_ctx->desc = desc;
718 [ # # # # ]: 61402 : unmap_ctx->ch = ch;
719 [ # # # # ]: 61402 : unmap_ctx->bdev = bdev;
720 : :
721 [ # # # # : 61402 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
722 [ # # # # : 61402 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
723 : 0 : } else {
724 [ # # ]: 61372 : unmap_ctx->count--; /* dequeued */
725 : : }
726 : :
727 [ # # # # : 122774 : spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
# # ]
728 : :
729 [ + + # # : 184176 : for (i = unmap_ctx->range_index; i < nr; i++) {
# # ]
730 : 122774 : struct spdk_nvme_dsm_range dsm_range = { 0 };
731 : :
732 : 122774 : spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range));
733 : :
734 [ # # ]: 122774 : lba = dsm_range.starting_lba;
735 [ # # ]: 122774 : lba_count = dsm_range.length;
736 [ - + - - : 122774 : if (max_discard_size > 0 && lba_count > (max_discard_size << 10) / block_size) {
- - # # ]
737 [ # # ]: 0 : SPDK_ERRLOG("invalid unmap size %" PRIu32 " blocks, should not exceed %" PRIu64 " blocks\n",
738 : : lba_count, max_discard_size << 1);
739 [ # # # # : 0 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
740 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
741 : 0 : break;
742 : : }
743 : :
744 [ # # ]: 122774 : unmap_ctx->count++;
745 : :
746 : 122774 : rc = spdk_bdev_unmap_blocks(desc, ch, lba, lba_count,
747 : 0 : nvmf_bdev_ctrlr_unmap_cpl, unmap_ctx);
748 [ + + ]: 122774 : if (rc) {
749 [ + - ]: 61372 : if (rc == -ENOMEM) {
750 : 61372 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_bdev_ctrlr_unmap_resubmit, unmap_ctx);
751 : : /* Unmap was not yet submitted to bdev */
752 : : /* unmap_ctx->count will be decremented when the request is dequeued */
753 : 61372 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
754 : : }
755 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
756 [ # # ]: 0 : unmap_ctx->count--;
757 : : /* We can't return here - we may have to wait for any other
758 : : * unmaps already sent to complete */
759 : 0 : break;
760 : : }
761 [ # # ]: 61402 : unmap_ctx->range_index++;
762 : 0 : }
763 : :
764 [ - + # # : 61402 : if (unmap_ctx->count == 0) {
# # ]
765 : 0 : free(unmap_ctx);
766 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
767 : : }
768 : :
769 : 61402 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
770 : 0 : }
771 : :
772 : : int
773 : 61402 : nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
774 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
775 : : {
776 [ # # # # : 61402 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
777 [ # # # # : 61402 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
778 : :
779 [ + - # # : 61402 : if (cmd->cdw11_bits.dsm.ad) {
# # # # #
# ]
780 : 61402 : return nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, NULL);
781 : : }
782 : :
783 [ # # # # : 0 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
784 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
785 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
786 : 0 : }
787 : :
788 : : int
789 : 29 : nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
790 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
791 : : {
792 [ # # # # : 29 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
793 [ # # # # : 29 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
794 [ # # # # : 29 : uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10;
# # # # #
# # # #
# ]
795 : 29 : struct spdk_nvme_scc_source_range range = { 0 };
796 : 15 : struct spdk_iov_xfer ix;
797 : : int rc;
798 : :
799 [ + + - + : 29 : SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, "
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
800 : : "DTYPE %u, STCW %u, PRINFOW %u, FUA %u, LR %u\n",
801 : : sdlba,
802 : : cmd->cdw12_bits.copy.nr,
803 : : cmd->cdw12_bits.copy.df,
804 : : cmd->cdw12_bits.copy.prinfor,
805 : : cmd->cdw12_bits.copy.dtype,
806 : : cmd->cdw12_bits.copy.stcw,
807 : : cmd->cdw12_bits.copy.prinfow,
808 : : cmd->cdw12_bits.copy.fua,
809 : : cmd->cdw12_bits.copy.lr);
810 : :
811 [ - + # # : 29 : if (spdk_unlikely(req->length != (cmd->cdw12_bits.copy.nr + 1) *
# # # # #
# # # # #
# # ]
812 : : sizeof(struct spdk_nvme_scc_source_range))) {
813 [ # # # # : 0 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
814 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
815 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
816 : : }
817 : :
818 : : /*
819 : : * We support only one source range, and rely on this with the xfer
820 : : * below.
821 : : */
822 [ + + # # : 29 : if (cmd->cdw12_bits.copy.nr > 0) {
# # # # #
# ]
823 [ # # # # : 4 : response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
# # ]
824 [ # # # # : 4 : response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED;
# # ]
825 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
826 : : }
827 : :
828 [ + + # # : 25 : if (cmd->cdw12_bits.copy.df != 0) {
# # # # #
# ]
829 [ # # # # : 4 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
830 [ # # # # : 4 : response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
831 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
832 : : }
833 : :
834 [ # # # # : 21 : spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
# # ]
835 : 21 : spdk_iov_xfer_to_buf(&ix, &range, sizeof(range));
836 : :
837 [ # # # # : 24 : rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1,
# # ]
838 : 3 : nvmf_bdev_ctrlr_complete_cmd, req);
839 [ + + ]: 21 : if (spdk_unlikely(rc)) {
840 [ - + ]: 4 : if (rc == -ENOMEM) {
841 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
842 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
843 : : }
844 : :
845 [ # # # # : 4 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
846 [ # # # # : 4 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
847 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
848 : : }
849 : :
850 : 17 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
851 : 5 : }
852 : :
853 : : int
854 : 272929 : nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
855 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
856 : : {
857 : : int rc;
858 : :
859 [ # # # # : 272933 : rc = spdk_bdev_nvme_iov_passthru_md(desc, ch, &req->cmd->nvme_cmd, req->iov, req->iovcnt,
# # # # #
# # # ]
860 [ # # # # ]: 272929 : req->length, NULL, 0, nvmf_bdev_ctrlr_complete_cmd, req);
861 : :
862 [ + + ]: 272929 : if (spdk_unlikely(rc)) {
863 [ + + ]: 272921 : if (rc == -ENOMEM) {
864 : 4 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
865 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
866 : : }
867 [ # # # # : 272917 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
868 [ # # # # : 272917 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
# # # # #
# # # ]
869 [ # # # # : 272917 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
870 : 272917 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
871 : : }
872 : :
873 : 8 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
874 : 4 : }
875 : :
876 : : int
877 : 110 : spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
878 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
879 : : spdk_nvmf_nvme_passthru_cmd_cb cb_fn)
880 : : {
881 : : int rc;
882 : :
883 [ - + # # : 110 : if (spdk_unlikely(req->iovcnt > 1)) {
# # ]
884 [ # # # # : 0 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
885 [ # # # # : 0 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # # # #
# # # ]
886 [ # # # # : 0 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
887 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
888 : : }
889 : :
890 [ # # # # ]: 110 : req->cmd_cb_fn = cb_fn;
891 : :
892 [ # # # # : 114 : rc = spdk_bdev_nvme_admin_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length,
# # # # #
# # # # #
# # # # #
# ]
893 : 4 : nvmf_bdev_ctrlr_complete_admin_cmd, req);
894 [ + + ]: 110 : if (spdk_unlikely(rc)) {
895 [ + + ]: 8 : if (rc == -ENOMEM) {
896 : 4 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req);
897 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
898 : : }
899 [ # # # # : 4 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
900 [ + - ]: 4 : if (rc == -ENOTSUP) {
901 [ # # # # : 4 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
# # # # #
# # # ]
902 : 1 : } else {
903 [ # # # # : 0 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # # # #
# # # ]
904 : : }
905 : :
906 [ # # # # : 4 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
907 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
908 : : }
909 : :
910 : 102 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
911 : 4 : }
912 : :
913 : : static void
914 : 99770 : nvmf_bdev_ctrlr_complete_abort_cmd(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
915 : : {
916 : 99770 : struct spdk_nvmf_request *req = cb_arg;
917 : :
918 [ + + # # ]: 99770 : if (success) {
919 [ # # # # : 97375 : req->rsp->nvme_cpl.cdw0 &= ~1U;
# # # # #
# ]
920 : 0 : }
921 : :
922 : 99770 : spdk_nvmf_request_complete(req);
923 : 99770 : spdk_bdev_free_io(bdev_io);
924 : 99770 : }
925 : :
926 : : int
927 : 99770 : spdk_nvmf_bdev_ctrlr_abort_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
928 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
929 : : struct spdk_nvmf_request *req_to_abort)
930 : : {
931 : : int rc;
932 : :
933 [ - + # # : 99770 : assert((req->rsp->nvme_cpl.cdw0 & 1U) != 0);
# # # # #
# # # #
# ]
934 : :
935 : 99770 : rc = spdk_bdev_abort(desc, ch, req_to_abort, nvmf_bdev_ctrlr_complete_abort_cmd, req);
936 [ + - ]: 99770 : if (spdk_likely(rc == 0)) {
937 : 99770 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
938 [ # # ]: 0 : } else if (rc == -ENOMEM) {
939 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req);
940 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
941 : : } else {
942 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
943 : : }
944 : 0 : }
945 : :
946 : : bool
947 : 572609 : nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev_desc *desc, struct spdk_nvme_cmd *cmd,
948 : : struct spdk_dif_ctx *dif_ctx)
949 : : {
950 : 572609 : uint32_t init_ref_tag, dif_check_flags = 0;
951 : : int rc;
952 : 6 : struct spdk_dif_ctx_init_ext_opts dif_opts;
953 : :
954 [ + + ]: 572609 : if (spdk_bdev_desc_get_md_size(desc) == 0) {
955 : 4 : return false;
956 : : }
957 : :
958 : : /* Initial Reference Tag is the lower 32 bits of the start LBA. */
959 [ # # # # ]: 572605 : init_ref_tag = (uint32_t)from_le64(&cmd->cdw10);
960 : :
961 [ + + ]: 572605 : if (spdk_bdev_desc_is_dif_check_enabled(desc, SPDK_DIF_CHECK_TYPE_REFTAG)) {
962 [ # # ]: 540255 : dif_check_flags |= SPDK_DIF_FLAGS_REFTAG_CHECK;
963 : 0 : }
964 : :
965 [ + + ]: 572605 : if (spdk_bdev_desc_is_dif_check_enabled(desc, SPDK_DIF_CHECK_TYPE_GUARD)) {
966 [ # # ]: 572601 : dif_check_flags |= SPDK_DIF_FLAGS_GUARD_CHECK;
967 : 0 : }
968 : :
969 : 572605 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
970 [ # # ]: 572605 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
971 : 1145210 : rc = spdk_dif_ctx_init(dif_ctx,
972 : 1 : spdk_bdev_desc_get_block_size(desc),
973 : 1 : spdk_bdev_desc_get_md_size(desc),
974 : 572605 : spdk_bdev_desc_is_md_interleaved(desc),
975 : 572605 : spdk_bdev_desc_is_dif_head_of_md(desc),
976 : 1 : spdk_bdev_desc_get_dif_type(desc),
977 : 1 : dif_check_flags,
978 : 1 : init_ref_tag, 0, 0, 0, 0, &dif_opts);
979 : :
980 : 572605 : return (rc == 0) ? true : false;
981 : 2 : }
982 : :
983 : : static void
984 : 1084285 : nvmf_bdev_ctrlr_zcopy_start_complete(struct spdk_bdev_io *bdev_io, bool success,
985 : : void *cb_arg)
986 : : {
987 : 1084285 : struct spdk_nvmf_request *req = cb_arg;
988 : 0 : struct iovec *iov;
989 : 1084285 : int iovcnt = 0;
990 : :
991 [ + + # # ]: 1084285 : if (spdk_unlikely(!success)) {
992 : 30916 : int sc = 0, sct = 0;
993 : 30916 : uint32_t cdw0 = 0;
994 [ # # # # : 30916 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
995 : 30916 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
996 : :
997 [ # # # # ]: 30916 : response->cdw0 = cdw0;
998 [ # # # # : 30916 : response->status.sc = sc;
# # ]
999 [ # # # # : 30916 : response->status.sct = sct;
# # ]
1000 : :
1001 : 30916 : spdk_bdev_free_io(bdev_io);
1002 : 30916 : spdk_nvmf_request_complete(req);
1003 : 30916 : return;
1004 : : }
1005 : :
1006 : 1053369 : spdk_bdev_io_get_iovec(bdev_io, &iov, &iovcnt);
1007 : :
1008 [ - + # # ]: 1053369 : assert(iovcnt <= NVMF_REQ_MAX_BUFFERS);
1009 [ - + # # ]: 1053369 : assert(iovcnt > 0);
1010 : :
1011 [ # # # # ]: 1053369 : req->iovcnt = iovcnt;
1012 : :
1013 [ - + # # : 1053369 : assert(req->iov == iov);
# # ]
1014 : :
1015 [ # # # # ]: 1053369 : req->zcopy_bdev_io = bdev_io; /* Preserve the bdev_io for the end zcopy */
1016 : :
1017 : 1053369 : spdk_nvmf_request_complete(req);
1018 : : /* Don't free the bdev_io here as it is needed for the END ZCOPY */
1019 : 0 : }
1020 : :
1021 : : int
1022 : 1084297 : nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev,
1023 : : struct spdk_bdev_desc *desc,
1024 : : struct spdk_io_channel *ch,
1025 : : struct spdk_nvmf_request *req)
1026 : : {
1027 [ # # # # : 1084297 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
1028 : 1084297 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
1029 : 1084297 : uint32_t block_size = spdk_bdev_desc_get_block_size(desc);
1030 : 9 : uint64_t start_lba;
1031 : 9 : uint64_t num_blocks;
1032 : : int rc;
1033 : :
1034 [ # # # # : 1084297 : nvmf_bdev_ctrlr_get_rw_params(&req->cmd->nvme_cmd, &start_lba, &num_blocks);
# # ]
1035 : :
1036 [ + + ]: 1084297 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
1037 : 4 : SPDK_ERRLOG("end of media\n");
1038 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1039 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
1040 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1041 : : }
1042 : :
1043 [ + + # # : 1084293 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
# # ]
1044 [ # # # # ]: 4 : SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
1045 : : num_blocks, block_size, req->length);
1046 [ # # # # : 4 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1047 [ # # # # : 4 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
1048 : 4 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1049 : : }
1050 : :
1051 [ # # # # : 1084289 : bool populate = (req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_READ) ? true : false;
# # # # ]
1052 : :
1053 [ # # # # : 1084290 : rc = spdk_bdev_zcopy_start(desc, ch, req->iov, req->iovcnt, start_lba,
# # ]
1054 [ # # ]: 1 : num_blocks, populate, nvmf_bdev_ctrlr_zcopy_start_complete, req);
1055 [ - + ]: 1084289 : if (spdk_unlikely(rc != 0)) {
1056 [ # # ]: 0 : if (rc == -ENOMEM) {
1057 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
1058 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
1059 : : }
1060 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1061 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
1062 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1063 : : }
1064 : :
1065 : 1084289 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
1066 : 3 : }
1067 : :
1068 : : static void
1069 : 1053369 : nvmf_bdev_ctrlr_zcopy_end_complete(struct spdk_bdev_io *bdev_io, bool success,
1070 : : void *cb_arg)
1071 : : {
1072 : 1053369 : struct spdk_nvmf_request *req = cb_arg;
1073 : :
1074 [ + + # # ]: 1053369 : if (spdk_unlikely(!success)) {
1075 : 30483 : int sc = 0, sct = 0;
1076 : 30483 : uint32_t cdw0 = 0;
1077 [ # # # # : 30483 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
1078 : 30483 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
1079 : :
1080 [ # # # # ]: 30483 : response->cdw0 = cdw0;
1081 [ # # # # : 30483 : response->status.sc = sc;
# # ]
1082 [ # # # # : 30483 : response->status.sct = sct;
# # ]
1083 : 0 : }
1084 : :
1085 : 1053369 : spdk_bdev_free_io(bdev_io);
1086 [ # # # # ]: 1053369 : req->zcopy_bdev_io = NULL;
1087 : 1053369 : spdk_nvmf_request_complete(req);
1088 : 1053369 : }
1089 : :
1090 : : void
1091 : 1053369 : nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit)
1092 : : {
1093 : : int rc __attribute__((unused));
1094 : :
1095 [ # # # # : 1053369 : rc = spdk_bdev_zcopy_end(req->zcopy_bdev_io, commit, nvmf_bdev_ctrlr_zcopy_end_complete, req);
# # ]
1096 : :
1097 : : /* The only way spdk_bdev_zcopy_end() can fail is if we pass a bdev_io type that isn't ZCOPY */
1098 [ - + # # ]: 1053369 : assert(rc == 0);
1099 : 1053369 : }
|