Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
5 : * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6 : */
7 :
8 : /*
9 : * NVMe transport abstraction
10 : */
11 :
12 : #include "nvme_internal.h"
13 : #include "spdk/queue.h"
14 :
15 : #define SPDK_MAX_NUM_OF_TRANSPORTS 16
16 :
17 : struct spdk_nvme_transport {
18 : struct spdk_nvme_transport_ops ops;
19 : TAILQ_ENTRY(spdk_nvme_transport) link;
20 : };
21 :
22 : TAILQ_HEAD(nvme_transport_list, spdk_nvme_transport) g_spdk_nvme_transports =
23 : TAILQ_HEAD_INITIALIZER(g_spdk_nvme_transports);
24 :
25 : static struct spdk_nvme_transport g_transports[SPDK_MAX_NUM_OF_TRANSPORTS] = {};
26 : static int g_current_transport_index = 0;
27 :
28 : struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts = {
29 : .rdma_srq_size = 0,
30 : .rdma_max_cq_size = 0,
31 : .rdma_cm_event_timeout_ms = 1000
32 : };
33 :
34 : const struct spdk_nvme_transport *
35 0 : nvme_get_first_transport(void)
36 : {
37 0 : return TAILQ_FIRST(&g_spdk_nvme_transports);
38 : }
39 :
40 : const struct spdk_nvme_transport *
41 0 : nvme_get_next_transport(const struct spdk_nvme_transport *transport)
42 : {
43 0 : return TAILQ_NEXT(transport, link);
44 : }
45 :
46 : /*
47 : * Unfortunately, due to NVMe PCIe multiprocess support, we cannot store the
48 : * transport object in either the controller struct or the admin qpair. This means
49 : * that a lot of admin related transport calls will have to call nvme_get_transport
50 : * in order to know which functions to call.
51 : * In the I/O path, we have the ability to store the transport struct in the I/O
52 : * qpairs to avoid taking a performance hit.
53 : */
54 : const struct spdk_nvme_transport *
55 4 : nvme_get_transport(const char *transport_name)
56 : {
57 : struct spdk_nvme_transport *registered_transport;
58 :
59 4 : TAILQ_FOREACH(registered_transport, &g_spdk_nvme_transports, link) {
60 3 : if (strcasecmp(transport_name, registered_transport->ops.name) == 0) {
61 3 : return registered_transport;
62 : }
63 0 : }
64 :
65 1 : return NULL;
66 4 : }
67 :
68 : bool
69 0 : spdk_nvme_transport_available(enum spdk_nvme_transport_type trtype)
70 : {
71 0 : return nvme_get_transport(spdk_nvme_transport_id_trtype_str(trtype)) == NULL ? false : true;
72 : }
73 :
74 : bool
75 0 : spdk_nvme_transport_available_by_name(const char *transport_name)
76 : {
77 0 : return nvme_get_transport(transport_name) == NULL ? false : true;
78 : }
79 :
80 : void
81 0 : spdk_nvme_transport_register(const struct spdk_nvme_transport_ops *ops)
82 : {
83 : struct spdk_nvme_transport *new_transport;
84 :
85 0 : if (nvme_get_transport(ops->name)) {
86 0 : SPDK_ERRLOG("Double registering NVMe transport %s is prohibited.\n", ops->name);
87 0 : assert(false);
88 : }
89 :
90 0 : if (g_current_transport_index == SPDK_MAX_NUM_OF_TRANSPORTS) {
91 0 : SPDK_ERRLOG("Unable to register new NVMe transport.\n");
92 0 : assert(false);
93 : return;
94 : }
95 0 : new_transport = &g_transports[g_current_transport_index++];
96 :
97 0 : new_transport->ops = *ops;
98 0 : TAILQ_INSERT_TAIL(&g_spdk_nvme_transports, new_transport, link);
99 0 : }
100 :
101 0 : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
102 : const struct spdk_nvme_ctrlr_opts *opts,
103 : void *devhandle)
104 : {
105 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(trid->trstring);
106 : struct spdk_nvme_ctrlr *ctrlr;
107 :
108 0 : if (transport == NULL) {
109 0 : SPDK_ERRLOG("Transport %s doesn't exist.", trid->trstring);
110 0 : return NULL;
111 : }
112 :
113 0 : ctrlr = transport->ops.ctrlr_construct(trid, opts, devhandle);
114 :
115 0 : return ctrlr;
116 0 : }
117 :
118 : int
119 0 : nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
120 : bool direct_connect)
121 : {
122 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
123 :
124 0 : if (transport == NULL) {
125 0 : SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
126 0 : return -ENOENT;
127 : }
128 :
129 0 : return transport->ops.ctrlr_scan(probe_ctx, direct_connect);
130 0 : }
131 :
132 : int
133 0 : nvme_transport_ctrlr_scan_attached(struct spdk_nvme_probe_ctx *probe_ctx)
134 : {
135 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
136 :
137 0 : if (transport == NULL) {
138 0 : SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
139 0 : return -ENOENT;
140 : }
141 :
142 0 : if (transport->ops.ctrlr_scan_attached != NULL) {
143 0 : return transport->ops.ctrlr_scan_attached(probe_ctx);
144 : }
145 0 : SPDK_ERRLOG("Transport %s does not support ctrlr_scan_attached callback\n",
146 : probe_ctx->trid.trstring);
147 0 : return -ENOTSUP;
148 0 : }
149 :
150 : int
151 0 : nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
152 : {
153 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
154 :
155 0 : assert(transport != NULL);
156 0 : return transport->ops.ctrlr_destruct(ctrlr);
157 : }
158 :
159 : int
160 0 : nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
161 : {
162 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
163 :
164 0 : assert(transport != NULL);
165 0 : return transport->ops.ctrlr_enable(ctrlr);
166 : }
167 :
168 : int
169 0 : nvme_transport_ctrlr_enable_interrupts(struct spdk_nvme_ctrlr *ctrlr)
170 : {
171 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
172 :
173 0 : assert(transport != NULL);
174 0 : if (transport->ops.ctrlr_enable_interrupts != NULL) {
175 0 : return transport->ops.ctrlr_enable_interrupts(ctrlr);
176 : }
177 :
178 0 : return -ENOTSUP;
179 0 : }
180 :
181 : int
182 0 : nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr)
183 : {
184 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
185 :
186 0 : assert(transport != NULL);
187 0 : if (transport->ops.ctrlr_ready) {
188 0 : return transport->ops.ctrlr_ready(ctrlr);
189 : }
190 :
191 0 : return 0;
192 0 : }
193 :
194 : int
195 0 : nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
196 : {
197 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
198 :
199 0 : assert(transport != NULL);
200 0 : return transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
201 : }
202 :
203 : int
204 0 : nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
205 : {
206 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
207 :
208 0 : assert(transport != NULL);
209 0 : return transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
210 : }
211 :
212 : int
213 0 : nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
214 : {
215 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
216 :
217 0 : assert(transport != NULL);
218 0 : return transport->ops.ctrlr_get_reg_4(ctrlr, offset, value);
219 : }
220 :
221 : int
222 0 : nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
223 : {
224 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
225 :
226 0 : assert(transport != NULL);
227 0 : return transport->ops.ctrlr_get_reg_8(ctrlr, offset, value);
228 : }
229 :
230 : static int
231 0 : nvme_queue_register_operation_completion(struct spdk_nvme_ctrlr *ctrlr, uint64_t value,
232 : spdk_nvme_reg_cb cb_fn, void *cb_ctx)
233 : {
234 : struct nvme_register_completion *ctx;
235 :
236 0 : ctx = spdk_zmalloc(sizeof(*ctx), 0, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
237 0 : if (ctx == NULL) {
238 0 : return -ENOMEM;
239 : }
240 :
241 0 : ctx->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
242 0 : ctx->cpl.status.sc = SPDK_NVME_SC_SUCCESS;
243 0 : ctx->cb_fn = cb_fn;
244 0 : ctx->cb_ctx = cb_ctx;
245 0 : ctx->value = value;
246 0 : ctx->pid = getpid();
247 :
248 0 : nvme_ctrlr_lock(ctrlr);
249 0 : STAILQ_INSERT_TAIL(&ctrlr->register_operations, ctx, stailq);
250 0 : nvme_ctrlr_unlock(ctrlr);
251 :
252 0 : return 0;
253 0 : }
254 :
255 : int
256 0 : nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value,
257 : spdk_nvme_reg_cb cb_fn, void *cb_arg)
258 : {
259 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
260 : int rc;
261 :
262 0 : assert(transport != NULL);
263 0 : if (transport->ops.ctrlr_set_reg_4_async == NULL) {
264 0 : rc = transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
265 0 : if (rc != 0) {
266 0 : return rc;
267 : }
268 :
269 0 : return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
270 : }
271 :
272 0 : return transport->ops.ctrlr_set_reg_4_async(ctrlr, offset, value, cb_fn, cb_arg);
273 0 : }
274 :
275 : int
276 0 : nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value,
277 : spdk_nvme_reg_cb cb_fn, void *cb_arg)
278 :
279 : {
280 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
281 : int rc;
282 :
283 0 : assert(transport != NULL);
284 0 : if (transport->ops.ctrlr_set_reg_8_async == NULL) {
285 0 : rc = transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
286 0 : if (rc != 0) {
287 0 : return rc;
288 : }
289 :
290 0 : return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
291 : }
292 :
293 0 : return transport->ops.ctrlr_set_reg_8_async(ctrlr, offset, value, cb_fn, cb_arg);
294 0 : }
295 :
296 : int
297 0 : nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
298 : spdk_nvme_reg_cb cb_fn, void *cb_arg)
299 : {
300 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
301 : uint32_t value;
302 : int rc;
303 :
304 0 : assert(transport != NULL);
305 0 : if (transport->ops.ctrlr_get_reg_4_async == NULL) {
306 0 : rc = transport->ops.ctrlr_get_reg_4(ctrlr, offset, &value);
307 0 : if (rc != 0) {
308 0 : return rc;
309 : }
310 :
311 0 : return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
312 : }
313 :
314 0 : return transport->ops.ctrlr_get_reg_4_async(ctrlr, offset, cb_fn, cb_arg);
315 0 : }
316 :
317 : int
318 0 : nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
319 : spdk_nvme_reg_cb cb_fn, void *cb_arg)
320 : {
321 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
322 : uint64_t value;
323 : int rc;
324 :
325 0 : assert(transport != NULL);
326 0 : if (transport->ops.ctrlr_get_reg_8_async == NULL) {
327 0 : rc = transport->ops.ctrlr_get_reg_8(ctrlr, offset, &value);
328 0 : if (rc != 0) {
329 0 : return rc;
330 : }
331 :
332 0 : return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
333 : }
334 :
335 0 : return transport->ops.ctrlr_get_reg_8_async(ctrlr, offset, cb_fn, cb_arg);
336 0 : }
337 :
338 : uint32_t
339 0 : nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
340 : {
341 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
342 :
343 0 : assert(transport != NULL);
344 0 : return transport->ops.ctrlr_get_max_xfer_size(ctrlr);
345 : }
346 :
347 : uint16_t
348 0 : nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
349 : {
350 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
351 :
352 0 : assert(transport != NULL);
353 0 : return transport->ops.ctrlr_get_max_sges(ctrlr);
354 : }
355 :
356 : int
357 0 : nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
358 : {
359 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
360 :
361 0 : assert(transport != NULL);
362 0 : if (transport->ops.ctrlr_reserve_cmb != NULL) {
363 0 : return transport->ops.ctrlr_reserve_cmb(ctrlr);
364 : }
365 :
366 0 : return -ENOTSUP;
367 0 : }
368 :
369 : void *
370 0 : nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
371 : {
372 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
373 :
374 0 : assert(transport != NULL);
375 0 : if (transport->ops.ctrlr_map_cmb != NULL) {
376 0 : return transport->ops.ctrlr_map_cmb(ctrlr, size);
377 : }
378 :
379 0 : return NULL;
380 0 : }
381 :
382 : int
383 0 : nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
384 : {
385 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
386 :
387 0 : assert(transport != NULL);
388 0 : if (transport->ops.ctrlr_unmap_cmb != NULL) {
389 0 : return transport->ops.ctrlr_unmap_cmb(ctrlr);
390 : }
391 :
392 0 : return 0;
393 0 : }
394 :
395 : int
396 0 : nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr)
397 : {
398 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
399 :
400 0 : assert(transport != NULL);
401 0 : if (transport->ops.ctrlr_enable_pmr != NULL) {
402 0 : return transport->ops.ctrlr_enable_pmr(ctrlr);
403 : }
404 :
405 0 : return -ENOSYS;
406 0 : }
407 :
408 : int
409 0 : nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr)
410 : {
411 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
412 :
413 0 : assert(transport != NULL);
414 0 : if (transport->ops.ctrlr_disable_pmr != NULL) {
415 0 : return transport->ops.ctrlr_disable_pmr(ctrlr);
416 : }
417 :
418 0 : return -ENOSYS;
419 0 : }
420 :
421 : void *
422 0 : nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
423 : {
424 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
425 :
426 0 : assert(transport != NULL);
427 0 : if (transport->ops.ctrlr_map_pmr != NULL) {
428 0 : return transport->ops.ctrlr_map_pmr(ctrlr, size);
429 : }
430 :
431 0 : return NULL;
432 0 : }
433 :
434 : int
435 0 : nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr)
436 : {
437 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
438 :
439 0 : assert(transport != NULL);
440 0 : if (transport->ops.ctrlr_unmap_pmr != NULL) {
441 0 : return transport->ops.ctrlr_unmap_pmr(ctrlr);
442 : }
443 :
444 0 : return -ENOSYS;
445 0 : }
446 :
447 : struct spdk_nvme_qpair *
448 0 : nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
449 : const struct spdk_nvme_io_qpair_opts *opts)
450 : {
451 : struct spdk_nvme_qpair *qpair;
452 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
453 :
454 0 : assert(transport != NULL);
455 0 : qpair = transport->ops.ctrlr_create_io_qpair(ctrlr, qid, opts);
456 0 : if (qpair != NULL && !nvme_qpair_is_admin_queue(qpair)) {
457 0 : qpair->transport = transport;
458 0 : }
459 :
460 0 : return qpair;
461 : }
462 :
463 : void
464 0 : nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
465 : {
466 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
467 : int rc;
468 :
469 0 : assert(transport != NULL);
470 :
471 : /* Do not rely on qpair->transport. For multi-process cases, a foreign process may delete
472 : * the IO qpair, in which case the transport object would be invalid (each process has their
473 : * own unique transport objects since they contain function pointers). So we look up the
474 : * transport object in the delete_io_qpair case.
475 : */
476 0 : rc = transport->ops.ctrlr_delete_io_qpair(ctrlr, qpair);
477 0 : if (rc != 0) {
478 0 : SPDK_ERRLOG("transport %s returned non-zero for ctrlr_delete_io_qpair op\n",
479 : transport->ops.name);
480 0 : assert(false);
481 : }
482 0 : }
483 :
484 : static void
485 0 : nvme_transport_connect_qpair_fail(struct spdk_nvme_qpair *qpair, void *unused)
486 : {
487 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
488 :
489 : /* If the qpair was unable to reconnect, restore the original failure reason */
490 0 : qpair->transport_failure_reason = qpair->last_transport_failure_reason;
491 0 : nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
492 0 : }
493 :
494 : int
495 0 : nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
496 : {
497 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
498 : int rc;
499 :
500 0 : assert(transport != NULL);
501 0 : if (!nvme_qpair_is_admin_queue(qpair) && qpair->transport == NULL) {
502 0 : qpair->transport = transport;
503 0 : }
504 :
505 0 : qpair->last_transport_failure_reason = qpair->transport_failure_reason;
506 0 : qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_NONE;
507 :
508 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTING);
509 0 : rc = transport->ops.ctrlr_connect_qpair(ctrlr, qpair);
510 0 : if (rc != 0) {
511 0 : goto err;
512 : }
513 :
514 0 : if (qpair->poll_group) {
515 0 : rc = nvme_poll_group_connect_qpair(qpair);
516 0 : if (rc) {
517 0 : goto err;
518 : }
519 0 : }
520 :
521 0 : if (!qpair->async) {
522 : /* Busy wait until the qpair exits the connecting state */
523 0 : while (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
524 0 : if (qpair->poll_group && spdk_nvme_ctrlr_is_fabrics(ctrlr)) {
525 0 : rc = spdk_nvme_poll_group_process_completions(
526 0 : qpair->poll_group->group, 0,
527 : nvme_transport_connect_qpair_fail);
528 0 : } else {
529 0 : rc = spdk_nvme_qpair_process_completions(qpair, 0);
530 : }
531 :
532 0 : if (rc < 0) {
533 0 : goto err;
534 : }
535 : }
536 0 : }
537 :
538 0 : return 0;
539 : err:
540 0 : nvme_transport_connect_qpair_fail(qpair, NULL);
541 0 : if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {
542 0 : assert(qpair->async == true);
543 : /* Let the caller to poll the qpair until it is actually disconnected. */
544 0 : return 0;
545 : }
546 :
547 0 : return rc;
548 0 : }
549 :
550 : void
551 0 : nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
552 : {
553 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
554 :
555 0 : if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING ||
556 0 : nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTED) {
557 0 : return;
558 : }
559 :
560 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTING);
561 0 : assert(transport != NULL);
562 :
563 0 : if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) {
564 0 : nvme_poll_group_disconnect_qpair(qpair);
565 0 : }
566 :
567 0 : transport->ops.ctrlr_disconnect_qpair(ctrlr, qpair);
568 0 : }
569 :
570 : void
571 0 : nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair)
572 : {
573 0 : if (qpair->active_proc == nvme_ctrlr_get_current_process(qpair->ctrlr) ||
574 0 : nvme_qpair_is_admin_queue(qpair)) {
575 0 : nvme_qpair_abort_all_queued_reqs(qpair);
576 0 : }
577 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
578 0 : }
579 :
580 : int
581 2 : nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
582 : struct spdk_memory_domain **domains, int array_size)
583 : {
584 2 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
585 :
586 2 : assert(transport != NULL);
587 2 : if (transport->ops.ctrlr_get_memory_domains) {
588 1 : return transport->ops.ctrlr_get_memory_domains(ctrlr, domains, array_size);
589 : }
590 :
591 1 : return 0;
592 2 : }
593 :
594 : void
595 0 : nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair)
596 : {
597 : const struct spdk_nvme_transport *transport;
598 :
599 0 : if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
600 0 : qpair->transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
601 0 : } else {
602 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
603 0 : assert(transport != NULL);
604 0 : transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
605 : }
606 0 : }
607 :
608 : int
609 0 : nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair)
610 : {
611 : const struct spdk_nvme_transport *transport;
612 :
613 0 : if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
614 0 : return qpair->transport->ops.qpair_reset(qpair);
615 : }
616 :
617 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
618 0 : assert(transport != NULL);
619 0 : return transport->ops.qpair_reset(qpair);
620 0 : }
621 :
622 : int
623 0 : nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
624 : {
625 : const struct spdk_nvme_transport *transport;
626 :
627 0 : if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
628 0 : return qpair->transport->ops.qpair_submit_request(qpair, req);
629 : }
630 :
631 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
632 0 : assert(transport != NULL);
633 0 : return transport->ops.qpair_submit_request(qpair, req);
634 0 : }
635 :
636 : int32_t
637 0 : nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
638 : {
639 : const struct spdk_nvme_transport *transport;
640 :
641 0 : if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
642 0 : return qpair->transport->ops.qpair_process_completions(qpair, max_completions);
643 : }
644 :
645 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
646 0 : assert(transport != NULL);
647 0 : return transport->ops.qpair_process_completions(qpair, max_completions);
648 0 : }
649 :
650 : int
651 0 : nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
652 : int (*iter_fn)(struct nvme_request *req, void *arg),
653 : void *arg)
654 : {
655 : const struct spdk_nvme_transport *transport;
656 :
657 0 : if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
658 0 : return qpair->transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
659 : }
660 :
661 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
662 0 : assert(transport != NULL);
663 0 : return transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
664 0 : }
665 :
666 : int
667 0 : nvme_transport_qpair_authenticate(struct spdk_nvme_qpair *qpair)
668 : {
669 : const struct spdk_nvme_transport *transport;
670 :
671 0 : transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
672 0 : if (transport->ops.qpair_authenticate == NULL) {
673 0 : return -ENOTSUP;
674 : }
675 :
676 0 : return transport->ops.qpair_authenticate(qpair);
677 0 : }
678 :
679 : void
680 0 : nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
681 : {
682 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
683 :
684 0 : assert(transport != NULL);
685 0 : transport->ops.admin_qpair_abort_aers(qpair);
686 0 : }
687 :
688 : struct spdk_nvme_transport_poll_group *
689 0 : nvme_transport_poll_group_create(const struct spdk_nvme_transport *transport)
690 : {
691 0 : struct spdk_nvme_transport_poll_group *group = NULL;
692 :
693 0 : group = transport->ops.poll_group_create();
694 0 : if (group) {
695 0 : group->transport = transport;
696 0 : STAILQ_INIT(&group->connected_qpairs);
697 0 : STAILQ_INIT(&group->disconnected_qpairs);
698 0 : group->num_connected_qpairs = 0;
699 0 : }
700 :
701 0 : return group;
702 : }
703 :
704 : struct spdk_nvme_transport_poll_group *
705 0 : nvme_transport_qpair_get_optimal_poll_group(const struct spdk_nvme_transport *transport,
706 : struct spdk_nvme_qpair *qpair)
707 : {
708 0 : if (transport->ops.qpair_get_optimal_poll_group) {
709 0 : return transport->ops.qpair_get_optimal_poll_group(qpair);
710 : } else {
711 0 : return NULL;
712 : }
713 0 : }
714 :
715 : int
716 1 : nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
717 : struct spdk_nvme_qpair *qpair)
718 : {
719 : int rc;
720 :
721 1 : rc = tgroup->transport->ops.poll_group_add(tgroup, qpair);
722 1 : if (rc == 0) {
723 1 : qpair->poll_group = tgroup;
724 1 : assert(nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTED);
725 1 : qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
726 1 : STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
727 1 : }
728 :
729 1 : return rc;
730 : }
731 :
732 : int
733 3 : nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
734 : struct spdk_nvme_qpair *qpair)
735 : {
736 : int rc __attribute__((unused));
737 :
738 3 : if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
739 1 : return -EINVAL;
740 2 : } else if (qpair->poll_group_tailq_head != &tgroup->disconnected_qpairs) {
741 1 : return -ENOENT;
742 : }
743 :
744 1 : rc = tgroup->transport->ops.poll_group_remove(tgroup, qpair);
745 1 : assert(rc == 0);
746 :
747 1 : STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
748 :
749 1 : qpair->poll_group = NULL;
750 1 : qpair->poll_group_tailq_head = NULL;
751 :
752 1 : return 0;
753 3 : }
754 :
755 : int64_t
756 0 : nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
757 : uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
758 : {
759 0 : return tgroup->transport->ops.poll_group_process_completions(tgroup, completions_per_qpair,
760 0 : disconnected_qpair_cb);
761 : }
762 :
763 : int
764 0 : nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
765 : {
766 0 : return tgroup->transport->ops.poll_group_destroy(tgroup);
767 : }
768 :
769 : int
770 3 : nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
771 : {
772 : struct spdk_nvme_transport_poll_group *tgroup;
773 : int rc __attribute__((unused));
774 :
775 3 : tgroup = qpair->poll_group;
776 :
777 3 : if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
778 1 : return 0;
779 : }
780 :
781 2 : if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
782 1 : rc = tgroup->transport->ops.poll_group_disconnect_qpair(qpair);
783 1 : assert(rc == 0);
784 :
785 1 : qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
786 1 : STAILQ_REMOVE(&tgroup->connected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
787 1 : assert(tgroup->num_connected_qpairs > 0);
788 1 : tgroup->num_connected_qpairs--;
789 1 : STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
790 :
791 1 : return 0;
792 : }
793 :
794 1 : return -EINVAL;
795 3 : }
796 :
797 : int
798 3 : nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
799 : {
800 : struct spdk_nvme_transport_poll_group *tgroup;
801 : int rc;
802 :
803 3 : tgroup = qpair->poll_group;
804 :
805 3 : if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
806 1 : return 0;
807 : }
808 :
809 2 : if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
810 1 : rc = tgroup->transport->ops.poll_group_connect_qpair(qpair);
811 1 : if (rc == 0) {
812 1 : qpair->poll_group_tailq_head = &tgroup->connected_qpairs;
813 1 : STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
814 1 : STAILQ_INSERT_TAIL(&tgroup->connected_qpairs, qpair, poll_group_stailq);
815 1 : tgroup->num_connected_qpairs++;
816 1 : }
817 :
818 1 : return rc == -EINPROGRESS ? 0 : rc;
819 : }
820 :
821 :
822 1 : return -EINVAL;
823 3 : }
824 :
825 : int
826 0 : nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
827 : struct spdk_nvme_transport_poll_group_stat **stats)
828 : {
829 0 : if (tgroup->transport->ops.poll_group_get_stats) {
830 0 : return tgroup->transport->ops.poll_group_get_stats(tgroup, stats);
831 : }
832 0 : return -ENOTSUP;
833 0 : }
834 :
835 : void
836 0 : nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
837 : struct spdk_nvme_transport_poll_group_stat *stats)
838 : {
839 0 : if (tgroup->transport->ops.poll_group_free_stats) {
840 0 : tgroup->transport->ops.poll_group_free_stats(tgroup, stats);
841 0 : }
842 0 : }
843 :
844 : spdk_nvme_transport_type_t
845 0 : nvme_transport_get_trtype(const struct spdk_nvme_transport *transport)
846 : {
847 0 : return transport->ops.type;
848 : }
849 :
850 : void
851 0 : spdk_nvme_transport_get_opts(struct spdk_nvme_transport_opts *opts, size_t opts_size)
852 : {
853 0 : if (opts == NULL) {
854 0 : SPDK_ERRLOG("opts should not be NULL.\n");
855 0 : return;
856 : }
857 :
858 0 : if (opts_size == 0) {
859 0 : SPDK_ERRLOG("opts_size should not be zero.\n");
860 0 : return;
861 : }
862 :
863 0 : opts->opts_size = opts_size;
864 :
865 : #define SET_FIELD(field) \
866 : if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts_size) { \
867 : opts->field = g_spdk_nvme_transport_opts.field; \
868 : } \
869 :
870 0 : SET_FIELD(rdma_srq_size);
871 0 : SET_FIELD(rdma_max_cq_size);
872 0 : SET_FIELD(rdma_cm_event_timeout_ms);
873 :
874 : /* Do not remove this statement, you should always update this statement when you adding a new field,
875 : * and do not forget to add the SET_FIELD statement for your added field. */
876 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 24, "Incorrect size");
877 :
878 : #undef SET_FIELD
879 0 : }
880 :
881 : int
882 0 : spdk_nvme_transport_set_opts(const struct spdk_nvme_transport_opts *opts, size_t opts_size)
883 : {
884 0 : if (opts == NULL) {
885 0 : SPDK_ERRLOG("opts should not be NULL.\n");
886 0 : return -EINVAL;
887 : }
888 :
889 0 : if (opts_size == 0) {
890 0 : SPDK_ERRLOG("opts_size should not be zero.\n");
891 0 : return -EINVAL;
892 : }
893 :
894 : #define SET_FIELD(field) \
895 : if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
896 : g_spdk_nvme_transport_opts.field = opts->field; \
897 : } \
898 :
899 0 : SET_FIELD(rdma_srq_size);
900 0 : SET_FIELD(rdma_max_cq_size);
901 0 : SET_FIELD(rdma_cm_event_timeout_ms);
902 :
903 0 : g_spdk_nvme_transport_opts.opts_size = opts->opts_size;
904 :
905 : #undef SET_FIELD
906 :
907 0 : return 0;
908 0 : }
909 :
910 : volatile struct spdk_nvme_registers *
911 0 : spdk_nvme_ctrlr_get_registers(struct spdk_nvme_ctrlr *ctrlr)
912 : {
913 0 : const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
914 :
915 0 : if (transport == NULL) {
916 : /* Transport does not exist. */
917 0 : return NULL;
918 : }
919 :
920 0 : if (transport->ops.ctrlr_get_registers) {
921 0 : return transport->ops.ctrlr_get_registers(ctrlr);
922 : }
923 :
924 0 : return NULL;
925 0 : }
|