Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved.
4 : * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/bdev.h"
10 : #include "spdk/bit_array.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/nvmf.h"
13 : #include "spdk/endian.h"
14 : #include "spdk/string.h"
15 : #include "spdk/log.h"
16 : #include "spdk_internal/usdt.h"
17 :
18 : #include "nvmf_internal.h"
19 : #include "transport.h"
20 :
21 1 : SPDK_LOG_REGISTER_COMPONENT(nvmf)
22 :
23 : #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024
24 :
25 : static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts);
26 :
27 : typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status);
28 :
29 : /* supplied to a single call to nvmf_qpair_disconnect */
30 : struct nvmf_qpair_disconnect_ctx {
31 : struct spdk_nvmf_qpair *qpair;
32 : struct spdk_nvmf_ctrlr *ctrlr;
33 : uint16_t qid;
34 : };
35 :
36 : /*
37 : * There are several times when we need to iterate through the list of all qpairs and selectively delete them.
38 : * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from
39 : * to enable calling nvmf_qpair_disconnect on the next desired qpair.
40 : */
41 : struct nvmf_qpair_disconnect_many_ctx {
42 : struct spdk_nvmf_subsystem *subsystem;
43 : struct spdk_nvmf_poll_group *group;
44 : spdk_nvmf_poll_group_mod_done cpl_fn;
45 : void *cpl_ctx;
46 : };
47 :
48 : static struct spdk_nvmf_referral *
49 0 : nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt,
50 : const struct spdk_nvme_transport_id *trid)
51 : {
52 : struct spdk_nvmf_referral *referral;
53 :
54 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
55 0 : if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) {
56 0 : return referral;
57 : }
58 : }
59 :
60 0 : return NULL;
61 : }
62 :
63 : int
64 0 : spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt,
65 : const struct spdk_nvmf_referral_opts *uopts)
66 : {
67 : struct spdk_nvmf_referral *referral;
68 0 : struct spdk_nvmf_referral_opts opts = {};
69 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
70 :
71 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
72 0 : if (trid->subnqn[0] == '\0') {
73 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
74 : }
75 :
76 0 : if (!nvmf_nqn_is_valid(trid->subnqn)) {
77 0 : SPDK_ERRLOG("Invalid subsystem NQN\n");
78 0 : return -EINVAL;
79 : }
80 :
81 : /* If the entry already exists, just ignore it. */
82 0 : if (nvmf_tgt_find_referral(tgt, trid)) {
83 0 : return 0;
84 : }
85 :
86 0 : referral = calloc(1, sizeof(*referral));
87 0 : if (!referral) {
88 0 : SPDK_ERRLOG("Failed to allocate memory for a referral\n");
89 0 : return -ENOMEM;
90 : }
91 :
92 0 : referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ?
93 : SPDK_NVMF_SUBTYPE_DISCOVERY :
94 : SPDK_NVMF_SUBTYPE_NVME;
95 0 : referral->entry.treq.secure_channel = opts.secure_channel ?
96 0 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED :
97 : SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED;
98 0 : referral->entry.cntlid = 0xffff;
99 0 : referral->entry.trtype = trid->trtype;
100 0 : referral->entry.adrfam = trid->adrfam;
101 0 : referral->allow_any_host = opts.allow_any_host;
102 0 : memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id));
103 0 : spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0');
104 0 : spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' ');
105 0 : spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' ');
106 :
107 0 : TAILQ_INSERT_HEAD(&tgt->referrals, referral, link);
108 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
109 :
110 0 : return 0;
111 : }
112 :
113 : int
114 0 : spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt,
115 : const struct spdk_nvmf_referral_opts *uopts)
116 : {
117 : struct spdk_nvmf_referral *referral;
118 0 : struct spdk_nvmf_referral_opts opts = {};
119 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
120 :
121 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
122 0 : if (trid->subnqn[0] == '\0') {
123 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
124 : }
125 :
126 0 : referral = nvmf_tgt_find_referral(tgt, &opts.trid);
127 0 : if (referral == NULL) {
128 0 : return -ENOENT;
129 : }
130 :
131 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
132 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
133 :
134 0 : free(referral);
135 :
136 0 : return 0;
137 : }
138 :
139 : void
140 0 : nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair,
141 : enum spdk_nvmf_qpair_state state)
142 : {
143 0 : assert(qpair != NULL);
144 0 : assert(qpair->group->thread == spdk_get_thread());
145 :
146 0 : qpair->state = state;
147 0 : }
148 :
149 : static int
150 0 : nvmf_poll_group_poll(void *ctx)
151 : {
152 0 : struct spdk_nvmf_poll_group *group = ctx;
153 : int rc;
154 0 : int count = 0;
155 : struct spdk_nvmf_transport_poll_group *tgroup;
156 :
157 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
158 0 : rc = nvmf_transport_poll_group_poll(tgroup);
159 0 : if (rc < 0) {
160 0 : return SPDK_POLLER_BUSY;
161 : }
162 0 : count += rc;
163 : }
164 :
165 0 : return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
166 : }
167 :
168 : /*
169 : * Reset and clean up the poll group (I/O channel code will actually free the
170 : * group).
171 : */
172 : static void
173 1 : nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group)
174 : {
175 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
176 : struct spdk_nvmf_subsystem_poll_group *sgroup;
177 : uint32_t sid, nsid;
178 :
179 2 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
180 1 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
181 1 : nvmf_transport_poll_group_destroy(tgroup);
182 : }
183 :
184 2 : for (sid = 0; sid < group->num_sgroups; sid++) {
185 1 : sgroup = &group->sgroups[sid];
186 :
187 1 : assert(sgroup != NULL);
188 :
189 2 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
190 1 : if (sgroup->ns_info[nsid].channel) {
191 1 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
192 1 : sgroup->ns_info[nsid].channel = NULL;
193 : }
194 : }
195 :
196 1 : free(sgroup->ns_info);
197 : }
198 :
199 1 : free(group->sgroups);
200 :
201 1 : spdk_poller_unregister(&group->poller);
202 :
203 1 : if (group->destroy_cb_fn) {
204 0 : group->destroy_cb_fn(group->destroy_cb_arg, 0);
205 : }
206 1 : }
207 :
208 : /*
209 : * Callback to unregister a poll group from the target, and clean up its state.
210 : */
211 : static void
212 1 : nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
213 : {
214 1 : struct spdk_nvmf_tgt *tgt = io_device;
215 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
216 :
217 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread));
218 :
219 1 : pthread_mutex_lock(&tgt->mutex);
220 1 : TAILQ_REMOVE(&tgt->poll_groups, group, link);
221 1 : tgt->num_poll_groups--;
222 1 : pthread_mutex_unlock(&tgt->mutex);
223 :
224 1 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
225 1 : nvmf_tgt_cleanup_poll_group(group);
226 1 : }
227 :
228 : static int
229 1 : nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group,
230 : struct spdk_nvmf_transport *transport)
231 : {
232 1 : struct spdk_nvmf_transport_poll_group *tgroup = nvmf_get_transport_poll_group(group, transport);
233 :
234 1 : if (tgroup != NULL) {
235 : /* Transport already in the poll group */
236 0 : return 0;
237 : }
238 :
239 1 : tgroup = nvmf_transport_poll_group_create(transport, group);
240 1 : if (!tgroup) {
241 0 : SPDK_ERRLOG("Unable to create poll group for transport\n");
242 0 : return -1;
243 : }
244 : SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport,
245 : spdk_thread_get_id(group->thread));
246 :
247 1 : tgroup->group = group;
248 1 : TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
249 :
250 1 : return 0;
251 : }
252 :
253 : static int
254 1 : nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
255 : {
256 1 : struct spdk_nvmf_tgt *tgt = io_device;
257 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
258 : struct spdk_nvmf_transport *transport;
259 : struct spdk_nvmf_subsystem *subsystem;
260 1 : struct spdk_thread *thread = spdk_get_thread();
261 : uint32_t i;
262 : int rc;
263 :
264 1 : group->tgt = tgt;
265 1 : TAILQ_INIT(&group->tgroups);
266 1 : TAILQ_INIT(&group->qpairs);
267 1 : group->thread = thread;
268 1 : pthread_mutex_init(&group->mutex, NULL);
269 :
270 1 : group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0);
271 1 : spdk_poller_register_interrupt(group->poller, NULL, NULL);
272 :
273 : SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread));
274 :
275 2 : TAILQ_FOREACH(transport, &tgt->transports, link) {
276 1 : rc = nvmf_poll_group_add_transport(group, transport);
277 1 : if (rc != 0) {
278 0 : nvmf_tgt_cleanup_poll_group(group);
279 0 : return rc;
280 : }
281 : }
282 :
283 1 : group->num_sgroups = tgt->max_subsystems;
284 1 : group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group));
285 1 : if (!group->sgroups) {
286 0 : nvmf_tgt_cleanup_poll_group(group);
287 0 : return -ENOMEM;
288 : }
289 :
290 2 : for (i = 0; i < tgt->max_subsystems; i++) {
291 1 : TAILQ_INIT(&group->sgroups[i].queued);
292 : }
293 :
294 1 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt);
295 2 : subsystem != NULL;
296 1 : subsystem = spdk_nvmf_subsystem_get_next(subsystem)) {
297 1 : if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) {
298 0 : nvmf_tgt_cleanup_poll_group(group);
299 0 : return -1;
300 : }
301 : }
302 :
303 1 : pthread_mutex_lock(&tgt->mutex);
304 1 : tgt->num_poll_groups++;
305 1 : TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link);
306 1 : pthread_mutex_unlock(&tgt->mutex);
307 :
308 1 : return 0;
309 : }
310 :
311 : static void
312 0 : _nvmf_tgt_disconnect_qpairs(void *ctx)
313 : {
314 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
315 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
316 0 : struct spdk_nvmf_poll_group *group = qpair_ctx->group;
317 : struct spdk_io_channel *ch;
318 : int rc;
319 :
320 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
321 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
322 0 : if (rc && rc != -EINPROGRESS) {
323 0 : break;
324 : }
325 : }
326 :
327 0 : if (TAILQ_EMPTY(&group->qpairs)) {
328 : /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */
329 0 : ch = spdk_io_channel_from_ctx(group);
330 0 : spdk_put_io_channel(ch);
331 0 : free(qpair_ctx);
332 0 : return;
333 : }
334 :
335 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
336 0 : spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx);
337 : }
338 :
339 : static void
340 0 : nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group)
341 : {
342 : struct nvmf_qpair_disconnect_many_ctx *ctx;
343 :
344 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread));
345 :
346 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
347 0 : if (!ctx) {
348 0 : SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n");
349 0 : return;
350 : }
351 :
352 0 : ctx->group = group;
353 0 : _nvmf_tgt_disconnect_qpairs(ctx);
354 : }
355 :
356 : struct spdk_nvmf_tgt *
357 0 : spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *_opts)
358 : {
359 : struct spdk_nvmf_tgt *tgt, *tmp_tgt;
360 0 : struct spdk_nvmf_target_opts opts = {
361 : .max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS,
362 : .discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY,
363 : };
364 :
365 0 : memcpy(&opts, _opts, _opts->size);
366 0 : if (strnlen(opts.name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) {
367 0 : SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH);
368 0 : return NULL;
369 : }
370 :
371 0 : TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) {
372 0 : if (!strncmp(opts.name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) {
373 0 : SPDK_ERRLOG("Provided target name must be unique.\n");
374 0 : return NULL;
375 : }
376 : }
377 :
378 0 : tgt = calloc(1, sizeof(*tgt));
379 0 : if (!tgt) {
380 0 : return NULL;
381 : }
382 :
383 0 : snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts.name);
384 :
385 0 : if (!opts.max_subsystems) {
386 0 : tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS;
387 : } else {
388 0 : tgt->max_subsystems = opts.max_subsystems;
389 : }
390 :
391 0 : tgt->crdt[0] = opts.crdt[0];
392 0 : tgt->crdt[1] = opts.crdt[1];
393 0 : tgt->crdt[2] = opts.crdt[2];
394 0 : tgt->discovery_filter = opts.discovery_filter;
395 0 : tgt->discovery_genctr = 0;
396 0 : tgt->dhchap_digests = opts.dhchap_digests;
397 0 : tgt->dhchap_dhgroups = opts.dhchap_dhgroups;
398 0 : TAILQ_INIT(&tgt->transports);
399 0 : TAILQ_INIT(&tgt->poll_groups);
400 0 : TAILQ_INIT(&tgt->referrals);
401 0 : tgt->num_poll_groups = 0;
402 :
403 0 : tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems);
404 0 : if (tgt->subsystem_ids == NULL) {
405 0 : free(tgt);
406 0 : return NULL;
407 : }
408 :
409 0 : RB_INIT(&tgt->subsystems);
410 :
411 0 : pthread_mutex_init(&tgt->mutex, NULL);
412 :
413 0 : spdk_io_device_register(tgt,
414 : nvmf_tgt_create_poll_group,
415 : nvmf_tgt_destroy_poll_group,
416 : sizeof(struct spdk_nvmf_poll_group),
417 0 : tgt->name);
418 :
419 0 : tgt->state = NVMF_TGT_RUNNING;
420 :
421 0 : TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link);
422 :
423 0 : return tgt;
424 : }
425 :
426 : static void
427 0 : _nvmf_tgt_destroy_next_transport(void *ctx)
428 : {
429 0 : struct spdk_nvmf_tgt *tgt = ctx;
430 : struct spdk_nvmf_transport *transport;
431 :
432 0 : if (!TAILQ_EMPTY(&tgt->transports)) {
433 0 : transport = TAILQ_FIRST(&tgt->transports);
434 0 : TAILQ_REMOVE(&tgt->transports, transport, link);
435 0 : spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt);
436 : } else {
437 0 : spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn;
438 0 : void *destroy_cb_arg = tgt->destroy_cb_arg;
439 :
440 0 : pthread_mutex_destroy(&tgt->mutex);
441 0 : free(tgt);
442 :
443 0 : if (destroy_cb_fn) {
444 0 : destroy_cb_fn(destroy_cb_arg, 0);
445 : }
446 : }
447 0 : }
448 :
449 : static void
450 0 : nvmf_tgt_destroy_cb(void *io_device)
451 : {
452 0 : struct spdk_nvmf_tgt *tgt = io_device;
453 : struct spdk_nvmf_subsystem *subsystem, *subsystem_next;
454 : int rc;
455 : struct spdk_nvmf_referral *referral;
456 :
457 0 : while ((referral = TAILQ_FIRST(&tgt->referrals))) {
458 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
459 0 : free(referral);
460 : }
461 :
462 0 : nvmf_tgt_stop_mdns_prr(tgt);
463 :
464 : /* We will be freeing subsystems in this loop, so we always need to get the next one
465 : * ahead of time, since we can't call get_next() on a subsystem that's been freed.
466 : */
467 0 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt),
468 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem);
469 0 : subsystem != NULL;
470 0 : subsystem = subsystem_next,
471 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) {
472 0 : nvmf_subsystem_remove_all_listeners(subsystem, true);
473 :
474 0 : rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt);
475 0 : if (rc) {
476 0 : if (rc == -EINPROGRESS) {
477 : /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i
478 : * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */
479 0 : return;
480 : } else {
481 0 : SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc);
482 : }
483 : }
484 : }
485 0 : spdk_bit_array_free(&tgt->subsystem_ids);
486 0 : _nvmf_tgt_destroy_next_transport(tgt);
487 : }
488 :
489 : void
490 0 : spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt,
491 : spdk_nvmf_tgt_destroy_done_fn cb_fn,
492 : void *cb_arg)
493 : {
494 0 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
495 :
496 0 : tgt->destroy_cb_fn = cb_fn;
497 0 : tgt->destroy_cb_arg = cb_arg;
498 :
499 0 : TAILQ_REMOVE(&g_nvmf_tgts, tgt, link);
500 :
501 0 : spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb);
502 0 : }
503 :
504 : const char *
505 0 : spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt)
506 : {
507 0 : return tgt->name;
508 : }
509 :
510 : struct spdk_nvmf_tgt *
511 0 : spdk_nvmf_get_tgt(const char *name)
512 : {
513 : struct spdk_nvmf_tgt *tgt;
514 0 : uint32_t num_targets = 0;
515 :
516 0 : TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) {
517 0 : if (name) {
518 0 : if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) {
519 0 : return tgt;
520 : }
521 : }
522 0 : num_targets++;
523 : }
524 :
525 : /*
526 : * special case. If there is only one target and
527 : * no name was specified, return the only available
528 : * target. If there is more than one target, name must
529 : * be specified.
530 : */
531 0 : if (!name && num_targets == 1) {
532 0 : return TAILQ_FIRST(&g_nvmf_tgts);
533 : }
534 :
535 0 : return NULL;
536 : }
537 :
538 : struct spdk_nvmf_tgt *
539 0 : spdk_nvmf_get_first_tgt(void)
540 : {
541 0 : return TAILQ_FIRST(&g_nvmf_tgts);
542 : }
543 :
544 : struct spdk_nvmf_tgt *
545 0 : spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev)
546 : {
547 0 : return TAILQ_NEXT(prev, link);
548 : }
549 :
550 : static void
551 0 : nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w,
552 : struct spdk_nvmf_subsystem *subsystem)
553 : {
554 : struct spdk_nvmf_host *host;
555 : struct spdk_nvmf_ns *ns;
556 0 : struct spdk_nvmf_ns_opts ns_opts;
557 : uint32_t max_namespaces;
558 : struct spdk_nvmf_transport *transport;
559 :
560 0 : assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME);
561 :
562 : /* { */
563 0 : spdk_json_write_object_begin(w);
564 0 : spdk_json_write_named_string(w, "method", "nvmf_create_subsystem");
565 :
566 : /* "params" : { */
567 0 : spdk_json_write_named_object_begin(w, "params");
568 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
569 0 : spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem));
570 0 : spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem));
571 0 : spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem));
572 :
573 0 : max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem);
574 0 : if (max_namespaces != 0) {
575 0 : spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces);
576 : }
577 :
578 0 : spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem));
579 0 : spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem));
580 0 : spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem));
581 :
582 : /* } "params" */
583 0 : spdk_json_write_object_end(w);
584 :
585 : /* } */
586 0 : spdk_json_write_object_end(w);
587 :
588 0 : for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL;
589 0 : host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) {
590 :
591 0 : spdk_json_write_object_begin(w);
592 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host");
593 :
594 : /* "params" : { */
595 0 : spdk_json_write_named_object_begin(w, "params");
596 :
597 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
598 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
599 0 : if (host->dhchap_key != NULL) {
600 0 : spdk_json_write_named_string(w, "dhchap_key",
601 : spdk_key_get_name(host->dhchap_key));
602 : }
603 0 : if (host->dhchap_ctrlr_key != NULL) {
604 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
605 : spdk_key_get_name(host->dhchap_ctrlr_key));
606 : }
607 0 : TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) {
608 0 : if (transport->ops->subsystem_dump_host != NULL) {
609 0 : transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w);
610 : }
611 : }
612 :
613 : /* } "params" */
614 0 : spdk_json_write_object_end(w);
615 :
616 : /* } */
617 0 : spdk_json_write_object_end(w);
618 : }
619 :
620 0 : for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
621 0 : ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
622 0 : spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts));
623 :
624 0 : spdk_json_write_object_begin(w);
625 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns");
626 :
627 : /* "params" : { */
628 0 : spdk_json_write_named_object_begin(w, "params");
629 :
630 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
631 :
632 : /* "namespace" : { */
633 0 : spdk_json_write_named_object_begin(w, "namespace");
634 :
635 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
636 0 : spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns)));
637 :
638 0 : if (ns->ptpl_file != NULL) {
639 0 : spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file);
640 : }
641 :
642 0 : if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) {
643 : SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch");
644 0 : spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]),
645 : from_be64(&ns_opts.nguid[8]));
646 : }
647 :
648 0 : if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) {
649 : SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch");
650 0 : spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64));
651 : }
652 :
653 0 : if (!spdk_uuid_is_null(&ns_opts.uuid)) {
654 0 : spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid);
655 : }
656 :
657 0 : if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) {
658 0 : spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid);
659 : }
660 :
661 0 : spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible);
662 :
663 : /* "namespace" */
664 0 : spdk_json_write_object_end(w);
665 :
666 : /* } "params" */
667 0 : spdk_json_write_object_end(w);
668 :
669 : /* } */
670 0 : spdk_json_write_object_end(w);
671 :
672 0 : TAILQ_FOREACH(host, &ns->hosts, link) {
673 0 : spdk_json_write_object_begin(w);
674 0 : spdk_json_write_named_string(w, "method", "nvmf_ns_add_host");
675 0 : spdk_json_write_named_object_begin(w, "params");
676 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
677 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
678 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
679 0 : spdk_json_write_object_end(w);
680 0 : spdk_json_write_object_end(w);
681 : }
682 : }
683 0 : }
684 :
685 : static void
686 0 : nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w,
687 : struct spdk_nvmf_subsystem *subsystem)
688 : {
689 : struct spdk_nvmf_subsystem_listener *listener;
690 : struct spdk_nvmf_transport *transport;
691 : const struct spdk_nvme_transport_id *trid;
692 :
693 0 : if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) {
694 0 : nvmf_write_nvme_subsystem_config(w, subsystem);
695 : }
696 :
697 0 : for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL;
698 0 : listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) {
699 0 : transport = listener->transport;
700 0 : trid = spdk_nvmf_subsystem_listener_get_trid(listener);
701 :
702 0 : spdk_json_write_object_begin(w);
703 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener");
704 :
705 : /* "params" : { */
706 0 : spdk_json_write_named_object_begin(w, "params");
707 :
708 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
709 :
710 0 : spdk_json_write_named_object_begin(w, "listen_address");
711 0 : nvmf_transport_listen_dump_trid(trid, w);
712 0 : spdk_json_write_object_end(w);
713 0 : if (transport->ops->listen_dump_opts) {
714 0 : transport->ops->listen_dump_opts(transport, trid, w);
715 : }
716 :
717 0 : spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel);
718 :
719 0 : if (listener->opts.sock_impl) {
720 0 : spdk_json_write_named_string(w, "sock_impl", listener->opts.sock_impl);
721 : }
722 :
723 : /* } "params" */
724 0 : spdk_json_write_object_end(w);
725 :
726 : /* } */
727 0 : spdk_json_write_object_end(w);
728 : }
729 0 : }
730 :
731 : void
732 0 : spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt)
733 : {
734 : struct spdk_nvmf_subsystem *subsystem;
735 : struct spdk_nvmf_transport *transport;
736 : struct spdk_nvmf_referral *referral;
737 : struct spdk_nvmf_host *host;
738 :
739 0 : spdk_json_write_object_begin(w);
740 0 : spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems");
741 :
742 0 : spdk_json_write_named_object_begin(w, "params");
743 0 : spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems);
744 0 : spdk_json_write_object_end(w);
745 :
746 0 : spdk_json_write_object_end(w);
747 :
748 0 : spdk_json_write_object_begin(w);
749 0 : spdk_json_write_named_string(w, "method", "nvmf_set_crdt");
750 0 : spdk_json_write_named_object_begin(w, "params");
751 0 : spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]);
752 0 : spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]);
753 0 : spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]);
754 0 : spdk_json_write_object_end(w);
755 0 : spdk_json_write_object_end(w);
756 :
757 : /* write transports */
758 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
759 0 : spdk_json_write_object_begin(w);
760 0 : spdk_json_write_named_string(w, "method", "nvmf_create_transport");
761 0 : nvmf_transport_dump_opts(transport, w, true);
762 0 : spdk_json_write_object_end(w);
763 : }
764 :
765 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
766 0 : spdk_json_write_object_begin(w);
767 0 : spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral");
768 :
769 0 : spdk_json_write_named_object_begin(w, "params");
770 0 : spdk_json_write_named_object_begin(w, "address");
771 0 : nvmf_transport_listen_dump_trid(&referral->trid, w);
772 0 : spdk_json_write_object_end(w);
773 0 : spdk_json_write_named_bool(w, "secure_channel",
774 0 : referral->entry.treq.secure_channel ==
775 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED);
776 0 : spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn);
777 0 : spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_referral_get_allow_any_host(referral));
778 0 : spdk_json_write_object_end(w);
779 :
780 0 : spdk_json_write_object_end(w);
781 : }
782 0 : for (host = spdk_nvmf_referral_get_first_host(referral); host != NULL;
783 0 : host = spdk_nvmf_referral_get_next_host(referral, host)) {
784 :
785 0 : spdk_json_write_object_begin(w);
786 0 : spdk_json_write_named_string(w, "method", "nvmf_discovery_referral_add_host");
787 :
788 : /* "params" : { */
789 0 : spdk_json_write_named_object_begin(w, "params");
790 :
791 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_referral_get_nqn(referral));
792 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
793 0 : if (host->dhchap_key != NULL) {
794 0 : spdk_json_write_named_string(w, "dhchap_key",
795 : spdk_key_get_name(host->dhchap_key));
796 : }
797 0 : if (host->dhchap_ctrlr_key != NULL) {
798 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
799 : spdk_key_get_name(host->dhchap_ctrlr_key));
800 : }
801 0 : TAILQ_FOREACH(transport, &referral->tgt->transports, link) {
802 0 : if (transport->ops->referral_dump_host != NULL) {
803 0 : transport->ops->referral_dump_host(transport, referral, host->nqn, w);
804 : }
805 : }
806 :
807 : /* } "params" */
808 0 : spdk_json_write_object_end(w);
809 :
810 : /* } */
811 0 : spdk_json_write_object_end(w);
812 : }
813 :
814 0 : subsystem = spdk_nvmf_subsystem_get_first(tgt);
815 0 : while (subsystem) {
816 0 : nvmf_write_subsystem_config_json(w, subsystem);
817 0 : subsystem = spdk_nvmf_subsystem_get_next(subsystem);
818 : }
819 0 : }
820 :
821 : static void
822 0 : nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts,
823 : const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size)
824 : {
825 0 : assert(opts);
826 0 : assert(opts_src);
827 :
828 0 : opts->opts_size = opts_size;
829 :
830 : #define SET_FIELD(field) \
831 : if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \
832 : opts->field = opts_src->field; \
833 : } \
834 :
835 0 : SET_FIELD(transport_specific);
836 0 : SET_FIELD(secure_channel);
837 0 : SET_FIELD(ana_state);
838 0 : SET_FIELD(sock_impl);
839 : #undef SET_FIELD
840 :
841 : /* Do not remove this statement, you should always update this statement when you adding a new field,
842 : * and do not forget to add the SET_FIELD statement for your added field. */
843 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 32, "Incorrect size");
844 0 : }
845 :
846 : void
847 0 : spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size)
848 : {
849 0 : struct spdk_nvmf_listen_opts opts_local = {};
850 :
851 : /* local version of opts should have defaults set here */
852 0 : opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
853 0 : nvmf_listen_opts_copy(opts, &opts_local, opts_size);
854 0 : }
855 :
856 : int
857 0 : spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid,
858 : struct spdk_nvmf_listen_opts *opts)
859 : {
860 : struct spdk_nvmf_transport *transport;
861 : int rc;
862 0 : struct spdk_nvmf_listen_opts opts_local = {};
863 :
864 0 : if (!opts) {
865 0 : SPDK_ERRLOG("opts should not be NULL\n");
866 0 : return -EINVAL;
867 : }
868 :
869 0 : if (!opts->opts_size) {
870 0 : SPDK_ERRLOG("The opts_size in opts structure should not be zero\n");
871 0 : return -EINVAL;
872 : }
873 :
874 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
875 0 : if (!transport) {
876 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
877 : trid->trstring);
878 0 : return -EINVAL;
879 : }
880 :
881 0 : nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size);
882 0 : rc = spdk_nvmf_transport_listen(transport, trid, &opts_local);
883 0 : if (rc < 0) {
884 0 : SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr);
885 : }
886 :
887 0 : return rc;
888 : }
889 :
890 : int
891 0 : spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt,
892 : struct spdk_nvme_transport_id *trid)
893 : {
894 : struct spdk_nvmf_transport *transport;
895 : int rc;
896 :
897 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
898 0 : if (!transport) {
899 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
900 : trid->trstring);
901 0 : return -EINVAL;
902 : }
903 :
904 0 : rc = spdk_nvmf_transport_stop_listen(transport, trid);
905 0 : if (rc < 0) {
906 0 : SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr);
907 0 : return rc;
908 : }
909 0 : return 0;
910 : }
911 :
912 : struct spdk_nvmf_tgt_add_transport_ctx {
913 : struct spdk_nvmf_tgt *tgt;
914 : struct spdk_nvmf_transport *transport;
915 : spdk_nvmf_tgt_add_transport_done_fn cb_fn;
916 : void *cb_arg;
917 : int status;
918 : };
919 :
920 : static void
921 0 : _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status)
922 : {
923 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
924 :
925 0 : ctx->cb_fn(ctx->cb_arg, ctx->status);
926 0 : free(ctx);
927 0 : }
928 :
929 : static void
930 0 : _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i)
931 : {
932 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
933 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
934 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
935 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
936 :
937 0 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
938 0 : if (tgroup->transport == ctx->transport) {
939 0 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
940 0 : nvmf_transport_poll_group_destroy(tgroup);
941 : }
942 : }
943 :
944 0 : spdk_for_each_channel_continue(i, 0);
945 0 : }
946 :
947 : static void
948 0 : _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status)
949 : {
950 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
951 :
952 0 : if (status) {
953 0 : ctx->status = status;
954 0 : spdk_for_each_channel(ctx->tgt,
955 : _nvmf_tgt_remove_transport,
956 : ctx,
957 : _nvmf_tgt_remove_transport_done);
958 0 : return;
959 : }
960 :
961 0 : ctx->transport->tgt = ctx->tgt;
962 0 : TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link);
963 0 : ctx->cb_fn(ctx->cb_arg, status);
964 0 : free(ctx);
965 : }
966 :
967 : static void
968 0 : _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i)
969 : {
970 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
971 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
972 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
973 : int rc;
974 :
975 0 : rc = nvmf_poll_group_add_transport(group, ctx->transport);
976 0 : spdk_for_each_channel_continue(i, rc);
977 0 : }
978 :
979 : void
980 0 : spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt,
981 : struct spdk_nvmf_transport *transport,
982 : spdk_nvmf_tgt_add_transport_done_fn cb_fn,
983 : void *cb_arg)
984 : {
985 : struct spdk_nvmf_tgt_add_transport_ctx *ctx;
986 :
987 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name);
988 :
989 0 : if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) {
990 0 : cb_fn(cb_arg, -EEXIST);
991 0 : return; /* transport already created */
992 : }
993 :
994 0 : ctx = calloc(1, sizeof(*ctx));
995 0 : if (!ctx) {
996 0 : cb_fn(cb_arg, -ENOMEM);
997 0 : return;
998 : }
999 :
1000 0 : ctx->tgt = tgt;
1001 0 : ctx->transport = transport;
1002 0 : ctx->cb_fn = cb_fn;
1003 0 : ctx->cb_arg = cb_arg;
1004 :
1005 0 : spdk_for_each_channel(tgt,
1006 : _nvmf_tgt_add_transport,
1007 : ctx,
1008 : _nvmf_tgt_add_transport_done);
1009 : }
1010 :
1011 : struct nvmf_tgt_pause_ctx {
1012 : struct spdk_nvmf_tgt *tgt;
1013 : spdk_nvmf_tgt_pause_polling_cb_fn cb_fn;
1014 : void *cb_arg;
1015 : };
1016 :
1017 : static void
1018 0 : _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status)
1019 : {
1020 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
1021 :
1022 0 : ctx->tgt->state = NVMF_TGT_PAUSED;
1023 :
1024 0 : ctx->cb_fn(ctx->cb_arg, status);
1025 0 : free(ctx);
1026 0 : }
1027 :
1028 : static void
1029 0 : _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i)
1030 : {
1031 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1032 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
1033 :
1034 0 : spdk_poller_unregister(&group->poller);
1035 :
1036 0 : spdk_for_each_channel_continue(i, 0);
1037 0 : }
1038 :
1039 : int
1040 0 : spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn,
1041 : void *cb_arg)
1042 : {
1043 : struct nvmf_tgt_pause_ctx *ctx;
1044 :
1045 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name);
1046 :
1047 0 : switch (tgt->state) {
1048 0 : case NVMF_TGT_PAUSING:
1049 : case NVMF_TGT_RESUMING:
1050 0 : return -EBUSY;
1051 0 : case NVMF_TGT_RUNNING:
1052 0 : break;
1053 0 : default:
1054 0 : return -EINVAL;
1055 : }
1056 :
1057 0 : ctx = calloc(1, sizeof(*ctx));
1058 0 : if (!ctx) {
1059 0 : return -ENOMEM;
1060 : }
1061 :
1062 :
1063 0 : tgt->state = NVMF_TGT_PAUSING;
1064 :
1065 0 : ctx->tgt = tgt;
1066 0 : ctx->cb_fn = cb_fn;
1067 0 : ctx->cb_arg = cb_arg;
1068 :
1069 0 : spdk_for_each_channel(tgt,
1070 : _nvmf_tgt_pause_polling,
1071 : ctx,
1072 : _nvmf_tgt_pause_polling_done);
1073 0 : return 0;
1074 : }
1075 :
1076 : static void
1077 0 : _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status)
1078 : {
1079 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
1080 :
1081 0 : ctx->tgt->state = NVMF_TGT_RUNNING;
1082 :
1083 0 : ctx->cb_fn(ctx->cb_arg, status);
1084 0 : free(ctx);
1085 0 : }
1086 :
1087 : static void
1088 0 : _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i)
1089 : {
1090 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1091 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
1092 :
1093 0 : assert(group->poller == NULL);
1094 0 : group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0);
1095 :
1096 0 : spdk_for_each_channel_continue(i, 0);
1097 0 : }
1098 :
1099 : int
1100 0 : spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn,
1101 : void *cb_arg)
1102 : {
1103 : struct nvmf_tgt_pause_ctx *ctx;
1104 :
1105 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name);
1106 :
1107 0 : switch (tgt->state) {
1108 0 : case NVMF_TGT_PAUSING:
1109 : case NVMF_TGT_RESUMING:
1110 0 : return -EBUSY;
1111 0 : case NVMF_TGT_PAUSED:
1112 0 : break;
1113 0 : default:
1114 0 : return -EINVAL;
1115 : }
1116 :
1117 0 : ctx = calloc(1, sizeof(*ctx));
1118 0 : if (!ctx) {
1119 0 : return -ENOMEM;
1120 : }
1121 :
1122 0 : tgt->state = NVMF_TGT_RESUMING;
1123 :
1124 0 : ctx->tgt = tgt;
1125 0 : ctx->cb_fn = cb_fn;
1126 0 : ctx->cb_arg = cb_arg;
1127 :
1128 0 : spdk_for_each_channel(tgt,
1129 : _nvmf_tgt_resume_polling,
1130 : ctx,
1131 : _nvmf_tgt_resume_polling_done);
1132 0 : return 0;
1133 : }
1134 :
1135 : struct spdk_nvmf_subsystem *
1136 0 : spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn)
1137 : {
1138 0 : struct spdk_nvmf_subsystem subsystem;
1139 :
1140 0 : if (!subnqn) {
1141 0 : return NULL;
1142 : }
1143 :
1144 : /* Ensure that subnqn is null terminated */
1145 0 : if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) {
1146 0 : SPDK_ERRLOG("Connect SUBNQN is not null terminated\n");
1147 0 : return NULL;
1148 : }
1149 :
1150 0 : snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn);
1151 0 : return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem);
1152 : }
1153 :
1154 : struct spdk_nvmf_referral *
1155 0 : spdk_nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt, const char *subnqn)
1156 : {
1157 : struct spdk_nvmf_referral *referral;
1158 :
1159 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
1160 0 : if (!strncmp(referral->trid.subnqn, subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
1161 0 : return referral;
1162 : }
1163 : }
1164 :
1165 0 : return 0;
1166 :
1167 : }
1168 :
1169 : struct spdk_nvmf_transport *
1170 0 : spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name)
1171 : {
1172 : struct spdk_nvmf_transport *transport;
1173 :
1174 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
1175 0 : if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) {
1176 0 : return transport;
1177 : }
1178 : }
1179 0 : return NULL;
1180 : }
1181 :
1182 : struct nvmf_new_qpair_ctx {
1183 : struct spdk_nvmf_qpair *qpair;
1184 : struct spdk_nvmf_poll_group *group;
1185 : };
1186 :
1187 : static void
1188 0 : _nvmf_poll_group_add(void *_ctx)
1189 : {
1190 0 : struct nvmf_new_qpair_ctx *ctx = _ctx;
1191 0 : struct spdk_nvmf_qpair *qpair = ctx->qpair;
1192 0 : struct spdk_nvmf_poll_group *group = ctx->group;
1193 :
1194 0 : free(_ctx);
1195 :
1196 0 : if (spdk_nvmf_poll_group_add(group, qpair) != 0) {
1197 0 : SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
1198 0 : spdk_nvmf_qpair_disconnect(qpair);
1199 : }
1200 0 : }
1201 :
1202 : void
1203 0 : spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)
1204 : {
1205 : struct spdk_nvmf_poll_group *group;
1206 : struct nvmf_new_qpair_ctx *ctx;
1207 :
1208 0 : group = spdk_nvmf_get_optimal_poll_group(qpair);
1209 0 : if (group == NULL) {
1210 0 : if (tgt->next_poll_group == NULL) {
1211 0 : tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups);
1212 0 : if (tgt->next_poll_group == NULL) {
1213 0 : SPDK_ERRLOG("No poll groups exist.\n");
1214 0 : spdk_nvmf_qpair_disconnect(qpair);
1215 0 : return;
1216 : }
1217 : }
1218 0 : group = tgt->next_poll_group;
1219 0 : tgt->next_poll_group = TAILQ_NEXT(group, link);
1220 : }
1221 :
1222 0 : ctx = calloc(1, sizeof(*ctx));
1223 0 : if (!ctx) {
1224 0 : SPDK_ERRLOG("Unable to send message to poll group.\n");
1225 0 : spdk_nvmf_qpair_disconnect(qpair);
1226 0 : return;
1227 : }
1228 :
1229 0 : ctx->qpair = qpair;
1230 0 : ctx->group = group;
1231 :
1232 0 : pthread_mutex_lock(&group->mutex);
1233 0 : group->current_unassociated_qpairs++;
1234 0 : pthread_mutex_unlock(&group->mutex);
1235 :
1236 0 : spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx);
1237 : }
1238 :
1239 : struct spdk_nvmf_poll_group *
1240 0 : spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt)
1241 : {
1242 : struct spdk_io_channel *ch;
1243 :
1244 0 : ch = spdk_get_io_channel(tgt);
1245 0 : if (!ch) {
1246 0 : SPDK_ERRLOG("Unable to get I/O channel for target\n");
1247 0 : return NULL;
1248 : }
1249 :
1250 0 : return spdk_io_channel_get_ctx(ch);
1251 : }
1252 :
1253 : void
1254 0 : spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
1255 : spdk_nvmf_poll_group_destroy_done_fn cb_fn,
1256 : void *cb_arg)
1257 : {
1258 0 : assert(group->destroy_cb_fn == NULL);
1259 0 : group->destroy_cb_fn = cb_fn;
1260 0 : group->destroy_cb_arg = cb_arg;
1261 :
1262 : /* This function will put the io_channel associated with this poll group */
1263 0 : nvmf_tgt_destroy_poll_group_qpairs(group);
1264 0 : }
1265 :
1266 : int
1267 0 : spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
1268 : struct spdk_nvmf_qpair *qpair)
1269 : {
1270 : int rc;
1271 : struct spdk_nvmf_transport_poll_group *tgroup;
1272 :
1273 0 : TAILQ_INIT(&qpair->outstanding);
1274 0 : qpair->group = group;
1275 0 : qpair->ctrlr = NULL;
1276 0 : qpair->disconnect_started = false;
1277 :
1278 0 : tgroup = nvmf_get_transport_poll_group(group, qpair->transport);
1279 0 : if (tgroup == NULL) {
1280 0 : return -1;
1281 : }
1282 :
1283 0 : rc = nvmf_transport_poll_group_add(tgroup, qpair);
1284 :
1285 : /* We add the qpair to the group only it is successfully added into the tgroup */
1286 0 : if (rc == 0) {
1287 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread));
1288 0 : TAILQ_INSERT_TAIL(&group->qpairs, qpair, link);
1289 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_CONNECTING);
1290 : }
1291 :
1292 0 : return rc;
1293 : }
1294 :
1295 : static void
1296 0 : _nvmf_ctrlr_destruct(void *ctx)
1297 : {
1298 0 : struct spdk_nvmf_ctrlr *ctrlr = ctx;
1299 :
1300 0 : nvmf_ctrlr_destruct(ctrlr);
1301 0 : }
1302 :
1303 : static void
1304 0 : _nvmf_ctrlr_free_from_qpair(void *ctx)
1305 : {
1306 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1307 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr;
1308 : uint32_t count;
1309 :
1310 0 : spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid);
1311 0 : SPDK_DEBUGLOG(nvmf, "qpair_mask cleared, qid %u\n", qpair_ctx->qid);
1312 0 : count = spdk_bit_array_count_set(ctrlr->qpair_mask);
1313 0 : if (count == 0) {
1314 0 : assert(!ctrlr->in_destruct);
1315 0 : SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid);
1316 0 : ctrlr->in_destruct = true;
1317 0 : spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr);
1318 : }
1319 0 : free(qpair_ctx);
1320 0 : }
1321 :
1322 : static void
1323 0 : _nvmf_transport_qpair_fini_complete(void *cb_ctx)
1324 : {
1325 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx;
1326 : struct spdk_nvmf_ctrlr *ctrlr;
1327 :
1328 0 : ctrlr = qpair_ctx->ctrlr;
1329 0 : SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid);
1330 :
1331 0 : if (ctrlr) {
1332 0 : if (qpair_ctx->qid == 0) {
1333 : /* Admin qpair is removed, so set the pointer to NULL.
1334 : * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same
1335 : * as controller's thread */
1336 0 : assert(ctrlr->thread == spdk_get_thread());
1337 0 : ctrlr->admin_qpair = NULL;
1338 : }
1339 : /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */
1340 0 : if (ctrlr->thread) {
1341 0 : spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx);
1342 : } else {
1343 0 : _nvmf_ctrlr_free_from_qpair(qpair_ctx);
1344 : }
1345 : } else {
1346 0 : free(qpair_ctx);
1347 : }
1348 0 : }
1349 :
1350 : void
1351 0 : spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair)
1352 : {
1353 : struct spdk_nvmf_transport_poll_group *tgroup;
1354 : int rc;
1355 :
1356 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair,
1357 : spdk_thread_get_id(qpair->group->thread));
1358 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR);
1359 :
1360 : /* Find the tgroup and remove the qpair from the tgroup */
1361 0 : tgroup = nvmf_get_transport_poll_group(qpair->group, qpair->transport);
1362 0 : if (tgroup != NULL) {
1363 0 : rc = nvmf_transport_poll_group_remove(tgroup, qpair);
1364 0 : if (rc && (rc != ENOTSUP)) {
1365 0 : SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n",
1366 : qpair, tgroup);
1367 : }
1368 : }
1369 :
1370 0 : TAILQ_REMOVE(&qpair->group->qpairs, qpair, link);
1371 0 : qpair->group = NULL;
1372 0 : }
1373 :
1374 : static void
1375 0 : _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup,
1376 : const struct spdk_nvmf_qpair *qpair)
1377 : {
1378 : struct spdk_nvmf_request *req, *tmp;
1379 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1380 0 : if (req->qpair == qpair) {
1381 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1382 0 : if (nvmf_transport_req_free(req)) {
1383 0 : SPDK_ERRLOG("Transport request free error!\n");
1384 : }
1385 : }
1386 : }
1387 0 : }
1388 :
1389 : static void
1390 0 : _nvmf_qpair_destroy(void *ctx, int status)
1391 : {
1392 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1393 0 : struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair;
1394 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr;
1395 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1396 : uint32_t sid;
1397 :
1398 0 : assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING);
1399 0 : qpair_ctx->qid = qpair->qid;
1400 :
1401 0 : if (qpair->connect_received) {
1402 0 : if (0 == qpair->qid) {
1403 0 : assert(qpair->group->stat.current_admin_qpairs > 0);
1404 0 : qpair->group->stat.current_admin_qpairs--;
1405 : } else {
1406 0 : assert(qpair->group->stat.current_io_qpairs > 0);
1407 0 : qpair->group->stat.current_io_qpairs--;
1408 : }
1409 : } else {
1410 0 : pthread_mutex_lock(&qpair->group->mutex);
1411 0 : qpair->group->current_unassociated_qpairs--;
1412 0 : pthread_mutex_unlock(&qpair->group->mutex);
1413 : }
1414 :
1415 0 : if (ctrlr) {
1416 0 : sgroup = &qpair->group->sgroups[ctrlr->subsys->id];
1417 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1418 : } else {
1419 0 : for (sid = 0; sid < qpair->group->num_sgroups; sid++) {
1420 0 : sgroup = &qpair->group->sgroups[sid];
1421 0 : assert(sgroup != NULL);
1422 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1423 : }
1424 : }
1425 :
1426 0 : nvmf_qpair_auth_destroy(qpair);
1427 0 : qpair_ctx->ctrlr = ctrlr;
1428 0 : spdk_nvmf_poll_group_remove(qpair);
1429 0 : nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx);
1430 0 : }
1431 :
1432 : static void
1433 0 : _nvmf_qpair_disconnect_msg(void *ctx)
1434 : {
1435 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1436 :
1437 0 : spdk_nvmf_qpair_disconnect(qpair_ctx->qpair);
1438 0 : free(ctx);
1439 0 : }
1440 :
1441 : int
1442 0 : spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair)
1443 : {
1444 0 : struct spdk_nvmf_poll_group *group = qpair->group;
1445 : struct nvmf_qpair_disconnect_ctx *qpair_ctx;
1446 :
1447 0 : if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) {
1448 0 : return -EINPROGRESS;
1449 : }
1450 :
1451 : /* If we get a qpair in the uninitialized state, we can just destroy it immediately */
1452 0 : if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {
1453 0 : nvmf_transport_qpair_fini(qpair, NULL, NULL);
1454 0 : return 0;
1455 : }
1456 :
1457 0 : assert(group != NULL);
1458 0 : if (spdk_get_thread() != group->thread) {
1459 : /* clear the atomic so we can set it on the next call on the proper thread. */
1460 0 : __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED);
1461 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1462 0 : if (!qpair_ctx) {
1463 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1464 0 : return -ENOMEM;
1465 : }
1466 0 : qpair_ctx->qpair = qpair;
1467 0 : spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx);
1468 0 : return 0;
1469 : }
1470 :
1471 : SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread));
1472 0 : assert(spdk_nvmf_qpair_is_active(qpair));
1473 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING);
1474 :
1475 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1476 0 : if (!qpair_ctx) {
1477 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1478 0 : return -ENOMEM;
1479 : }
1480 :
1481 0 : qpair_ctx->qpair = qpair;
1482 :
1483 : /* Check for outstanding I/O */
1484 0 : if (!TAILQ_EMPTY(&qpair->outstanding)) {
1485 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread));
1486 0 : qpair->state_cb = _nvmf_qpair_destroy;
1487 0 : qpair->state_cb_arg = qpair_ctx;
1488 0 : nvmf_qpair_abort_pending_zcopy_reqs(qpair);
1489 0 : nvmf_qpair_free_aer(qpair);
1490 0 : return 0;
1491 : }
1492 :
1493 0 : _nvmf_qpair_destroy(qpair_ctx, 0);
1494 :
1495 0 : return 0;
1496 : }
1497 :
1498 : int
1499 0 : spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair,
1500 : struct spdk_nvme_transport_id *trid)
1501 : {
1502 0 : memset(trid, 0, sizeof(*trid));
1503 0 : return nvmf_transport_qpair_get_peer_trid(qpair, trid);
1504 : }
1505 :
1506 : int
1507 0 : spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair,
1508 : struct spdk_nvme_transport_id *trid)
1509 : {
1510 0 : memset(trid, 0, sizeof(*trid));
1511 0 : return nvmf_transport_qpair_get_local_trid(qpair, trid);
1512 : }
1513 :
1514 : int
1515 0 : spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair,
1516 : struct spdk_nvme_transport_id *trid)
1517 : {
1518 0 : memset(trid, 0, sizeof(*trid));
1519 0 : return nvmf_transport_qpair_get_listen_trid(qpair, trid);
1520 : }
1521 :
1522 : static int
1523 1 : poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1524 : struct spdk_nvmf_subsystem *subsystem)
1525 : {
1526 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1527 : uint32_t i, j;
1528 : struct spdk_nvmf_ns *ns;
1529 : struct spdk_nvmf_registrant *reg, *tmp;
1530 : struct spdk_io_channel *ch;
1531 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
1532 : struct spdk_nvmf_ctrlr *ctrlr;
1533 : bool ns_changed;
1534 :
1535 : /* Make sure our poll group has memory for this subsystem allocated */
1536 1 : if (subsystem->id >= group->num_sgroups) {
1537 0 : return -ENOMEM;
1538 : }
1539 :
1540 1 : sgroup = &group->sgroups[subsystem->id];
1541 :
1542 : /* Make sure the array of namespace information is the correct size */
1543 1 : if (sgroup->num_ns == 0 && subsystem->max_nsid > 0) {
1544 : /* First allocation */
1545 1 : sgroup->ns_info = calloc(subsystem->max_nsid, sizeof(struct spdk_nvmf_subsystem_pg_ns_info));
1546 1 : if (!sgroup->ns_info) {
1547 0 : return -ENOMEM;
1548 : }
1549 1 : sgroup->num_ns = subsystem->max_nsid;
1550 : }
1551 :
1552 1 : ns_changed = false;
1553 :
1554 : /* Detect bdevs that were added or removed */
1555 2 : for (i = 0; i < sgroup->num_ns; i++) {
1556 1 : ns = subsystem->ns[i];
1557 1 : ns_info = &sgroup->ns_info[i];
1558 1 : ch = ns_info->channel;
1559 :
1560 1 : if (ns == NULL && ch == NULL) {
1561 : /* Both NULL. Leave empty */
1562 1 : } else if (ns == NULL && ch != NULL) {
1563 : /* There was a channel here, but the namespace is gone. */
1564 0 : ns_changed = true;
1565 0 : spdk_put_io_channel(ch);
1566 0 : ns_info->channel = NULL;
1567 1 : } else if (ns != NULL && ch == NULL) {
1568 : /* A namespace appeared but there is no channel yet */
1569 1 : ns_changed = true;
1570 1 : ch = spdk_bdev_get_io_channel(ns->desc);
1571 1 : if (ch == NULL) {
1572 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1573 0 : return -ENOMEM;
1574 : }
1575 1 : ns_info->channel = ch;
1576 0 : } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) {
1577 : /* A namespace was here before, but was replaced by a new one. */
1578 0 : ns_changed = true;
1579 0 : spdk_put_io_channel(ns_info->channel);
1580 0 : memset(ns_info, 0, sizeof(*ns_info));
1581 :
1582 0 : ch = spdk_bdev_get_io_channel(ns->desc);
1583 0 : if (ch == NULL) {
1584 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1585 0 : return -ENOMEM;
1586 : }
1587 0 : ns_info->channel = ch;
1588 0 : } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) {
1589 : /* Namespace is still there but size has changed */
1590 0 : SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u,"
1591 : " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n",
1592 : subsystem->id,
1593 : ns->nsid,
1594 : group,
1595 : ns_info->num_blocks,
1596 : spdk_bdev_get_num_blocks(ns->bdev));
1597 0 : ns_changed = true;
1598 0 : } else if (ns_info->anagrpid != ns->anagrpid) {
1599 : /* Namespace is still there but ANA group ID has changed */
1600 0 : SPDK_DEBUGLOG(nvmf, "ANA group ID changed: subsystem_id %u,"
1601 : "nsid %u, pg %p, old %u, new %u\n",
1602 : subsystem->id,
1603 : ns->nsid,
1604 : group,
1605 : ns_info->anagrpid,
1606 : ns->anagrpid);
1607 0 : ns_changed = true;
1608 : }
1609 :
1610 1 : if (ns == NULL) {
1611 0 : memset(ns_info, 0, sizeof(*ns_info));
1612 : } else {
1613 1 : ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev);
1614 1 : ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev);
1615 1 : ns_info->anagrpid = ns->anagrpid;
1616 1 : ns_info->crkey = ns->crkey;
1617 1 : ns_info->rtype = ns->rtype;
1618 1 : if (ns->holder) {
1619 0 : ns_info->holder_id = ns->holder->hostid;
1620 : }
1621 :
1622 1 : memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid));
1623 1 : j = 0;
1624 1 : TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {
1625 0 : if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) {
1626 0 : SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS);
1627 0 : return -EINVAL;
1628 : }
1629 0 : ns_info->reg_hostid[j++] = reg->hostid;
1630 : }
1631 : }
1632 : }
1633 :
1634 1 : if (ns_changed) {
1635 1 : TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {
1636 0 : if (ctrlr->thread != spdk_get_thread()) {
1637 0 : continue;
1638 : }
1639 : /* It is possible that a ctrlr was added but the admin_qpair hasn't been
1640 : * assigned yet.
1641 : */
1642 0 : if (!ctrlr->admin_qpair) {
1643 0 : continue;
1644 : }
1645 0 : if (ctrlr->admin_qpair->group == group) {
1646 0 : nvmf_ctrlr_async_event_ns_notice(ctrlr);
1647 0 : nvmf_ctrlr_async_event_ana_change_notice(ctrlr);
1648 : }
1649 : }
1650 : }
1651 :
1652 1 : return 0;
1653 : }
1654 :
1655 : int
1656 0 : nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1657 : struct spdk_nvmf_subsystem *subsystem)
1658 : {
1659 0 : return poll_group_update_subsystem(group, subsystem);
1660 : }
1661 :
1662 : int
1663 1 : nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
1664 : struct spdk_nvmf_subsystem *subsystem,
1665 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1666 : {
1667 1 : int rc = 0;
1668 1 : struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id];
1669 : struct spdk_nvmf_request *req, *tmp;
1670 : uint32_t i;
1671 :
1672 1 : if (!TAILQ_EMPTY(&sgroup->queued)) {
1673 0 : SPDK_ERRLOG("sgroup->queued not empty when adding subsystem\n");
1674 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1675 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1676 0 : if (nvmf_transport_req_free(req)) {
1677 0 : SPDK_ERRLOG("Transport request free error!\n");
1678 : }
1679 : }
1680 : }
1681 :
1682 1 : rc = poll_group_update_subsystem(group, subsystem);
1683 1 : if (rc) {
1684 0 : nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL);
1685 0 : goto fini;
1686 : }
1687 :
1688 1 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1689 :
1690 2 : for (i = 0; i < sgroup->num_ns; i++) {
1691 1 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1692 : }
1693 :
1694 1 : fini:
1695 1 : if (cb_fn) {
1696 0 : cb_fn(cb_arg, rc);
1697 : }
1698 :
1699 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread),
1700 : subsystem->subnqn);
1701 :
1702 1 : return rc;
1703 : }
1704 :
1705 : static void
1706 0 : _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status)
1707 : {
1708 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1709 : struct spdk_nvmf_subsystem *subsystem;
1710 : struct spdk_nvmf_poll_group *group;
1711 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1712 0 : spdk_nvmf_poll_group_mod_done cpl_fn = NULL;
1713 0 : void *cpl_ctx = NULL;
1714 : uint32_t nsid;
1715 :
1716 0 : group = qpair_ctx->group;
1717 0 : subsystem = qpair_ctx->subsystem;
1718 0 : cpl_fn = qpair_ctx->cpl_fn;
1719 0 : cpl_ctx = qpair_ctx->cpl_ctx;
1720 0 : sgroup = &group->sgroups[subsystem->id];
1721 :
1722 0 : if (status) {
1723 0 : goto fini;
1724 : }
1725 :
1726 0 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
1727 0 : if (sgroup->ns_info[nsid].channel) {
1728 0 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
1729 0 : sgroup->ns_info[nsid].channel = NULL;
1730 : }
1731 : }
1732 :
1733 0 : sgroup->num_ns = 0;
1734 0 : free(sgroup->ns_info);
1735 0 : sgroup->ns_info = NULL;
1736 0 : fini:
1737 0 : free(qpair_ctx);
1738 0 : if (cpl_fn) {
1739 0 : cpl_fn(cpl_ctx, status);
1740 : }
1741 0 : }
1742 :
1743 : static void nvmf_poll_group_remove_subsystem_msg(void *ctx);
1744 :
1745 : static void
1746 0 : nvmf_poll_group_remove_subsystem_msg(void *ctx)
1747 : {
1748 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
1749 : struct spdk_nvmf_subsystem *subsystem;
1750 : struct spdk_nvmf_poll_group *group;
1751 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1752 0 : bool qpairs_found = false;
1753 0 : int rc = 0;
1754 :
1755 0 : group = qpair_ctx->group;
1756 0 : subsystem = qpair_ctx->subsystem;
1757 :
1758 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
1759 0 : if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) {
1760 0 : qpairs_found = true;
1761 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
1762 0 : if (rc && rc != -EINPROGRESS) {
1763 0 : break;
1764 : }
1765 : }
1766 : }
1767 :
1768 0 : if (!qpairs_found) {
1769 0 : _nvmf_poll_group_remove_subsystem_cb(ctx, 0);
1770 0 : return;
1771 : }
1772 :
1773 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
1774 0 : spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx);
1775 : }
1776 :
1777 : void
1778 0 : nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
1779 : struct spdk_nvmf_subsystem *subsystem,
1780 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1781 : {
1782 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1783 : struct nvmf_qpair_disconnect_many_ctx *ctx;
1784 : uint32_t i;
1785 :
1786 : SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread),
1787 : subsystem->subnqn);
1788 :
1789 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
1790 0 : if (!ctx) {
1791 0 : SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n");
1792 0 : if (cb_fn) {
1793 0 : cb_fn(cb_arg, -1);
1794 : }
1795 0 : return;
1796 : }
1797 :
1798 0 : ctx->group = group;
1799 0 : ctx->subsystem = subsystem;
1800 0 : ctx->cpl_fn = cb_fn;
1801 0 : ctx->cpl_ctx = cb_arg;
1802 :
1803 0 : sgroup = &group->sgroups[subsystem->id];
1804 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1805 :
1806 0 : for (i = 0; i < sgroup->num_ns; i++) {
1807 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1808 : }
1809 :
1810 0 : nvmf_poll_group_remove_subsystem_msg(ctx);
1811 : }
1812 :
1813 : void
1814 0 : nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group,
1815 : struct spdk_nvmf_subsystem *subsystem,
1816 : uint32_t nsid,
1817 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1818 : {
1819 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1820 0 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL;
1821 0 : int rc = 0;
1822 : uint32_t i;
1823 :
1824 0 : if (subsystem->id >= group->num_sgroups) {
1825 0 : rc = -1;
1826 0 : goto fini;
1827 : }
1828 :
1829 0 : sgroup = &group->sgroups[subsystem->id];
1830 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) {
1831 0 : goto fini;
1832 : }
1833 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1834 :
1835 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1836 0 : for (i = 0; i < sgroup->num_ns; i++) {
1837 0 : ns_info = &sgroup->ns_info[i];
1838 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1839 : }
1840 : } else {
1841 : /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */
1842 0 : if (nsid - 1 < sgroup->num_ns) {
1843 0 : ns_info = &sgroup->ns_info[nsid - 1];
1844 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1845 : }
1846 : }
1847 :
1848 0 : if (sgroup->mgmt_io_outstanding > 0) {
1849 0 : assert(sgroup->cb_fn == NULL);
1850 0 : sgroup->cb_fn = cb_fn;
1851 0 : assert(sgroup->cb_arg == NULL);
1852 0 : sgroup->cb_arg = cb_arg;
1853 0 : return;
1854 : }
1855 :
1856 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1857 0 : for (i = 0; i < sgroup->num_ns; i++) {
1858 0 : ns_info = &sgroup->ns_info[i];
1859 :
1860 0 : if (ns_info->io_outstanding > 0) {
1861 0 : assert(sgroup->cb_fn == NULL);
1862 0 : sgroup->cb_fn = cb_fn;
1863 0 : assert(sgroup->cb_arg == NULL);
1864 0 : sgroup->cb_arg = cb_arg;
1865 0 : return;
1866 : }
1867 : }
1868 : } else {
1869 0 : if (ns_info != NULL && ns_info->io_outstanding > 0) {
1870 0 : assert(sgroup->cb_fn == NULL);
1871 0 : sgroup->cb_fn = cb_fn;
1872 0 : assert(sgroup->cb_arg == NULL);
1873 0 : sgroup->cb_arg = cb_arg;
1874 0 : return;
1875 : }
1876 : }
1877 :
1878 0 : assert(sgroup->mgmt_io_outstanding == 0);
1879 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED;
1880 0 : fini:
1881 0 : if (cb_fn) {
1882 0 : cb_fn(cb_arg, rc);
1883 : }
1884 : }
1885 :
1886 : void
1887 0 : nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group,
1888 : struct spdk_nvmf_subsystem *subsystem,
1889 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1890 : {
1891 : struct spdk_nvmf_request *req, *tmp;
1892 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1893 0 : int rc = 0;
1894 : uint32_t i;
1895 :
1896 0 : if (subsystem->id >= group->num_sgroups) {
1897 0 : rc = -1;
1898 0 : goto fini;
1899 : }
1900 :
1901 0 : sgroup = &group->sgroups[subsystem->id];
1902 :
1903 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) {
1904 0 : goto fini;
1905 : }
1906 :
1907 0 : rc = poll_group_update_subsystem(group, subsystem);
1908 0 : if (rc) {
1909 0 : goto fini;
1910 : }
1911 :
1912 0 : for (i = 0; i < sgroup->num_ns; i++) {
1913 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1914 : }
1915 :
1916 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1917 :
1918 : /* Release all queued requests */
1919 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1920 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1921 0 : if (spdk_nvmf_request_using_zcopy(req)) {
1922 0 : spdk_nvmf_request_zcopy_start(req);
1923 : } else {
1924 0 : spdk_nvmf_request_exec(req);
1925 : }
1926 :
1927 : }
1928 0 : fini:
1929 0 : if (cb_fn) {
1930 0 : cb_fn(cb_arg, rc);
1931 : }
1932 0 : }
1933 :
1934 :
1935 : struct spdk_nvmf_poll_group *
1936 0 : spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair)
1937 : {
1938 : struct spdk_nvmf_transport_poll_group *tgroup;
1939 :
1940 0 : tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair);
1941 :
1942 0 : if (tgroup == NULL) {
1943 0 : return NULL;
1944 : }
1945 :
1946 0 : return tgroup->group;
1947 : }
1948 :
1949 : void
1950 0 : spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w)
1951 : {
1952 : struct spdk_nvmf_transport_poll_group *tgroup;
1953 :
1954 0 : spdk_json_write_object_begin(w);
1955 :
1956 0 : spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread()));
1957 0 : spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs);
1958 0 : spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs);
1959 0 : spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs);
1960 0 : spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs);
1961 0 : spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io);
1962 0 : spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io);
1963 :
1964 0 : spdk_json_write_named_array_begin(w, "transports");
1965 :
1966 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
1967 0 : spdk_json_write_object_begin(w);
1968 : /*
1969 : * The trtype field intentionally contains a transport name as this is more informative.
1970 : * The field has not been renamed for backward compatibility.
1971 : */
1972 0 : spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport));
1973 :
1974 0 : if (tgroup->transport->ops->poll_group_dump_stat) {
1975 0 : tgroup->transport->ops->poll_group_dump_stat(tgroup, w);
1976 : }
1977 :
1978 0 : spdk_json_write_object_end(w);
1979 : }
1980 :
1981 0 : spdk_json_write_array_end(w);
1982 0 : spdk_json_write_object_end(w);
1983 0 : }
|
