Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2021 Mellanox Technologies LTD.
4 : * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
5 : * All rights reserved.
6 : */
7 :
8 : #include "nvme_internal.h"
9 :
10 : struct spdk_nvme_poll_group *
11 9 : spdk_nvme_poll_group_create(void *ctx, struct spdk_nvme_accel_fn_table *table)
12 : {
13 : struct spdk_nvme_poll_group *group;
14 : int rc;
15 :
16 9 : group = calloc(1, sizeof(*group));
17 9 : if (group == NULL) {
18 1 : return NULL;
19 : }
20 :
21 8 : group->accel_fn_table.table_size = sizeof(struct spdk_nvme_accel_fn_table);
22 8 : if (table && table->table_size != 0) {
23 0 : group->accel_fn_table.table_size = table->table_size;
24 : #define SET_FIELD(field) \
25 : if (offsetof(struct spdk_nvme_accel_fn_table, field) + sizeof(table->field) <= table->table_size) { \
26 : group->accel_fn_table.field = table->field; \
27 : } \
28 :
29 0 : SET_FIELD(append_crc32c);
30 0 : SET_FIELD(append_copy);
31 0 : SET_FIELD(finish_sequence);
32 0 : SET_FIELD(reverse_sequence);
33 0 : SET_FIELD(abort_sequence);
34 : /* Do not remove this statement, you should always update this statement when you adding a new field,
35 : * and do not forget to add the SET_FIELD statement for your added field. */
36 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_accel_fn_table) == 56, "Incorrect size");
37 :
38 : #undef SET_FIELD
39 : }
40 :
41 : /* Make sure either all or none of the sequence manipulation callbacks are implemented */
42 8 : if ((group->accel_fn_table.finish_sequence && group->accel_fn_table.reverse_sequence &&
43 8 : group->accel_fn_table.abort_sequence) !=
44 16 : (group->accel_fn_table.finish_sequence || group->accel_fn_table.reverse_sequence ||
45 8 : group->accel_fn_table.abort_sequence)) {
46 0 : SPDK_ERRLOG("Invalid accel_fn_table configuration: either all or none of the "
47 : "sequence callbacks must be provided\n");
48 0 : free(group);
49 0 : return NULL;
50 : }
51 :
52 : /* Make sure that sequence callbacks are implemented if append* callbacks are provided */
53 8 : if ((group->accel_fn_table.append_crc32c || group->accel_fn_table.append_copy) &&
54 0 : !group->accel_fn_table.finish_sequence) {
55 0 : SPDK_ERRLOG("Invalid accel_fn_table configuration: append_crc32c and/or append_copy require sequence "
56 : "callbacks to be provided\n");
57 0 : free(group);
58 0 : return NULL;
59 : }
60 :
61 : /* If interrupt is enabled, this fd_group will be used to manage events triggerd on file
62 : * descriptors of all the qpairs in this poll group */
63 8 : rc = spdk_fd_group_create(&group->fgrp);
64 8 : if (rc) {
65 : /* Ignore this for non-Linux platforms, as fd_groups aren't supported there. */
66 : #if defined(__linux__)
67 0 : SPDK_ERRLOG("Cannot create fd group for the nvme poll group\n");
68 0 : free(group);
69 0 : return NULL;
70 : #endif
71 : }
72 :
73 8 : group->disconnect_qpair_fd = -1;
74 8 : group->ctx = ctx;
75 8 : STAILQ_INIT(&group->tgroups);
76 :
77 8 : return group;
78 : }
79 :
80 : int
81 0 : spdk_nvme_poll_group_get_fd(struct spdk_nvme_poll_group *group)
82 : {
83 0 : if (!group->fgrp) {
84 0 : SPDK_ERRLOG("No fd group present for the nvme poll group.\n");
85 0 : assert(false);
86 : return -EINVAL;
87 : }
88 :
89 0 : return spdk_fd_group_get_fd(group->fgrp);
90 : }
91 :
92 : struct spdk_nvme_poll_group *
93 0 : spdk_nvme_qpair_get_optimal_poll_group(struct spdk_nvme_qpair *qpair)
94 : {
95 : struct spdk_nvme_transport_poll_group *tgroup;
96 :
97 0 : tgroup = nvme_transport_qpair_get_optimal_poll_group(qpair->transport, qpair);
98 :
99 0 : if (tgroup == NULL) {
100 0 : return NULL;
101 : }
102 :
103 0 : return tgroup->group;
104 : }
105 :
106 : #ifdef __linux__
107 : static int
108 0 : nvme_poll_group_read_disconnect_qpair_fd(void *arg)
109 : {
110 0 : return 0;
111 : }
112 :
113 : void
114 0 : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
115 : {
116 0 : uint64_t notify = 1;
117 : int rc;
118 :
119 0 : if (!group->enable_interrupts) {
120 0 : return;
121 : }
122 :
123 : /* Write to the disconnect qpair fd. This will generate event on the epoll fd of poll
124 : * group. We then check for disconnected qpairs in spdk_nvme_poll_group_wait() */
125 0 : rc = write(group->disconnect_qpair_fd, ¬ify, sizeof(notify));
126 0 : if (rc < 0) {
127 0 : SPDK_ERRLOG("failed to write the disconnect qpair fd: %s.\n", strerror(errno));
128 : }
129 : }
130 :
131 : static int
132 0 : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
133 : {
134 0 : struct spdk_event_handler_opts opts = {};
135 : int fd;
136 :
137 0 : fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
138 0 : if (fd < 0) {
139 0 : return fd;
140 : }
141 :
142 0 : assert(group->disconnect_qpair_fd == -1);
143 0 : group->disconnect_qpair_fd = fd;
144 :
145 0 : spdk_fd_group_get_default_event_handler_opts(&opts, sizeof(opts));
146 0 : opts.fd_type = SPDK_FD_TYPE_EVENTFD;
147 :
148 0 : return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_poll_group_read_disconnect_qpair_fd,
149 : group, &opts);
150 : }
151 :
152 : #else
153 :
154 : void
155 : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
156 : {
157 : }
158 :
159 : static int
160 : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
161 : {
162 : return -ENOTSUP;
163 : }
164 :
165 : #endif
166 :
167 : int
168 13 : spdk_nvme_poll_group_add(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
169 : {
170 : struct spdk_nvme_transport_poll_group *tgroup;
171 : const struct spdk_nvme_transport *transport;
172 : int rc;
173 :
174 13 : if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
175 1 : return -EINVAL;
176 : }
177 :
178 12 : if (!group->enable_interrupts_is_valid) {
179 4 : group->enable_interrupts_is_valid = true;
180 4 : group->enable_interrupts = qpair->ctrlr->opts.enable_interrupts;
181 4 : if (group->enable_interrupts) {
182 0 : rc = nvme_poll_group_add_disconnect_qpair_fd(group);
183 0 : if (rc != 0) {
184 0 : return rc;
185 : }
186 : }
187 8 : } else if (qpair->ctrlr->opts.enable_interrupts != group->enable_interrupts) {
188 1 : SPDK_ERRLOG("Queue pair %s interrupts cannot be added to poll group\n",
189 : qpair->ctrlr->opts.enable_interrupts ? "without" : "with");
190 1 : return -EINVAL;
191 : }
192 :
193 21 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
194 13 : if (tgroup->transport == qpair->transport) {
195 3 : break;
196 : }
197 : }
198 :
199 : /* See if a new transport has been added (dlopen style) and we need to update the poll group */
200 11 : if (!tgroup) {
201 8 : transport = nvme_get_first_transport();
202 18 : while (transport != NULL) {
203 16 : if (transport == qpair->transport) {
204 6 : tgroup = nvme_transport_poll_group_create(transport);
205 6 : if (tgroup == NULL) {
206 0 : return -ENOMEM;
207 : }
208 6 : tgroup->group = group;
209 6 : STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
210 6 : break;
211 : }
212 10 : transport = nvme_get_next_transport(transport);
213 : }
214 : }
215 :
216 11 : return tgroup ? nvme_transport_poll_group_add(tgroup, qpair) : -ENODEV;
217 : }
218 :
219 : int
220 9 : spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
221 : {
222 : struct spdk_nvme_transport_poll_group *tgroup;
223 :
224 17 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
225 16 : if (tgroup->transport == qpair->transport) {
226 8 : return nvme_transport_poll_group_remove(tgroup, qpair);
227 : }
228 : }
229 :
230 1 : return -ENODEV;
231 : }
232 :
233 : static int
234 0 : nvme_qpair_process_completion_wrapper(void *arg)
235 : {
236 0 : struct spdk_nvme_qpair *qpair = arg;
237 :
238 0 : return spdk_nvme_qpair_process_completions(qpair, 0);
239 : }
240 :
241 : static int
242 1 : nvme_poll_group_add_qpair_fd(struct spdk_nvme_qpair *qpair)
243 : {
244 : struct spdk_nvme_poll_group *group;
245 1 : struct spdk_event_handler_opts opts = {
246 : .opts_size = SPDK_SIZEOF(&opts, fd_type),
247 : };
248 : int fd;
249 :
250 1 : group = qpair->poll_group->group;
251 1 : if (group->enable_interrupts == false) {
252 1 : return 0;
253 : }
254 :
255 0 : fd = spdk_nvme_qpair_get_fd(qpair, &opts);
256 0 : if (fd < 0) {
257 0 : SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
258 0 : return -EINVAL;
259 : }
260 :
261 0 : return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_qpair_process_completion_wrapper,
262 : qpair, &opts);
263 : }
264 :
265 : static void
266 0 : nvme_poll_group_remove_qpair_fd(struct spdk_nvme_qpair *qpair)
267 : {
268 : struct spdk_nvme_poll_group *group;
269 : int fd;
270 :
271 0 : group = qpair->poll_group->group;
272 0 : if (group->enable_interrupts == false) {
273 0 : return;
274 : }
275 :
276 0 : fd = spdk_nvme_qpair_get_fd(qpair, NULL);
277 0 : if (fd < 0) {
278 0 : SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
279 0 : assert(false);
280 : return;
281 : }
282 :
283 0 : spdk_fd_group_remove(group->fgrp, fd);
284 : }
285 :
286 : int
287 1 : nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
288 : {
289 : int rc;
290 :
291 1 : rc = nvme_transport_poll_group_connect_qpair(qpair);
292 1 : if (rc != 0) {
293 0 : return rc;
294 : }
295 :
296 1 : rc = nvme_poll_group_add_qpair_fd(qpair);
297 1 : if (rc != 0) {
298 0 : nvme_transport_poll_group_disconnect_qpair(qpair);
299 0 : return rc;
300 : }
301 :
302 1 : return 0;
303 : }
304 :
305 : int
306 0 : nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
307 : {
308 0 : nvme_poll_group_remove_qpair_fd(qpair);
309 :
310 0 : return nvme_transport_poll_group_disconnect_qpair(qpair);
311 : }
312 :
313 : int
314 0 : spdk_nvme_poll_group_wait(struct spdk_nvme_poll_group *group,
315 : spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
316 : {
317 : struct spdk_nvme_transport_poll_group *tgroup;
318 0 : int num_events, timeout = -1;
319 :
320 0 : if (disconnected_qpair_cb == NULL) {
321 0 : return -EINVAL;
322 : }
323 :
324 0 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
325 0 : nvme_transport_poll_group_check_disconnected_qpairs(tgroup, disconnected_qpair_cb);
326 : }
327 :
328 0 : num_events = spdk_fd_group_wait(group->fgrp, timeout);
329 :
330 0 : return num_events;
331 : }
332 :
333 : int64_t
334 2 : spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,
335 : uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
336 : {
337 : struct spdk_nvme_transport_poll_group *tgroup;
338 2 : int64_t local_completions = 0, error_reason = 0, num_completions = 0;
339 :
340 2 : if (disconnected_qpair_cb == NULL) {
341 0 : return -EINVAL;
342 : }
343 :
344 2 : if (spdk_unlikely(group->in_process_completions)) {
345 0 : return 0;
346 : }
347 2 : group->in_process_completions = true;
348 :
349 3 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
350 1 : local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair,
351 : disconnected_qpair_cb);
352 1 : if (local_completions < 0 && error_reason == 0) {
353 0 : error_reason = local_completions;
354 : } else {
355 1 : num_completions += local_completions;
356 : /* Just to be safe */
357 1 : assert(num_completions >= 0);
358 : }
359 : }
360 2 : group->in_process_completions = false;
361 :
362 2 : return error_reason ? error_reason : num_completions;
363 : }
364 :
365 : int
366 0 : spdk_nvme_poll_group_all_connected(struct spdk_nvme_poll_group *group)
367 : {
368 : struct spdk_nvme_transport_poll_group *tgroup;
369 : struct spdk_nvme_qpair *qpair;
370 0 : int rc = 0;
371 :
372 0 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
373 0 : if (!STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
374 : /* Treat disconnected qpairs as highest priority for notification.
375 : * This means we can just return immediately here.
376 : */
377 0 : return -EIO;
378 : }
379 0 : STAILQ_FOREACH(qpair, &tgroup->connected_qpairs, poll_group_stailq) {
380 0 : if (nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTING) {
381 0 : return -EIO;
382 0 : } else if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
383 0 : rc = -EAGAIN;
384 : /* Break so that we can check the remaining transport groups,
385 : * in case any of them have a disconnected qpair.
386 : */
387 0 : break;
388 : }
389 : }
390 : }
391 :
392 0 : return rc;
393 : }
394 :
395 : void *
396 0 : spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group)
397 : {
398 0 : return group->ctx;
399 : }
400 :
401 : int
402 9 : spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group)
403 : {
404 : struct spdk_nvme_transport_poll_group *tgroup, *tmp_tgroup;
405 9 : struct spdk_fd_group *fgrp = group->fgrp;
406 :
407 10 : STAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp_tgroup) {
408 2 : STAILQ_REMOVE(&group->tgroups, tgroup, spdk_nvme_transport_poll_group, link);
409 2 : if (nvme_transport_poll_group_destroy(tgroup) != 0) {
410 1 : STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
411 1 : return -EBUSY;
412 : }
413 :
414 : }
415 :
416 8 : if (fgrp) {
417 8 : if (group->enable_interrupts) {
418 0 : spdk_fd_group_remove(fgrp, group->disconnect_qpair_fd);
419 0 : close(group->disconnect_qpair_fd);
420 : }
421 8 : spdk_fd_group_destroy(fgrp);
422 : }
423 :
424 8 : free(group);
425 :
426 8 : return 0;
427 : }
428 :
429 : int
430 2 : spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group,
431 : struct spdk_nvme_poll_group_stat **stats)
432 : {
433 : struct spdk_nvme_transport_poll_group *tgroup;
434 : struct spdk_nvme_poll_group_stat *result;
435 2 : uint32_t transports_count = 0;
436 : /* Not all transports used by this poll group may support statistics reporting */
437 2 : uint32_t reported_stats_count = 0;
438 : int rc;
439 :
440 2 : assert(group);
441 2 : assert(stats);
442 :
443 2 : result = calloc(1, sizeof(*result));
444 2 : if (!result) {
445 0 : SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
446 0 : return -ENOMEM;
447 : }
448 :
449 5 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
450 3 : transports_count++;
451 : }
452 :
453 2 : result->transport_stat = calloc(transports_count, sizeof(*result->transport_stat));
454 2 : if (!result->transport_stat) {
455 0 : SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
456 0 : free(result);
457 0 : return -ENOMEM;
458 : }
459 :
460 5 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
461 3 : rc = nvme_transport_poll_group_get_stats(tgroup, &result->transport_stat[reported_stats_count]);
462 3 : if (rc == 0) {
463 3 : reported_stats_count++;
464 : }
465 : }
466 :
467 2 : if (reported_stats_count == 0) {
468 1 : free(result->transport_stat);
469 1 : free(result);
470 1 : SPDK_DEBUGLOG(nvme, "No transport statistics available\n");
471 1 : return -ENOTSUP;
472 : }
473 :
474 1 : result->num_transports = reported_stats_count;
475 1 : *stats = result;
476 :
477 1 : return 0;
478 : }
479 :
480 : void
481 1 : spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group,
482 : struct spdk_nvme_poll_group_stat *stat)
483 : {
484 : struct spdk_nvme_transport_poll_group *tgroup;
485 : uint32_t i;
486 1 : uint32_t freed_stats __attribute__((unused)) = 0;
487 :
488 1 : assert(group);
489 1 : assert(stat);
490 :
491 4 : for (i = 0; i < stat->num_transports; i++) {
492 3 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
493 3 : if (nvme_transport_get_trtype(tgroup->transport) == stat->transport_stat[i]->trtype) {
494 3 : nvme_transport_poll_group_free_stats(tgroup, stat->transport_stat[i]);
495 3 : freed_stats++;
496 3 : break;
497 : }
498 : }
499 : }
500 :
501 1 : assert(freed_stats == stat->num_transports);
502 :
503 1 : free(stat->transport_stat);
504 1 : free(stat);
505 1 : }
|
