Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2017 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "bdev_malloc.h"
10 : #include "spdk/endian.h"
11 : #include "spdk/env.h"
12 : #include "spdk/accel.h"
13 : #include "spdk/dma.h"
14 : #include "spdk/likely.h"
15 : #include "spdk/string.h"
16 :
17 : #include "spdk/log.h"
18 :
19 : struct malloc_disk {
20 : struct spdk_bdev disk;
21 : void *malloc_buf;
22 : void *malloc_md_buf;
23 : TAILQ_ENTRY(malloc_disk) link;
24 : };
25 :
26 : struct malloc_task {
27 : struct iovec iov;
28 : int num_outstanding;
29 : enum spdk_bdev_io_status status;
30 : TAILQ_ENTRY(malloc_task) tailq;
31 : };
32 :
33 : struct malloc_channel {
34 : struct spdk_io_channel *accel_channel;
35 : struct spdk_poller *completion_poller;
36 : TAILQ_HEAD(, malloc_task) completed_tasks;
37 : };
38 :
39 : static int
40 0 : malloc_verify_pi(struct spdk_bdev_io *bdev_io)
41 : {
42 0 : struct spdk_bdev *bdev = bdev_io->bdev;
43 : struct spdk_dif_ctx dif_ctx;
44 : struct spdk_dif_error err_blk;
45 : int rc;
46 : struct spdk_dif_ctx_init_ext_opts dif_opts;
47 :
48 0 : assert(bdev_io->u.bdev.memory_domain == NULL);
49 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
50 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
51 0 : rc = spdk_dif_ctx_init(&dif_ctx,
52 0 : bdev->blocklen,
53 0 : bdev->md_len,
54 0 : bdev->md_interleave,
55 0 : bdev->dif_is_head_of_md,
56 0 : bdev->dif_type,
57 0 : bdev_io->u.bdev.dif_check_flags,
58 0 : bdev_io->u.bdev.offset_blocks & 0xFFFFFFFF,
59 : 0xFFFF, 0, 0, 0, &dif_opts);
60 0 : if (rc != 0) {
61 0 : SPDK_ERRLOG("Failed to initialize DIF/DIX context\n");
62 0 : return rc;
63 : }
64 :
65 0 : if (spdk_bdev_is_md_interleaved(bdev)) {
66 0 : rc = spdk_dif_verify(bdev_io->u.bdev.iovs,
67 0 : bdev_io->u.bdev.iovcnt,
68 0 : bdev_io->u.bdev.num_blocks,
69 : &dif_ctx,
70 : &err_blk);
71 0 : } else {
72 0 : struct iovec md_iov = {
73 0 : .iov_base = bdev_io->u.bdev.md_buf,
74 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
75 : };
76 :
77 0 : if (bdev_io->u.bdev.md_buf == NULL) {
78 0 : return 0;
79 : }
80 :
81 0 : rc = spdk_dix_verify(bdev_io->u.bdev.iovs,
82 0 : bdev_io->u.bdev.iovcnt,
83 : &md_iov,
84 0 : bdev_io->u.bdev.num_blocks,
85 : &dif_ctx,
86 : &err_blk);
87 : }
88 :
89 0 : if (rc != 0) {
90 0 : SPDK_ERRLOG("DIF/DIX verify failed: lba %" PRIu64 ", num_blocks %" PRIu64 ", "
91 : "err_type %u, expected %lu, actual %lu, err_offset %u\n",
92 : bdev_io->u.bdev.offset_blocks,
93 : bdev_io->u.bdev.num_blocks,
94 : err_blk.err_type,
95 : err_blk.expected,
96 : err_blk.actual,
97 : err_blk.err_offset);
98 0 : }
99 :
100 0 : return rc;
101 0 : }
102 :
103 : static int
104 0 : malloc_unmap_write_zeroes_generate_pi(struct spdk_bdev_io *bdev_io)
105 : {
106 0 : struct spdk_bdev *bdev = bdev_io->bdev;
107 0 : struct malloc_disk *mdisk = bdev_io->bdev->ctxt;
108 0 : uint32_t block_size = bdev_io->bdev->blocklen;
109 : uint32_t dif_check_flags;
110 : struct spdk_dif_ctx dif_ctx;
111 : struct spdk_dif_ctx_init_ext_opts dif_opts;
112 : int rc;
113 :
114 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
115 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
116 0 : dif_check_flags = bdev->dif_check_flags | SPDK_DIF_CHECK_TYPE_REFTAG |
117 : SPDK_DIF_FLAGS_APPTAG_CHECK;
118 0 : rc = spdk_dif_ctx_init(&dif_ctx,
119 0 : bdev->blocklen,
120 0 : bdev->md_len,
121 0 : bdev->md_interleave,
122 0 : bdev->dif_is_head_of_md,
123 0 : bdev->dif_type,
124 0 : dif_check_flags,
125 : SPDK_DIF_REFTAG_IGNORE,
126 : 0xFFFF, SPDK_DIF_APPTAG_IGNORE,
127 : 0, 0, &dif_opts);
128 0 : if (rc != 0) {
129 0 : SPDK_ERRLOG("Initialization of DIF/DIX context failed\n");
130 0 : return rc;
131 : }
132 :
133 0 : if (bdev->md_interleave) {
134 0 : struct iovec iov = {
135 0 : .iov_base = mdisk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size,
136 0 : .iov_len = bdev_io->u.bdev.num_blocks * block_size,
137 : };
138 :
139 0 : rc = spdk_dif_generate(&iov, 1, bdev_io->u.bdev.num_blocks, &dif_ctx);
140 0 : } else {
141 0 : struct iovec iov = {
142 0 : .iov_base = mdisk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size,
143 0 : .iov_len = bdev_io->u.bdev.num_blocks * block_size,
144 : };
145 :
146 0 : struct iovec md_iov = {
147 0 : .iov_base = mdisk->malloc_md_buf + bdev_io->u.bdev.offset_blocks * bdev->md_len,
148 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
149 : };
150 :
151 0 : rc = spdk_dix_generate(&iov, 1, &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx);
152 : }
153 :
154 0 : if (rc != 0) {
155 0 : SPDK_ERRLOG("Formatting by DIF/DIX failed\n");
156 0 : }
157 :
158 :
159 0 : return rc;
160 0 : }
161 :
162 : static void
163 0 : malloc_done(void *ref, int status)
164 : {
165 0 : struct malloc_task *task = (struct malloc_task *)ref;
166 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
167 : int rc;
168 :
169 0 : if (status != 0) {
170 0 : if (status == -ENOMEM) {
171 0 : if (task->status == SPDK_BDEV_IO_STATUS_SUCCESS) {
172 0 : task->status = SPDK_BDEV_IO_STATUS_NOMEM;
173 0 : }
174 0 : } else {
175 0 : task->status = SPDK_BDEV_IO_STATUS_FAILED;
176 : }
177 0 : }
178 :
179 0 : if (--task->num_outstanding != 0) {
180 0 : return;
181 : }
182 :
183 0 : if (bdev_io->bdev->dif_type != SPDK_DIF_DISABLE &&
184 0 : task->status == SPDK_BDEV_IO_STATUS_SUCCESS) {
185 0 : switch (bdev_io->type) {
186 : case SPDK_BDEV_IO_TYPE_READ:
187 0 : rc = malloc_verify_pi(bdev_io);
188 0 : break;
189 : case SPDK_BDEV_IO_TYPE_UNMAP:
190 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
191 0 : rc = malloc_unmap_write_zeroes_generate_pi(bdev_io);
192 0 : break;
193 : default:
194 0 : rc = 0;
195 0 : break;
196 : }
197 :
198 0 : if (rc != 0) {
199 0 : task->status = SPDK_BDEV_IO_STATUS_FAILED;
200 0 : }
201 0 : }
202 :
203 0 : assert(!bdev_io->u.bdev.accel_sequence || task->status == SPDK_BDEV_IO_STATUS_NOMEM);
204 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), task->status);
205 0 : }
206 :
207 : static void
208 0 : malloc_complete_task(struct malloc_task *task, struct malloc_channel *mch,
209 : enum spdk_bdev_io_status status)
210 : {
211 0 : task->status = status;
212 0 : TAILQ_INSERT_TAIL(&mch->completed_tasks, task, tailq);
213 0 : }
214 :
215 : static TAILQ_HEAD(, malloc_disk) g_malloc_disks = TAILQ_HEAD_INITIALIZER(g_malloc_disks);
216 :
217 : int malloc_disk_count = 0;
218 :
219 : static int bdev_malloc_initialize(void);
220 : static void bdev_malloc_deinitialize(void);
221 :
222 : static int
223 0 : bdev_malloc_get_ctx_size(void)
224 : {
225 0 : return sizeof(struct malloc_task);
226 : }
227 :
228 : static struct spdk_bdev_module malloc_if = {
229 : .name = "malloc",
230 : .module_init = bdev_malloc_initialize,
231 : .module_fini = bdev_malloc_deinitialize,
232 : .get_ctx_size = bdev_malloc_get_ctx_size,
233 :
234 : };
235 :
236 0 : SPDK_BDEV_MODULE_REGISTER(malloc, &malloc_if)
237 :
238 : static void
239 0 : malloc_disk_free(struct malloc_disk *malloc_disk)
240 : {
241 0 : if (!malloc_disk) {
242 0 : return;
243 : }
244 :
245 0 : free(malloc_disk->disk.name);
246 0 : spdk_free(malloc_disk->malloc_buf);
247 0 : spdk_free(malloc_disk->malloc_md_buf);
248 0 : free(malloc_disk);
249 0 : }
250 :
251 : static int
252 0 : bdev_malloc_destruct(void *ctx)
253 : {
254 0 : struct malloc_disk *malloc_disk = ctx;
255 :
256 0 : TAILQ_REMOVE(&g_malloc_disks, malloc_disk, link);
257 0 : malloc_disk_free(malloc_disk);
258 0 : return 0;
259 : }
260 :
261 : static int
262 0 : bdev_malloc_check_iov_len(struct iovec *iovs, int iovcnt, size_t nbytes)
263 : {
264 : int i;
265 :
266 0 : for (i = 0; i < iovcnt; i++) {
267 0 : if (nbytes < iovs[i].iov_len) {
268 0 : return 0;
269 : }
270 :
271 0 : nbytes -= iovs[i].iov_len;
272 0 : }
273 :
274 0 : return nbytes != 0;
275 0 : }
276 :
277 : static size_t
278 0 : malloc_get_md_len(struct spdk_bdev_io *bdev_io)
279 : {
280 0 : return bdev_io->u.bdev.num_blocks * bdev_io->bdev->md_len;
281 : }
282 :
283 : static uint64_t
284 0 : malloc_get_md_offset(struct spdk_bdev_io *bdev_io)
285 : {
286 0 : return bdev_io->u.bdev.offset_blocks * bdev_io->bdev->md_len;
287 : }
288 :
289 : static void *
290 0 : malloc_get_md_buf(struct spdk_bdev_io *bdev_io)
291 : {
292 0 : struct malloc_disk *mdisk = SPDK_CONTAINEROF(bdev_io->bdev, struct malloc_disk, disk);
293 :
294 0 : assert(spdk_bdev_is_md_separate(bdev_io->bdev));
295 :
296 0 : return (char *)mdisk->malloc_md_buf + malloc_get_md_offset(bdev_io);
297 : }
298 :
299 : static void
300 0 : malloc_sequence_fail(struct malloc_task *task, int status)
301 : {
302 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
303 :
304 : /* For ENOMEM, the IO will be retried by the bdev layer, so we don't abort the sequence */
305 0 : if (status != -ENOMEM) {
306 0 : spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence);
307 0 : bdev_io->u.bdev.accel_sequence = NULL;
308 0 : }
309 :
310 0 : malloc_done(task, status);
311 0 : }
312 :
313 : static void
314 0 : malloc_sequence_done(void *ctx, int status)
315 : {
316 0 : struct malloc_task *task = ctx;
317 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
318 :
319 0 : bdev_io->u.bdev.accel_sequence = NULL;
320 : /* Prevent bdev layer from retrying the request if the sequence failed with ENOMEM */
321 0 : malloc_done(task, status != -ENOMEM ? status : -EFAULT);
322 0 : }
323 :
324 : static void
325 0 : bdev_malloc_readv(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
326 : struct malloc_task *task, struct spdk_bdev_io *bdev_io)
327 : {
328 : uint64_t len, offset;
329 0 : int res = 0;
330 :
331 0 : len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
332 0 : offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen;
333 :
334 0 : if (bdev_malloc_check_iov_len(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, len)) {
335 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task),
336 : SPDK_BDEV_IO_STATUS_FAILED);
337 0 : return;
338 : }
339 :
340 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
341 0 : task->num_outstanding = 0;
342 0 : task->iov.iov_base = mdisk->malloc_buf + offset;
343 0 : task->iov.iov_len = len;
344 :
345 0 : SPDK_DEBUGLOG(bdev_malloc, "read %zu bytes from offset %#" PRIx64 ", iovcnt=%d\n",
346 : len, offset, bdev_io->u.bdev.iovcnt);
347 :
348 0 : task->num_outstanding++;
349 0 : res = spdk_accel_append_copy(&bdev_io->u.bdev.accel_sequence, ch,
350 0 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
351 0 : bdev_io->u.bdev.memory_domain,
352 0 : bdev_io->u.bdev.memory_domain_ctx,
353 0 : &task->iov, 1, NULL, NULL, NULL, NULL);
354 0 : if (spdk_unlikely(res != 0)) {
355 0 : malloc_sequence_fail(task, res);
356 0 : return;
357 : }
358 :
359 0 : spdk_accel_sequence_reverse(bdev_io->u.bdev.accel_sequence);
360 0 : spdk_accel_sequence_finish(bdev_io->u.bdev.accel_sequence, malloc_sequence_done, task);
361 :
362 0 : if (bdev_io->u.bdev.md_buf == NULL) {
363 0 : return;
364 : }
365 :
366 0 : SPDK_DEBUGLOG(bdev_malloc, "read metadata %zu bytes from offset%#" PRIx64 "\n",
367 : malloc_get_md_len(bdev_io), malloc_get_md_offset(bdev_io));
368 :
369 0 : task->num_outstanding++;
370 0 : res = spdk_accel_submit_copy(ch, bdev_io->u.bdev.md_buf, malloc_get_md_buf(bdev_io),
371 0 : malloc_get_md_len(bdev_io), malloc_done, task);
372 0 : if (res != 0) {
373 0 : malloc_done(task, res);
374 0 : }
375 0 : }
376 :
377 : static void
378 0 : bdev_malloc_writev(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
379 : struct malloc_task *task, struct spdk_bdev_io *bdev_io)
380 : {
381 : uint64_t len, offset;
382 0 : int res = 0;
383 :
384 0 : len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
385 0 : offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen;
386 :
387 0 : if (bdev_malloc_check_iov_len(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, len)) {
388 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task),
389 : SPDK_BDEV_IO_STATUS_FAILED);
390 0 : return;
391 : }
392 :
393 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
394 0 : task->num_outstanding = 0;
395 0 : task->iov.iov_base = mdisk->malloc_buf + offset;
396 0 : task->iov.iov_len = len;
397 :
398 0 : SPDK_DEBUGLOG(bdev_malloc, "write %zu bytes to offset %#" PRIx64 ", iovcnt=%d\n",
399 : len, offset, bdev_io->u.bdev.iovcnt);
400 :
401 0 : task->num_outstanding++;
402 0 : res = spdk_accel_append_copy(&bdev_io->u.bdev.accel_sequence, ch, &task->iov, 1, NULL, NULL,
403 0 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
404 0 : bdev_io->u.bdev.memory_domain,
405 0 : bdev_io->u.bdev.memory_domain_ctx, NULL, NULL);
406 0 : if (spdk_unlikely(res != 0)) {
407 0 : malloc_sequence_fail(task, res);
408 0 : return;
409 : }
410 :
411 0 : spdk_accel_sequence_finish(bdev_io->u.bdev.accel_sequence, malloc_sequence_done, task);
412 :
413 0 : if (bdev_io->u.bdev.md_buf == NULL) {
414 0 : return;
415 : }
416 :
417 0 : SPDK_DEBUGLOG(bdev_malloc, "write metadata %zu bytes to offset %#" PRIx64 "\n",
418 : malloc_get_md_len(bdev_io), malloc_get_md_offset(bdev_io));
419 :
420 0 : task->num_outstanding++;
421 0 : res = spdk_accel_submit_copy(ch, malloc_get_md_buf(bdev_io), bdev_io->u.bdev.md_buf,
422 0 : malloc_get_md_len(bdev_io), malloc_done, task);
423 0 : if (res != 0) {
424 0 : malloc_done(task, res);
425 0 : }
426 0 : }
427 :
428 : static int
429 0 : bdev_malloc_unmap(struct malloc_disk *mdisk,
430 : struct spdk_io_channel *ch,
431 : struct malloc_task *task,
432 : uint64_t offset,
433 : uint64_t byte_count)
434 : {
435 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
436 0 : task->num_outstanding = 1;
437 :
438 0 : return spdk_accel_submit_fill(ch, mdisk->malloc_buf + offset, 0,
439 0 : byte_count, malloc_done, task);
440 : }
441 :
442 : static void
443 0 : bdev_malloc_copy(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
444 : struct malloc_task *task,
445 : uint64_t dst_offset, uint64_t src_offset, size_t len)
446 : {
447 0 : int64_t res = 0;
448 0 : void *dst = mdisk->malloc_buf + dst_offset;
449 0 : void *src = mdisk->malloc_buf + src_offset;
450 :
451 0 : SPDK_DEBUGLOG(bdev_malloc, "Copy %zu bytes from offset %#" PRIx64 " to offset %#" PRIx64 "\n",
452 : len, src_offset, dst_offset);
453 :
454 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
455 0 : task->num_outstanding = 1;
456 :
457 0 : res = spdk_accel_submit_copy(ch, dst, src, len, malloc_done, task);
458 0 : if (res != 0) {
459 0 : malloc_done(task, res);
460 0 : }
461 0 : }
462 :
463 : static int
464 0 : _bdev_malloc_submit_request(struct malloc_channel *mch, struct spdk_bdev_io *bdev_io)
465 : {
466 0 : struct malloc_task *task = (struct malloc_task *)bdev_io->driver_ctx;
467 0 : struct malloc_disk *disk = bdev_io->bdev->ctxt;
468 0 : uint32_t block_size = bdev_io->bdev->blocklen;
469 : int rc;
470 :
471 0 : switch (bdev_io->type) {
472 : case SPDK_BDEV_IO_TYPE_READ:
473 0 : if (bdev_io->u.bdev.iovs[0].iov_base == NULL) {
474 0 : assert(bdev_io->u.bdev.iovcnt == 1);
475 0 : assert(bdev_io->u.bdev.memory_domain == NULL);
476 0 : bdev_io->u.bdev.iovs[0].iov_base =
477 0 : disk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size;
478 0 : bdev_io->u.bdev.iovs[0].iov_len = bdev_io->u.bdev.num_blocks * block_size;
479 0 : if (spdk_bdev_is_md_separate(bdev_io->bdev)) {
480 0 : spdk_bdev_io_set_md_buf(bdev_io, malloc_get_md_buf(bdev_io),
481 0 : malloc_get_md_len(bdev_io));
482 0 : }
483 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
484 0 : return 0;
485 : }
486 :
487 0 : bdev_malloc_readv(disk, mch->accel_channel, task, bdev_io);
488 0 : return 0;
489 :
490 : case SPDK_BDEV_IO_TYPE_WRITE:
491 0 : if (bdev_io->bdev->dif_type != SPDK_DIF_DISABLE) {
492 0 : rc = malloc_verify_pi(bdev_io);
493 0 : if (rc != 0) {
494 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_FAILED);
495 0 : return 0;
496 : }
497 0 : }
498 :
499 0 : bdev_malloc_writev(disk, mch->accel_channel, task, bdev_io);
500 0 : return 0;
501 :
502 : case SPDK_BDEV_IO_TYPE_RESET:
503 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
504 0 : return 0;
505 :
506 : case SPDK_BDEV_IO_TYPE_FLUSH:
507 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
508 0 : return 0;
509 :
510 : case SPDK_BDEV_IO_TYPE_UNMAP:
511 0 : return bdev_malloc_unmap(disk, mch->accel_channel, task,
512 0 : bdev_io->u.bdev.offset_blocks * block_size,
513 0 : bdev_io->u.bdev.num_blocks * block_size);
514 :
515 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
516 : /* bdev_malloc_unmap is implemented with a call to mem_cpy_fill which zeroes out all of the requested bytes. */
517 0 : return bdev_malloc_unmap(disk, mch->accel_channel, task,
518 0 : bdev_io->u.bdev.offset_blocks * block_size,
519 0 : bdev_io->u.bdev.num_blocks * block_size);
520 :
521 : case SPDK_BDEV_IO_TYPE_ZCOPY:
522 0 : if (bdev_io->u.bdev.zcopy.start) {
523 : void *buf;
524 : size_t len;
525 :
526 0 : buf = disk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size;
527 0 : len = bdev_io->u.bdev.num_blocks * block_size;
528 0 : spdk_bdev_io_set_buf(bdev_io, buf, len);
529 0 : if (spdk_bdev_is_md_separate(bdev_io->bdev)) {
530 0 : spdk_bdev_io_set_md_buf(bdev_io, malloc_get_md_buf(bdev_io),
531 0 : malloc_get_md_len(bdev_io));
532 0 : }
533 0 : }
534 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
535 0 : return 0;
536 : case SPDK_BDEV_IO_TYPE_ABORT:
537 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_FAILED);
538 0 : return 0;
539 : case SPDK_BDEV_IO_TYPE_COPY:
540 0 : bdev_malloc_copy(disk, mch->accel_channel, task,
541 0 : bdev_io->u.bdev.offset_blocks * block_size,
542 0 : bdev_io->u.bdev.copy.src_offset_blocks * block_size,
543 0 : bdev_io->u.bdev.num_blocks * block_size);
544 0 : return 0;
545 :
546 : default:
547 0 : return -1;
548 : }
549 : return 0;
550 0 : }
551 :
552 : static void
553 0 : bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
554 : {
555 0 : struct malloc_channel *mch = spdk_io_channel_get_ctx(ch);
556 :
557 0 : if (_bdev_malloc_submit_request(mch, bdev_io) != 0) {
558 0 : malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch,
559 : SPDK_BDEV_IO_STATUS_FAILED);
560 0 : }
561 0 : }
562 :
563 : static bool
564 0 : bdev_malloc_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
565 : {
566 0 : switch (io_type) {
567 : case SPDK_BDEV_IO_TYPE_READ:
568 : case SPDK_BDEV_IO_TYPE_WRITE:
569 : case SPDK_BDEV_IO_TYPE_FLUSH:
570 : case SPDK_BDEV_IO_TYPE_RESET:
571 : case SPDK_BDEV_IO_TYPE_UNMAP:
572 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
573 : case SPDK_BDEV_IO_TYPE_ZCOPY:
574 : case SPDK_BDEV_IO_TYPE_ABORT:
575 : case SPDK_BDEV_IO_TYPE_COPY:
576 0 : return true;
577 :
578 : default:
579 0 : return false;
580 : }
581 0 : }
582 :
583 : static struct spdk_io_channel *
584 0 : bdev_malloc_get_io_channel(void *ctx)
585 : {
586 0 : return spdk_get_io_channel(&g_malloc_disks);
587 : }
588 :
589 : static void
590 0 : bdev_malloc_write_json_config(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
591 : {
592 0 : spdk_json_write_object_begin(w);
593 :
594 0 : spdk_json_write_named_string(w, "method", "bdev_malloc_create");
595 :
596 0 : spdk_json_write_named_object_begin(w, "params");
597 0 : spdk_json_write_named_string(w, "name", bdev->name);
598 0 : spdk_json_write_named_uint64(w, "num_blocks", bdev->blockcnt);
599 0 : spdk_json_write_named_uint32(w, "block_size", bdev->blocklen);
600 0 : spdk_json_write_named_uint32(w, "physical_block_size", bdev->phys_blocklen);
601 0 : spdk_json_write_named_uuid(w, "uuid", &bdev->uuid);
602 0 : spdk_json_write_named_uint32(w, "optimal_io_boundary", bdev->optimal_io_boundary);
603 0 : spdk_json_write_named_uint32(w, "md_size", bdev->md_len);
604 0 : spdk_json_write_named_uint32(w, "dif_type", bdev->dif_type);
605 0 : spdk_json_write_named_bool(w, "dif_is_head_of_md", bdev->dif_is_head_of_md);
606 0 : spdk_json_write_named_uint32(w, "dif_pi_format", bdev->dif_pi_format);
607 :
608 0 : spdk_json_write_object_end(w);
609 :
610 0 : spdk_json_write_object_end(w);
611 0 : }
612 :
613 : static int
614 0 : bdev_malloc_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
615 : {
616 0 : struct malloc_disk *malloc_disk = ctx;
617 : struct spdk_memory_domain *domain;
618 0 : int num_domains = 0;
619 :
620 0 : if (malloc_disk->disk.dif_type != SPDK_DIF_DISABLE) {
621 0 : return 0;
622 : }
623 :
624 : /* Report support for every memory domain */
625 0 : for (domain = spdk_memory_domain_get_first(NULL); domain != NULL;
626 0 : domain = spdk_memory_domain_get_next(domain, NULL)) {
627 0 : if (domains != NULL && num_domains < array_size) {
628 0 : domains[num_domains] = domain;
629 0 : }
630 0 : num_domains++;
631 0 : }
632 :
633 0 : return num_domains;
634 0 : }
635 :
636 : static bool
637 0 : bdev_malloc_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
638 : {
639 0 : struct malloc_disk *malloc_disk = ctx;
640 :
641 0 : if (malloc_disk->disk.dif_type != SPDK_DIF_DISABLE) {
642 0 : return false;
643 : }
644 :
645 0 : switch (type) {
646 : case SPDK_BDEV_IO_TYPE_READ:
647 : case SPDK_BDEV_IO_TYPE_WRITE:
648 0 : return true;
649 : default:
650 0 : return false;
651 : }
652 0 : }
653 :
654 : static const struct spdk_bdev_fn_table malloc_fn_table = {
655 : .destruct = bdev_malloc_destruct,
656 : .submit_request = bdev_malloc_submit_request,
657 : .io_type_supported = bdev_malloc_io_type_supported,
658 : .get_io_channel = bdev_malloc_get_io_channel,
659 : .write_config_json = bdev_malloc_write_json_config,
660 : .get_memory_domains = bdev_malloc_get_memory_domains,
661 : .accel_sequence_supported = bdev_malloc_accel_sequence_supported,
662 : };
663 :
664 : static int
665 0 : malloc_disk_setup_pi(struct malloc_disk *mdisk)
666 : {
667 0 : struct spdk_bdev *bdev = &mdisk->disk;
668 : struct spdk_dif_ctx dif_ctx;
669 : struct iovec iov, md_iov;
670 : uint32_t dif_check_flags;
671 : int rc;
672 : struct spdk_dif_ctx_init_ext_opts dif_opts;
673 :
674 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
675 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
676 : /* Set APPTAG|REFTAG_IGNORE to PI fields after creation of malloc bdev */
677 0 : dif_check_flags = bdev->dif_check_flags | SPDK_DIF_CHECK_TYPE_REFTAG |
678 : SPDK_DIF_FLAGS_APPTAG_CHECK;
679 0 : rc = spdk_dif_ctx_init(&dif_ctx,
680 0 : bdev->blocklen,
681 0 : bdev->md_len,
682 0 : bdev->md_interleave,
683 0 : bdev->dif_is_head_of_md,
684 0 : bdev->dif_type,
685 0 : dif_check_flags,
686 : SPDK_DIF_REFTAG_IGNORE,
687 : 0xFFFF, SPDK_DIF_APPTAG_IGNORE,
688 : 0, 0, &dif_opts);
689 0 : if (rc != 0) {
690 0 : SPDK_ERRLOG("Initialization of DIF/DIX context failed\n");
691 0 : return rc;
692 : }
693 :
694 0 : iov.iov_base = mdisk->malloc_buf;
695 0 : iov.iov_len = bdev->blockcnt * bdev->blocklen;
696 :
697 0 : if (mdisk->disk.md_interleave) {
698 0 : rc = spdk_dif_generate(&iov, 1, bdev->blockcnt, &dif_ctx);
699 0 : } else {
700 0 : md_iov.iov_base = mdisk->malloc_md_buf;
701 0 : md_iov.iov_len = bdev->blockcnt * bdev->md_len;
702 :
703 0 : rc = spdk_dix_generate(&iov, 1, &md_iov, bdev->blockcnt, &dif_ctx);
704 : }
705 :
706 0 : if (rc != 0) {
707 0 : SPDK_ERRLOG("Formatting by DIF/DIX failed\n");
708 0 : }
709 :
710 0 : return rc;
711 0 : }
712 :
713 : int
714 0 : create_malloc_disk(struct spdk_bdev **bdev, const struct malloc_bdev_opts *opts)
715 : {
716 : struct malloc_disk *mdisk;
717 : uint32_t block_size;
718 : int rc;
719 :
720 0 : assert(opts != NULL);
721 :
722 0 : if (opts->num_blocks == 0) {
723 0 : SPDK_ERRLOG("Disk num_blocks must be greater than 0");
724 0 : return -EINVAL;
725 : }
726 :
727 0 : if (opts->block_size % 512) {
728 0 : SPDK_ERRLOG("Data block size must be 512 bytes aligned\n");
729 0 : return -EINVAL;
730 : }
731 :
732 0 : if (opts->physical_block_size % 512) {
733 0 : SPDK_ERRLOG("Physical block must be 512 bytes aligned\n");
734 0 : return -EINVAL;
735 : }
736 :
737 0 : switch (opts->md_size) {
738 : case 0:
739 : case 8:
740 : case 16:
741 : case 32:
742 : case 64:
743 : case 128:
744 0 : break;
745 : default:
746 0 : SPDK_ERRLOG("metadata size %u is not supported\n", opts->md_size);
747 0 : return -EINVAL;
748 : }
749 :
750 0 : if (opts->md_interleave) {
751 0 : block_size = opts->block_size + opts->md_size;
752 0 : } else {
753 0 : block_size = opts->block_size;
754 : }
755 :
756 0 : mdisk = calloc(1, sizeof(*mdisk));
757 0 : if (!mdisk) {
758 0 : SPDK_ERRLOG("mdisk calloc() failed\n");
759 0 : return -ENOMEM;
760 : }
761 :
762 : /*
763 : * Allocate the large backend memory buffer from pinned memory.
764 : *
765 : * TODO: need to pass a hint so we know which socket to allocate
766 : * from on multi-socket systems.
767 : */
768 0 : mdisk->malloc_buf = spdk_zmalloc(opts->num_blocks * block_size, 2 * 1024 * 1024, NULL,
769 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
770 0 : if (!mdisk->malloc_buf) {
771 0 : SPDK_ERRLOG("malloc_buf spdk_zmalloc() failed\n");
772 0 : malloc_disk_free(mdisk);
773 0 : return -ENOMEM;
774 : }
775 :
776 0 : if (!opts->md_interleave && opts->md_size != 0) {
777 0 : mdisk->malloc_md_buf = spdk_zmalloc(opts->num_blocks * opts->md_size, 2 * 1024 * 1024, NULL,
778 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
779 0 : if (!mdisk->malloc_md_buf) {
780 0 : SPDK_ERRLOG("malloc_md_buf spdk_zmalloc() failed\n");
781 0 : malloc_disk_free(mdisk);
782 0 : return -ENOMEM;
783 : }
784 0 : }
785 :
786 0 : if (opts->name) {
787 0 : mdisk->disk.name = strdup(opts->name);
788 0 : } else {
789 : /* Auto-generate a name */
790 0 : mdisk->disk.name = spdk_sprintf_alloc("Malloc%d", malloc_disk_count);
791 0 : malloc_disk_count++;
792 : }
793 0 : if (!mdisk->disk.name) {
794 0 : malloc_disk_free(mdisk);
795 0 : return -ENOMEM;
796 : }
797 0 : mdisk->disk.product_name = "Malloc disk";
798 :
799 0 : mdisk->disk.write_cache = 1;
800 0 : mdisk->disk.blocklen = block_size;
801 0 : mdisk->disk.phys_blocklen = opts->physical_block_size;
802 0 : mdisk->disk.blockcnt = opts->num_blocks;
803 0 : mdisk->disk.md_len = opts->md_size;
804 0 : mdisk->disk.md_interleave = opts->md_interleave;
805 0 : mdisk->disk.dif_type = opts->dif_type;
806 0 : mdisk->disk.dif_is_head_of_md = opts->dif_is_head_of_md;
807 : /* Current block device layer API does not propagate
808 : * any DIF related information from user. So, we can
809 : * not generate or verify Application Tag.
810 : */
811 0 : switch (opts->dif_type) {
812 : case SPDK_DIF_TYPE1:
813 : case SPDK_DIF_TYPE2:
814 0 : mdisk->disk.dif_check_flags = SPDK_DIF_FLAGS_GUARD_CHECK |
815 : SPDK_DIF_FLAGS_REFTAG_CHECK;
816 0 : break;
817 : case SPDK_DIF_TYPE3:
818 0 : mdisk->disk.dif_check_flags = SPDK_DIF_FLAGS_GUARD_CHECK;
819 0 : break;
820 : case SPDK_DIF_DISABLE:
821 0 : break;
822 : }
823 0 : mdisk->disk.dif_pi_format = opts->dif_pi_format;
824 :
825 0 : if (opts->dif_type != SPDK_DIF_DISABLE) {
826 0 : rc = malloc_disk_setup_pi(mdisk);
827 0 : if (rc) {
828 0 : SPDK_ERRLOG("Failed to set up protection information.\n");
829 0 : malloc_disk_free(mdisk);
830 0 : return rc;
831 : }
832 0 : }
833 :
834 0 : if (opts->optimal_io_boundary) {
835 0 : mdisk->disk.optimal_io_boundary = opts->optimal_io_boundary;
836 0 : mdisk->disk.split_on_optimal_io_boundary = true;
837 0 : }
838 0 : if (!spdk_uuid_is_null(&opts->uuid)) {
839 0 : spdk_uuid_copy(&mdisk->disk.uuid, &opts->uuid);
840 0 : }
841 :
842 0 : mdisk->disk.max_copy = 0;
843 0 : mdisk->disk.ctxt = mdisk;
844 0 : mdisk->disk.fn_table = &malloc_fn_table;
845 0 : mdisk->disk.module = &malloc_if;
846 :
847 0 : rc = spdk_bdev_register(&mdisk->disk);
848 0 : if (rc) {
849 0 : malloc_disk_free(mdisk);
850 0 : return rc;
851 : }
852 :
853 0 : *bdev = &(mdisk->disk);
854 :
855 0 : TAILQ_INSERT_TAIL(&g_malloc_disks, mdisk, link);
856 :
857 0 : return rc;
858 0 : }
859 :
860 : void
861 0 : delete_malloc_disk(const char *name, spdk_delete_malloc_complete cb_fn, void *cb_arg)
862 : {
863 : int rc;
864 :
865 0 : rc = spdk_bdev_unregister_by_name(name, &malloc_if, cb_fn, cb_arg);
866 0 : if (rc != 0) {
867 0 : cb_fn(cb_arg, rc);
868 0 : }
869 0 : }
870 :
871 : static int
872 0 : malloc_completion_poller(void *ctx)
873 : {
874 0 : struct malloc_channel *ch = ctx;
875 : struct malloc_task *task;
876 : TAILQ_HEAD(, malloc_task) completed_tasks;
877 0 : uint32_t num_completions = 0;
878 :
879 0 : TAILQ_INIT(&completed_tasks);
880 0 : TAILQ_SWAP(&completed_tasks, &ch->completed_tasks, malloc_task, tailq);
881 :
882 0 : while (!TAILQ_EMPTY(&completed_tasks)) {
883 0 : task = TAILQ_FIRST(&completed_tasks);
884 0 : TAILQ_REMOVE(&completed_tasks, task, tailq);
885 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), task->status);
886 0 : num_completions++;
887 : }
888 :
889 0 : return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
890 : }
891 :
892 : static int
893 0 : malloc_create_channel_cb(void *io_device, void *ctx)
894 : {
895 0 : struct malloc_channel *ch = ctx;
896 :
897 0 : ch->accel_channel = spdk_accel_get_io_channel();
898 0 : if (!ch->accel_channel) {
899 0 : SPDK_ERRLOG("Failed to get accel framework's IO channel\n");
900 0 : return -ENOMEM;
901 : }
902 :
903 0 : ch->completion_poller = SPDK_POLLER_REGISTER(malloc_completion_poller, ch, 0);
904 0 : if (!ch->completion_poller) {
905 0 : SPDK_ERRLOG("Failed to register malloc completion poller\n");
906 0 : spdk_put_io_channel(ch->accel_channel);
907 0 : return -ENOMEM;
908 : }
909 :
910 0 : TAILQ_INIT(&ch->completed_tasks);
911 :
912 0 : return 0;
913 0 : }
914 :
915 : static void
916 0 : malloc_destroy_channel_cb(void *io_device, void *ctx)
917 : {
918 0 : struct malloc_channel *ch = ctx;
919 :
920 0 : assert(TAILQ_EMPTY(&ch->completed_tasks));
921 :
922 0 : spdk_put_io_channel(ch->accel_channel);
923 0 : spdk_poller_unregister(&ch->completion_poller);
924 0 : }
925 :
926 : static int
927 0 : bdev_malloc_initialize(void)
928 : {
929 : /* This needs to be reset for each reinitialization of submodules.
930 : * Otherwise after enough devices or reinitializations the value gets too high.
931 : * TODO: Make malloc bdev name mandatory and remove this counter. */
932 0 : malloc_disk_count = 0;
933 :
934 0 : spdk_io_device_register(&g_malloc_disks, malloc_create_channel_cb,
935 : malloc_destroy_channel_cb, sizeof(struct malloc_channel),
936 : "bdev_malloc");
937 :
938 0 : return 0;
939 : }
940 :
941 : static void
942 0 : bdev_malloc_deinitialize(void)
943 : {
944 0 : spdk_io_device_unregister(&g_malloc_disks, NULL);
945 0 : }
946 :
947 0 : SPDK_LOG_REGISTER_COMPONENT(bdev_malloc)
|