Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2018 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
5 : * All rights reserved.
6 : */
7 :
8 : #include "vbdev_crypto.h"
9 :
10 : #include "spdk_internal/assert.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/bdev_module.h"
13 : #include "spdk/likely.h"
14 :
15 : /* This namespace UUID was generated using uuid_generate() method. */
16 : #define BDEV_CRYPTO_NAMESPACE_UUID "078e3cf7-f4b4-4545-b2c3-d40045a64ae2"
17 :
18 : struct bdev_names {
19 : struct vbdev_crypto_opts *opts;
20 : TAILQ_ENTRY(bdev_names) link;
21 : };
22 :
23 : /* List of crypto_bdev names and their base bdevs via configuration file. */
24 : static TAILQ_HEAD(, bdev_names) g_bdev_names = TAILQ_HEAD_INITIALIZER(g_bdev_names);
25 :
26 : struct vbdev_crypto {
27 : struct spdk_bdev *base_bdev; /* the thing we're attaching to */
28 : struct spdk_bdev_desc *base_desc; /* its descriptor we get from open */
29 : struct spdk_bdev crypto_bdev; /* the crypto virtual bdev */
30 : struct vbdev_crypto_opts *opts; /* crypto options such as names and DEK */
31 : TAILQ_ENTRY(vbdev_crypto) link;
32 : struct spdk_thread *thread; /* thread where base device is opened */
33 : };
34 :
35 : /* List of virtual bdevs and associated info for each. We keep the device friendly name here even
36 : * though its also in the device struct because we use it early on.
37 : */
38 : static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto);
39 :
40 : /* The crypto vbdev channel struct. It is allocated and freed on my behalf by the io channel code.
41 : * We store things in here that are needed on per thread basis like the base_channel for this thread.
42 : */
43 : struct crypto_io_channel {
44 : struct spdk_io_channel *base_ch; /* IO channel of base device */
45 : struct spdk_io_channel *accel_channel; /* Accel engine channel used for crypto ops */
46 : struct spdk_accel_crypto_key *crypto_key;
47 : };
48 :
49 : enum crypto_io_resubmit_state {
50 : CRYPTO_IO_DECRYPT_DONE, /* Appended decrypt, need to read */
51 : CRYPTO_IO_ENCRYPT_DONE, /* Need to write */
52 : };
53 :
54 : /* This is the crypto per IO context that the bdev layer allocates for us opaquely and attaches to
55 : * each IO for us.
56 : */
57 : struct crypto_bdev_io {
58 : struct crypto_io_channel *crypto_ch; /* need to store for crypto completion handling */
59 : struct vbdev_crypto *crypto_bdev; /* the crypto node struct associated with this IO */
60 : /* Used for the single contiguous buffer that serves as the crypto destination target for writes */
61 : uint64_t aux_num_blocks; /* num of blocks for the contiguous buffer */
62 : uint64_t aux_offset_blocks; /* block offset on media */
63 : void *aux_buf_raw; /* raw buffer that the bdev layer gave us for write buffer */
64 : struct iovec aux_buf_iov; /* iov representing aligned contig write buffer */
65 : struct spdk_memory_domain *aux_domain; /* memory domain of the aux buf */
66 : void *aux_domain_ctx; /* memory domain ctx of the aux buf */
67 : struct spdk_accel_sequence *seq; /* sequence of accel operations */
68 :
69 : /* for bdev_io_wait */
70 : struct spdk_bdev_io_wait_entry bdev_io_wait;
71 : enum crypto_io_resubmit_state resubmit_state;
72 : };
73 :
74 : static void vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io,
75 : enum crypto_io_resubmit_state state);
76 : static void _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
77 : static void vbdev_crypto_examine(struct spdk_bdev *bdev);
78 : static int vbdev_crypto_claim(const char *bdev_name);
79 : static void vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io);
80 :
81 : static void
82 7 : crypto_io_fail(struct crypto_bdev_io *crypto_io)
83 : {
84 7 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(crypto_io);
85 7 : struct crypto_io_channel *crypto_ch = crypto_io->crypto_ch;
86 :
87 7 : if (crypto_io->aux_buf_raw) {
88 4 : spdk_accel_put_buf(crypto_ch->accel_channel, crypto_io->aux_buf_raw,
89 2 : crypto_io->aux_domain, crypto_io->aux_domain_ctx);
90 2 : }
91 :
92 : /* This function can only be used to fail an IO that hasn't been sent to the base bdev,
93 : * otherwise accel sequence might have already been executed/aborted. */
94 7 : spdk_accel_sequence_abort(crypto_io->seq);
95 7 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
96 7 : }
97 :
98 : static void
99 6 : crypto_write(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io)
100 : {
101 6 : struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto,
102 : crypto_bdev);
103 6 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
104 6 : struct spdk_bdev_ext_io_opts opts = {};
105 6 : int rc;
106 :
107 6 : opts.size = sizeof(opts);
108 6 : opts.accel_sequence = crypto_io->seq;
109 6 : opts.memory_domain = crypto_io->aux_domain;
110 6 : opts.memory_domain_ctx = crypto_io->aux_domain_ctx;
111 :
112 : /* Write the encrypted data. */
113 12 : rc = spdk_bdev_writev_blocks_ext(crypto_bdev->base_desc, crypto_ch->base_ch,
114 6 : &crypto_io->aux_buf_iov, 1, crypto_io->aux_offset_blocks,
115 6 : crypto_io->aux_num_blocks, _complete_internal_io,
116 6 : bdev_io, &opts);
117 6 : if (spdk_unlikely(rc != 0)) {
118 3 : if (rc == -ENOMEM) {
119 1 : SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n");
120 1 : vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_ENCRYPT_DONE);
121 1 : } else {
122 2 : SPDK_ERRLOG("Failed to submit bdev_io!\n");
123 2 : crypto_io_fail(crypto_io);
124 : }
125 3 : }
126 6 : }
127 :
128 : /* We're either encrypting on the way down or decrypting on the way back. */
129 : static void
130 6 : crypto_encrypt(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io)
131 : {
132 6 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
133 6 : uint32_t crypto_len = crypto_io->crypto_bdev->crypto_bdev.blocklen;
134 6 : uint64_t total_length;
135 6 : uint64_t alignment;
136 6 : void *aux_buf = crypto_io->aux_buf_raw;
137 6 : int rc;
138 :
139 : /* For encryption, we need to prepare a single contiguous buffer as the encryption
140 : * destination, we'll then pass that along for the write after encryption is done.
141 : * This is done to avoiding encrypting the provided write buffer which may be
142 : * undesirable in some use cases.
143 : */
144 6 : total_length = bdev_io->u.bdev.num_blocks * crypto_len;
145 6 : alignment = spdk_bdev_get_buf_align(&crypto_io->crypto_bdev->crypto_bdev);
146 6 : crypto_io->aux_buf_iov.iov_len = total_length;
147 12 : crypto_io->aux_buf_iov.iov_base = (void *)(((uintptr_t)aux_buf + (alignment - 1)) & ~
148 6 : (alignment - 1));
149 6 : crypto_io->aux_offset_blocks = bdev_io->u.bdev.offset_blocks;
150 6 : crypto_io->aux_num_blocks = bdev_io->u.bdev.num_blocks;
151 :
152 12 : rc = spdk_accel_append_encrypt(&crypto_io->seq, crypto_ch->accel_channel,
153 6 : crypto_ch->crypto_key, &crypto_io->aux_buf_iov, 1,
154 6 : crypto_io->aux_domain, crypto_io->aux_domain_ctx,
155 6 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
156 6 : bdev_io->u.bdev.memory_domain,
157 6 : bdev_io->u.bdev.memory_domain_ctx,
158 6 : bdev_io->u.bdev.offset_blocks, crypto_len, 0,
159 : NULL, NULL);
160 6 : if (spdk_unlikely(rc != 0)) {
161 4 : spdk_accel_put_buf(crypto_ch->accel_channel, crypto_io->aux_buf_raw,
162 2 : crypto_io->aux_domain, crypto_io->aux_domain_ctx);
163 2 : if (rc == -ENOMEM) {
164 1 : SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n");
165 1 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
166 1 : } else {
167 1 : SPDK_ERRLOG("Failed to submit bdev_io!\n");
168 1 : crypto_io_fail(crypto_io);
169 : }
170 :
171 2 : return;
172 : }
173 :
174 4 : crypto_write(crypto_ch, bdev_io);
175 6 : }
176 :
177 : static void
178 8 : _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
179 : {
180 8 : struct spdk_bdev_io *orig_io = cb_arg;
181 8 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)orig_io->driver_ctx;
182 8 : struct crypto_io_channel *crypto_ch = crypto_io->crypto_ch;
183 8 : int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;
184 :
185 8 : if (crypto_io->aux_buf_raw) {
186 4 : spdk_accel_put_buf(crypto_ch->accel_channel, crypto_io->aux_buf_raw,
187 2 : crypto_io->aux_domain, crypto_io->aux_domain_ctx);
188 2 : }
189 :
190 8 : spdk_bdev_io_complete(orig_io, status);
191 8 : spdk_bdev_free_io(bdev_io);
192 8 : }
193 :
194 : static void crypto_read(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io);
195 :
196 : static void
197 0 : vbdev_crypto_resubmit_io(void *arg)
198 : {
199 0 : struct spdk_bdev_io *bdev_io = (struct spdk_bdev_io *)arg;
200 0 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
201 :
202 0 : switch (crypto_io->resubmit_state) {
203 : case CRYPTO_IO_ENCRYPT_DONE:
204 0 : crypto_write(crypto_io->crypto_ch, bdev_io);
205 0 : break;
206 : case CRYPTO_IO_DECRYPT_DONE:
207 0 : crypto_read(crypto_io->crypto_ch, bdev_io);
208 0 : break;
209 : default:
210 0 : SPDK_UNREACHABLE();
211 : }
212 0 : }
213 :
214 : static void
215 2 : vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io, enum crypto_io_resubmit_state state)
216 : {
217 2 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
218 2 : int rc;
219 :
220 2 : crypto_io->bdev_io_wait.bdev = bdev_io->bdev;
221 2 : crypto_io->bdev_io_wait.cb_fn = vbdev_crypto_resubmit_io;
222 2 : crypto_io->bdev_io_wait.cb_arg = bdev_io;
223 2 : crypto_io->resubmit_state = state;
224 :
225 2 : rc = spdk_bdev_queue_io_wait(bdev_io->bdev, crypto_io->crypto_ch->base_ch,
226 2 : &crypto_io->bdev_io_wait);
227 2 : if (rc != 0) {
228 0 : SPDK_ERRLOG("Queue io failed in vbdev_crypto_queue_io, rc=%d.\n", rc);
229 0 : crypto_io_fail(crypto_io);
230 0 : }
231 2 : }
232 :
233 : static void
234 4 : crypto_read(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io)
235 : {
236 4 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
237 4 : struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto,
238 : crypto_bdev);
239 4 : struct spdk_bdev_ext_io_opts opts = {};
240 4 : int rc;
241 :
242 4 : opts.size = sizeof(opts);
243 4 : opts.accel_sequence = crypto_io->seq;
244 4 : opts.memory_domain = bdev_io->u.bdev.memory_domain;
245 4 : opts.memory_domain_ctx = bdev_io->u.bdev.memory_domain_ctx;
246 :
247 8 : rc = spdk_bdev_readv_blocks_ext(crypto_bdev->base_desc, crypto_ch->base_ch,
248 4 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
249 4 : bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
250 4 : _complete_internal_io, bdev_io, &opts);
251 4 : if (rc != 0) {
252 2 : if (rc == -ENOMEM) {
253 1 : SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n");
254 1 : vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_DECRYPT_DONE);
255 1 : } else {
256 1 : SPDK_ERRLOG("Failed to submit bdev_io!\n");
257 1 : crypto_io_fail(crypto_io);
258 : }
259 2 : }
260 4 : }
261 :
262 : /* Callback for getting a buf from the bdev pool in the event that the caller passed
263 : * in NULL, we need to own the buffer so it doesn't get freed by another vbdev module
264 : * beneath us before we're done with it.
265 : */
266 : static void
267 5 : crypto_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
268 : bool success)
269 : {
270 5 : struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch);
271 5 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
272 5 : uint32_t blocklen = crypto_io->crypto_bdev->crypto_bdev.blocklen;
273 5 : int rc;
274 :
275 5 : if (!success) {
276 0 : crypto_io_fail(crypto_io);
277 0 : return;
278 : }
279 :
280 10 : rc = spdk_accel_append_decrypt(&crypto_io->seq, crypto_ch->accel_channel,
281 5 : crypto_ch->crypto_key,
282 5 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
283 5 : bdev_io->u.bdev.memory_domain,
284 5 : bdev_io->u.bdev.memory_domain_ctx,
285 5 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
286 5 : bdev_io->u.bdev.memory_domain,
287 5 : bdev_io->u.bdev.memory_domain_ctx,
288 5 : bdev_io->u.bdev.offset_blocks, blocklen, 0,
289 : NULL, NULL);
290 5 : if (rc != 0) {
291 1 : if (rc == -ENOMEM) {
292 1 : SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n");
293 1 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
294 1 : } else {
295 0 : SPDK_ERRLOG("Failed to submit bdev_io!\n");
296 0 : crypto_io_fail(crypto_io);
297 : }
298 :
299 1 : return;
300 : }
301 :
302 4 : crypto_read(crypto_ch, bdev_io);
303 5 : }
304 :
305 : /* Called when someone submits IO to this crypto vbdev. For IO's not relevant to crypto,
306 : * we're simply passing it on here via SPDK IO calls which in turn allocate another bdev IO
307 : * and call our cpl callback provided below along with the original bdev_io so that we can
308 : * complete it once this IO completes. For crypto operations, we'll either encrypt it first
309 : * (writes) then call back into bdev to submit it or we'll submit a read and then catch it
310 : * on the way back for decryption.
311 : */
312 : static void
313 16 : vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
314 : {
315 16 : struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto,
316 : crypto_bdev);
317 16 : struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch);
318 16 : struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx;
319 16 : int rc = 0;
320 :
321 16 : memset(crypto_io, 0, sizeof(struct crypto_bdev_io));
322 16 : crypto_io->crypto_bdev = crypto_bdev;
323 16 : crypto_io->crypto_ch = crypto_ch;
324 16 : crypto_io->seq = bdev_io->u.bdev.accel_sequence;
325 :
326 16 : switch (bdev_io->type) {
327 : case SPDK_BDEV_IO_TYPE_READ:
328 10 : spdk_bdev_io_get_buf(bdev_io, crypto_read_get_buf_cb,
329 5 : bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
330 5 : break;
331 : case SPDK_BDEV_IO_TYPE_WRITE:
332 : /* For encryption we don't want to encrypt the data in place as the host isn't
333 : * expecting us to mangle its data buffers so we need to encrypt into the aux accel
334 : * buffer, then we can use that as the source for the disk data transfer.
335 : */
336 12 : rc = spdk_accel_get_buf(crypto_ch->accel_channel,
337 6 : bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen,
338 6 : &crypto_io->aux_buf_raw, &crypto_io->aux_domain,
339 6 : &crypto_io->aux_domain_ctx);
340 6 : if (rc == 0) {
341 6 : crypto_encrypt(crypto_ch, bdev_io);
342 6 : }
343 6 : break;
344 : case SPDK_BDEV_IO_TYPE_UNMAP:
345 4 : rc = spdk_bdev_unmap_blocks(crypto_bdev->base_desc, crypto_ch->base_ch,
346 2 : bdev_io->u.bdev.offset_blocks,
347 2 : bdev_io->u.bdev.num_blocks,
348 2 : _complete_internal_io, bdev_io);
349 2 : break;
350 : case SPDK_BDEV_IO_TYPE_FLUSH:
351 4 : rc = spdk_bdev_flush_blocks(crypto_bdev->base_desc, crypto_ch->base_ch,
352 2 : bdev_io->u.bdev.offset_blocks,
353 2 : bdev_io->u.bdev.num_blocks,
354 2 : _complete_internal_io, bdev_io);
355 2 : break;
356 : case SPDK_BDEV_IO_TYPE_RESET:
357 0 : rc = spdk_bdev_reset(crypto_bdev->base_desc, crypto_ch->base_ch,
358 0 : _complete_internal_io, bdev_io);
359 0 : break;
360 1 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
361 : default:
362 1 : SPDK_ERRLOG("crypto: unknown I/O type %d\n", bdev_io->type);
363 1 : rc = -EINVAL;
364 1 : break;
365 : }
366 :
367 16 : if (rc != 0) {
368 3 : if (rc == -ENOMEM) {
369 0 : SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n");
370 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
371 0 : } else {
372 3 : SPDK_ERRLOG("Failed to submit bdev_io!\n");
373 3 : crypto_io_fail(crypto_io);
374 : }
375 3 : }
376 16 : }
377 :
378 : /* We'll just call the base bdev and let it answer except for WZ command which
379 : * we always say we don't support so that the bdev layer will actually send us
380 : * real writes that we can encrypt.
381 : */
382 : static bool
383 1 : vbdev_crypto_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
384 : {
385 1 : struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx;
386 :
387 1 : switch (io_type) {
388 : case SPDK_BDEV_IO_TYPE_WRITE:
389 : case SPDK_BDEV_IO_TYPE_UNMAP:
390 : case SPDK_BDEV_IO_TYPE_RESET:
391 : case SPDK_BDEV_IO_TYPE_READ:
392 : case SPDK_BDEV_IO_TYPE_FLUSH:
393 0 : return spdk_bdev_io_type_supported(crypto_bdev->base_bdev, io_type);
394 1 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
395 : /* Force the bdev layer to issue actual writes of zeroes so we can
396 : * encrypt them as regular writes.
397 : */
398 : default:
399 1 : return false;
400 : }
401 1 : }
402 :
403 : /* Callback for unregistering the IO device. */
404 : static void
405 0 : _device_unregister_cb(void *io_device)
406 : {
407 0 : struct vbdev_crypto *crypto_bdev = io_device;
408 :
409 : /* Done with this crypto_bdev. */
410 0 : crypto_bdev->opts = NULL;
411 :
412 0 : spdk_bdev_destruct_done(&crypto_bdev->crypto_bdev, 0);
413 0 : free(crypto_bdev->crypto_bdev.name);
414 0 : free(crypto_bdev);
415 0 : }
416 :
417 : /* Wrapper for the bdev close operation. */
418 : static void
419 0 : _vbdev_crypto_destruct(void *ctx)
420 : {
421 0 : struct spdk_bdev_desc *desc = ctx;
422 :
423 0 : spdk_bdev_close(desc);
424 0 : }
425 :
426 : /* Called after we've unregistered following a hot remove callback.
427 : * Our finish entry point will be called next.
428 : */
429 : static int
430 0 : vbdev_crypto_destruct(void *ctx)
431 : {
432 0 : struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx;
433 :
434 : /* Remove this device from the internal list */
435 0 : TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link);
436 :
437 : /* Unclaim the underlying bdev. */
438 0 : spdk_bdev_module_release_bdev(crypto_bdev->base_bdev);
439 :
440 : /* Close the underlying bdev on its same opened thread. */
441 0 : if (crypto_bdev->thread && crypto_bdev->thread != spdk_get_thread()) {
442 0 : spdk_thread_send_msg(crypto_bdev->thread, _vbdev_crypto_destruct, crypto_bdev->base_desc);
443 0 : } else {
444 0 : spdk_bdev_close(crypto_bdev->base_desc);
445 : }
446 :
447 : /* Unregister the io_device. */
448 0 : spdk_io_device_unregister(crypto_bdev, _device_unregister_cb);
449 :
450 0 : return 1;
451 0 : }
452 :
453 : /* We supplied this as an entry point for upper layers who want to communicate to this
454 : * bdev. This is how they get a channel. We are passed the same context we provided when
455 : * we created our crypto vbdev in examine() which, for this bdev, is the address of one of
456 : * our context nodes. From here we'll ask the SPDK channel code to fill out our channel
457 : * struct and we'll keep it in our crypto node.
458 : */
459 : static struct spdk_io_channel *
460 0 : vbdev_crypto_get_io_channel(void *ctx)
461 : {
462 0 : struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx;
463 :
464 : /* The IO channel code will allocate a channel for us which consists of
465 : * the SPDK channel structure plus the size of our crypto_io_channel struct
466 : * that we passed in when we registered our IO device. It will then call
467 : * our channel create callback to populate any elements that we need to
468 : * update.
469 : */
470 0 : return spdk_get_io_channel(crypto_bdev);
471 0 : }
472 :
473 : /* This is the output for bdev_get_bdevs() for this vbdev */
474 : static int
475 0 : vbdev_crypto_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
476 : {
477 0 : struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx;
478 :
479 0 : spdk_json_write_name(w, "crypto");
480 0 : spdk_json_write_object_begin(w);
481 0 : spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev));
482 0 : spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev));
483 0 : spdk_json_write_named_string(w, "key_name", crypto_bdev->opts->key->param.key_name);
484 0 : spdk_json_write_object_end(w);
485 :
486 0 : return 0;
487 0 : }
488 :
489 : static int
490 0 : vbdev_crypto_config_json(struct spdk_json_write_ctx *w)
491 : {
492 0 : struct vbdev_crypto *crypto_bdev;
493 :
494 0 : TAILQ_FOREACH(crypto_bdev, &g_vbdev_crypto, link) {
495 0 : spdk_json_write_object_begin(w);
496 0 : spdk_json_write_named_string(w, "method", "bdev_crypto_create");
497 0 : spdk_json_write_named_object_begin(w, "params");
498 0 : spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev));
499 0 : spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev));
500 0 : spdk_json_write_named_string(w, "key_name", crypto_bdev->opts->key->param.key_name);
501 0 : spdk_json_write_object_end(w);
502 0 : spdk_json_write_object_end(w);
503 0 : }
504 0 : return 0;
505 0 : }
506 :
507 : /* We provide this callback for the SPDK channel code to create a channel using
508 : * the channel struct we provided in our module get_io_channel() entry point. Here
509 : * we get and save off an underlying base channel of the device below us so that
510 : * we can communicate with the base bdev on a per channel basis. We also register the
511 : * poller used to complete crypto operations from the device.
512 : */
513 : static int
514 0 : crypto_bdev_ch_create_cb(void *io_device, void *ctx_buf)
515 : {
516 0 : struct crypto_io_channel *crypto_ch = ctx_buf;
517 0 : struct vbdev_crypto *crypto_bdev = io_device;
518 :
519 0 : crypto_ch->base_ch = spdk_bdev_get_io_channel(crypto_bdev->base_desc);
520 0 : if (crypto_ch->base_ch == NULL) {
521 0 : SPDK_ERRLOG("Failed to get base bdev IO channel (bdev: %s)\n",
522 : crypto_bdev->crypto_bdev.name);
523 0 : return -ENOMEM;
524 : }
525 :
526 0 : crypto_ch->accel_channel = spdk_accel_get_io_channel();
527 0 : if (crypto_ch->accel_channel == NULL) {
528 0 : SPDK_ERRLOG("Failed to get accel IO channel (bdev: %s)\n",
529 : crypto_bdev->crypto_bdev.name);
530 0 : spdk_put_io_channel(crypto_ch->base_ch);
531 0 : return -ENOMEM;
532 : }
533 :
534 0 : crypto_ch->crypto_key = crypto_bdev->opts->key;
535 :
536 0 : return 0;
537 0 : }
538 :
539 : /* We provide this callback for the SPDK channel code to destroy a channel
540 : * created with our create callback. We just need to undo anything we did
541 : * when we created.
542 : */
543 : static void
544 0 : crypto_bdev_ch_destroy_cb(void *io_device, void *ctx_buf)
545 : {
546 0 : struct crypto_io_channel *crypto_ch = ctx_buf;
547 :
548 0 : spdk_put_io_channel(crypto_ch->base_ch);
549 0 : spdk_put_io_channel(crypto_ch->accel_channel);
550 0 : }
551 :
552 : /* Create the association from the bdev and vbdev name and insert
553 : * on the global list. */
554 : static int
555 0 : vbdev_crypto_insert_name(struct vbdev_crypto_opts *opts, struct bdev_names **out)
556 : {
557 0 : struct bdev_names *name;
558 :
559 0 : assert(opts);
560 0 : assert(out);
561 :
562 0 : TAILQ_FOREACH(name, &g_bdev_names, link) {
563 0 : if (strcmp(opts->vbdev_name, name->opts->vbdev_name) == 0) {
564 0 : SPDK_ERRLOG("Crypto bdev %s already exists\n", opts->vbdev_name);
565 0 : return -EEXIST;
566 : }
567 0 : }
568 :
569 0 : name = calloc(1, sizeof(struct bdev_names));
570 0 : if (!name) {
571 0 : SPDK_ERRLOG("Failed to allocate memory for bdev_names.\n");
572 0 : return -ENOMEM;
573 : }
574 :
575 0 : name->opts = opts;
576 0 : TAILQ_INSERT_TAIL(&g_bdev_names, name, link);
577 0 : *out = name;
578 :
579 0 : return 0;
580 0 : }
581 :
582 : void
583 0 : free_crypto_opts(struct vbdev_crypto_opts *opts)
584 : {
585 0 : free(opts->bdev_name);
586 0 : free(opts->vbdev_name);
587 0 : free(opts);
588 0 : }
589 :
590 : static void
591 0 : vbdev_crypto_delete_name(struct bdev_names *name)
592 : {
593 0 : TAILQ_REMOVE(&g_bdev_names, name, link);
594 0 : if (name->opts) {
595 0 : if (name->opts->key_owner && name->opts->key) {
596 0 : spdk_accel_crypto_key_destroy(name->opts->key);
597 0 : }
598 0 : free_crypto_opts(name->opts);
599 0 : name->opts = NULL;
600 0 : }
601 0 : free(name);
602 0 : }
603 :
604 : /* RPC entry point for crypto creation. */
605 : int
606 0 : create_crypto_disk(struct vbdev_crypto_opts *opts)
607 : {
608 0 : struct bdev_names *name = NULL;
609 0 : int rc;
610 :
611 0 : rc = vbdev_crypto_insert_name(opts, &name);
612 0 : if (rc) {
613 0 : return rc;
614 : }
615 :
616 0 : rc = vbdev_crypto_claim(opts->bdev_name);
617 0 : if (rc == -ENODEV) {
618 0 : SPDK_NOTICELOG("vbdev creation deferred pending base bdev arrival\n");
619 0 : rc = 0;
620 0 : }
621 :
622 0 : if (rc) {
623 0 : assert(name != NULL);
624 : /* In case of error we let the caller function to deallocate @opts
625 : * since it is its responsibility. Setting name->opts = NULL let's
626 : * vbdev_crypto_delete_name() know it does not have to do anything
627 : * about @opts.
628 : */
629 0 : name->opts = NULL;
630 0 : vbdev_crypto_delete_name(name);
631 0 : }
632 0 : return rc;
633 0 : }
634 :
635 : /* Called at driver init time, parses config file to prepare for examine calls,
636 : * also fully initializes the crypto drivers.
637 : */
638 : static int
639 0 : vbdev_crypto_init(void)
640 : {
641 0 : return 0;
642 : }
643 :
644 : /* Called when the entire module is being torn down. */
645 : static void
646 0 : vbdev_crypto_finish(void)
647 : {
648 0 : struct bdev_names *name;
649 :
650 0 : while ((name = TAILQ_FIRST(&g_bdev_names))) {
651 0 : vbdev_crypto_delete_name(name);
652 : }
653 0 : }
654 :
655 : /* During init we'll be asked how much memory we'd like passed to us
656 : * in bev_io structures as context. Here's where we specify how
657 : * much context we want per IO.
658 : */
659 : static int
660 0 : vbdev_crypto_get_ctx_size(void)
661 : {
662 0 : return sizeof(struct crypto_bdev_io);
663 : }
664 :
665 : static void
666 0 : vbdev_crypto_base_bdev_hotremove_cb(struct spdk_bdev *bdev_find)
667 : {
668 0 : struct vbdev_crypto *crypto_bdev, *tmp;
669 :
670 0 : TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) {
671 0 : if (bdev_find == crypto_bdev->base_bdev) {
672 0 : spdk_bdev_unregister(&crypto_bdev->crypto_bdev, NULL, NULL);
673 0 : }
674 0 : }
675 0 : }
676 :
677 : /* Called when the underlying base bdev triggers asynchronous event such as bdev removal. */
678 : static void
679 0 : vbdev_crypto_base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
680 : void *event_ctx)
681 : {
682 0 : switch (type) {
683 : case SPDK_BDEV_EVENT_REMOVE:
684 0 : vbdev_crypto_base_bdev_hotremove_cb(bdev);
685 0 : break;
686 : default:
687 0 : SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
688 0 : break;
689 : }
690 0 : }
691 :
692 : static int
693 0 : vbdev_crypto_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
694 : {
695 0 : struct vbdev_crypto *crypto_bdev = ctx;
696 0 : int num_domains;
697 :
698 : /* Report base bdev's memory domains plus accel memory domain */
699 0 : num_domains = spdk_bdev_get_memory_domains(crypto_bdev->base_bdev, domains, array_size);
700 0 : if (domains != NULL && num_domains < array_size) {
701 0 : domains[num_domains] = spdk_accel_get_memory_domain();
702 0 : }
703 :
704 0 : return num_domains + 1;
705 0 : }
706 :
707 : static bool
708 0 : vbdev_crypto_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
709 : {
710 0 : switch (type) {
711 : case SPDK_BDEV_IO_TYPE_READ:
712 : case SPDK_BDEV_IO_TYPE_WRITE:
713 0 : return true;
714 : default:
715 0 : return false;
716 : }
717 0 : }
718 :
719 : /* When we register our bdev this is how we specify our entry points. */
720 : static const struct spdk_bdev_fn_table vbdev_crypto_fn_table = {
721 : .destruct = vbdev_crypto_destruct,
722 : .submit_request = vbdev_crypto_submit_request,
723 : .io_type_supported = vbdev_crypto_io_type_supported,
724 : .get_io_channel = vbdev_crypto_get_io_channel,
725 : .dump_info_json = vbdev_crypto_dump_info_json,
726 : .get_memory_domains = vbdev_crypto_get_memory_domains,
727 : .accel_sequence_supported = vbdev_crypto_sequence_supported,
728 : };
729 :
730 : static struct spdk_bdev_module crypto_if = {
731 : .name = "crypto",
732 : .module_init = vbdev_crypto_init,
733 : .get_ctx_size = vbdev_crypto_get_ctx_size,
734 : .examine_config = vbdev_crypto_examine,
735 : .module_fini = vbdev_crypto_finish,
736 : .config_json = vbdev_crypto_config_json
737 : };
738 :
739 1 : SPDK_BDEV_MODULE_REGISTER(crypto, &crypto_if)
740 :
741 : static int
742 0 : vbdev_crypto_claim(const char *bdev_name)
743 : {
744 0 : struct bdev_names *name;
745 0 : struct vbdev_crypto *vbdev;
746 0 : struct spdk_bdev *bdev;
747 0 : struct spdk_iobuf_opts iobuf_opts;
748 0 : struct spdk_accel_operation_exec_ctx opctx = {};
749 0 : struct spdk_uuid ns_uuid;
750 0 : int rc = 0;
751 :
752 0 : spdk_uuid_parse(&ns_uuid, BDEV_CRYPTO_NAMESPACE_UUID);
753 :
754 : /* Limit the max IO size by some reasonable value. Since in write operation we use aux buffer,
755 : * let's set the limit to the large_bufsize value */
756 0 : spdk_iobuf_get_opts(&iobuf_opts);
757 :
758 : /* Check our list of names from config versus this bdev and if
759 : * there's a match, create the crypto_bdev & bdev accordingly.
760 : */
761 0 : TAILQ_FOREACH(name, &g_bdev_names, link) {
762 0 : if (strcmp(name->opts->bdev_name, bdev_name) != 0) {
763 0 : continue;
764 : }
765 0 : SPDK_DEBUGLOG(vbdev_crypto, "Match on %s\n", bdev_name);
766 :
767 0 : vbdev = calloc(1, sizeof(struct vbdev_crypto));
768 0 : if (!vbdev) {
769 0 : SPDK_ERRLOG("Failed to allocate memory for crypto_bdev.\n");
770 0 : return -ENOMEM;
771 : }
772 0 : vbdev->crypto_bdev.product_name = "crypto";
773 :
774 0 : vbdev->crypto_bdev.name = strdup(name->opts->vbdev_name);
775 0 : if (!vbdev->crypto_bdev.name) {
776 0 : SPDK_ERRLOG("Failed to allocate memory for crypto_bdev name.\n");
777 0 : rc = -ENOMEM;
778 0 : goto error_bdev_name;
779 : }
780 :
781 0 : rc = spdk_bdev_open_ext(bdev_name, true, vbdev_crypto_base_bdev_event_cb,
782 0 : NULL, &vbdev->base_desc);
783 0 : if (rc) {
784 0 : if (rc != -ENODEV) {
785 0 : SPDK_ERRLOG("Failed to open bdev %s: error %d\n", bdev_name, rc);
786 0 : }
787 0 : goto error_open;
788 : }
789 :
790 0 : bdev = spdk_bdev_desc_get_bdev(vbdev->base_desc);
791 0 : vbdev->base_bdev = bdev;
792 :
793 0 : vbdev->crypto_bdev.write_cache = bdev->write_cache;
794 0 : vbdev->crypto_bdev.optimal_io_boundary = bdev->optimal_io_boundary;
795 0 : vbdev->crypto_bdev.max_rw_size = spdk_min(
796 : bdev->max_rw_size ? bdev->max_rw_size : UINT32_MAX,
797 : iobuf_opts.large_bufsize / bdev->blocklen);
798 :
799 0 : opctx.size = SPDK_SIZEOF(&opctx, block_size);
800 0 : opctx.block_size = bdev->blocklen;
801 0 : vbdev->crypto_bdev.required_alignment =
802 0 : spdk_max(bdev->required_alignment,
803 : spdk_max(spdk_accel_get_buf_align(SPDK_ACCEL_OPC_ENCRYPT, &opctx),
804 : spdk_accel_get_buf_align(SPDK_ACCEL_OPC_DECRYPT, &opctx)));
805 :
806 0 : vbdev->crypto_bdev.blocklen = bdev->blocklen;
807 0 : vbdev->crypto_bdev.blockcnt = bdev->blockcnt;
808 :
809 : /* This is the context that is passed to us when the bdev
810 : * layer calls in so we'll save our crypto_bdev node here.
811 : */
812 0 : vbdev->crypto_bdev.ctxt = vbdev;
813 0 : vbdev->crypto_bdev.fn_table = &vbdev_crypto_fn_table;
814 0 : vbdev->crypto_bdev.module = &crypto_if;
815 :
816 : /* Assign crypto opts from the name. The pointer is valid up to the point
817 : * the module is unloaded and all names removed from the list. */
818 0 : vbdev->opts = name->opts;
819 :
820 : /* Generate UUID based on namespace UUID + base bdev UUID */
821 0 : rc = spdk_uuid_generate_sha1(&vbdev->crypto_bdev.uuid, &ns_uuid,
822 0 : (const char *)&vbdev->base_bdev->uuid, sizeof(struct spdk_uuid));
823 0 : if (rc) {
824 0 : SPDK_ERRLOG("Unable to generate new UUID for crypto bdev\n");
825 0 : goto error_uuid;
826 : }
827 :
828 0 : TAILQ_INSERT_TAIL(&g_vbdev_crypto, vbdev, link);
829 :
830 0 : spdk_io_device_register(vbdev, crypto_bdev_ch_create_cb, crypto_bdev_ch_destroy_cb,
831 0 : sizeof(struct crypto_io_channel), vbdev->crypto_bdev.name);
832 :
833 : /* Save the thread where the base device is opened */
834 0 : vbdev->thread = spdk_get_thread();
835 :
836 0 : rc = spdk_bdev_module_claim_bdev(bdev, vbdev->base_desc, vbdev->crypto_bdev.module);
837 0 : if (rc) {
838 0 : SPDK_ERRLOG("Failed to claim bdev %s\n", spdk_bdev_get_name(bdev));
839 0 : goto error_claim;
840 : }
841 :
842 0 : rc = spdk_bdev_register(&vbdev->crypto_bdev);
843 0 : if (rc < 0) {
844 0 : SPDK_ERRLOG("Failed to register vbdev: error %d\n", rc);
845 0 : rc = -EINVAL;
846 0 : goto error_bdev_register;
847 : }
848 0 : SPDK_DEBUGLOG(vbdev_crypto, "Registered io_device and virtual bdev for: %s\n",
849 : vbdev->opts->vbdev_name);
850 0 : break;
851 : }
852 :
853 0 : return rc;
854 :
855 : /* Error cleanup paths. */
856 : error_bdev_register:
857 0 : spdk_bdev_module_release_bdev(vbdev->base_bdev);
858 : error_claim:
859 0 : TAILQ_REMOVE(&g_vbdev_crypto, vbdev, link);
860 0 : spdk_io_device_unregister(vbdev, NULL);
861 : error_uuid:
862 0 : spdk_bdev_close(vbdev->base_desc);
863 : error_open:
864 0 : free(vbdev->crypto_bdev.name);
865 : error_bdev_name:
866 0 : free(vbdev);
867 :
868 0 : return rc;
869 0 : }
870 :
871 : struct crypto_delete_disk_ctx {
872 : spdk_delete_crypto_complete cb_fn;
873 : void *cb_arg;
874 : char *bdev_name;
875 : };
876 :
877 : static void
878 0 : delete_crypto_disk_bdev_name(void *ctx, int rc)
879 : {
880 0 : struct bdev_names *name;
881 0 : struct crypto_delete_disk_ctx *disk_ctx = ctx;
882 :
883 : /* Remove the association (vbdev, bdev) from g_bdev_names. This is required so that the
884 : * vbdev does not get re-created if the same bdev is constructed at some other time,
885 : * unless the underlying bdev was hot-removed. */
886 0 : TAILQ_FOREACH(name, &g_bdev_names, link) {
887 0 : if (strcmp(name->opts->vbdev_name, disk_ctx->bdev_name) == 0) {
888 0 : vbdev_crypto_delete_name(name);
889 0 : break;
890 : }
891 0 : }
892 :
893 0 : disk_ctx->cb_fn(disk_ctx->cb_arg, rc);
894 :
895 0 : free(disk_ctx->bdev_name);
896 0 : free(disk_ctx);
897 0 : }
898 :
899 : /* RPC entry for deleting a crypto vbdev. */
900 : void
901 0 : delete_crypto_disk(const char *bdev_name, spdk_delete_crypto_complete cb_fn,
902 : void *cb_arg)
903 : {
904 0 : int rc;
905 0 : struct crypto_delete_disk_ctx *ctx;
906 :
907 0 : ctx = calloc(1, sizeof(struct crypto_delete_disk_ctx));
908 0 : if (!ctx) {
909 0 : SPDK_ERRLOG("Failed to allocate delete crypto disk ctx\n");
910 0 : cb_fn(cb_arg, -ENOMEM);
911 0 : return;
912 : }
913 :
914 0 : ctx->bdev_name = strdup(bdev_name);
915 0 : if (!ctx->bdev_name) {
916 0 : SPDK_ERRLOG("Failed to copy bdev_name\n");
917 0 : free(ctx);
918 0 : cb_fn(cb_arg, -ENOMEM);
919 0 : return;
920 : }
921 0 : ctx->cb_arg = cb_arg;
922 0 : ctx->cb_fn = cb_fn;
923 : /* Some cleanup happens in the destruct callback. */
924 0 : rc = spdk_bdev_unregister_by_name(bdev_name, &crypto_if, delete_crypto_disk_bdev_name, ctx);
925 0 : if (rc != 0) {
926 0 : SPDK_ERRLOG("Encountered an error during bdev unregistration\n");
927 0 : cb_fn(cb_arg, rc);
928 0 : free(ctx->bdev_name);
929 0 : free(ctx);
930 0 : }
931 0 : }
932 :
933 : /* Because we specified this function in our crypto bdev function table when we
934 : * registered our crypto bdev, we'll get this call anytime a new bdev shows up.
935 : * Here we need to decide if we care about it and if so what to do. We
936 : * parsed the config file at init so we check the new bdev against the list
937 : * we built up at that time and if the user configured us to attach to this
938 : * bdev, here's where we do it.
939 : */
940 : static void
941 0 : vbdev_crypto_examine(struct spdk_bdev *bdev)
942 : {
943 0 : vbdev_crypto_claim(spdk_bdev_get_name(bdev));
944 0 : spdk_bdev_module_examine_done(&crypto_if);
945 0 : }
946 :
947 1 : SPDK_LOG_REGISTER_COMPONENT(vbdev_crypto)
|