Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/accel_module.h"
10 :
11 : #include "accel_internal.h"
12 :
13 : #include "spdk/dma.h"
14 : #include "spdk/env.h"
15 : #include "spdk/likely.h"
16 : #include "spdk/log.h"
17 : #include "spdk/thread.h"
18 : #include "spdk/json.h"
19 : #include "spdk/crc32.h"
20 : #include "spdk/util.h"
21 : #include "spdk/hexlify.h"
22 : #include "spdk/string.h"
23 :
24 : /* Accelerator Framework: The following provides a top level
25 : * generic API for the accelerator functions defined here. Modules,
26 : * such as the one in /module/accel/ioat, supply the implementation
27 : * with the exception of the pure software implementation contained
28 : * later in this file.
29 : */
30 :
31 : #define ALIGN_4K 0x1000
32 : #define ACCEL_TASKS_PER_CHANNEL 2048
33 : #define ACCEL_SMALL_CACHE_SIZE 128
34 : #define ACCEL_LARGE_CACHE_SIZE 16
35 : /* Set MSB, so we don't return NULL pointers as buffers */
36 : #define ACCEL_BUFFER_BASE ((void *)(1ull << 63))
37 : #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1)
38 :
39 : #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA
40 : #define ACCEL_TASKS_IN_SEQUENCE_LIMIT 8
41 :
42 : struct accel_module {
43 : struct spdk_accel_module_if *module;
44 : bool supports_memory_domains;
45 : };
46 :
47 : /* Largest context size for all accel modules */
48 : static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
49 :
50 : static struct spdk_accel_module_if *g_accel_module = NULL;
51 : static spdk_accel_fini_cb g_fini_cb_fn = NULL;
52 : static void *g_fini_cb_arg = NULL;
53 : static bool g_modules_started = false;
54 : static struct spdk_memory_domain *g_accel_domain;
55 :
56 : /* Global list of registered accelerator modules */
57 : static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
58 : TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
59 :
60 : /* Crypto keyring */
61 : static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring);
62 : static struct spdk_spinlock g_keyring_spin;
63 :
64 : /* Global array mapping capabilities to modules */
65 : static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {};
66 : static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {};
67 : TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers);
68 : static struct spdk_accel_driver *g_accel_driver;
69 : static struct spdk_accel_opts g_opts = {
70 : .small_cache_size = ACCEL_SMALL_CACHE_SIZE,
71 : .large_cache_size = ACCEL_LARGE_CACHE_SIZE,
72 : .task_count = ACCEL_TASKS_PER_CHANNEL,
73 : .sequence_count = ACCEL_TASKS_PER_CHANNEL,
74 : .buf_count = ACCEL_TASKS_PER_CHANNEL,
75 : };
76 : static struct accel_stats g_stats;
77 : static struct spdk_spinlock g_stats_lock;
78 :
79 : static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = {
80 : "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c",
81 : "compress", "decompress", "encrypt", "decrypt", "xor",
82 : "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy",
83 : "dix_generate", "dix_verify"
84 : };
85 :
86 : enum accel_sequence_state {
87 : ACCEL_SEQUENCE_STATE_INIT,
88 : ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF,
89 : ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF,
90 : ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF,
91 : ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF,
92 : ACCEL_SEQUENCE_STATE_PULL_DATA,
93 : ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA,
94 : ACCEL_SEQUENCE_STATE_EXEC_TASK,
95 : ACCEL_SEQUENCE_STATE_AWAIT_TASK,
96 : ACCEL_SEQUENCE_STATE_COMPLETE_TASK,
97 : ACCEL_SEQUENCE_STATE_NEXT_TASK,
98 : ACCEL_SEQUENCE_STATE_PUSH_DATA,
99 : ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA,
100 : ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS,
101 : ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS,
102 : ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS,
103 : ACCEL_SEQUENCE_STATE_ERROR,
104 : ACCEL_SEQUENCE_STATE_MAX,
105 : };
106 :
107 : static const char *g_seq_states[]
108 : __attribute__((unused)) = {
109 : [ACCEL_SEQUENCE_STATE_INIT] = "init",
110 : [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf",
111 : [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf",
112 : [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf",
113 : [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf",
114 : [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data",
115 : [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data",
116 : [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task",
117 : [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task",
118 : [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task",
119 : [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task",
120 : [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data",
121 : [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data",
122 : [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks",
123 : [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks",
124 : [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks",
125 : [ACCEL_SEQUENCE_STATE_ERROR] = "error",
126 : [ACCEL_SEQUENCE_STATE_MAX] = "",
127 : };
128 :
129 : #define ACCEL_SEQUENCE_STATE_STRING(s) \
130 : (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \
131 : ? g_seq_states[s] : "unknown")
132 :
133 : struct accel_buffer {
134 : struct spdk_accel_sequence *seq;
135 : void *buf;
136 : uint64_t len;
137 : struct spdk_iobuf_entry iobuf;
138 : spdk_accel_sequence_get_buf_cb cb_fn;
139 : void *cb_ctx;
140 : SLIST_ENTRY(accel_buffer) link;
141 : struct accel_io_channel *ch;
142 : };
143 :
144 : struct accel_io_channel {
145 : struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST];
146 : struct spdk_io_channel *driver_channel;
147 : void *task_pool_base;
148 : struct spdk_accel_sequence *seq_pool_base;
149 : struct accel_buffer *buf_pool_base;
150 : struct spdk_accel_task_aux_data *task_aux_data_base;
151 : STAILQ_HEAD(, spdk_accel_task) task_pool;
152 : SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool;
153 : SLIST_HEAD(, spdk_accel_sequence) seq_pool;
154 : SLIST_HEAD(, accel_buffer) buf_pool;
155 : struct spdk_iobuf_channel iobuf;
156 : struct accel_stats stats;
157 : };
158 :
159 : TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task);
160 :
161 : struct spdk_accel_sequence {
162 : struct accel_io_channel *ch;
163 : struct accel_sequence_tasks tasks;
164 : SLIST_HEAD(, accel_buffer) bounce_bufs;
165 : int status;
166 : /* state uses enum accel_sequence_state */
167 : uint8_t state;
168 : bool in_process_sequence;
169 : spdk_accel_completion_cb cb_fn;
170 : void *cb_arg;
171 : SLIST_ENTRY(spdk_accel_sequence) link;
172 : };
173 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size");
174 :
175 : #define accel_update_stats(ch, event, v) \
176 : do { \
177 : (ch)->stats.event += (v); \
178 : } while (0)
179 :
180 : #define accel_update_task_stats(ch, task, event, v) \
181 : accel_update_stats(ch, operations[(task)->op_code].event, v)
182 :
183 : static inline void accel_sequence_task_cb(struct spdk_accel_sequence *seq,
184 : struct spdk_accel_task *task, int status);
185 :
186 : static inline void
187 888 : accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state)
188 : {
189 888 : SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq,
190 : ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state));
191 888 : assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR);
192 888 : seq->state = state;
193 888 : }
194 :
195 : static void
196 9 : accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status)
197 : {
198 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
199 9 : assert(status != 0);
200 9 : seq->status = status;
201 9 : }
202 :
203 : int
204 17 : spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name)
205 : {
206 17 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
207 : /* invalid opcode */
208 0 : return -EINVAL;
209 : }
210 :
211 17 : if (g_modules_opc[opcode].module) {
212 17 : *module_name = g_modules_opc[opcode].module->name;
213 : } else {
214 0 : return -ENOENT;
215 : }
216 :
217 17 : return 0;
218 : }
219 :
220 : void
221 0 : _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn)
222 : {
223 : struct spdk_accel_module_if *accel_module;
224 : enum spdk_accel_opcode opcode;
225 0 : int j = 0;
226 :
227 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
228 0 : for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) {
229 0 : if (accel_module->supports_opcode(opcode)) {
230 0 : info->ops[j] = opcode;
231 0 : j++;
232 : }
233 : }
234 0 : info->name = accel_module->name;
235 0 : info->num_ops = j;
236 0 : fn(info);
237 0 : j = 0;
238 : }
239 0 : }
240 :
241 : const char *
242 0 : spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode)
243 : {
244 0 : if (opcode < SPDK_ACCEL_OPC_LAST) {
245 0 : return g_opcode_strings[opcode];
246 : }
247 :
248 0 : return NULL;
249 : }
250 :
251 : int
252 0 : spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name)
253 : {
254 : char *copy;
255 :
256 0 : if (g_modules_started == true) {
257 : /* we don't allow re-assignment once things have started */
258 0 : return -EINVAL;
259 : }
260 :
261 0 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
262 : /* invalid opcode */
263 0 : return -EINVAL;
264 : }
265 :
266 0 : copy = strdup(name);
267 0 : if (copy == NULL) {
268 0 : return -ENOMEM;
269 : }
270 :
271 : /* module selection will be validated after the framework starts. */
272 0 : free(g_modules_opc_override[opcode]);
273 0 : g_modules_opc_override[opcode] = copy;
274 :
275 0 : return 0;
276 : }
277 :
278 : inline static struct spdk_accel_task *
279 191 : _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg)
280 : {
281 : struct spdk_accel_task *accel_task;
282 :
283 191 : accel_task = STAILQ_FIRST(&accel_ch->task_pool);
284 191 : if (spdk_unlikely(accel_task == NULL)) {
285 11 : accel_update_stats(accel_ch, retry.task, 1);
286 11 : return NULL;
287 : }
288 :
289 180 : accel_update_stats(accel_ch, task_outstanding, 1);
290 180 : STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);
291 180 : accel_task->link.stqe_next = NULL;
292 :
293 180 : accel_task->cb_fn = cb_fn;
294 180 : accel_task->cb_arg = cb_arg;
295 180 : accel_task->accel_ch = accel_ch;
296 180 : accel_task->s.iovs = NULL;
297 180 : accel_task->d.iovs = NULL;
298 :
299 180 : return accel_task;
300 : }
301 :
302 : static void
303 172 : _put_task(struct accel_io_channel *ch, struct spdk_accel_task *task)
304 : {
305 172 : STAILQ_INSERT_HEAD(&ch->task_pool, task, link);
306 172 : accel_update_stats(ch, task_outstanding, -1);
307 172 : }
308 :
309 : void
310 122 : spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status)
311 : {
312 122 : struct accel_io_channel *accel_ch = accel_task->accel_ch;
313 : spdk_accel_completion_cb cb_fn;
314 : void *cb_arg;
315 :
316 122 : accel_update_task_stats(accel_ch, accel_task, executed, 1);
317 122 : accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes);
318 122 : if (spdk_unlikely(status != 0)) {
319 3 : accel_update_task_stats(accel_ch, accel_task, failed, 1);
320 : }
321 :
322 122 : if (accel_task->seq) {
323 118 : accel_sequence_task_cb(accel_task->seq, accel_task, status);
324 118 : return;
325 : }
326 :
327 4 : cb_fn = accel_task->cb_fn;
328 4 : cb_arg = accel_task->cb_arg;
329 :
330 4 : if (accel_task->has_aux) {
331 2 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link);
332 2 : accel_task->aux = NULL;
333 2 : accel_task->has_aux = false;
334 : }
335 :
336 : /* We should put the accel_task into the list firstly in order to avoid
337 : * the accel task list is exhausted when there is recursive call to
338 : * allocate accel_task in user's call back function (cb_fn)
339 : */
340 4 : _put_task(accel_ch, accel_task);
341 :
342 4 : cb_fn(cb_arg, status);
343 : }
344 :
345 : static inline int
346 119 : accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task)
347 : {
348 119 : struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code];
349 119 : struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module;
350 : int rc;
351 :
352 119 : rc = module->submit_tasks(module_ch, task);
353 119 : if (spdk_unlikely(rc != 0)) {
354 2 : accel_update_task_stats(accel_ch, task, failed, 1);
355 : }
356 :
357 119 : return rc;
358 : }
359 :
360 : static inline uint64_t
361 141 : accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)
362 : {
363 141 : uint64_t result = 0;
364 : uint32_t i;
365 :
366 313 : for (i = 0; i < iovcnt; ++i) {
367 172 : result += iovs[i].iov_len;
368 : }
369 :
370 141 : return result;
371 : }
372 :
373 : #define ACCEL_TASK_ALLOC_AUX_BUF(task) \
374 : do { \
375 : (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \
376 : if (spdk_unlikely(!(task)->aux)) { \
377 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \
378 : _put_task(task->accel_ch, task); \
379 : assert(0); \
380 : return -ENOMEM; \
381 : } \
382 : SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \
383 : (task)->has_aux = true; \
384 : } while (0)
385 :
386 : /* Accel framework public API for copy function */
387 : int
388 2 : spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
389 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
390 : {
391 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
392 : struct spdk_accel_task *accel_task;
393 :
394 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
395 2 : if (spdk_unlikely(accel_task == NULL)) {
396 1 : return -ENOMEM;
397 : }
398 :
399 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
400 :
401 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
402 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
403 1 : accel_task->d.iovs[0].iov_base = dst;
404 1 : accel_task->d.iovs[0].iov_len = nbytes;
405 1 : accel_task->d.iovcnt = 1;
406 1 : accel_task->s.iovs[0].iov_base = src;
407 1 : accel_task->s.iovs[0].iov_len = nbytes;
408 1 : accel_task->s.iovcnt = 1;
409 1 : accel_task->nbytes = nbytes;
410 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY;
411 1 : accel_task->src_domain = NULL;
412 1 : accel_task->dst_domain = NULL;
413 :
414 1 : return accel_submit_task(accel_ch, accel_task);
415 : }
416 :
417 : /* Accel framework public API for dual cast copy function */
418 : int
419 4 : spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
420 : void *dst2, void *src, uint64_t nbytes,
421 : spdk_accel_completion_cb cb_fn, void *cb_arg)
422 : {
423 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
424 : struct spdk_accel_task *accel_task;
425 :
426 4 : if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
427 2 : SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
428 2 : return -EINVAL;
429 : }
430 :
431 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
432 2 : if (spdk_unlikely(accel_task == NULL)) {
433 1 : return -ENOMEM;
434 : }
435 :
436 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
437 :
438 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
439 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
440 1 : accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2];
441 1 : accel_task->d.iovs[0].iov_base = dst1;
442 1 : accel_task->d.iovs[0].iov_len = nbytes;
443 1 : accel_task->d.iovcnt = 1;
444 1 : accel_task->d2.iovs[0].iov_base = dst2;
445 1 : accel_task->d2.iovs[0].iov_len = nbytes;
446 1 : accel_task->d2.iovcnt = 1;
447 1 : accel_task->s.iovs[0].iov_base = src;
448 1 : accel_task->s.iovs[0].iov_len = nbytes;
449 1 : accel_task->s.iovcnt = 1;
450 1 : accel_task->nbytes = nbytes;
451 1 : accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST;
452 1 : accel_task->src_domain = NULL;
453 1 : accel_task->dst_domain = NULL;
454 :
455 1 : return accel_submit_task(accel_ch, accel_task);
456 : }
457 :
458 : /* Accel framework public API for compare function */
459 :
460 : int
461 2 : spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
462 : void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
463 : void *cb_arg)
464 : {
465 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
466 : struct spdk_accel_task *accel_task;
467 :
468 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
469 2 : if (spdk_unlikely(accel_task == NULL)) {
470 1 : return -ENOMEM;
471 : }
472 :
473 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
474 :
475 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
476 1 : accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2];
477 1 : accel_task->s.iovs[0].iov_base = src1;
478 1 : accel_task->s.iovs[0].iov_len = nbytes;
479 1 : accel_task->s.iovcnt = 1;
480 1 : accel_task->s2.iovs[0].iov_base = src2;
481 1 : accel_task->s2.iovs[0].iov_len = nbytes;
482 1 : accel_task->s2.iovcnt = 1;
483 1 : accel_task->nbytes = nbytes;
484 1 : accel_task->op_code = SPDK_ACCEL_OPC_COMPARE;
485 1 : accel_task->src_domain = NULL;
486 1 : accel_task->dst_domain = NULL;
487 :
488 1 : return accel_submit_task(accel_ch, accel_task);
489 : }
490 :
491 : /* Accel framework public API for fill function */
492 : int
493 2 : spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
494 : uint8_t fill, uint64_t nbytes,
495 : spdk_accel_completion_cb cb_fn, void *cb_arg)
496 : {
497 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
498 : struct spdk_accel_task *accel_task;
499 :
500 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
501 2 : if (spdk_unlikely(accel_task == NULL)) {
502 1 : return -ENOMEM;
503 : }
504 :
505 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
506 :
507 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
508 1 : accel_task->d.iovs[0].iov_base = dst;
509 1 : accel_task->d.iovs[0].iov_len = nbytes;
510 1 : accel_task->d.iovcnt = 1;
511 1 : accel_task->nbytes = nbytes;
512 1 : memset(&accel_task->fill_pattern, fill, sizeof(uint64_t));
513 1 : accel_task->op_code = SPDK_ACCEL_OPC_FILL;
514 1 : accel_task->src_domain = NULL;
515 1 : accel_task->dst_domain = NULL;
516 :
517 1 : return accel_submit_task(accel_ch, accel_task);
518 : }
519 :
520 : /* Accel framework public API for CRC-32C function */
521 : int
522 2 : spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
523 : void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
524 : void *cb_arg)
525 : {
526 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
527 : struct spdk_accel_task *accel_task;
528 :
529 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
530 2 : if (spdk_unlikely(accel_task == NULL)) {
531 1 : return -ENOMEM;
532 : }
533 :
534 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
535 :
536 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
537 1 : accel_task->s.iovs[0].iov_base = src;
538 1 : accel_task->s.iovs[0].iov_len = nbytes;
539 1 : accel_task->s.iovcnt = 1;
540 1 : accel_task->nbytes = nbytes;
541 1 : accel_task->crc_dst = crc_dst;
542 1 : accel_task->seed = seed;
543 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
544 1 : accel_task->src_domain = NULL;
545 1 : accel_task->dst_domain = NULL;
546 :
547 1 : return accel_submit_task(accel_ch, accel_task);
548 : }
549 :
550 : /* Accel framework public API for chained CRC-32C function */
551 : int
552 1 : spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
553 : struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
554 : spdk_accel_completion_cb cb_fn, void *cb_arg)
555 : {
556 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
557 : struct spdk_accel_task *accel_task;
558 :
559 1 : if (iov == NULL) {
560 0 : SPDK_ERRLOG("iov should not be NULL");
561 0 : return -EINVAL;
562 : }
563 :
564 1 : if (!iov_cnt) {
565 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
566 0 : return -EINVAL;
567 : }
568 :
569 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
570 1 : if (spdk_unlikely(accel_task == NULL)) {
571 0 : SPDK_ERRLOG("no memory\n");
572 0 : assert(0);
573 : return -ENOMEM;
574 : }
575 :
576 1 : accel_task->s.iovs = iov;
577 1 : accel_task->s.iovcnt = iov_cnt;
578 1 : accel_task->nbytes = accel_get_iovlen(iov, iov_cnt);
579 1 : accel_task->crc_dst = crc_dst;
580 1 : accel_task->seed = seed;
581 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
582 1 : accel_task->src_domain = NULL;
583 1 : accel_task->dst_domain = NULL;
584 :
585 1 : return accel_submit_task(accel_ch, accel_task);
586 : }
587 :
588 : /* Accel framework public API for copy with CRC-32C function */
589 : int
590 2 : spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
591 : void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
592 : spdk_accel_completion_cb cb_fn, void *cb_arg)
593 : {
594 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
595 : struct spdk_accel_task *accel_task;
596 :
597 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
598 2 : if (spdk_unlikely(accel_task == NULL)) {
599 1 : return -ENOMEM;
600 : }
601 :
602 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
603 :
604 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
605 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
606 1 : accel_task->d.iovs[0].iov_base = dst;
607 1 : accel_task->d.iovs[0].iov_len = nbytes;
608 1 : accel_task->d.iovcnt = 1;
609 1 : accel_task->s.iovs[0].iov_base = src;
610 1 : accel_task->s.iovs[0].iov_len = nbytes;
611 1 : accel_task->s.iovcnt = 1;
612 1 : accel_task->nbytes = nbytes;
613 1 : accel_task->crc_dst = crc_dst;
614 1 : accel_task->seed = seed;
615 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
616 1 : accel_task->src_domain = NULL;
617 1 : accel_task->dst_domain = NULL;
618 :
619 1 : return accel_submit_task(accel_ch, accel_task);
620 : }
621 :
622 : /* Accel framework public API for chained copy + CRC-32C function */
623 : int
624 0 : spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
625 : struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
626 : uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
627 : {
628 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
629 : struct spdk_accel_task *accel_task;
630 : uint64_t nbytes;
631 :
632 0 : if (src_iovs == NULL) {
633 0 : SPDK_ERRLOG("iov should not be NULL");
634 0 : return -EINVAL;
635 : }
636 :
637 0 : if (!iov_cnt) {
638 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
639 0 : return -EINVAL;
640 : }
641 :
642 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
643 0 : if (spdk_unlikely(accel_task == NULL)) {
644 0 : SPDK_ERRLOG("no memory\n");
645 0 : assert(0);
646 : return -ENOMEM;
647 : }
648 :
649 0 : nbytes = accel_get_iovlen(src_iovs, iov_cnt);
650 :
651 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
652 :
653 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
654 0 : accel_task->d.iovs[0].iov_base = dst;
655 0 : accel_task->d.iovs[0].iov_len = nbytes;
656 0 : accel_task->d.iovcnt = 1;
657 0 : accel_task->s.iovs = src_iovs;
658 0 : accel_task->s.iovcnt = iov_cnt;
659 0 : accel_task->nbytes = nbytes;
660 0 : accel_task->crc_dst = crc_dst;
661 0 : accel_task->seed = seed;
662 0 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
663 0 : accel_task->src_domain = NULL;
664 0 : accel_task->dst_domain = NULL;
665 :
666 0 : return accel_submit_task(accel_ch, accel_task);
667 : }
668 :
669 : int
670 0 : spdk_accel_get_compress_level_range(enum spdk_accel_comp_algo comp_algo,
671 : uint32_t *min_level, uint32_t *max_level)
672 : {
673 0 : struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module;
674 :
675 0 : if (module->get_compress_level_range == NULL) {
676 0 : SPDK_ERRLOG("Module %s doesn't implement callback fn get_compress_level_range.\n", module->name);
677 0 : return -ENOTSUP;
678 : }
679 :
680 0 : return module->get_compress_level_range(comp_algo, min_level, max_level);
681 : }
682 :
683 : static int
684 49 : _accel_check_comp_algo(enum spdk_accel_comp_algo comp_algo)
685 : {
686 49 : struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module;
687 :
688 49 : if (!module->compress_supports_algo || !module->compress_supports_algo(comp_algo)) {
689 0 : SPDK_ERRLOG("Module %s doesn't support compression algo %d\n", module->name, comp_algo);
690 0 : return -ENOTSUP;
691 : }
692 :
693 49 : return 0;
694 : }
695 :
696 : int
697 2 : spdk_accel_submit_compress_ext(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
698 : struct iovec *src_iovs, size_t src_iovcnt,
699 : enum spdk_accel_comp_algo comp_algo, uint32_t comp_level,
700 : uint32_t *output_size, spdk_accel_completion_cb cb_fn, void *cb_arg)
701 : {
702 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
703 : struct spdk_accel_task *accel_task;
704 : int rc;
705 :
706 2 : rc = _accel_check_comp_algo(comp_algo);
707 2 : if (spdk_unlikely(rc != 0)) {
708 0 : return rc;
709 : }
710 :
711 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
712 2 : if (spdk_unlikely(accel_task == NULL)) {
713 0 : return -ENOMEM;
714 : }
715 :
716 2 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
717 :
718 2 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
719 2 : accel_task->d.iovs[0].iov_base = dst;
720 2 : accel_task->d.iovs[0].iov_len = nbytes;
721 2 : accel_task->d.iovcnt = 1;
722 2 : accel_task->output_size = output_size;
723 2 : accel_task->s.iovs = src_iovs;
724 2 : accel_task->s.iovcnt = src_iovcnt;
725 2 : accel_task->nbytes = nbytes;
726 2 : accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS;
727 2 : accel_task->src_domain = NULL;
728 2 : accel_task->dst_domain = NULL;
729 2 : accel_task->comp.algo = comp_algo;
730 2 : accel_task->comp.level = comp_level;
731 :
732 2 : return accel_submit_task(accel_ch, accel_task);
733 : }
734 :
735 : int
736 0 : spdk_accel_submit_decompress_ext(struct spdk_io_channel *ch, struct iovec *dst_iovs,
737 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
738 : enum spdk_accel_comp_algo decomp_algo, uint32_t *output_size,
739 : spdk_accel_completion_cb cb_fn, void *cb_arg)
740 : {
741 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
742 : struct spdk_accel_task *accel_task;
743 : int rc;
744 :
745 0 : rc = _accel_check_comp_algo(decomp_algo);
746 0 : if (spdk_unlikely(rc != 0)) {
747 0 : return rc;
748 : }
749 :
750 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
751 0 : if (spdk_unlikely(accel_task == NULL)) {
752 0 : return -ENOMEM;
753 : }
754 :
755 0 : accel_task->output_size = output_size;
756 0 : accel_task->s.iovs = src_iovs;
757 0 : accel_task->s.iovcnt = src_iovcnt;
758 0 : accel_task->d.iovs = dst_iovs;
759 0 : accel_task->d.iovcnt = dst_iovcnt;
760 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
761 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
762 0 : accel_task->src_domain = NULL;
763 0 : accel_task->dst_domain = NULL;
764 0 : accel_task->comp.algo = decomp_algo;
765 :
766 0 : return accel_submit_task(accel_ch, accel_task);
767 : }
768 :
769 : int
770 2 : spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
771 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size,
772 : spdk_accel_completion_cb cb_fn, void *cb_arg)
773 : {
774 2 : return spdk_accel_submit_compress_ext(ch, dst, nbytes, src_iovs, src_iovcnt,
775 : SPDK_ACCEL_COMP_ALGO_DEFLATE, 1, output_size, cb_fn, cb_arg);
776 : }
777 :
778 : int
779 0 : spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
780 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
781 : uint32_t *output_size, spdk_accel_completion_cb cb_fn,
782 : void *cb_arg)
783 : {
784 0 : return spdk_accel_submit_decompress_ext(ch, dst_iovs, dst_iovcnt, src_iovs, src_iovcnt,
785 : SPDK_ACCEL_COMP_ALGO_DEFLATE, output_size, cb_fn, cb_arg);
786 : }
787 :
788 : int
789 1 : spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
790 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
791 : struct iovec *src_iovs, uint32_t src_iovcnt,
792 : uint64_t iv, uint32_t block_size,
793 : spdk_accel_completion_cb cb_fn, void *cb_arg)
794 : {
795 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
796 : struct spdk_accel_task *accel_task;
797 :
798 1 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
799 0 : return -EINVAL;
800 : }
801 :
802 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
803 1 : if (spdk_unlikely(accel_task == NULL)) {
804 0 : return -ENOMEM;
805 : }
806 :
807 1 : accel_task->crypto_key = key;
808 1 : accel_task->s.iovs = src_iovs;
809 1 : accel_task->s.iovcnt = src_iovcnt;
810 1 : accel_task->d.iovs = dst_iovs;
811 1 : accel_task->d.iovcnt = dst_iovcnt;
812 1 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
813 1 : accel_task->iv = iv;
814 1 : accel_task->block_size = block_size;
815 1 : accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
816 1 : accel_task->src_domain = NULL;
817 1 : accel_task->dst_domain = NULL;
818 :
819 1 : return accel_submit_task(accel_ch, accel_task);
820 : }
821 :
822 : int
823 0 : spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
824 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
825 : struct iovec *src_iovs, uint32_t src_iovcnt,
826 : uint64_t iv, uint32_t block_size,
827 : spdk_accel_completion_cb cb_fn, void *cb_arg)
828 : {
829 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
830 : struct spdk_accel_task *accel_task;
831 :
832 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
833 0 : return -EINVAL;
834 : }
835 :
836 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
837 0 : if (spdk_unlikely(accel_task == NULL)) {
838 0 : return -ENOMEM;
839 : }
840 :
841 0 : accel_task->crypto_key = key;
842 0 : accel_task->s.iovs = src_iovs;
843 0 : accel_task->s.iovcnt = src_iovcnt;
844 0 : accel_task->d.iovs = dst_iovs;
845 0 : accel_task->d.iovcnt = dst_iovcnt;
846 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
847 0 : accel_task->iv = iv;
848 0 : accel_task->block_size = block_size;
849 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT;
850 0 : accel_task->src_domain = NULL;
851 0 : accel_task->dst_domain = NULL;
852 :
853 0 : return accel_submit_task(accel_ch, accel_task);
854 : }
855 :
856 : int
857 2 : spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs,
858 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
859 : {
860 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
861 : struct spdk_accel_task *accel_task;
862 :
863 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
864 2 : if (spdk_unlikely(accel_task == NULL)) {
865 1 : return -ENOMEM;
866 : }
867 :
868 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
869 :
870 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
871 1 : accel_task->nsrcs.srcs = sources;
872 1 : accel_task->nsrcs.cnt = nsrcs;
873 1 : accel_task->d.iovs[0].iov_base = dst;
874 1 : accel_task->d.iovs[0].iov_len = nbytes;
875 1 : accel_task->d.iovcnt = 1;
876 1 : accel_task->nbytes = nbytes;
877 1 : accel_task->op_code = SPDK_ACCEL_OPC_XOR;
878 1 : accel_task->src_domain = NULL;
879 1 : accel_task->dst_domain = NULL;
880 :
881 1 : return accel_submit_task(accel_ch, accel_task);
882 : }
883 :
884 : int
885 0 : spdk_accel_submit_dif_verify(struct spdk_io_channel *ch,
886 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
887 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
888 : spdk_accel_completion_cb cb_fn, void *cb_arg)
889 : {
890 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
891 : struct spdk_accel_task *accel_task;
892 :
893 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
894 0 : if (accel_task == NULL) {
895 0 : return -ENOMEM;
896 : }
897 :
898 0 : accel_task->s.iovs = iovs;
899 0 : accel_task->s.iovcnt = iovcnt;
900 0 : accel_task->dif.ctx = ctx;
901 0 : accel_task->dif.err = err;
902 0 : accel_task->dif.num_blocks = num_blocks;
903 0 : accel_task->nbytes = num_blocks * ctx->block_size;
904 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
905 0 : accel_task->src_domain = NULL;
906 0 : accel_task->dst_domain = NULL;
907 :
908 0 : return accel_submit_task(accel_ch, accel_task);
909 : }
910 :
911 : int
912 0 : spdk_accel_submit_dif_generate(struct spdk_io_channel *ch,
913 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
914 : const struct spdk_dif_ctx *ctx,
915 : spdk_accel_completion_cb cb_fn, void *cb_arg)
916 : {
917 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
918 : struct spdk_accel_task *accel_task;
919 :
920 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
921 0 : if (accel_task == NULL) {
922 0 : return -ENOMEM;
923 : }
924 :
925 0 : accel_task->s.iovs = iovs;
926 0 : accel_task->s.iovcnt = iovcnt;
927 0 : accel_task->dif.ctx = ctx;
928 0 : accel_task->dif.num_blocks = num_blocks;
929 0 : accel_task->nbytes = num_blocks * ctx->block_size;
930 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
931 0 : accel_task->src_domain = NULL;
932 0 : accel_task->dst_domain = NULL;
933 :
934 0 : return accel_submit_task(accel_ch, accel_task);
935 : }
936 :
937 : int
938 0 : spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs,
939 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
940 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
941 : spdk_accel_completion_cb cb_fn, void *cb_arg)
942 : {
943 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
944 : struct spdk_accel_task *accel_task;
945 :
946 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
947 0 : if (accel_task == NULL) {
948 0 : return -ENOMEM;
949 : }
950 :
951 0 : accel_task->s.iovs = src_iovs;
952 0 : accel_task->s.iovcnt = src_iovcnt;
953 0 : accel_task->d.iovs = dst_iovs;
954 0 : accel_task->d.iovcnt = dst_iovcnt;
955 0 : accel_task->dif.ctx = ctx;
956 0 : accel_task->dif.num_blocks = num_blocks;
957 0 : accel_task->nbytes = num_blocks * ctx->block_size;
958 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
959 0 : accel_task->src_domain = NULL;
960 0 : accel_task->dst_domain = NULL;
961 :
962 0 : return accel_submit_task(accel_ch, accel_task);
963 : }
964 :
965 : int
966 0 : spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch,
967 : struct iovec *dst_iovs, size_t dst_iovcnt,
968 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks,
969 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
970 : spdk_accel_completion_cb cb_fn, void *cb_arg)
971 : {
972 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
973 : struct spdk_accel_task *accel_task;
974 :
975 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
976 0 : if (accel_task == NULL) {
977 0 : return -ENOMEM;
978 : }
979 :
980 0 : accel_task->s.iovs = src_iovs;
981 0 : accel_task->s.iovcnt = src_iovcnt;
982 0 : accel_task->d.iovs = dst_iovs;
983 0 : accel_task->d.iovcnt = dst_iovcnt;
984 0 : accel_task->dif.ctx = ctx;
985 0 : accel_task->dif.err = err;
986 0 : accel_task->dif.num_blocks = num_blocks;
987 0 : accel_task->nbytes = num_blocks * ctx->block_size;
988 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
989 0 : accel_task->src_domain = NULL;
990 0 : accel_task->dst_domain = NULL;
991 :
992 0 : return accel_submit_task(accel_ch, accel_task);
993 : }
994 :
995 : int
996 0 : spdk_accel_submit_dix_generate(struct spdk_io_channel *ch, struct iovec *iovs,
997 : size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks,
998 : const struct spdk_dif_ctx *ctx, spdk_accel_completion_cb cb_fn,
999 : void *cb_arg)
1000 : {
1001 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1002 : struct spdk_accel_task *accel_task;
1003 :
1004 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
1005 0 : if (accel_task == NULL) {
1006 0 : return -ENOMEM;
1007 : }
1008 :
1009 0 : accel_task->s.iovs = iovs;
1010 0 : accel_task->s.iovcnt = iovcnt;
1011 0 : accel_task->d.iovs = md_iov;
1012 0 : accel_task->d.iovcnt = 1;
1013 0 : accel_task->dif.ctx = ctx;
1014 0 : accel_task->dif.num_blocks = num_blocks;
1015 0 : accel_task->nbytes = num_blocks * ctx->block_size;
1016 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE;
1017 0 : accel_task->src_domain = NULL;
1018 0 : accel_task->dst_domain = NULL;
1019 :
1020 0 : return accel_submit_task(accel_ch, accel_task);
1021 : }
1022 :
1023 : int
1024 0 : spdk_accel_submit_dix_verify(struct spdk_io_channel *ch, struct iovec *iovs,
1025 : size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks,
1026 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1027 : spdk_accel_completion_cb cb_fn, void *cb_arg)
1028 : {
1029 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1030 : struct spdk_accel_task *accel_task;
1031 :
1032 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
1033 0 : if (accel_task == NULL) {
1034 0 : return -ENOMEM;
1035 : }
1036 :
1037 0 : accel_task->s.iovs = iovs;
1038 0 : accel_task->s.iovcnt = iovcnt;
1039 0 : accel_task->d.iovs = md_iov;
1040 0 : accel_task->d.iovcnt = 1;
1041 0 : accel_task->dif.ctx = ctx;
1042 0 : accel_task->dif.err = err;
1043 0 : accel_task->dif.num_blocks = num_blocks;
1044 0 : accel_task->nbytes = num_blocks * ctx->block_size;
1045 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY;
1046 0 : accel_task->src_domain = NULL;
1047 0 : accel_task->dst_domain = NULL;
1048 :
1049 0 : return accel_submit_task(accel_ch, accel_task);
1050 : }
1051 :
1052 : static inline struct accel_buffer *
1053 33 : accel_get_buf(struct accel_io_channel *ch, uint64_t len)
1054 : {
1055 : struct accel_buffer *buf;
1056 :
1057 33 : buf = SLIST_FIRST(&ch->buf_pool);
1058 33 : if (spdk_unlikely(buf == NULL)) {
1059 0 : accel_update_stats(ch, retry.bufdesc, 1);
1060 0 : return NULL;
1061 : }
1062 :
1063 33 : SLIST_REMOVE_HEAD(&ch->buf_pool, link);
1064 33 : buf->len = len;
1065 33 : buf->buf = NULL;
1066 33 : buf->seq = NULL;
1067 33 : buf->cb_fn = NULL;
1068 :
1069 33 : return buf;
1070 : }
1071 :
1072 : static inline void
1073 33 : accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
1074 : {
1075 33 : if (buf->buf != NULL) {
1076 29 : spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len);
1077 : }
1078 :
1079 33 : SLIST_INSERT_HEAD(&ch->buf_pool, buf, link);
1080 33 : }
1081 :
1082 : static inline struct spdk_accel_sequence *
1083 73 : accel_sequence_get(struct accel_io_channel *ch)
1084 : {
1085 : struct spdk_accel_sequence *seq;
1086 :
1087 73 : assert(g_opts.task_count >= ch->stats.task_outstanding);
1088 :
1089 : /* Sequence cannot be allocated if number of available task objects cannot satisfy required limit.
1090 : * This is to prevent potential dead lock when few requests are pending task resource and none can
1091 : * advance the processing. This solution should work only if there is single async operation after
1092 : * sequence obj obtained, so assume that is possible to happen with io buffer allocation now, if
1093 : * there are more async operations then solution should be improved. */
1094 73 : if (spdk_unlikely(g_opts.task_count - ch->stats.task_outstanding < ACCEL_TASKS_IN_SEQUENCE_LIMIT)) {
1095 0 : return NULL;
1096 : }
1097 :
1098 73 : seq = SLIST_FIRST(&ch->seq_pool);
1099 73 : if (spdk_unlikely(seq == NULL)) {
1100 3 : accel_update_stats(ch, retry.sequence, 1);
1101 3 : return NULL;
1102 : }
1103 :
1104 70 : accel_update_stats(ch, sequence_outstanding, 1);
1105 70 : SLIST_REMOVE_HEAD(&ch->seq_pool, link);
1106 :
1107 70 : TAILQ_INIT(&seq->tasks);
1108 70 : SLIST_INIT(&seq->bounce_bufs);
1109 :
1110 70 : seq->ch = ch;
1111 70 : seq->status = 0;
1112 70 : seq->state = ACCEL_SEQUENCE_STATE_INIT;
1113 70 : seq->in_process_sequence = false;
1114 :
1115 70 : return seq;
1116 : }
1117 :
1118 : static inline void
1119 70 : accel_sequence_put(struct spdk_accel_sequence *seq)
1120 : {
1121 70 : struct accel_io_channel *ch = seq->ch;
1122 : struct accel_buffer *buf;
1123 :
1124 89 : while (!SLIST_EMPTY(&seq->bounce_bufs)) {
1125 19 : buf = SLIST_FIRST(&seq->bounce_bufs);
1126 19 : SLIST_REMOVE_HEAD(&seq->bounce_bufs, link);
1127 19 : accel_put_buf(seq->ch, buf);
1128 : }
1129 :
1130 70 : assert(TAILQ_EMPTY(&seq->tasks));
1131 70 : seq->ch = NULL;
1132 :
1133 70 : SLIST_INSERT_HEAD(&ch->seq_pool, seq, link);
1134 70 : accel_update_stats(ch, sequence_outstanding, -1);
1135 70 : }
1136 :
1137 : static inline struct spdk_accel_task *
1138 171 : accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq,
1139 : spdk_accel_step_cb cb_fn, void *cb_arg)
1140 : {
1141 : struct spdk_accel_task *task;
1142 :
1143 171 : task = _get_task(ch, NULL, NULL);
1144 171 : if (spdk_unlikely(task == NULL)) {
1145 3 : return task;
1146 : }
1147 :
1148 168 : task->step_cb_fn = cb_fn;
1149 168 : task->cb_arg = cb_arg;
1150 168 : task->seq = seq;
1151 :
1152 168 : return task;
1153 : }
1154 :
1155 : int
1156 48 : spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1157 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1158 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1159 : struct iovec *src_iovs, uint32_t src_iovcnt,
1160 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1161 : spdk_accel_step_cb cb_fn, void *cb_arg)
1162 : {
1163 48 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1164 : struct spdk_accel_task *task;
1165 48 : struct spdk_accel_sequence *seq = *pseq;
1166 :
1167 48 : if (seq == NULL) {
1168 17 : seq = accel_sequence_get(accel_ch);
1169 17 : if (spdk_unlikely(seq == NULL)) {
1170 1 : return -ENOMEM;
1171 : }
1172 : }
1173 :
1174 47 : assert(seq->ch == accel_ch);
1175 47 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1176 47 : if (spdk_unlikely(task == NULL)) {
1177 1 : if (*pseq == NULL) {
1178 1 : accel_sequence_put(seq);
1179 : }
1180 :
1181 1 : return -ENOMEM;
1182 : }
1183 :
1184 46 : task->dst_domain = dst_domain;
1185 46 : task->dst_domain_ctx = dst_domain_ctx;
1186 46 : task->d.iovs = dst_iovs;
1187 46 : task->d.iovcnt = dst_iovcnt;
1188 46 : task->src_domain = src_domain;
1189 46 : task->src_domain_ctx = src_domain_ctx;
1190 46 : task->s.iovs = src_iovs;
1191 46 : task->s.iovcnt = src_iovcnt;
1192 46 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1193 46 : task->op_code = SPDK_ACCEL_OPC_COPY;
1194 :
1195 46 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1196 46 : *pseq = seq;
1197 :
1198 46 : return 0;
1199 : }
1200 :
1201 : int
1202 42 : spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1203 : void *buf, uint64_t len,
1204 : struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
1205 : spdk_accel_step_cb cb_fn, void *cb_arg)
1206 : {
1207 42 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1208 : struct spdk_accel_task *task;
1209 42 : struct spdk_accel_sequence *seq = *pseq;
1210 :
1211 42 : if (seq == NULL) {
1212 24 : seq = accel_sequence_get(accel_ch);
1213 24 : if (spdk_unlikely(seq == NULL)) {
1214 1 : return -ENOMEM;
1215 : }
1216 : }
1217 :
1218 41 : assert(seq->ch == accel_ch);
1219 41 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1220 41 : if (spdk_unlikely(task == NULL)) {
1221 1 : if (*pseq == NULL) {
1222 1 : accel_sequence_put(seq);
1223 : }
1224 :
1225 1 : return -ENOMEM;
1226 : }
1227 :
1228 40 : memset(&task->fill_pattern, pattern, sizeof(uint64_t));
1229 :
1230 40 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1231 40 : if (spdk_unlikely(!task->aux)) {
1232 0 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");
1233 0 : if (*pseq == NULL) {
1234 0 : accel_sequence_put((seq));
1235 : }
1236 :
1237 0 : task->seq = NULL;
1238 0 : _put_task(task->accel_ch, task);
1239 0 : assert(0);
1240 : return -ENOMEM;
1241 : }
1242 40 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1243 40 : task->has_aux = true;
1244 :
1245 40 : task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
1246 40 : task->d.iovs[0].iov_base = buf;
1247 40 : task->d.iovs[0].iov_len = len;
1248 40 : task->d.iovcnt = 1;
1249 40 : task->nbytes = len;
1250 40 : task->src_domain = NULL;
1251 40 : task->dst_domain = domain;
1252 40 : task->dst_domain_ctx = domain_ctx;
1253 40 : task->op_code = SPDK_ACCEL_OPC_FILL;
1254 :
1255 40 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1256 40 : *pseq = seq;
1257 :
1258 40 : return 0;
1259 : }
1260 :
1261 : int
1262 47 : spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1263 : struct iovec *dst_iovs, size_t dst_iovcnt,
1264 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1265 : struct iovec *src_iovs, size_t src_iovcnt,
1266 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1267 : spdk_accel_step_cb cb_fn, void *cb_arg)
1268 : {
1269 47 : return spdk_accel_append_decompress_ext(pseq, ch, dst_iovs, dst_iovcnt, dst_domain,
1270 : dst_domain_ctx, src_iovs, src_iovcnt, src_domain,
1271 : src_domain_ctx, SPDK_ACCEL_COMP_ALGO_DEFLATE,
1272 : cb_fn, cb_arg);
1273 : }
1274 :
1275 : int
1276 47 : spdk_accel_append_decompress_ext(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1277 : struct iovec *dst_iovs, size_t dst_iovcnt,
1278 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1279 : struct iovec *src_iovs, size_t src_iovcnt,
1280 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1281 : enum spdk_accel_comp_algo decomp_algo,
1282 : spdk_accel_step_cb cb_fn, void *cb_arg)
1283 : {
1284 47 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1285 : struct spdk_accel_task *task;
1286 47 : struct spdk_accel_sequence *seq = *pseq;
1287 : int rc;
1288 :
1289 47 : rc = _accel_check_comp_algo(decomp_algo);
1290 47 : if (spdk_unlikely(rc != 0)) {
1291 0 : return rc;
1292 : }
1293 :
1294 47 : if (seq == NULL) {
1295 21 : seq = accel_sequence_get(accel_ch);
1296 21 : if (spdk_unlikely(seq == NULL)) {
1297 1 : return -ENOMEM;
1298 : }
1299 : }
1300 :
1301 46 : assert(seq->ch == accel_ch);
1302 46 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1303 46 : if (spdk_unlikely(task == NULL)) {
1304 1 : if (*pseq == NULL) {
1305 1 : accel_sequence_put(seq);
1306 : }
1307 :
1308 1 : return -ENOMEM;
1309 : }
1310 :
1311 : /* TODO: support output_size for chaining */
1312 45 : task->output_size = NULL;
1313 45 : task->dst_domain = dst_domain;
1314 45 : task->dst_domain_ctx = dst_domain_ctx;
1315 45 : task->d.iovs = dst_iovs;
1316 45 : task->d.iovcnt = dst_iovcnt;
1317 45 : task->src_domain = src_domain;
1318 45 : task->src_domain_ctx = src_domain_ctx;
1319 45 : task->s.iovs = src_iovs;
1320 45 : task->s.iovcnt = src_iovcnt;
1321 45 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1322 45 : task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
1323 45 : task->comp.algo = decomp_algo;
1324 :
1325 45 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1326 45 : *pseq = seq;
1327 :
1328 45 : return 0;
1329 : }
1330 :
1331 : int
1332 11 : spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1333 : struct spdk_accel_crypto_key *key,
1334 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1335 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1336 : struct iovec *src_iovs, uint32_t src_iovcnt,
1337 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1338 : uint64_t iv, uint32_t block_size,
1339 : spdk_accel_step_cb cb_fn, void *cb_arg)
1340 : {
1341 11 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1342 : struct spdk_accel_task *task;
1343 11 : struct spdk_accel_sequence *seq = *pseq;
1344 :
1345 11 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1346 :
1347 11 : if (seq == NULL) {
1348 7 : seq = accel_sequence_get(accel_ch);
1349 7 : if (spdk_unlikely(seq == NULL)) {
1350 0 : return -ENOMEM;
1351 : }
1352 : }
1353 :
1354 11 : assert(seq->ch == accel_ch);
1355 11 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1356 11 : if (spdk_unlikely(task == NULL)) {
1357 0 : if (*pseq == NULL) {
1358 0 : accel_sequence_put(seq);
1359 : }
1360 :
1361 0 : return -ENOMEM;
1362 : }
1363 :
1364 11 : task->crypto_key = key;
1365 11 : task->src_domain = src_domain;
1366 11 : task->src_domain_ctx = src_domain_ctx;
1367 11 : task->s.iovs = src_iovs;
1368 11 : task->s.iovcnt = src_iovcnt;
1369 11 : task->dst_domain = dst_domain;
1370 11 : task->dst_domain_ctx = dst_domain_ctx;
1371 11 : task->d.iovs = dst_iovs;
1372 11 : task->d.iovcnt = dst_iovcnt;
1373 11 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1374 11 : task->iv = iv;
1375 11 : task->block_size = block_size;
1376 11 : task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
1377 :
1378 11 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1379 11 : *pseq = seq;
1380 :
1381 11 : return 0;
1382 : }
1383 :
1384 : int
1385 12 : spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1386 : struct spdk_accel_crypto_key *key,
1387 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1388 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1389 : struct iovec *src_iovs, uint32_t src_iovcnt,
1390 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1391 : uint64_t iv, uint32_t block_size,
1392 : spdk_accel_step_cb cb_fn, void *cb_arg)
1393 : {
1394 12 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1395 : struct spdk_accel_task *task;
1396 12 : struct spdk_accel_sequence *seq = *pseq;
1397 :
1398 12 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1399 :
1400 12 : if (seq == NULL) {
1401 1 : seq = accel_sequence_get(accel_ch);
1402 1 : if (spdk_unlikely(seq == NULL)) {
1403 0 : return -ENOMEM;
1404 : }
1405 : }
1406 :
1407 12 : assert(seq->ch == accel_ch);
1408 12 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1409 12 : if (spdk_unlikely(task == NULL)) {
1410 0 : if (*pseq == NULL) {
1411 0 : accel_sequence_put(seq);
1412 : }
1413 :
1414 0 : return -ENOMEM;
1415 : }
1416 :
1417 12 : task->crypto_key = key;
1418 12 : task->src_domain = src_domain;
1419 12 : task->src_domain_ctx = src_domain_ctx;
1420 12 : task->s.iovs = src_iovs;
1421 12 : task->s.iovcnt = src_iovcnt;
1422 12 : task->dst_domain = dst_domain;
1423 12 : task->dst_domain_ctx = dst_domain_ctx;
1424 12 : task->d.iovs = dst_iovs;
1425 12 : task->d.iovcnt = dst_iovcnt;
1426 12 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1427 12 : task->iv = iv;
1428 12 : task->block_size = block_size;
1429 12 : task->op_code = SPDK_ACCEL_OPC_DECRYPT;
1430 :
1431 12 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1432 12 : *pseq = seq;
1433 :
1434 12 : return 0;
1435 : }
1436 :
1437 : int
1438 6 : spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1439 : uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
1440 : struct spdk_memory_domain *domain, void *domain_ctx,
1441 : uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)
1442 : {
1443 6 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1444 : struct spdk_accel_task *task;
1445 6 : struct spdk_accel_sequence *seq = *pseq;
1446 :
1447 6 : if (seq == NULL) {
1448 2 : seq = accel_sequence_get(accel_ch);
1449 2 : if (spdk_unlikely(seq == NULL)) {
1450 0 : return -ENOMEM;
1451 : }
1452 : }
1453 :
1454 6 : assert(seq->ch == accel_ch);
1455 6 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1456 6 : if (spdk_unlikely(task == NULL)) {
1457 0 : if (*pseq == NULL) {
1458 0 : accel_sequence_put(seq);
1459 : }
1460 :
1461 0 : return -ENOMEM;
1462 : }
1463 :
1464 6 : task->s.iovs = iovs;
1465 6 : task->s.iovcnt = iovcnt;
1466 6 : task->src_domain = domain;
1467 6 : task->src_domain_ctx = domain_ctx;
1468 6 : task->nbytes = accel_get_iovlen(iovs, iovcnt);
1469 6 : task->crc_dst = dst;
1470 6 : task->seed = seed;
1471 6 : task->op_code = SPDK_ACCEL_OPC_CRC32C;
1472 6 : task->dst_domain = NULL;
1473 :
1474 6 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1475 6 : *pseq = seq;
1476 :
1477 6 : return 0;
1478 : }
1479 :
1480 : int
1481 0 : spdk_accel_append_dif_verify(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1482 : struct iovec *iovs, size_t iovcnt,
1483 : struct spdk_memory_domain *domain, void *domain_ctx,
1484 : uint32_t num_blocks,
1485 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1486 : spdk_accel_step_cb cb_fn, void *cb_arg)
1487 : {
1488 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1489 : struct spdk_accel_task *task;
1490 0 : struct spdk_accel_sequence *seq = *pseq;
1491 :
1492 0 : if (seq == NULL) {
1493 0 : seq = accel_sequence_get(accel_ch);
1494 0 : if (spdk_unlikely(seq == NULL)) {
1495 0 : return -ENOMEM;
1496 : }
1497 : }
1498 :
1499 0 : assert(seq->ch == accel_ch);
1500 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1501 0 : if (spdk_unlikely(task == NULL)) {
1502 0 : if (*pseq == NULL) {
1503 0 : accel_sequence_put(seq);
1504 : }
1505 :
1506 0 : return -ENOMEM;
1507 : }
1508 :
1509 0 : task->s.iovs = iovs;
1510 0 : task->s.iovcnt = iovcnt;
1511 0 : task->src_domain = domain;
1512 0 : task->src_domain_ctx = domain_ctx;
1513 0 : task->dst_domain = NULL;
1514 0 : task->dif.ctx = ctx;
1515 0 : task->dif.err = err;
1516 0 : task->dif.num_blocks = num_blocks;
1517 0 : task->nbytes = num_blocks * ctx->block_size;
1518 0 : task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
1519 :
1520 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1521 0 : *pseq = seq;
1522 :
1523 0 : return 0;
1524 : }
1525 :
1526 : int
1527 0 : spdk_accel_append_dif_verify_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1528 : struct iovec *dst_iovs, size_t dst_iovcnt,
1529 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1530 : struct iovec *src_iovs, size_t src_iovcnt,
1531 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1532 : uint32_t num_blocks,
1533 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1534 : spdk_accel_step_cb cb_fn, void *cb_arg)
1535 : {
1536 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1537 : struct spdk_accel_task *task;
1538 0 : struct spdk_accel_sequence *seq = *pseq;
1539 :
1540 0 : if (seq == NULL) {
1541 0 : seq = accel_sequence_get(accel_ch);
1542 0 : if (spdk_unlikely(seq == NULL)) {
1543 0 : return -ENOMEM;
1544 : }
1545 : }
1546 :
1547 0 : assert(seq->ch == accel_ch);
1548 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1549 0 : if (spdk_unlikely(task == NULL)) {
1550 0 : if (*pseq == NULL) {
1551 0 : accel_sequence_put(seq);
1552 : }
1553 :
1554 0 : return -ENOMEM;
1555 : }
1556 :
1557 0 : task->dst_domain = dst_domain;
1558 0 : task->dst_domain_ctx = dst_domain_ctx;
1559 0 : task->d.iovs = dst_iovs;
1560 0 : task->d.iovcnt = dst_iovcnt;
1561 0 : task->src_domain = src_domain;
1562 0 : task->src_domain_ctx = src_domain_ctx;
1563 0 : task->s.iovs = src_iovs;
1564 0 : task->s.iovcnt = src_iovcnt;
1565 0 : task->dif.ctx = ctx;
1566 0 : task->dif.err = err;
1567 0 : task->dif.num_blocks = num_blocks;
1568 0 : task->nbytes = num_blocks * ctx->block_size;
1569 0 : task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
1570 :
1571 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1572 0 : *pseq = seq;
1573 :
1574 0 : return 0;
1575 : }
1576 :
1577 : int
1578 0 : spdk_accel_append_dif_generate(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1579 : struct iovec *iovs, size_t iovcnt,
1580 : struct spdk_memory_domain *domain, void *domain_ctx,
1581 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1582 : spdk_accel_step_cb cb_fn, void *cb_arg)
1583 : {
1584 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1585 : struct spdk_accel_task *task;
1586 0 : struct spdk_accel_sequence *seq = *pseq;
1587 :
1588 0 : if (seq == NULL) {
1589 0 : seq = accel_sequence_get(accel_ch);
1590 0 : if (spdk_unlikely(seq == NULL)) {
1591 0 : return -ENOMEM;
1592 : }
1593 : }
1594 :
1595 0 : assert(seq->ch == accel_ch);
1596 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1597 0 : if (spdk_unlikely(task == NULL)) {
1598 0 : if (*pseq == NULL) {
1599 0 : accel_sequence_put(seq);
1600 : }
1601 :
1602 0 : return -ENOMEM;
1603 : }
1604 :
1605 0 : task->s.iovs = iovs;
1606 0 : task->s.iovcnt = iovcnt;
1607 0 : task->src_domain = domain;
1608 0 : task->src_domain_ctx = domain_ctx;
1609 0 : task->dst_domain = NULL;
1610 0 : task->dif.ctx = ctx;
1611 0 : task->dif.num_blocks = num_blocks;
1612 0 : task->nbytes = num_blocks * ctx->block_size;
1613 0 : task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
1614 :
1615 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1616 0 : *pseq = seq;
1617 :
1618 0 : return 0;
1619 : }
1620 :
1621 : int
1622 0 : spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1623 : struct iovec *dst_iovs, size_t dst_iovcnt,
1624 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1625 : struct iovec *src_iovs, size_t src_iovcnt,
1626 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1627 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1628 : spdk_accel_step_cb cb_fn, void *cb_arg)
1629 : {
1630 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1631 : struct spdk_accel_task *task;
1632 0 : struct spdk_accel_sequence *seq = *pseq;
1633 :
1634 0 : if (seq == NULL) {
1635 0 : seq = accel_sequence_get(accel_ch);
1636 0 : if (spdk_unlikely(seq == NULL)) {
1637 0 : return -ENOMEM;
1638 : }
1639 : }
1640 :
1641 0 : assert(seq->ch == accel_ch);
1642 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1643 0 : if (spdk_unlikely(task == NULL)) {
1644 0 : if (*pseq == NULL) {
1645 0 : accel_sequence_put(seq);
1646 : }
1647 :
1648 0 : return -ENOMEM;
1649 : }
1650 :
1651 0 : task->dst_domain = dst_domain;
1652 0 : task->dst_domain_ctx = dst_domain_ctx;
1653 0 : task->d.iovs = dst_iovs;
1654 0 : task->d.iovcnt = dst_iovcnt;
1655 0 : task->src_domain = src_domain;
1656 0 : task->src_domain_ctx = src_domain_ctx;
1657 0 : task->s.iovs = src_iovs;
1658 0 : task->s.iovcnt = src_iovcnt;
1659 0 : task->nbytes = num_blocks * ctx->block_size;
1660 0 : task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
1661 :
1662 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1663 0 : *pseq = seq;
1664 :
1665 0 : return 0;
1666 : }
1667 :
1668 : int
1669 4 : spdk_accel_append_dix_generate(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
1670 : struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,
1671 : void *domain_ctx, struct iovec *md_iov,
1672 : struct spdk_memory_domain *md_domain, void *md_domain_ctx,
1673 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1674 : spdk_accel_step_cb cb_fn, void *cb_arg)
1675 : {
1676 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1677 : struct spdk_accel_task *task;
1678 4 : struct spdk_accel_sequence *pseq = *seq;
1679 :
1680 4 : if (pseq == NULL) {
1681 1 : pseq = accel_sequence_get(accel_ch);
1682 1 : if (spdk_unlikely(pseq == NULL)) {
1683 0 : return -ENOMEM;
1684 : }
1685 : }
1686 :
1687 4 : assert(pseq->ch == accel_ch);
1688 4 : task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);
1689 4 : if (spdk_unlikely(task == NULL)) {
1690 0 : if (*seq == NULL) {
1691 0 : accel_sequence_put(pseq);
1692 : }
1693 :
1694 0 : return -ENOMEM;
1695 : }
1696 :
1697 4 : task->d.iovs = md_iov;
1698 4 : task->d.iovcnt = 1;
1699 4 : task->dst_domain = md_domain;
1700 4 : task->dst_domain_ctx = md_domain_ctx;
1701 4 : task->s.iovs = iovs;
1702 4 : task->s.iovcnt = iovcnt;
1703 4 : task->src_domain = domain;
1704 4 : task->src_domain_ctx = domain_ctx;
1705 4 : task->dif.ctx = ctx;
1706 4 : task->dif.num_blocks = num_blocks;
1707 4 : task->nbytes = num_blocks * ctx->block_size;
1708 4 : task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE;
1709 :
1710 4 : TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);
1711 4 : *seq = pseq;
1712 :
1713 4 : return 0;
1714 : }
1715 :
1716 : int
1717 4 : spdk_accel_append_dix_verify(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
1718 : struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,
1719 : void *domain_ctx, struct iovec *md_iov,
1720 : struct spdk_memory_domain *md_domain, void *md_domain_ctx,
1721 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1722 : struct spdk_dif_error *err, spdk_accel_step_cb cb_fn, void *cb_arg)
1723 : {
1724 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1725 : struct spdk_accel_task *task;
1726 4 : struct spdk_accel_sequence *pseq = *seq;
1727 :
1728 4 : if (pseq == NULL) {
1729 0 : pseq = accel_sequence_get(accel_ch);
1730 0 : if (spdk_unlikely(pseq == NULL)) {
1731 0 : return -ENOMEM;
1732 : }
1733 : }
1734 :
1735 4 : assert(pseq->ch == accel_ch);
1736 4 : task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);
1737 4 : if (spdk_unlikely(task == NULL)) {
1738 0 : if (*seq == NULL) {
1739 0 : accel_sequence_put(pseq);
1740 : }
1741 :
1742 0 : return -ENOMEM;
1743 : }
1744 :
1745 4 : task->d.iovs = md_iov;
1746 4 : task->d.iovcnt = 1;
1747 4 : task->dst_domain = md_domain;
1748 4 : task->dst_domain_ctx = md_domain_ctx;
1749 4 : task->s.iovs = iovs;
1750 4 : task->s.iovcnt = iovcnt;
1751 4 : task->src_domain = domain;
1752 4 : task->src_domain_ctx = domain_ctx;
1753 4 : task->dif.ctx = ctx;
1754 4 : task->dif.err = err;
1755 4 : task->dif.num_blocks = num_blocks;
1756 4 : task->nbytes = num_blocks * ctx->block_size;
1757 4 : task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY;
1758 :
1759 4 : TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);
1760 4 : *seq = pseq;
1761 :
1762 4 : return 0;
1763 : }
1764 :
1765 : int
1766 14 : spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
1767 : struct spdk_memory_domain **domain, void **domain_ctx)
1768 : {
1769 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1770 : struct accel_buffer *accel_buf;
1771 :
1772 14 : accel_buf = accel_get_buf(accel_ch, len);
1773 14 : if (spdk_unlikely(accel_buf == NULL)) {
1774 0 : return -ENOMEM;
1775 : }
1776 :
1777 14 : accel_buf->ch = accel_ch;
1778 :
1779 : /* We always return the same pointer and identify the buffers through domain_ctx */
1780 14 : *buf = ACCEL_BUFFER_BASE;
1781 14 : *domain_ctx = accel_buf;
1782 14 : *domain = g_accel_domain;
1783 :
1784 14 : return 0;
1785 : }
1786 :
1787 : void
1788 14 : spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
1789 : struct spdk_memory_domain *domain, void *domain_ctx)
1790 : {
1791 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1792 14 : struct accel_buffer *accel_buf = domain_ctx;
1793 :
1794 14 : assert(domain == g_accel_domain);
1795 14 : assert(buf == ACCEL_BUFFER_BASE);
1796 :
1797 14 : accel_put_buf(accel_ch, accel_buf);
1798 14 : }
1799 :
1800 : static void
1801 168 : accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1802 : {
1803 168 : struct accel_io_channel *ch = seq->ch;
1804 : spdk_accel_step_cb cb_fn;
1805 : void *cb_arg;
1806 :
1807 168 : TAILQ_REMOVE(&seq->tasks, task, seq_link);
1808 168 : cb_fn = task->step_cb_fn;
1809 168 : cb_arg = task->cb_arg;
1810 168 : task->seq = NULL;
1811 168 : if (task->has_aux) {
1812 61 : SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link);
1813 61 : task->aux = NULL;
1814 61 : task->has_aux = false;
1815 : }
1816 :
1817 168 : _put_task(ch, task);
1818 :
1819 168 : if (cb_fn != NULL) {
1820 168 : cb_fn(cb_arg);
1821 : }
1822 168 : }
1823 :
1824 : static void
1825 67 : accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
1826 : {
1827 : struct spdk_accel_task *task;
1828 :
1829 85 : while (!TAILQ_EMPTY(&seq->tasks)) {
1830 18 : task = TAILQ_FIRST(&seq->tasks);
1831 18 : accel_sequence_complete_task(seq, task);
1832 : }
1833 67 : }
1834 :
1835 : static void
1836 65 : accel_sequence_complete(struct spdk_accel_sequence *seq)
1837 : {
1838 65 : spdk_accel_completion_cb cb_fn = seq->cb_fn;
1839 65 : void *cb_arg = seq->cb_arg;
1840 65 : int status = seq->status;
1841 :
1842 65 : SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status);
1843 :
1844 65 : accel_update_stats(seq->ch, sequence_executed, 1);
1845 65 : if (spdk_unlikely(status != 0)) {
1846 10 : accel_update_stats(seq->ch, sequence_failed, 1);
1847 : }
1848 :
1849 : /* First notify all users that appended operations to this sequence */
1850 65 : accel_sequence_complete_tasks(seq);
1851 65 : accel_sequence_put(seq);
1852 :
1853 : /* Then notify the user that finished the sequence */
1854 65 : cb_fn(cb_arg, status);
1855 65 : }
1856 :
1857 : static void
1858 28 : accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf)
1859 : {
1860 : uintptr_t offset;
1861 :
1862 28 : offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK;
1863 28 : assert(offset < accel_buf->len);
1864 :
1865 28 : diov->iov_base = (char *)accel_buf->buf + offset;
1866 28 : diov->iov_len = siov->iov_len;
1867 28 : }
1868 :
1869 : static void
1870 10 : accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf)
1871 : {
1872 : struct spdk_accel_task *task;
1873 : struct iovec *iov;
1874 :
1875 : /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks
1876 : * in a sequence that were using it.
1877 : */
1878 38 : TAILQ_FOREACH(task, &seq->tasks, seq_link) {
1879 28 : if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) {
1880 11 : if (!task->has_aux) {
1881 11 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1882 11 : assert(task->aux && "Can't allocate aux data structure");
1883 11 : task->has_aux = true;
1884 11 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1885 : }
1886 :
1887 11 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC];
1888 11 : assert(task->s.iovcnt == 1);
1889 11 : accel_update_virt_iov(iov, &task->s.iovs[0], buf);
1890 11 : task->src_domain = NULL;
1891 11 : task->s.iovs = iov;
1892 : }
1893 28 : if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) {
1894 17 : if (!task->has_aux) {
1895 2 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1896 2 : assert(task->aux && "Can't allocate aux data structure");
1897 2 : task->has_aux = true;
1898 2 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1899 : }
1900 :
1901 17 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST];
1902 17 : assert(task->d.iovcnt == 1);
1903 17 : accel_update_virt_iov(iov, &task->d.iovs[0], buf);
1904 17 : task->dst_domain = NULL;
1905 17 : task->d.iovs = iov;
1906 : }
1907 : }
1908 10 : }
1909 :
1910 : static void accel_process_sequence(struct spdk_accel_sequence *seq);
1911 :
1912 : static void
1913 3 : accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf)
1914 : {
1915 : struct accel_buffer *accel_buf;
1916 :
1917 3 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1918 :
1919 3 : assert(accel_buf->seq != NULL);
1920 3 : assert(accel_buf->buf == NULL);
1921 3 : accel_buf->buf = buf;
1922 :
1923 3 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1924 3 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1925 3 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1926 3 : accel_process_sequence(accel_buf->seq);
1927 3 : }
1928 :
1929 : static bool
1930 29 : accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf,
1931 : spdk_iobuf_get_cb cb_fn)
1932 : {
1933 29 : struct accel_io_channel *ch = seq->ch;
1934 :
1935 29 : assert(buf->seq == NULL);
1936 :
1937 29 : buf->seq = seq;
1938 :
1939 : /* Buffer might be already allocated by memory domain translation. */
1940 29 : if (buf->buf) {
1941 0 : return true;
1942 : }
1943 :
1944 29 : buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn);
1945 29 : if (spdk_unlikely(buf->buf == NULL)) {
1946 5 : accel_update_stats(ch, retry.iobuf, 1);
1947 5 : return false;
1948 : }
1949 :
1950 24 : return true;
1951 : }
1952 :
1953 : static bool
1954 113 : accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1955 : {
1956 : /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to
1957 : * NULL */
1958 113 : if (task->src_domain == g_accel_domain) {
1959 0 : if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx,
1960 : accel_iobuf_get_virtbuf_cb)) {
1961 0 : return false;
1962 : }
1963 :
1964 0 : accel_sequence_set_virtbuf(seq, task->src_domain_ctx);
1965 : }
1966 :
1967 113 : if (task->dst_domain == g_accel_domain) {
1968 10 : if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx,
1969 : accel_iobuf_get_virtbuf_cb)) {
1970 3 : return false;
1971 : }
1972 :
1973 7 : accel_sequence_set_virtbuf(seq, task->dst_domain_ctx);
1974 : }
1975 :
1976 110 : return true;
1977 : }
1978 :
1979 : static void
1980 0 : accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf)
1981 : {
1982 : struct accel_buffer *accel_buf;
1983 :
1984 0 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1985 :
1986 0 : assert(accel_buf->seq != NULL);
1987 0 : assert(accel_buf->buf == NULL);
1988 0 : accel_buf->buf = buf;
1989 :
1990 0 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1991 0 : accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx);
1992 0 : }
1993 :
1994 : bool
1995 0 : spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf,
1996 : struct spdk_memory_domain *domain, void *domain_ctx,
1997 : spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx)
1998 : {
1999 0 : struct accel_buffer *accel_buf = domain_ctx;
2000 :
2001 0 : assert(domain == g_accel_domain);
2002 0 : accel_buf->cb_fn = cb_fn;
2003 0 : accel_buf->cb_ctx = cb_ctx;
2004 :
2005 0 : if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) {
2006 0 : return false;
2007 : }
2008 :
2009 0 : accel_sequence_set_virtbuf(seq, accel_buf);
2010 :
2011 0 : return true;
2012 : }
2013 :
2014 : struct spdk_accel_task *
2015 24 : spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq)
2016 : {
2017 24 : return TAILQ_FIRST(&seq->tasks);
2018 : }
2019 :
2020 : struct spdk_accel_task *
2021 0 : spdk_accel_sequence_next_task(struct spdk_accel_task *task)
2022 : {
2023 0 : return TAILQ_NEXT(task, seq_link);
2024 : }
2025 :
2026 : static inline void
2027 19 : accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,
2028 : uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,
2029 : struct accel_buffer *buf)
2030 : {
2031 19 : bounce->orig_iovs = *iovs;
2032 19 : bounce->orig_iovcnt = *iovcnt;
2033 19 : bounce->orig_domain = *domain;
2034 19 : bounce->orig_domain_ctx = *domain_ctx;
2035 19 : bounce->iov.iov_base = buf->buf;
2036 19 : bounce->iov.iov_len = buf->len;
2037 :
2038 19 : *iovs = &bounce->iov;
2039 19 : *iovcnt = 1;
2040 19 : *domain = NULL;
2041 19 : }
2042 :
2043 : static void
2044 1 : accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
2045 : {
2046 : struct spdk_accel_task *task;
2047 : struct accel_buffer *accel_buf;
2048 :
2049 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
2050 1 : assert(accel_buf->buf == NULL);
2051 1 : accel_buf->buf = buf;
2052 :
2053 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
2054 1 : assert(task != NULL);
2055 :
2056 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2057 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2058 1 : assert(task->aux);
2059 1 : assert(task->has_aux);
2060 1 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,
2061 : &task->src_domain_ctx, accel_buf);
2062 1 : accel_process_sequence(accel_buf->seq);
2063 1 : }
2064 :
2065 : static void
2066 1 : accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
2067 : {
2068 : struct spdk_accel_task *task;
2069 : struct accel_buffer *accel_buf;
2070 :
2071 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
2072 1 : assert(accel_buf->buf == NULL);
2073 1 : accel_buf->buf = buf;
2074 :
2075 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
2076 1 : assert(task != NULL);
2077 :
2078 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2079 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2080 1 : assert(task->aux);
2081 1 : assert(task->has_aux);
2082 1 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,
2083 : &task->dst_domain_ctx, accel_buf);
2084 1 : accel_process_sequence(accel_buf->seq);
2085 1 : }
2086 :
2087 : static int
2088 97 : accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2089 : {
2090 : struct accel_buffer *buf;
2091 :
2092 97 : if (task->src_domain != NULL) {
2093 : /* By the time we're here, accel buffers should have been allocated */
2094 9 : assert(task->src_domain != g_accel_domain);
2095 :
2096 9 : if (!task->has_aux) {
2097 8 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
2098 8 : if (spdk_unlikely(!task->aux)) {
2099 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
2100 0 : assert(0);
2101 : return -EAGAIN;
2102 : }
2103 8 : task->has_aux = true;
2104 8 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
2105 : }
2106 9 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));
2107 9 : if (buf == NULL) {
2108 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
2109 0 : return -ENOMEM;
2110 : }
2111 :
2112 9 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
2113 9 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {
2114 1 : return -EAGAIN;
2115 : }
2116 :
2117 8 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt,
2118 : &task->src_domain, &task->src_domain_ctx, buf);
2119 : }
2120 :
2121 96 : if (task->dst_domain != NULL) {
2122 : /* By the time we're here, accel buffers should have been allocated */
2123 10 : assert(task->dst_domain != g_accel_domain);
2124 :
2125 10 : if (!task->has_aux) {
2126 0 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
2127 0 : if (spdk_unlikely(!task->aux)) {
2128 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
2129 0 : assert(0);
2130 : return -EAGAIN;
2131 : }
2132 0 : task->has_aux = true;
2133 0 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
2134 : }
2135 10 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));
2136 10 : if (buf == NULL) {
2137 : /* The src buffer will be released when a sequence is completed */
2138 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
2139 0 : return -ENOMEM;
2140 : }
2141 :
2142 10 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
2143 10 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {
2144 1 : return -EAGAIN;
2145 : }
2146 :
2147 9 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt,
2148 : &task->dst_domain, &task->dst_domain_ctx, buf);
2149 : }
2150 :
2151 95 : return 0;
2152 : }
2153 :
2154 : static void
2155 8 : accel_task_pull_data_cb(void *ctx, int status)
2156 : {
2157 8 : struct spdk_accel_sequence *seq = ctx;
2158 :
2159 8 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
2160 8 : if (spdk_likely(status == 0)) {
2161 7 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2162 : } else {
2163 1 : accel_sequence_set_fail(seq, status);
2164 : }
2165 :
2166 8 : accel_process_sequence(seq);
2167 8 : }
2168 :
2169 : static void
2170 9 : accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2171 : {
2172 : int rc;
2173 :
2174 9 : assert(task->has_aux);
2175 9 : assert(task->aux);
2176 9 : assert(task->aux->bounce.s.orig_iovs != NULL);
2177 9 : assert(task->aux->bounce.s.orig_domain != NULL);
2178 9 : assert(task->aux->bounce.s.orig_domain != g_accel_domain);
2179 9 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
2180 :
2181 36 : rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain,
2182 9 : task->aux->bounce.s.orig_domain_ctx,
2183 18 : task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt,
2184 : task->s.iovs, task->s.iovcnt,
2185 : accel_task_pull_data_cb, seq);
2186 9 : if (spdk_unlikely(rc != 0)) {
2187 1 : SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n",
2188 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
2189 1 : accel_sequence_set_fail(seq, rc);
2190 : }
2191 9 : }
2192 :
2193 : static void
2194 7 : accel_task_push_data_cb(void *ctx, int status)
2195 : {
2196 7 : struct spdk_accel_sequence *seq = ctx;
2197 :
2198 7 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
2199 7 : if (spdk_likely(status == 0)) {
2200 6 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
2201 : } else {
2202 1 : accel_sequence_set_fail(seq, status);
2203 : }
2204 :
2205 7 : accel_process_sequence(seq);
2206 7 : }
2207 :
2208 : static void
2209 8 : accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2210 : {
2211 : int rc;
2212 :
2213 8 : assert(task->has_aux);
2214 8 : assert(task->aux);
2215 8 : assert(task->aux->bounce.d.orig_iovs != NULL);
2216 8 : assert(task->aux->bounce.d.orig_domain != NULL);
2217 8 : assert(task->aux->bounce.d.orig_domain != g_accel_domain);
2218 8 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
2219 :
2220 32 : rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain,
2221 8 : task->aux->bounce.d.orig_domain_ctx,
2222 16 : task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt,
2223 : task->d.iovs, task->d.iovcnt,
2224 : accel_task_push_data_cb, seq);
2225 8 : if (spdk_unlikely(rc != 0)) {
2226 1 : SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n",
2227 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
2228 1 : accel_sequence_set_fail(seq, rc);
2229 : }
2230 8 : }
2231 :
2232 : static void
2233 203 : accel_process_sequence(struct spdk_accel_sequence *seq)
2234 : {
2235 203 : struct accel_io_channel *accel_ch = seq->ch;
2236 : struct spdk_accel_task *task;
2237 : enum accel_sequence_state state;
2238 : int rc;
2239 :
2240 : /* Prevent recursive calls to this function */
2241 203 : if (spdk_unlikely(seq->in_process_sequence)) {
2242 78 : return;
2243 : }
2244 125 : seq->in_process_sequence = true;
2245 :
2246 125 : task = TAILQ_FIRST(&seq->tasks);
2247 : do {
2248 467 : state = seq->state;
2249 467 : switch (state) {
2250 115 : case ACCEL_SEQUENCE_STATE_INIT:
2251 115 : if (g_accel_driver != NULL) {
2252 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS);
2253 13 : break;
2254 : }
2255 : /* Fall through */
2256 : case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF:
2257 113 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
2258 113 : if (!accel_sequence_check_virtbuf(seq, task)) {
2259 : /* We couldn't allocate a buffer, wait until one is available */
2260 3 : break;
2261 : }
2262 110 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2263 : /* Fall through */
2264 112 : case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF:
2265 : /* If a module supports memory domains, we don't need to allocate bounce
2266 : * buffers */
2267 112 : if (g_modules_opc[task->op_code].supports_memory_domains) {
2268 15 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2269 15 : break;
2270 : }
2271 97 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2272 97 : rc = accel_sequence_check_bouncebuf(seq, task);
2273 97 : if (spdk_unlikely(rc != 0)) {
2274 : /* We couldn't allocate a buffer, wait until one is available */
2275 2 : if (rc == -EAGAIN) {
2276 2 : break;
2277 : }
2278 0 : accel_sequence_set_fail(seq, rc);
2279 0 : break;
2280 : }
2281 95 : if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) {
2282 9 : assert(task->aux->bounce.s.orig_iovs);
2283 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA);
2284 9 : break;
2285 : }
2286 86 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2287 : /* Fall through */
2288 108 : case ACCEL_SEQUENCE_STATE_EXEC_TASK:
2289 108 : SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",
2290 : g_opcode_strings[task->op_code], seq);
2291 :
2292 108 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
2293 108 : rc = accel_submit_task(accel_ch, task);
2294 108 : if (spdk_unlikely(rc != 0)) {
2295 2 : SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n",
2296 : g_opcode_strings[task->op_code], seq);
2297 2 : accel_sequence_set_fail(seq, rc);
2298 : }
2299 108 : break;
2300 9 : case ACCEL_SEQUENCE_STATE_PULL_DATA:
2301 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
2302 9 : accel_task_pull_data(seq, task);
2303 9 : break;
2304 104 : case ACCEL_SEQUENCE_STATE_COMPLETE_TASK:
2305 104 : if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) {
2306 8 : assert(task->aux->bounce.d.orig_iovs);
2307 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA);
2308 8 : break;
2309 : }
2310 96 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
2311 96 : break;
2312 8 : case ACCEL_SEQUENCE_STATE_PUSH_DATA:
2313 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
2314 8 : accel_task_push_data(seq, task);
2315 8 : break;
2316 102 : case ACCEL_SEQUENCE_STATE_NEXT_TASK:
2317 102 : accel_sequence_complete_task(seq, task);
2318 : /* Check if there are any remaining tasks */
2319 102 : task = TAILQ_FIRST(&seq->tasks);
2320 102 : if (task == NULL) {
2321 : /* Immediately return here to make sure we don't touch the sequence
2322 : * after it's completed */
2323 52 : accel_sequence_complete(seq);
2324 52 : return;
2325 : }
2326 50 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT);
2327 50 : break;
2328 13 : case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS:
2329 13 : assert(!TAILQ_EMPTY(&seq->tasks));
2330 :
2331 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
2332 13 : rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq);
2333 13 : if (spdk_unlikely(rc != 0)) {
2334 1 : SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n",
2335 : seq, g_accel_driver->name);
2336 1 : accel_sequence_set_fail(seq, rc);
2337 : }
2338 13 : break;
2339 11 : case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS:
2340 : /* Get the task again, as the driver might have completed some tasks
2341 : * synchronously */
2342 11 : task = TAILQ_FIRST(&seq->tasks);
2343 11 : if (task == NULL) {
2344 : /* Immediately return here to make sure we don't touch the sequence
2345 : * after it's completed */
2346 3 : accel_sequence_complete(seq);
2347 3 : return;
2348 : }
2349 : /* We don't want to execute the next task through the driver, so we
2350 : * explicitly omit the INIT state here */
2351 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
2352 8 : break;
2353 10 : case ACCEL_SEQUENCE_STATE_ERROR:
2354 : /* Immediately return here to make sure we don't touch the sequence
2355 : * after it's completed */
2356 10 : assert(seq->status != 0);
2357 10 : accel_sequence_complete(seq);
2358 10 : return;
2359 60 : case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF:
2360 : case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF:
2361 : case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA:
2362 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
2363 : case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA:
2364 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
2365 60 : break;
2366 0 : default:
2367 0 : assert(0 && "bad state");
2368 : break;
2369 : }
2370 402 : } while (seq->state != state);
2371 :
2372 60 : seq->in_process_sequence = false;
2373 : }
2374 :
2375 : static void
2376 118 : accel_sequence_task_cb(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, int status)
2377 : {
2378 118 : switch (seq->state) {
2379 106 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
2380 106 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK);
2381 106 : if (spdk_unlikely(status != 0)) {
2382 2 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n",
2383 : g_opcode_strings[task->op_code], seq);
2384 2 : accel_sequence_set_fail(seq, status);
2385 : }
2386 :
2387 106 : accel_process_sequence(seq);
2388 106 : break;
2389 12 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
2390 12 : assert(g_accel_driver != NULL);
2391 : /* Immediately remove the task from the outstanding list to make sure the next call
2392 : * to spdk_accel_sequence_first_task() doesn't return it */
2393 12 : accel_sequence_complete_task(seq, task);
2394 12 : if (spdk_unlikely(status != 0)) {
2395 1 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through "
2396 : "driver: %s\n", g_opcode_strings[task->op_code], seq,
2397 : g_accel_driver->name);
2398 : /* Update status without using accel_sequence_set_fail() to avoid changing
2399 : * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */
2400 1 : seq->status = status;
2401 : }
2402 12 : break;
2403 0 : default:
2404 0 : assert(0 && "bad state");
2405 : break;
2406 : }
2407 118 : }
2408 :
2409 : void
2410 12 : spdk_accel_sequence_continue(struct spdk_accel_sequence *seq)
2411 : {
2412 12 : assert(g_accel_driver != NULL);
2413 12 : assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
2414 :
2415 12 : if (spdk_likely(seq->status == 0)) {
2416 11 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS);
2417 : } else {
2418 1 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
2419 : }
2420 :
2421 12 : accel_process_sequence(seq);
2422 12 : }
2423 :
2424 : static bool
2425 43 : accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt)
2426 : {
2427 : /* For now, just do a dumb check that the iovecs arrays are exactly the same */
2428 43 : if (iovacnt != iovbcnt) {
2429 0 : return false;
2430 : }
2431 :
2432 43 : return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0;
2433 : }
2434 :
2435 : static bool
2436 30 : accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)
2437 : {
2438 : struct spdk_accel_task *prev;
2439 :
2440 30 : switch (task->op_code) {
2441 24 : case SPDK_ACCEL_OPC_DECOMPRESS:
2442 : case SPDK_ACCEL_OPC_FILL:
2443 : case SPDK_ACCEL_OPC_ENCRYPT:
2444 : case SPDK_ACCEL_OPC_DECRYPT:
2445 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
2446 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
2447 24 : if (task->dst_domain != next->src_domain) {
2448 0 : return false;
2449 : }
2450 24 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2451 : next->s.iovs, next->s.iovcnt)) {
2452 1 : return false;
2453 : }
2454 23 : task->d.iovs = next->d.iovs;
2455 23 : task->d.iovcnt = next->d.iovcnt;
2456 23 : task->dst_domain = next->dst_domain;
2457 23 : task->dst_domain_ctx = next->dst_domain_ctx;
2458 23 : break;
2459 6 : case SPDK_ACCEL_OPC_CRC32C:
2460 : case SPDK_ACCEL_OPC_DIX_GENERATE:
2461 : case SPDK_ACCEL_OPC_DIX_VERIFY:
2462 : /* crc32 and dix_generate/verify are special, because they do not have a dst buffer */
2463 6 : if (task->src_domain != next->src_domain) {
2464 0 : return false;
2465 : }
2466 6 : if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,
2467 : next->s.iovs, next->s.iovcnt)) {
2468 1 : return false;
2469 : }
2470 : /* We can only change operation's buffer if we can change previous task's buffer */
2471 5 : prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);
2472 5 : if (prev == NULL) {
2473 1 : return false;
2474 : }
2475 4 : if (!accel_task_set_dstbuf(prev, next)) {
2476 0 : return false;
2477 : }
2478 4 : task->s.iovs = next->d.iovs;
2479 4 : task->s.iovcnt = next->d.iovcnt;
2480 4 : task->src_domain = next->dst_domain;
2481 4 : task->src_domain_ctx = next->dst_domain_ctx;
2482 4 : break;
2483 0 : default:
2484 0 : return false;
2485 : }
2486 :
2487 27 : return true;
2488 : }
2489 :
2490 : static void
2491 96 : accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task,
2492 : struct spdk_accel_task **next_task)
2493 : {
2494 96 : struct spdk_accel_task *next = *next_task;
2495 :
2496 96 : switch (task->op_code) {
2497 15 : case SPDK_ACCEL_OPC_COPY:
2498 : /* We only allow changing src of operations that actually have a src, e.g. we never
2499 : * do it for fill. Theoretically, it is possible, but we'd have to be careful to
2500 : * change the src of the operation after fill (which in turn could also be a fill).
2501 : * So, for the sake of simplicity, skip this type of operations for now.
2502 : */
2503 15 : if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS &&
2504 9 : next->op_code != SPDK_ACCEL_OPC_COPY &&
2505 8 : next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&
2506 5 : next->op_code != SPDK_ACCEL_OPC_DECRYPT &&
2507 2 : next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C &&
2508 2 : next->op_code != SPDK_ACCEL_OPC_DIF_GENERATE_COPY &&
2509 2 : next->op_code != SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
2510 2 : break;
2511 : }
2512 13 : if (task->dst_domain != next->src_domain) {
2513 0 : break;
2514 : }
2515 13 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2516 : next->s.iovs, next->s.iovcnt)) {
2517 0 : break;
2518 : }
2519 13 : next->s.iovs = task->s.iovs;
2520 13 : next->s.iovcnt = task->s.iovcnt;
2521 13 : next->src_domain = task->src_domain;
2522 13 : next->src_domain_ctx = task->src_domain_ctx;
2523 13 : accel_sequence_complete_task(seq, task);
2524 13 : break;
2525 81 : case SPDK_ACCEL_OPC_DECOMPRESS:
2526 : case SPDK_ACCEL_OPC_FILL:
2527 : case SPDK_ACCEL_OPC_ENCRYPT:
2528 : case SPDK_ACCEL_OPC_DECRYPT:
2529 : case SPDK_ACCEL_OPC_CRC32C:
2530 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
2531 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
2532 : case SPDK_ACCEL_OPC_DIX_GENERATE:
2533 : case SPDK_ACCEL_OPC_DIX_VERIFY:
2534 : /* We can only merge tasks when one of them is a copy */
2535 81 : if (next->op_code != SPDK_ACCEL_OPC_COPY) {
2536 55 : break;
2537 : }
2538 26 : if (!accel_task_set_dstbuf(task, next)) {
2539 3 : break;
2540 : }
2541 : /* We're removing next_task from the tasks queue, so we need to update its pointer,
2542 : * so that the TAILQ_FOREACH_SAFE() loop below works correctly */
2543 23 : *next_task = TAILQ_NEXT(next, seq_link);
2544 23 : accel_sequence_complete_task(seq, next);
2545 23 : break;
2546 0 : default:
2547 0 : assert(0 && "bad opcode");
2548 : break;
2549 : }
2550 96 : }
2551 :
2552 : void
2553 65 : spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
2554 : spdk_accel_completion_cb cb_fn, void *cb_arg)
2555 : {
2556 65 : struct spdk_accel_task *task, *next;
2557 :
2558 : /* Try to remove any copy operations if possible */
2559 161 : TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) {
2560 140 : if (next == NULL) {
2561 44 : break;
2562 : }
2563 96 : accel_sequence_merge_tasks(seq, task, &next);
2564 : }
2565 :
2566 65 : seq->cb_fn = cb_fn;
2567 65 : seq->cb_arg = cb_arg;
2568 :
2569 65 : accel_process_sequence(seq);
2570 65 : }
2571 :
2572 : void
2573 5 : spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq)
2574 : {
2575 5 : struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks);
2576 : struct spdk_accel_task *task;
2577 :
2578 5 : TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link);
2579 :
2580 17 : while (!TAILQ_EMPTY(&tasks)) {
2581 12 : task = TAILQ_FIRST(&tasks);
2582 12 : TAILQ_REMOVE(&tasks, task, seq_link);
2583 12 : TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link);
2584 : }
2585 5 : }
2586 :
2587 : void
2588 3 : spdk_accel_sequence_abort(struct spdk_accel_sequence *seq)
2589 : {
2590 3 : if (seq == NULL) {
2591 1 : return;
2592 : }
2593 :
2594 2 : accel_sequence_complete_tasks(seq);
2595 2 : accel_sequence_put(seq);
2596 : }
2597 :
2598 : struct spdk_memory_domain *
2599 0 : spdk_accel_get_memory_domain(void)
2600 : {
2601 0 : return g_accel_domain;
2602 : }
2603 :
2604 : static struct spdk_accel_module_if *
2605 7 : _module_find_by_name(const char *name)
2606 : {
2607 7 : struct spdk_accel_module_if *accel_module = NULL;
2608 :
2609 16 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2610 10 : if (strcmp(name, accel_module->name) == 0) {
2611 1 : break;
2612 : }
2613 : }
2614 :
2615 7 : return accel_module;
2616 : }
2617 :
2618 : static inline struct spdk_accel_crypto_key *
2619 6 : _accel_crypto_key_get(const char *name)
2620 : {
2621 : struct spdk_accel_crypto_key *key;
2622 :
2623 6 : assert(spdk_spin_held(&g_keyring_spin));
2624 :
2625 6 : TAILQ_FOREACH(key, &g_keyring, link) {
2626 4 : if (strcmp(name, key->param.key_name) == 0) {
2627 4 : return key;
2628 : }
2629 : }
2630 :
2631 2 : return NULL;
2632 : }
2633 :
2634 : static void
2635 2 : accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key)
2636 : {
2637 2 : if (key->param.hex_key) {
2638 2 : spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2);
2639 2 : free(key->param.hex_key);
2640 : }
2641 2 : if (key->param.hex_key2) {
2642 2 : spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2);
2643 2 : free(key->param.hex_key2);
2644 : }
2645 2 : free(key->param.tweak_mode);
2646 2 : free(key->param.key_name);
2647 2 : free(key->param.cipher);
2648 2 : if (key->key) {
2649 2 : spdk_memset_s(key->key, key->key_size, 0, key->key_size);
2650 2 : free(key->key);
2651 : }
2652 2 : if (key->key2) {
2653 2 : spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size);
2654 2 : free(key->key2);
2655 : }
2656 2 : free(key);
2657 2 : }
2658 :
2659 : static void
2660 2 : accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key)
2661 : {
2662 2 : assert(key->module_if);
2663 2 : assert(key->module_if->crypto_key_deinit);
2664 :
2665 2 : key->module_if->crypto_key_deinit(key);
2666 2 : accel_crypto_key_free_mem(key);
2667 2 : }
2668 :
2669 : /*
2670 : * This function mitigates a timing side channel which could be caused by using strcmp()
2671 : * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in
2672 : * the article [1] for more details
2673 : * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html
2674 : */
2675 : static bool
2676 2 : accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len)
2677 : {
2678 : size_t i;
2679 2 : volatile size_t x = k1_len ^ k2_len;
2680 :
2681 34 : for (i = 0; ((i < k1_len) & (i < k2_len)); i++) {
2682 32 : x |= k1[i] ^ k2[i];
2683 : }
2684 :
2685 2 : return x == 0;
2686 : }
2687 :
2688 : static const char *g_tweak_modes[] = {
2689 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA",
2690 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA",
2691 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA",
2692 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA",
2693 : };
2694 :
2695 : static const char *g_ciphers[] = {
2696 : [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC",
2697 : [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS",
2698 : };
2699 :
2700 : int
2701 2 : spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param)
2702 : {
2703 : struct spdk_accel_module_if *module;
2704 : struct spdk_accel_crypto_key *key;
2705 : size_t hex_key_size, hex_key2_size;
2706 2 : bool found = false;
2707 : size_t i;
2708 : int rc;
2709 :
2710 2 : if (!param || !param->hex_key || !param->cipher || !param->key_name) {
2711 0 : return -EINVAL;
2712 : }
2713 :
2714 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2715 : /* hardly ever possible, but let's check and warn the user */
2716 0 : SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
2717 : }
2718 2 : module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module;
2719 :
2720 2 : if (!module) {
2721 0 : SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
2722 0 : return -ENOENT;
2723 : }
2724 :
2725 2 : if (!module->crypto_key_init || !module->crypto_supports_cipher) {
2726 0 : SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name);
2727 0 : return -ENOTSUP;
2728 : }
2729 :
2730 2 : key = calloc(1, sizeof(*key));
2731 2 : if (!key) {
2732 0 : return -ENOMEM;
2733 : }
2734 :
2735 2 : key->param.key_name = strdup(param->key_name);
2736 2 : if (!key->param.key_name) {
2737 0 : rc = -ENOMEM;
2738 0 : goto error;
2739 : }
2740 :
2741 4 : for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) {
2742 4 : assert(g_ciphers[i]);
2743 :
2744 4 : if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) {
2745 2 : key->cipher = i;
2746 2 : found = true;
2747 2 : break;
2748 : }
2749 : }
2750 :
2751 2 : if (!found) {
2752 0 : SPDK_ERRLOG("Failed to parse cipher\n");
2753 0 : rc = -EINVAL;
2754 0 : goto error;
2755 : }
2756 :
2757 2 : key->param.cipher = strdup(param->cipher);
2758 2 : if (!key->param.cipher) {
2759 0 : rc = -ENOMEM;
2760 0 : goto error;
2761 : }
2762 :
2763 2 : hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2764 2 : if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2765 0 : SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2766 0 : rc = -EINVAL;
2767 0 : goto error;
2768 : }
2769 :
2770 2 : if (hex_key_size == 0) {
2771 0 : SPDK_ERRLOG("key1 size cannot be 0\n");
2772 0 : rc = -EINVAL;
2773 0 : goto error;
2774 : }
2775 :
2776 2 : key->param.hex_key = strdup(param->hex_key);
2777 2 : if (!key->param.hex_key) {
2778 0 : rc = -ENOMEM;
2779 0 : goto error;
2780 : }
2781 :
2782 2 : key->key_size = hex_key_size / 2;
2783 2 : key->key = spdk_unhexlify(key->param.hex_key);
2784 2 : if (!key->key) {
2785 0 : SPDK_ERRLOG("Failed to unhexlify key1\n");
2786 0 : rc = -EINVAL;
2787 0 : goto error;
2788 : }
2789 :
2790 2 : if (param->hex_key2) {
2791 2 : hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2792 2 : if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2793 0 : SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2794 0 : rc = -EINVAL;
2795 0 : goto error;
2796 : }
2797 :
2798 2 : if (hex_key2_size == 0) {
2799 0 : SPDK_ERRLOG("key2 size cannot be 0\n");
2800 0 : rc = -EINVAL;
2801 0 : goto error;
2802 : }
2803 :
2804 2 : key->param.hex_key2 = strdup(param->hex_key2);
2805 2 : if (!key->param.hex_key2) {
2806 0 : rc = -ENOMEM;
2807 0 : goto error;
2808 : }
2809 :
2810 2 : key->key2_size = hex_key2_size / 2;
2811 2 : key->key2 = spdk_unhexlify(key->param.hex_key2);
2812 2 : if (!key->key2) {
2813 0 : SPDK_ERRLOG("Failed to unhexlify key2\n");
2814 0 : rc = -EINVAL;
2815 0 : goto error;
2816 : }
2817 : }
2818 :
2819 2 : key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT;
2820 2 : if (param->tweak_mode) {
2821 0 : found = false;
2822 :
2823 0 : key->param.tweak_mode = strdup(param->tweak_mode);
2824 0 : if (!key->param.tweak_mode) {
2825 0 : rc = -ENOMEM;
2826 0 : goto error;
2827 : }
2828 :
2829 0 : for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) {
2830 0 : assert(g_tweak_modes[i]);
2831 :
2832 0 : if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) {
2833 0 : key->tweak_mode = i;
2834 0 : found = true;
2835 0 : break;
2836 : }
2837 : }
2838 :
2839 0 : if (!found) {
2840 0 : SPDK_ERRLOG("Failed to parse tweak mode\n");
2841 0 : rc = -EINVAL;
2842 0 : goto error;
2843 : }
2844 : }
2845 :
2846 2 : if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) ||
2847 2 : (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) {
2848 0 : SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name,
2849 : g_tweak_modes[key->tweak_mode]);
2850 0 : rc = -EINVAL;
2851 0 : goto error;
2852 : }
2853 :
2854 2 : if (!module->crypto_supports_cipher(key->cipher, key->key_size)) {
2855 0 : SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name,
2856 : g_ciphers[key->cipher], key->key_size);
2857 0 : rc = -EINVAL;
2858 0 : goto error;
2859 : }
2860 :
2861 2 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
2862 2 : if (!key->key2) {
2863 0 : SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]);
2864 0 : rc = -EINVAL;
2865 0 : goto error;
2866 : }
2867 :
2868 2 : if (key->key_size != key->key2_size) {
2869 0 : SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher],
2870 : key->key_size,
2871 : key->key2_size);
2872 0 : rc = -EINVAL;
2873 0 : goto error;
2874 : }
2875 :
2876 2 : if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) {
2877 0 : SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]);
2878 0 : rc = -EINVAL;
2879 0 : goto error;
2880 : }
2881 : }
2882 :
2883 2 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) {
2884 0 : if (key->key2_size) {
2885 0 : SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]);
2886 0 : rc = -EINVAL;
2887 0 : goto error;
2888 : }
2889 : }
2890 :
2891 2 : key->module_if = module;
2892 :
2893 2 : spdk_spin_lock(&g_keyring_spin);
2894 2 : if (_accel_crypto_key_get(param->key_name)) {
2895 0 : rc = -EEXIST;
2896 : } else {
2897 2 : rc = module->crypto_key_init(key);
2898 2 : if (rc) {
2899 0 : SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name);
2900 : } else {
2901 2 : TAILQ_INSERT_TAIL(&g_keyring, key, link);
2902 : }
2903 : }
2904 2 : spdk_spin_unlock(&g_keyring_spin);
2905 :
2906 2 : if (rc) {
2907 0 : goto error;
2908 : }
2909 :
2910 2 : return 0;
2911 :
2912 0 : error:
2913 0 : accel_crypto_key_free_mem(key);
2914 0 : return rc;
2915 : }
2916 :
2917 : int
2918 2 : spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key)
2919 : {
2920 2 : if (!key || !key->module_if) {
2921 0 : return -EINVAL;
2922 : }
2923 :
2924 2 : spdk_spin_lock(&g_keyring_spin);
2925 2 : if (!_accel_crypto_key_get(key->param.key_name)) {
2926 0 : spdk_spin_unlock(&g_keyring_spin);
2927 0 : return -ENOENT;
2928 : }
2929 2 : TAILQ_REMOVE(&g_keyring, key, link);
2930 2 : spdk_spin_unlock(&g_keyring_spin);
2931 :
2932 2 : accel_crypto_key_destroy_unsafe(key);
2933 :
2934 2 : return 0;
2935 : }
2936 :
2937 : struct spdk_accel_crypto_key *
2938 2 : spdk_accel_crypto_key_get(const char *name)
2939 : {
2940 : struct spdk_accel_crypto_key *key;
2941 :
2942 2 : spdk_spin_lock(&g_keyring_spin);
2943 2 : key = _accel_crypto_key_get(name);
2944 2 : spdk_spin_unlock(&g_keyring_spin);
2945 :
2946 2 : return key;
2947 : }
2948 :
2949 : /* Helper function when accel modules register with the framework. */
2950 : void
2951 5 : spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module)
2952 : {
2953 : struct spdk_accel_module_if *tmp;
2954 :
2955 5 : if (_module_find_by_name(accel_module->name)) {
2956 0 : SPDK_NOTICELOG("Module %s already registered\n", accel_module->name);
2957 0 : assert(false);
2958 : return;
2959 : }
2960 :
2961 8 : TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) {
2962 5 : if (accel_module->priority < tmp->priority) {
2963 2 : break;
2964 : }
2965 : }
2966 :
2967 5 : if (tmp != NULL) {
2968 2 : TAILQ_INSERT_BEFORE(tmp, accel_module, tailq);
2969 : } else {
2970 3 : TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq);
2971 : }
2972 : }
2973 :
2974 : /* Framework level channel create callback. */
2975 : static int
2976 16 : accel_create_channel(void *io_device, void *ctx_buf)
2977 : {
2978 16 : struct accel_io_channel *accel_ch = ctx_buf;
2979 : struct spdk_accel_task *accel_task;
2980 : struct spdk_accel_task_aux_data *accel_task_aux;
2981 : struct spdk_accel_sequence *seq;
2982 : struct accel_buffer *buf;
2983 : size_t task_size_aligned;
2984 : uint8_t *task_mem;
2985 16 : uint32_t i = 0, j;
2986 : int rc;
2987 :
2988 16 : task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE);
2989 16 : accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2990 16 : g_opts.task_count * task_size_aligned);
2991 16 : if (!accel_ch->task_pool_base) {
2992 0 : return -ENOMEM;
2993 : }
2994 16 : memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned);
2995 :
2996 16 : accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2997 16 : g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
2998 16 : if (accel_ch->seq_pool_base == NULL) {
2999 0 : goto err;
3000 : }
3001 16 : memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
3002 :
3003 16 : accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data));
3004 16 : if (accel_ch->task_aux_data_base == NULL) {
3005 0 : goto err;
3006 : }
3007 :
3008 16 : accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer));
3009 16 : if (accel_ch->buf_pool_base == NULL) {
3010 0 : goto err;
3011 : }
3012 :
3013 16 : STAILQ_INIT(&accel_ch->task_pool);
3014 16 : SLIST_INIT(&accel_ch->task_aux_data_pool);
3015 16 : SLIST_INIT(&accel_ch->seq_pool);
3016 16 : SLIST_INIT(&accel_ch->buf_pool);
3017 :
3018 16 : task_mem = accel_ch->task_pool_base;
3019 32784 : for (i = 0; i < g_opts.task_count; i++) {
3020 32768 : accel_task = (struct spdk_accel_task *)task_mem;
3021 32768 : accel_task->aux = NULL;
3022 32768 : STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
3023 32768 : task_mem += task_size_aligned;
3024 32768 : accel_task_aux = &accel_ch->task_aux_data_base[i];
3025 32768 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link);
3026 : }
3027 32784 : for (i = 0; i < g_opts.sequence_count; i++) {
3028 32768 : seq = &accel_ch->seq_pool_base[i];
3029 32768 : SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link);
3030 : }
3031 32784 : for (i = 0; i < g_opts.buf_count; i++) {
3032 32768 : buf = &accel_ch->buf_pool_base[i];
3033 32768 : SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link);
3034 : }
3035 :
3036 : /* Assign modules and get IO channels for each */
3037 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3038 272 : accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
3039 : /* This can happen if idxd runs out of channels. */
3040 272 : if (accel_ch->module_ch[i] == NULL) {
3041 0 : SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name);
3042 0 : goto err;
3043 : }
3044 : }
3045 :
3046 16 : if (g_accel_driver != NULL) {
3047 0 : accel_ch->driver_channel = g_accel_driver->get_io_channel();
3048 0 : if (accel_ch->driver_channel == NULL) {
3049 0 : SPDK_ERRLOG("Failed to get driver's IO channel\n");
3050 0 : goto err;
3051 : }
3052 : }
3053 :
3054 16 : rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size,
3055 : g_opts.large_cache_size);
3056 16 : if (rc != 0) {
3057 0 : SPDK_ERRLOG("Failed to initialize iobuf accel channel\n");
3058 0 : goto err;
3059 : }
3060 :
3061 16 : return 0;
3062 0 : err:
3063 0 : if (accel_ch->driver_channel != NULL) {
3064 0 : spdk_put_io_channel(accel_ch->driver_channel);
3065 : }
3066 0 : for (j = 0; j < i; j++) {
3067 0 : spdk_put_io_channel(accel_ch->module_ch[j]);
3068 : }
3069 0 : free(accel_ch->task_pool_base);
3070 0 : free(accel_ch->task_aux_data_base);
3071 0 : free(accel_ch->seq_pool_base);
3072 0 : free(accel_ch->buf_pool_base);
3073 :
3074 0 : return -ENOMEM;
3075 : }
3076 :
3077 : static void
3078 16 : accel_add_stats(struct accel_stats *total, struct accel_stats *stats)
3079 : {
3080 : int i;
3081 :
3082 16 : total->sequence_executed += stats->sequence_executed;
3083 16 : total->sequence_failed += stats->sequence_failed;
3084 16 : total->sequence_outstanding += stats->sequence_outstanding;
3085 16 : total->task_outstanding += stats->task_outstanding;
3086 16 : total->retry.task += stats->retry.task;
3087 16 : total->retry.sequence += stats->retry.sequence;
3088 16 : total->retry.iobuf += stats->retry.iobuf;
3089 16 : total->retry.bufdesc += stats->retry.bufdesc;
3090 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {
3091 272 : total->operations[i].executed += stats->operations[i].executed;
3092 272 : total->operations[i].failed += stats->operations[i].failed;
3093 272 : total->operations[i].num_bytes += stats->operations[i].num_bytes;
3094 : }
3095 16 : }
3096 :
3097 : /* Framework level channel destroy callback. */
3098 : static void
3099 16 : accel_destroy_channel(void *io_device, void *ctx_buf)
3100 : {
3101 16 : struct accel_io_channel *accel_ch = ctx_buf;
3102 : int i;
3103 :
3104 16 : spdk_iobuf_channel_fini(&accel_ch->iobuf);
3105 :
3106 16 : if (accel_ch->driver_channel != NULL) {
3107 0 : spdk_put_io_channel(accel_ch->driver_channel);
3108 : }
3109 :
3110 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3111 272 : assert(accel_ch->module_ch[i] != NULL);
3112 272 : spdk_put_io_channel(accel_ch->module_ch[i]);
3113 272 : accel_ch->module_ch[i] = NULL;
3114 : }
3115 :
3116 : /* Update global stats to make sure channel's stats aren't lost after a channel is gone */
3117 16 : spdk_spin_lock(&g_stats_lock);
3118 16 : accel_add_stats(&g_stats, &accel_ch->stats);
3119 16 : spdk_spin_unlock(&g_stats_lock);
3120 :
3121 16 : free(accel_ch->task_pool_base);
3122 16 : free(accel_ch->task_aux_data_base);
3123 16 : free(accel_ch->seq_pool_base);
3124 16 : free(accel_ch->buf_pool_base);
3125 16 : }
3126 :
3127 : struct spdk_io_channel *
3128 16 : spdk_accel_get_io_channel(void)
3129 : {
3130 16 : return spdk_get_io_channel(&spdk_accel_module_list);
3131 : }
3132 :
3133 : static int
3134 2 : accel_module_initialize(void)
3135 : {
3136 : struct spdk_accel_module_if *accel_module, *tmp_module;
3137 2 : int rc = 0, module_rc;
3138 :
3139 7 : TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) {
3140 5 : module_rc = accel_module->module_init();
3141 5 : if (module_rc) {
3142 0 : TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq);
3143 0 : if (module_rc == -ENODEV) {
3144 0 : SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name);
3145 0 : } else if (!rc) {
3146 0 : SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc);
3147 0 : rc = module_rc;
3148 : }
3149 0 : continue;
3150 : }
3151 :
3152 5 : SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name);
3153 : }
3154 :
3155 2 : return rc;
3156 : }
3157 :
3158 : static void
3159 34 : accel_module_init_opcode(enum spdk_accel_opcode opcode)
3160 : {
3161 34 : struct accel_module *module = &g_modules_opc[opcode];
3162 34 : struct spdk_accel_module_if *module_if = module->module;
3163 :
3164 34 : if (module_if->get_memory_domains != NULL) {
3165 0 : module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0;
3166 : }
3167 34 : }
3168 :
3169 : static int
3170 0 : accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx,
3171 : struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx,
3172 : void *addr, size_t len, struct spdk_memory_domain_translation_result *result)
3173 : {
3174 0 : struct accel_buffer *buf = src_domain_ctx;
3175 :
3176 0 : SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len);
3177 :
3178 0 : assert(g_accel_domain == src_domain);
3179 0 : assert(spdk_memory_domain_get_system_domain() == dst_domain);
3180 0 : assert(buf->buf == NULL);
3181 0 : assert(addr == ACCEL_BUFFER_BASE);
3182 0 : assert(len == buf->len);
3183 :
3184 0 : buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL);
3185 0 : if (spdk_unlikely(buf->buf == NULL)) {
3186 0 : return -ENOMEM;
3187 : }
3188 :
3189 0 : result->iov_count = 1;
3190 0 : result->iov.iov_base = buf->buf;
3191 0 : result->iov.iov_len = buf->len;
3192 0 : SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base);
3193 0 : return 0;
3194 : }
3195 :
3196 : static void
3197 0 : accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx,
3198 : struct iovec *iov, uint32_t iovcnt)
3199 : {
3200 0 : struct accel_buffer *buf = domain_ctx;
3201 :
3202 0 : SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len);
3203 :
3204 0 : assert(g_accel_domain == domain);
3205 0 : assert(iovcnt == 1);
3206 0 : assert(buf->buf != NULL);
3207 0 : assert(iov[0].iov_base == buf->buf);
3208 0 : assert(iov[0].iov_len == buf->len);
3209 :
3210 0 : spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len);
3211 0 : buf->buf = NULL;
3212 0 : }
3213 :
3214 : int
3215 2 : spdk_accel_initialize(void)
3216 : {
3217 : enum spdk_accel_opcode op;
3218 2 : struct spdk_accel_module_if *accel_module = NULL;
3219 : int rc;
3220 :
3221 : /*
3222 : * We need a unique identifier for the accel framework, so use the
3223 : * spdk_accel_module_list address for this purpose.
3224 : */
3225 2 : spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel,
3226 : sizeof(struct accel_io_channel), "accel");
3227 :
3228 2 : spdk_spin_init(&g_keyring_spin);
3229 2 : spdk_spin_init(&g_stats_lock);
3230 :
3231 2 : rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
3232 : "SPDK_ACCEL_DMA_DEVICE");
3233 2 : if (rc != 0) {
3234 0 : SPDK_ERRLOG("Failed to create accel memory domain\n");
3235 0 : return rc;
3236 : }
3237 :
3238 2 : spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate);
3239 2 : spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate);
3240 :
3241 2 : g_modules_started = true;
3242 2 : rc = accel_module_initialize();
3243 2 : if (rc) {
3244 0 : return rc;
3245 : }
3246 :
3247 2 : if (g_accel_driver != NULL && g_accel_driver->init != NULL) {
3248 0 : rc = g_accel_driver->init();
3249 0 : if (rc != 0) {
3250 0 : SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name,
3251 : spdk_strerror(-rc));
3252 0 : return rc;
3253 : }
3254 : }
3255 :
3256 : /* The module list is order by priority, with the highest priority modules being at the end
3257 : * of the list. The software module should be somewhere at the beginning of the list,
3258 : * before all HW modules.
3259 : * NOTE: all opcodes must be supported by software in the event that no HW modules are
3260 : * initialized to support the operation.
3261 : */
3262 7 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
3263 90 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3264 85 : if (accel_module->supports_opcode(op)) {
3265 85 : g_modules_opc[op].module = accel_module;
3266 85 : SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
3267 : }
3268 : }
3269 :
3270 5 : if (accel_module->get_ctx_size != NULL) {
3271 1 : g_max_accel_module_size = spdk_max(g_max_accel_module_size,
3272 : accel_module->get_ctx_size());
3273 : }
3274 : }
3275 :
3276 : /* Now lets check for overrides and apply all that exist */
3277 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3278 34 : if (g_modules_opc_override[op] != NULL) {
3279 0 : accel_module = _module_find_by_name(g_modules_opc_override[op]);
3280 0 : if (accel_module == NULL) {
3281 0 : SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
3282 0 : return -EINVAL;
3283 : }
3284 0 : if (accel_module->supports_opcode(op) == false) {
3285 0 : SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
3286 0 : return -EINVAL;
3287 : }
3288 0 : g_modules_opc[op].module = accel_module;
3289 : }
3290 : }
3291 :
3292 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
3293 0 : SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
3294 0 : return -EINVAL;
3295 : }
3296 2 : if (g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module !=
3297 2 : g_modules_opc[SPDK_ACCEL_OPC_DECOMPRESS].module) {
3298 0 : SPDK_ERRLOG("Different accel modules are assigned to compress and decompress operations");
3299 0 : return -EINVAL;
3300 : }
3301 :
3302 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3303 34 : assert(g_modules_opc[op].module != NULL);
3304 34 : accel_module_init_opcode(op);
3305 : }
3306 :
3307 2 : rc = spdk_iobuf_register_module("accel");
3308 2 : if (rc != 0) {
3309 0 : SPDK_ERRLOG("Failed to register accel iobuf module\n");
3310 0 : return rc;
3311 : }
3312 :
3313 2 : return 0;
3314 : }
3315 :
3316 : static void
3317 2 : accel_module_finish_cb(void)
3318 : {
3319 2 : spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
3320 :
3321 2 : cb_fn(g_fini_cb_arg);
3322 2 : g_fini_cb_fn = NULL;
3323 2 : g_fini_cb_arg = NULL;
3324 2 : }
3325 :
3326 : static void
3327 0 : accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str,
3328 : const char *module_str)
3329 : {
3330 0 : spdk_json_write_object_begin(w);
3331 0 : spdk_json_write_named_string(w, "method", "accel_assign_opc");
3332 0 : spdk_json_write_named_object_begin(w, "params");
3333 0 : spdk_json_write_named_string(w, "opname", opc_str);
3334 0 : spdk_json_write_named_string(w, "module", module_str);
3335 0 : spdk_json_write_object_end(w);
3336 0 : spdk_json_write_object_end(w);
3337 0 : }
3338 :
3339 : static void
3340 0 : __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
3341 : {
3342 0 : spdk_json_write_named_string(w, "name", key->param.key_name);
3343 0 : spdk_json_write_named_string(w, "cipher", key->param.cipher);
3344 0 : spdk_json_write_named_string(w, "key", key->param.hex_key);
3345 0 : if (key->param.hex_key2) {
3346 0 : spdk_json_write_named_string(w, "key2", key->param.hex_key2);
3347 : }
3348 :
3349 0 : if (key->param.tweak_mode) {
3350 0 : spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode);
3351 : }
3352 0 : }
3353 :
3354 : void
3355 0 : _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
3356 : {
3357 0 : spdk_json_write_object_begin(w);
3358 0 : __accel_crypto_key_dump_param(w, key);
3359 0 : spdk_json_write_object_end(w);
3360 0 : }
3361 :
3362 : static void
3363 0 : _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w,
3364 : struct spdk_accel_crypto_key *key)
3365 : {
3366 0 : spdk_json_write_object_begin(w);
3367 0 : spdk_json_write_named_string(w, "method", "accel_crypto_key_create");
3368 0 : spdk_json_write_named_object_begin(w, "params");
3369 0 : __accel_crypto_key_dump_param(w, key);
3370 0 : spdk_json_write_object_end(w);
3371 0 : spdk_json_write_object_end(w);
3372 0 : }
3373 :
3374 : static void
3375 0 : accel_write_options(struct spdk_json_write_ctx *w)
3376 : {
3377 0 : spdk_json_write_object_begin(w);
3378 0 : spdk_json_write_named_string(w, "method", "accel_set_options");
3379 0 : spdk_json_write_named_object_begin(w, "params");
3380 0 : spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size);
3381 0 : spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size);
3382 0 : spdk_json_write_named_uint32(w, "task_count", g_opts.task_count);
3383 0 : spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count);
3384 0 : spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count);
3385 0 : spdk_json_write_object_end(w);
3386 0 : spdk_json_write_object_end(w);
3387 0 : }
3388 :
3389 : static void
3390 0 : _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump)
3391 : {
3392 : struct spdk_accel_crypto_key *key;
3393 :
3394 0 : spdk_spin_lock(&g_keyring_spin);
3395 0 : TAILQ_FOREACH(key, &g_keyring, link) {
3396 0 : if (full_dump) {
3397 0 : _accel_crypto_key_write_config_json(w, key);
3398 : } else {
3399 0 : _accel_crypto_key_dump_param(w, key);
3400 : }
3401 : }
3402 0 : spdk_spin_unlock(&g_keyring_spin);
3403 0 : }
3404 :
3405 : void
3406 0 : _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w)
3407 : {
3408 0 : _accel_crypto_keys_write_config_json(w, false);
3409 0 : }
3410 :
3411 : void
3412 0 : spdk_accel_write_config_json(struct spdk_json_write_ctx *w)
3413 : {
3414 : struct spdk_accel_module_if *accel_module;
3415 : int i;
3416 :
3417 0 : spdk_json_write_array_begin(w);
3418 0 : accel_write_options(w);
3419 :
3420 0 : if (g_accel_driver != NULL) {
3421 0 : spdk_json_write_object_begin(w);
3422 0 : spdk_json_write_named_string(w, "method", "accel_set_driver");
3423 0 : spdk_json_write_named_object_begin(w, "params");
3424 0 : spdk_json_write_named_string(w, "name", g_accel_driver->name);
3425 0 : spdk_json_write_object_end(w);
3426 0 : spdk_json_write_object_end(w);
3427 : }
3428 :
3429 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
3430 0 : if (accel_module->write_config_json) {
3431 0 : accel_module->write_config_json(w);
3432 : }
3433 : }
3434 0 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3435 0 : if (g_modules_opc_override[i]) {
3436 0 : accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]);
3437 : }
3438 : }
3439 :
3440 0 : _accel_crypto_keys_write_config_json(w, true);
3441 :
3442 0 : spdk_json_write_array_end(w);
3443 0 : }
3444 :
3445 : void
3446 7 : spdk_accel_module_finish(void)
3447 : {
3448 7 : if (!g_accel_module) {
3449 2 : g_accel_module = TAILQ_FIRST(&spdk_accel_module_list);
3450 : } else {
3451 5 : g_accel_module = TAILQ_NEXT(g_accel_module, tailq);
3452 : }
3453 :
3454 7 : if (!g_accel_module) {
3455 2 : if (g_accel_driver != NULL && g_accel_driver->fini != NULL) {
3456 0 : g_accel_driver->fini();
3457 : }
3458 :
3459 2 : spdk_spin_destroy(&g_keyring_spin);
3460 2 : spdk_spin_destroy(&g_stats_lock);
3461 2 : if (g_accel_domain) {
3462 2 : spdk_memory_domain_destroy(g_accel_domain);
3463 2 : g_accel_domain = NULL;
3464 : }
3465 2 : accel_module_finish_cb();
3466 2 : return;
3467 : }
3468 :
3469 5 : if (g_accel_module->module_fini) {
3470 1 : spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL);
3471 : } else {
3472 4 : spdk_accel_module_finish();
3473 : }
3474 : }
3475 :
3476 : static void
3477 2 : accel_io_device_unregister_cb(void *io_device)
3478 : {
3479 : struct spdk_accel_crypto_key *key, *key_tmp;
3480 : enum spdk_accel_opcode op;
3481 :
3482 2 : spdk_spin_lock(&g_keyring_spin);
3483 2 : TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) {
3484 0 : accel_crypto_key_destroy_unsafe(key);
3485 : }
3486 2 : spdk_spin_unlock(&g_keyring_spin);
3487 :
3488 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3489 34 : if (g_modules_opc_override[op] != NULL) {
3490 0 : free(g_modules_opc_override[op]);
3491 0 : g_modules_opc_override[op] = NULL;
3492 : }
3493 34 : g_modules_opc[op].module = NULL;
3494 : }
3495 :
3496 2 : spdk_accel_module_finish();
3497 2 : }
3498 :
3499 : void
3500 2 : spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
3501 : {
3502 2 : assert(cb_fn != NULL);
3503 :
3504 2 : g_fini_cb_fn = cb_fn;
3505 2 : g_fini_cb_arg = cb_arg;
3506 :
3507 2 : spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb);
3508 2 : }
3509 :
3510 : static struct spdk_accel_driver *
3511 2 : accel_find_driver(const char *name)
3512 : {
3513 : struct spdk_accel_driver *driver;
3514 :
3515 2 : TAILQ_FOREACH(driver, &g_accel_drivers, tailq) {
3516 1 : if (strcmp(driver->name, name) == 0) {
3517 1 : return driver;
3518 : }
3519 : }
3520 :
3521 1 : return NULL;
3522 : }
3523 :
3524 : int
3525 1 : spdk_accel_set_driver(const char *name)
3526 : {
3527 1 : struct spdk_accel_driver *driver = NULL;
3528 :
3529 1 : if (name != NULL && name[0] != '\0') {
3530 1 : driver = accel_find_driver(name);
3531 1 : if (driver == NULL) {
3532 0 : SPDK_ERRLOG("Couldn't find driver named '%s'\n", name);
3533 0 : return -ENODEV;
3534 : }
3535 : }
3536 :
3537 1 : g_accel_driver = driver;
3538 :
3539 1 : return 0;
3540 : }
3541 :
3542 : const char *
3543 0 : spdk_accel_get_driver_name(void)
3544 : {
3545 0 : if (!g_accel_driver) {
3546 0 : return NULL;
3547 : }
3548 :
3549 0 : return g_accel_driver->name;
3550 : }
3551 :
3552 : void
3553 1 : spdk_accel_driver_register(struct spdk_accel_driver *driver)
3554 : {
3555 1 : if (accel_find_driver(driver->name)) {
3556 0 : SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name);
3557 0 : assert(0);
3558 : return;
3559 : }
3560 :
3561 1 : TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq);
3562 : }
3563 :
3564 : int
3565 0 : spdk_accel_set_opts(const struct spdk_accel_opts *opts)
3566 : {
3567 0 : if (!opts) {
3568 0 : SPDK_ERRLOG("opts cannot be NULL\n");
3569 0 : return -1;
3570 : }
3571 :
3572 0 : if (!opts->opts_size) {
3573 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
3574 0 : return -1;
3575 : }
3576 :
3577 0 : if (SPDK_GET_FIELD(opts, task_count, g_opts.task_count,
3578 0 : opts->opts_size) < ACCEL_TASKS_IN_SEQUENCE_LIMIT) {
3579 0 : return -EINVAL;
3580 : }
3581 :
3582 : #define SET_FIELD(field) \
3583 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
3584 : g_opts.field = opts->field; \
3585 : } \
3586 :
3587 0 : SET_FIELD(small_cache_size);
3588 0 : SET_FIELD(large_cache_size);
3589 0 : SET_FIELD(task_count);
3590 0 : SET_FIELD(sequence_count);
3591 0 : SET_FIELD(buf_count);
3592 :
3593 0 : g_opts.opts_size = opts->opts_size;
3594 :
3595 : #undef SET_FIELD
3596 :
3597 0 : return 0;
3598 : }
3599 :
3600 : void
3601 0 : spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size)
3602 : {
3603 0 : if (!opts) {
3604 0 : SPDK_ERRLOG("opts should not be NULL\n");
3605 0 : return;
3606 : }
3607 :
3608 0 : if (!opts_size) {
3609 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
3610 0 : return;
3611 : }
3612 :
3613 0 : opts->opts_size = opts_size;
3614 :
3615 : #define SET_FIELD(field) \
3616 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \
3617 : opts->field = g_opts.field; \
3618 : } \
3619 :
3620 0 : SET_FIELD(small_cache_size);
3621 0 : SET_FIELD(large_cache_size);
3622 0 : SET_FIELD(task_count);
3623 0 : SET_FIELD(sequence_count);
3624 0 : SET_FIELD(buf_count);
3625 :
3626 : #undef SET_FIELD
3627 :
3628 : /* Do not remove this statement, you should always update this statement when you adding a new field,
3629 : * and do not forget to add the SET_FIELD statement for your added field. */
3630 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size");
3631 : }
3632 :
3633 : struct accel_get_stats_ctx {
3634 : struct accel_stats stats;
3635 : accel_get_stats_cb cb_fn;
3636 : void *cb_arg;
3637 : };
3638 :
3639 : static void
3640 0 : accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status)
3641 : {
3642 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3643 :
3644 0 : ctx->cb_fn(&ctx->stats, ctx->cb_arg);
3645 0 : free(ctx);
3646 0 : }
3647 :
3648 : static void
3649 0 : accel_get_channel_stats(struct spdk_io_channel_iter *iter)
3650 : {
3651 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter);
3652 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3653 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3654 :
3655 0 : accel_add_stats(&ctx->stats, &accel_ch->stats);
3656 0 : spdk_for_each_channel_continue(iter, 0);
3657 0 : }
3658 :
3659 : int
3660 0 : accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg)
3661 : {
3662 : struct accel_get_stats_ctx *ctx;
3663 :
3664 0 : ctx = calloc(1, sizeof(*ctx));
3665 0 : if (ctx == NULL) {
3666 0 : return -ENOMEM;
3667 : }
3668 :
3669 0 : spdk_spin_lock(&g_stats_lock);
3670 0 : accel_add_stats(&ctx->stats, &g_stats);
3671 0 : spdk_spin_unlock(&g_stats_lock);
3672 :
3673 0 : ctx->cb_fn = cb_fn;
3674 0 : ctx->cb_arg = cb_arg;
3675 :
3676 0 : spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx,
3677 : accel_get_channel_stats_done);
3678 :
3679 0 : return 0;
3680 : }
3681 :
3682 : void
3683 0 : spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode,
3684 : struct spdk_accel_opcode_stats *stats, size_t size)
3685 : {
3686 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3687 :
3688 : #define FIELD_OK(field) \
3689 : offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size
3690 :
3691 : #define SET_FIELD(field, value) \
3692 : if (FIELD_OK(field)) { \
3693 : stats->field = value; \
3694 : }
3695 :
3696 0 : SET_FIELD(executed, accel_ch->stats.operations[opcode].executed);
3697 0 : SET_FIELD(failed, accel_ch->stats.operations[opcode].failed);
3698 0 : SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes);
3699 :
3700 : #undef FIELD_OK
3701 : #undef SET_FIELD
3702 0 : }
3703 :
3704 : uint8_t
3705 0 : spdk_accel_get_buf_align(enum spdk_accel_opcode opcode,
3706 : const struct spdk_accel_operation_exec_ctx *ctx)
3707 : {
3708 0 : struct spdk_accel_module_if *module = g_modules_opc[opcode].module;
3709 0 : struct spdk_accel_opcode_info modinfo = {}, drvinfo = {};
3710 :
3711 0 : if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) {
3712 0 : g_accel_driver->get_operation_info(opcode, ctx, &drvinfo);
3713 : }
3714 :
3715 0 : if (module->get_operation_info != NULL) {
3716 0 : module->get_operation_info(opcode, ctx, &modinfo);
3717 : }
3718 :
3719 : /* If a driver is set, it'll execute most of the operations, while the rest will usually
3720 : * fall back to accel_sw, which doesn't have any alignment requirements. However, to be
3721 : * extra safe, return the max(driver, module) if a driver delegates some operations to a
3722 : * hardware module. */
3723 0 : return spdk_max(modinfo.required_alignment, drvinfo.required_alignment);
3724 : }
3725 :
3726 : struct spdk_accel_module_if *
3727 0 : spdk_accel_get_module(const char *name)
3728 : {
3729 : struct spdk_accel_module_if *module;
3730 :
3731 0 : TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) {
3732 0 : if (strcmp(module->name, name) == 0) {
3733 0 : return module;
3734 : }
3735 : }
3736 :
3737 0 : return NULL;
3738 : }
3739 :
3740 : int
3741 0 : spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode,
3742 : struct spdk_memory_domain **domains,
3743 : int array_size)
3744 : {
3745 0 : assert(opcode < SPDK_ACCEL_OPC_LAST);
3746 :
3747 0 : if (g_modules_opc[opcode].module->get_memory_domains) {
3748 0 : return g_modules_opc[opcode].module->get_memory_domains(domains, array_size);
3749 : }
3750 :
3751 0 : return 0;
3752 : }
3753 :
3754 1 : SPDK_LOG_REGISTER_COMPONENT(accel)
|
