Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2022 Intel Corporation.
3 : * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
4 : * All rights reserved.
5 : */
6 :
7 : #include "accel_dsa.h"
8 :
9 : #include "spdk/stdinc.h"
10 :
11 : #include "spdk/accel_module.h"
12 : #include "spdk/log.h"
13 : #include "spdk_internal/idxd.h"
14 :
15 : #include "spdk/env.h"
16 : #include "spdk/event.h"
17 : #include "spdk/likely.h"
18 : #include "spdk/thread.h"
19 : #include "spdk/idxd.h"
20 : #include "spdk/util.h"
21 : #include "spdk/json.h"
22 : #include "spdk/trace.h"
23 : #include "spdk_internal/trace_defs.h"
24 :
25 : static bool g_dsa_enable = false;
26 : static bool g_kernel_mode = false;
27 :
28 : enum channel_state {
29 : IDXD_CHANNEL_ACTIVE,
30 : IDXD_CHANNEL_ERROR,
31 : };
32 :
33 : static bool g_dsa_initialized = false;
34 :
35 : struct idxd_device {
36 : struct spdk_idxd_device *dsa;
37 : TAILQ_ENTRY(idxd_device) tailq;
38 : };
39 : static TAILQ_HEAD(, idxd_device) g_dsa_devices = TAILQ_HEAD_INITIALIZER(g_dsa_devices);
40 : static struct idxd_device *g_next_dev = NULL;
41 : static uint32_t g_num_devices = 0;
42 : static pthread_mutex_t g_dev_lock = PTHREAD_MUTEX_INITIALIZER;
43 :
44 : struct idxd_task {
45 : struct spdk_accel_task task;
46 : struct idxd_io_channel *chan;
47 : };
48 :
49 : struct idxd_io_channel {
50 : struct spdk_idxd_io_channel *chan;
51 : struct idxd_device *dev;
52 : enum channel_state state;
53 : struct spdk_poller *poller;
54 : uint32_t num_outstanding;
55 : STAILQ_HEAD(, spdk_accel_task) queued_tasks;
56 : };
57 :
58 : static struct spdk_io_channel *dsa_get_io_channel(void);
59 :
60 : static struct idxd_device *
61 0 : idxd_select_device(struct idxd_io_channel *chan)
62 : {
63 0 : uint32_t count = 0;
64 : struct idxd_device *dev;
65 0 : uint32_t socket_id = spdk_env_get_socket_id(spdk_env_get_current_core());
66 :
67 : /*
68 : * We allow channels to share underlying devices,
69 : * selection is round-robin based with a limitation
70 : * on how many channel can share one device.
71 : */
72 : do {
73 : /* select next device */
74 0 : pthread_mutex_lock(&g_dev_lock);
75 0 : g_next_dev = TAILQ_NEXT(g_next_dev, tailq);
76 0 : if (g_next_dev == NULL) {
77 0 : g_next_dev = TAILQ_FIRST(&g_dsa_devices);
78 : }
79 0 : dev = g_next_dev;
80 0 : pthread_mutex_unlock(&g_dev_lock);
81 :
82 0 : if (socket_id != spdk_idxd_get_socket(dev->dsa)) {
83 0 : continue;
84 : }
85 :
86 : /*
87 : * Now see if a channel is available on this one. We only
88 : * allow a specific number of channels to share a device
89 : * to limit outstanding IO for flow control purposes.
90 : */
91 0 : chan->chan = spdk_idxd_get_channel(dev->dsa);
92 0 : if (chan->chan != NULL) {
93 0 : SPDK_DEBUGLOG(accel_dsa, "On socket %d using device on socket %d\n",
94 : socket_id, spdk_idxd_get_socket(dev->dsa));
95 0 : return dev;
96 : }
97 0 : } while (++count < g_num_devices);
98 :
99 : /* We are out of available channels and/or devices for the local socket. We fix the number
100 : * of channels that we allocate per device and only allocate devices on the same socket
101 : * that the current thread is on. If on a 2 socket system it may be possible to avoid
102 : * this situation by spreading threads across the sockets.
103 : */
104 0 : SPDK_ERRLOG("No more DSA devices available on the local socket.\n");
105 0 : return NULL;
106 : }
107 :
108 : static void
109 0 : dsa_done(void *cb_arg, int status)
110 : {
111 0 : struct idxd_task *idxd_task = cb_arg;
112 : struct idxd_io_channel *chan;
113 : int rc;
114 :
115 0 : chan = idxd_task->chan;
116 :
117 : /* If the DSA DIF Check operation detects an error, detailed info about
118 : * this error (like actual/expected values) needs to be obtained by
119 : * calling the software DIF Verify operation.
120 : */
121 0 : if (spdk_unlikely(status == -EIO)) {
122 0 : if (idxd_task->task.op_code == SPDK_ACCEL_OPC_DIF_VERIFY) {
123 0 : rc = spdk_dif_verify(idxd_task->task.s.iovs, idxd_task->task.s.iovcnt,
124 : idxd_task->task.dif.num_blocks,
125 : idxd_task->task.dif.ctx, idxd_task->task.dif.err);
126 0 : if (rc != 0) {
127 0 : SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
128 : idxd_task->task.dif.err->err_type,
129 : idxd_task->task.dif.err->err_offset);
130 : }
131 : }
132 : }
133 :
134 0 : assert(chan->num_outstanding > 0);
135 0 : spdk_trace_record(TRACE_ACCEL_DSA_OP_COMPLETE, 0, 0, 0, chan->num_outstanding - 1);
136 0 : chan->num_outstanding--;
137 :
138 0 : spdk_accel_task_complete(&idxd_task->task, status);
139 0 : }
140 :
141 : static int
142 0 : idxd_submit_dualcast(struct idxd_io_channel *ch, struct idxd_task *idxd_task, int flags)
143 : {
144 0 : struct spdk_accel_task *task = &idxd_task->task;
145 :
146 0 : if (spdk_unlikely(task->d.iovcnt != 1 || task->d2.iovcnt != 1 || task->s.iovcnt != 1)) {
147 0 : return -EINVAL;
148 : }
149 :
150 0 : if (spdk_unlikely(task->d.iovs[0].iov_len != task->s.iovs[0].iov_len ||
151 : task->d.iovs[0].iov_len != task->d2.iovs[0].iov_len)) {
152 0 : return -EINVAL;
153 : }
154 :
155 0 : return spdk_idxd_submit_dualcast(ch->chan, task->d.iovs[0].iov_base,
156 0 : task->d2.iovs[0].iov_base, task->s.iovs[0].iov_base,
157 0 : task->d.iovs[0].iov_len, flags, dsa_done, idxd_task);
158 : }
159 :
160 : static int
161 0 : _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
162 : {
163 0 : struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
164 : struct idxd_task *idxd_task;
165 0 : int rc = 0, flags = 0;
166 :
167 0 : idxd_task = SPDK_CONTAINEROF(task, struct idxd_task, task);
168 0 : idxd_task->chan = chan;
169 :
170 0 : switch (task->op_code) {
171 0 : case SPDK_ACCEL_OPC_COPY:
172 0 : rc = spdk_idxd_submit_copy(chan->chan, task->d.iovs, task->d.iovcnt,
173 : task->s.iovs, task->s.iovcnt, flags, dsa_done, idxd_task);
174 0 : break;
175 0 : case SPDK_ACCEL_OPC_DUALCAST:
176 0 : rc = idxd_submit_dualcast(chan, idxd_task, flags);
177 0 : break;
178 0 : case SPDK_ACCEL_OPC_COMPARE:
179 0 : rc = spdk_idxd_submit_compare(chan->chan, task->s.iovs, task->s.iovcnt,
180 0 : task->s2.iovs, task->s2.iovcnt, flags,
181 : dsa_done, idxd_task);
182 0 : break;
183 0 : case SPDK_ACCEL_OPC_FILL:
184 0 : rc = spdk_idxd_submit_fill(chan->chan, task->d.iovs, task->d.iovcnt,
185 : task->fill_pattern, flags, dsa_done, idxd_task);
186 0 : break;
187 0 : case SPDK_ACCEL_OPC_CRC32C:
188 0 : rc = spdk_idxd_submit_crc32c(chan->chan, task->s.iovs, task->s.iovcnt, task->seed,
189 : task->crc_dst, flags, dsa_done, idxd_task);
190 0 : break;
191 0 : case SPDK_ACCEL_OPC_COPY_CRC32C:
192 0 : rc = spdk_idxd_submit_copy_crc32c(chan->chan, task->d.iovs, task->d.iovcnt,
193 0 : task->s.iovs, task->s.iovcnt,
194 : task->seed, task->crc_dst, flags,
195 : dsa_done, idxd_task);
196 0 : break;
197 0 : case SPDK_ACCEL_OPC_DIF_VERIFY:
198 0 : rc = spdk_idxd_submit_dif_check(chan->chan,
199 0 : task->s.iovs, task->s.iovcnt,
200 : task->dif.num_blocks, task->dif.ctx, flags,
201 : dsa_done, idxd_task);
202 0 : break;
203 0 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
204 0 : rc = spdk_idxd_submit_dif_insert(chan->chan,
205 0 : task->d.iovs, task->d.iovcnt,
206 0 : task->s.iovs, task->s.iovcnt,
207 : task->dif.num_blocks, task->dif.ctx, flags,
208 : dsa_done, idxd_task);
209 0 : break;
210 0 : default:
211 0 : assert(false);
212 : rc = -EINVAL;
213 : break;
214 : }
215 :
216 0 : if (rc == 0) {
217 0 : chan->num_outstanding++;
218 0 : spdk_trace_record(TRACE_ACCEL_DSA_OP_SUBMIT, 0, 0, 0, chan->num_outstanding);
219 : }
220 :
221 0 : return rc;
222 : }
223 :
224 : static int
225 0 : dsa_submit_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
226 : {
227 0 : struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
228 0 : int rc = 0;
229 :
230 0 : assert(STAILQ_NEXT(task, link) == NULL);
231 :
232 0 : if (spdk_unlikely(chan->state == IDXD_CHANNEL_ERROR)) {
233 0 : spdk_accel_task_complete(task, -EINVAL);
234 0 : return 0;
235 : }
236 :
237 0 : if (!STAILQ_EMPTY(&chan->queued_tasks)) {
238 0 : STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
239 0 : return 0;
240 : }
241 :
242 0 : rc = _process_single_task(ch, task);
243 0 : if (rc == -EBUSY) {
244 0 : STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
245 0 : } else if (rc) {
246 0 : spdk_accel_task_complete(task, rc);
247 : }
248 :
249 0 : return 0;
250 : }
251 :
252 : static int
253 0 : dsa_submit_queued_tasks(struct idxd_io_channel *chan)
254 : {
255 : struct spdk_accel_task *task, *tmp;
256 0 : struct spdk_io_channel *ch = spdk_io_channel_from_ctx(chan);
257 0 : int rc = 0;
258 :
259 0 : if (spdk_unlikely(chan->state == IDXD_CHANNEL_ERROR)) {
260 : /* Complete queued tasks with error and clear the list */
261 0 : while ((task = STAILQ_FIRST(&chan->queued_tasks))) {
262 0 : STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);
263 0 : spdk_accel_task_complete(task, -EINVAL);
264 : }
265 0 : return 0;
266 : }
267 :
268 0 : STAILQ_FOREACH_SAFE(task, &chan->queued_tasks, link, tmp) {
269 0 : rc = _process_single_task(ch, task);
270 0 : if (rc == -EBUSY) {
271 0 : return rc;
272 : }
273 0 : STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);
274 0 : if (rc) {
275 0 : spdk_accel_task_complete(task, rc);
276 : }
277 : }
278 :
279 0 : return 0;
280 : }
281 :
282 : static int
283 0 : idxd_poll(void *arg)
284 : {
285 0 : struct idxd_io_channel *chan = arg;
286 : int count;
287 :
288 0 : count = spdk_idxd_process_events(chan->chan);
289 :
290 : /* Check if there are any pending ops to process if the channel is active */
291 0 : if (!STAILQ_EMPTY(&chan->queued_tasks)) {
292 0 : dsa_submit_queued_tasks(chan);
293 : }
294 :
295 0 : return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
296 : }
297 :
298 : static size_t
299 0 : accel_dsa_get_ctx_size(void)
300 : {
301 0 : return sizeof(struct idxd_task);
302 : }
303 :
304 : static bool
305 0 : dsa_supports_opcode(enum spdk_accel_opcode opc)
306 : {
307 0 : if (!g_dsa_initialized) {
308 0 : assert(0);
309 : return false;
310 : }
311 :
312 0 : switch (opc) {
313 0 : case SPDK_ACCEL_OPC_COPY:
314 : case SPDK_ACCEL_OPC_FILL:
315 : case SPDK_ACCEL_OPC_DUALCAST:
316 : case SPDK_ACCEL_OPC_COMPARE:
317 : case SPDK_ACCEL_OPC_CRC32C:
318 : case SPDK_ACCEL_OPC_COPY_CRC32C:
319 0 : return true;
320 0 : case SPDK_ACCEL_OPC_DIF_VERIFY:
321 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
322 : /* Supported only if the IOMMU is enabled */
323 0 : return spdk_iommu_is_enabled();
324 0 : default:
325 0 : return false;
326 : }
327 : }
328 :
329 : static int accel_dsa_init(void);
330 : static void accel_dsa_exit(void *ctx);
331 : static void accel_dsa_write_config_json(struct spdk_json_write_ctx *w);
332 :
333 : static struct spdk_accel_module_if g_dsa_module = {
334 : .module_init = accel_dsa_init,
335 : .module_fini = accel_dsa_exit,
336 : .write_config_json = accel_dsa_write_config_json,
337 : .get_ctx_size = accel_dsa_get_ctx_size,
338 : .name = "dsa",
339 : .supports_opcode = dsa_supports_opcode,
340 : .get_io_channel = dsa_get_io_channel,
341 : .submit_tasks = dsa_submit_task
342 : };
343 :
344 : static int
345 0 : dsa_create_cb(void *io_device, void *ctx_buf)
346 : {
347 0 : struct idxd_io_channel *chan = ctx_buf;
348 : struct idxd_device *dsa;
349 :
350 0 : dsa = idxd_select_device(chan);
351 0 : if (dsa == NULL) {
352 0 : SPDK_ERRLOG("Failed to get an idxd channel\n");
353 0 : return -EINVAL;
354 : }
355 :
356 0 : chan->dev = dsa;
357 0 : chan->poller = SPDK_POLLER_REGISTER(idxd_poll, chan, 0);
358 0 : STAILQ_INIT(&chan->queued_tasks);
359 0 : chan->num_outstanding = 0;
360 0 : chan->state = IDXD_CHANNEL_ACTIVE;
361 :
362 0 : return 0;
363 : }
364 :
365 : static void
366 0 : dsa_destroy_cb(void *io_device, void *ctx_buf)
367 : {
368 0 : struct idxd_io_channel *chan = ctx_buf;
369 :
370 0 : spdk_poller_unregister(&chan->poller);
371 0 : spdk_idxd_put_channel(chan->chan);
372 0 : }
373 :
374 : static struct spdk_io_channel *
375 0 : dsa_get_io_channel(void)
376 : {
377 0 : return spdk_get_io_channel(&g_dsa_module);
378 : }
379 :
380 : static void
381 0 : attach_cb(void *cb_ctx, struct spdk_idxd_device *idxd)
382 : {
383 : struct idxd_device *dev;
384 :
385 0 : dev = calloc(1, sizeof(*dev));
386 0 : if (dev == NULL) {
387 0 : SPDK_ERRLOG("Failed to allocate device struct\n");
388 0 : return;
389 : }
390 :
391 0 : dev->dsa = idxd;
392 0 : if (g_next_dev == NULL) {
393 0 : g_next_dev = dev;
394 : }
395 :
396 0 : TAILQ_INSERT_TAIL(&g_dsa_devices, dev, tailq);
397 0 : g_num_devices++;
398 : }
399 :
400 : int
401 0 : accel_dsa_enable_probe(bool kernel_mode)
402 : {
403 : int rc;
404 :
405 0 : if (g_dsa_enable) {
406 0 : return -EALREADY;
407 : }
408 :
409 0 : rc = spdk_idxd_set_config(kernel_mode);
410 0 : if (rc != 0) {
411 0 : return rc;
412 : }
413 :
414 0 : spdk_accel_module_list_add(&g_dsa_module);
415 0 : g_kernel_mode = kernel_mode;
416 0 : g_dsa_enable = true;
417 :
418 0 : return 0;
419 : }
420 :
421 : static bool
422 0 : probe_cb(void *cb_ctx, struct spdk_pci_device *dev)
423 : {
424 0 : if (dev->id.device_id == PCI_DEVICE_ID_INTEL_DSA) {
425 0 : return true;
426 : }
427 :
428 0 : return false;
429 : }
430 :
431 : static int
432 0 : accel_dsa_init(void)
433 : {
434 0 : if (!g_dsa_enable) {
435 0 : return -EINVAL;
436 : }
437 :
438 0 : if (spdk_idxd_probe(NULL, attach_cb, probe_cb) != 0) {
439 0 : SPDK_ERRLOG("spdk_idxd_probe() failed\n");
440 0 : return -EINVAL;
441 : }
442 :
443 0 : if (TAILQ_EMPTY(&g_dsa_devices)) {
444 0 : SPDK_NOTICELOG("no available dsa devices\n");
445 0 : return -EINVAL;
446 : }
447 :
448 0 : g_dsa_initialized = true;
449 0 : spdk_io_device_register(&g_dsa_module, dsa_create_cb, dsa_destroy_cb,
450 : sizeof(struct idxd_io_channel), "dsa_accel_module");
451 0 : return 0;
452 : }
453 :
454 : static void
455 0 : accel_dsa_exit(void *ctx)
456 : {
457 : struct idxd_device *dev;
458 :
459 0 : if (g_dsa_initialized) {
460 0 : spdk_io_device_unregister(&g_dsa_module, NULL);
461 0 : g_dsa_initialized = false;
462 : }
463 :
464 0 : while (!TAILQ_EMPTY(&g_dsa_devices)) {
465 0 : dev = TAILQ_FIRST(&g_dsa_devices);
466 0 : TAILQ_REMOVE(&g_dsa_devices, dev, tailq);
467 0 : spdk_idxd_detach(dev->dsa);
468 0 : free(dev);
469 : }
470 :
471 0 : spdk_accel_module_finish();
472 0 : }
473 :
474 : static void
475 0 : accel_dsa_write_config_json(struct spdk_json_write_ctx *w)
476 : {
477 0 : if (g_dsa_enable) {
478 0 : spdk_json_write_object_begin(w);
479 0 : spdk_json_write_named_string(w, "method", "dsa_scan_accel_module");
480 0 : spdk_json_write_named_object_begin(w, "params");
481 0 : spdk_json_write_named_bool(w, "config_kernel_mode", g_kernel_mode);
482 0 : spdk_json_write_object_end(w);
483 0 : spdk_json_write_object_end(w);
484 : }
485 0 : }
486 :
487 0 : SPDK_TRACE_REGISTER_FN(dsa_trace, "dsa", TRACE_GROUP_ACCEL_DSA)
488 : {
489 0 : spdk_trace_register_description("DSA_OP_SUBMIT", TRACE_ACCEL_DSA_OP_SUBMIT, OWNER_NONE, OBJECT_NONE,
490 : 0,
491 : SPDK_TRACE_ARG_TYPE_INT, "count");
492 0 : spdk_trace_register_description("DSA_OP_COMPLETE", TRACE_ACCEL_DSA_OP_COMPLETE, OWNER_NONE,
493 : OBJECT_NONE,
494 : 0, SPDK_TRACE_ARG_TYPE_INT, "count");
495 0 : }
496 :
497 0 : SPDK_LOG_REGISTER_COMPONENT(accel_dsa)
|