Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2019 Intel Corporation.
3 : * All rights reserved.
4 : */
5 : #include "spdk/stdinc.h"
6 : #include "spdk/config.h"
7 : #include "spdk/log.h"
8 : #include "spdk/nvme.h"
9 :
10 : #ifdef SPDK_CONFIG_NVME_CUSE
11 : #define FUSE_USE_VERSION 31
12 :
13 : #include <fuse3/cuse_lowlevel.h>
14 :
15 : #include <linux/nvme_ioctl.h>
16 : #include <linux/fs.h>
17 :
18 : #include "nvme_internal.h"
19 : #include "nvme_io_msg.h"
20 : #include "nvme_cuse.h"
21 :
22 : struct cuse_device {
23 : char dev_name[128];
24 : uint32_t index;
25 : int claim_fd;
26 : char lock_name[64];
27 :
28 : struct spdk_nvme_ctrlr *ctrlr; /**< NVMe controller */
29 : uint32_t nsid; /**< NVMe name space id, or 0 */
30 :
31 : pthread_t tid;
32 : struct fuse_session *session;
33 :
34 : struct cuse_device *ctrlr_device;
35 : TAILQ_HEAD(, cuse_device) ns_devices;
36 :
37 : TAILQ_ENTRY(cuse_device) tailq;
38 : };
39 :
40 : static pthread_mutex_t g_cuse_mtx = PTHREAD_MUTEX_INITIALIZER;
41 : static TAILQ_HEAD(, cuse_device) g_ctrlr_ctx_head = TAILQ_HEAD_INITIALIZER(g_ctrlr_ctx_head);
42 : static struct spdk_bit_array *g_ctrlr_started;
43 :
44 : struct cuse_io_ctx {
45 : struct spdk_nvme_cmd nvme_cmd;
46 : enum spdk_nvme_data_transfer data_transfer;
47 :
48 : uint64_t lba;
49 : uint32_t lba_count;
50 : uint16_t apptag;
51 : uint16_t appmask;
52 :
53 : void *data;
54 : void *metadata;
55 :
56 : int data_len;
57 : int metadata_len;
58 :
59 : fuse_req_t req;
60 : };
61 :
62 : static void
63 8 : cuse_io_ctx_free(struct cuse_io_ctx *ctx)
64 : {
65 8 : spdk_free(ctx->data);
66 8 : spdk_free(ctx->metadata);
67 8 : free(ctx);
68 8 : }
69 :
70 : #define FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, val) \
71 : if (out_bufsz == 0) { \
72 : struct iovec out_iov; \
73 : out_iov.iov_base = (void *)arg; \
74 : out_iov.iov_len = sizeof(val); \
75 : fuse_reply_ioctl_retry(req, NULL, 0, &out_iov, 1); \
76 : return; \
77 : }
78 :
79 : #define FUSE_MAX_SIZE 128*1024
80 :
81 : static bool
82 2 : fuse_check_req_size(fuse_req_t req, struct iovec iov[], int iovcnt)
83 : {
84 2 : int total_iov_len = 0;
85 5 : for (int i = 0; i < iovcnt; i++) {
86 3 : total_iov_len += iov[i].iov_len;
87 3 : if (total_iov_len > FUSE_MAX_SIZE) {
88 0 : fuse_reply_err(req, ENOMEM);
89 0 : SPDK_ERRLOG("FUSE request cannot be larger that %d\n", FUSE_MAX_SIZE);
90 0 : return false;
91 : }
92 : }
93 2 : return true;
94 : }
95 :
96 : static void
97 0 : cuse_nvme_passthru_cmd_cb(void *arg, const struct spdk_nvme_cpl *cpl)
98 : {
99 0 : struct cuse_io_ctx *ctx = arg;
100 0 : struct iovec out_iov[3];
101 0 : struct spdk_nvme_cpl _cpl;
102 0 : int out_iovcnt = 0;
103 0 : uint16_t status_field = cpl->status_raw >> 1; /* Drop out phase bit */
104 :
105 0 : memcpy(&_cpl, cpl, sizeof(struct spdk_nvme_cpl));
106 0 : out_iov[out_iovcnt].iov_base = &_cpl.cdw0;
107 0 : out_iov[out_iovcnt].iov_len = sizeof(_cpl.cdw0);
108 0 : out_iovcnt += 1;
109 :
110 0 : if (ctx->data_transfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {
111 0 : if (ctx->data_len > 0) {
112 0 : out_iov[out_iovcnt].iov_base = ctx->data;
113 0 : out_iov[out_iovcnt].iov_len = ctx->data_len;
114 0 : out_iovcnt += 1;
115 : }
116 0 : if (ctx->metadata_len > 0) {
117 0 : out_iov[out_iovcnt].iov_base = ctx->metadata;
118 0 : out_iov[out_iovcnt].iov_len = ctx->metadata_len;
119 0 : out_iovcnt += 1;
120 : }
121 : }
122 :
123 0 : fuse_reply_ioctl_iov(ctx->req, status_field, out_iov, out_iovcnt);
124 0 : cuse_io_ctx_free(ctx);
125 0 : }
126 :
127 : static void
128 0 : cuse_nvme_passthru_cmd_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
129 : {
130 : int rc;
131 0 : struct cuse_io_ctx *ctx = arg;
132 :
133 0 : if (nsid != 0) {
134 0 : rc = spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, ctrlr->external_io_msgs_qpair, &ctx->nvme_cmd,
135 : ctx->data,
136 0 : ctx->data_len, ctx->metadata, cuse_nvme_passthru_cmd_cb, (void *)ctx);
137 : } else {
138 0 : rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &ctx->nvme_cmd, ctx->data, ctx->data_len,
139 : cuse_nvme_passthru_cmd_cb, (void *)ctx);
140 : }
141 0 : if (rc < 0) {
142 0 : fuse_reply_err(ctx->req, EINVAL);
143 0 : cuse_io_ctx_free(ctx);
144 : }
145 0 : }
146 :
147 : static void
148 2 : cuse_nvme_passthru_cmd_send(fuse_req_t req, struct nvme_passthru_cmd *passthru_cmd,
149 : const void *data, const void *metadata, int cmd)
150 : {
151 : struct cuse_io_ctx *ctx;
152 2 : struct cuse_device *cuse_device = fuse_req_userdata(req);
153 : int rv;
154 :
155 2 : ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
156 2 : if (!ctx) {
157 0 : SPDK_ERRLOG("Cannot allocate memory for cuse_io_ctx\n");
158 0 : fuse_reply_err(req, ENOMEM);
159 0 : return;
160 : }
161 :
162 2 : ctx->req = req;
163 2 : ctx->data_transfer = spdk_nvme_opc_get_data_transfer(passthru_cmd->opcode);
164 :
165 2 : memset(&ctx->nvme_cmd, 0, sizeof(ctx->nvme_cmd));
166 2 : ctx->nvme_cmd.opc = passthru_cmd->opcode;
167 2 : ctx->nvme_cmd.nsid = passthru_cmd->nsid;
168 2 : ctx->nvme_cmd.cdw10 = passthru_cmd->cdw10;
169 2 : ctx->nvme_cmd.cdw11 = passthru_cmd->cdw11;
170 2 : ctx->nvme_cmd.cdw12 = passthru_cmd->cdw12;
171 2 : ctx->nvme_cmd.cdw13 = passthru_cmd->cdw13;
172 2 : ctx->nvme_cmd.cdw14 = passthru_cmd->cdw14;
173 2 : ctx->nvme_cmd.cdw15 = passthru_cmd->cdw15;
174 :
175 2 : ctx->data_len = passthru_cmd->data_len;
176 2 : ctx->metadata_len = passthru_cmd->metadata_len;
177 :
178 2 : if (ctx->data_len > 0) {
179 2 : ctx->data = spdk_malloc(ctx->data_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
180 2 : if (!ctx->data) {
181 0 : SPDK_ERRLOG("Cannot allocate memory for data\n");
182 0 : fuse_reply_err(req, ENOMEM);
183 0 : free(ctx);
184 0 : return;
185 : }
186 2 : if (data != NULL) {
187 0 : memcpy(ctx->data, data, ctx->data_len);
188 : }
189 : }
190 :
191 2 : if (ctx->metadata_len > 0) {
192 1 : ctx->metadata = spdk_malloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
193 1 : if (!ctx->metadata) {
194 0 : SPDK_ERRLOG("Cannot allocate memory for metadata\n");
195 0 : fuse_reply_err(req, ENOMEM);
196 0 : cuse_io_ctx_free(ctx);
197 0 : return;
198 : }
199 1 : if (metadata != NULL) {
200 0 : memcpy(ctx->metadata, metadata, ctx->metadata_len);
201 : }
202 : }
203 :
204 2 : if ((unsigned int)cmd != NVME_IOCTL_ADMIN_CMD) {
205 : /* Send NS for IO IOCTLs */
206 2 : rv = nvme_io_msg_send(cuse_device->ctrlr, passthru_cmd->nsid, cuse_nvme_passthru_cmd_execute, ctx);
207 : } else {
208 : /* NS == 0 for Admin IOCTLs */
209 0 : rv = nvme_io_msg_send(cuse_device->ctrlr, 0, cuse_nvme_passthru_cmd_execute, ctx);
210 : }
211 2 : if (rv) {
212 0 : SPDK_ERRLOG("Cannot send io msg to the controller\n");
213 0 : fuse_reply_err(req, -rv);
214 0 : cuse_io_ctx_free(ctx);
215 0 : return;
216 : }
217 : }
218 :
219 : static void
220 0 : cuse_nvme_passthru_cmd(fuse_req_t req, int cmd, void *arg,
221 : struct fuse_file_info *fi, unsigned flags,
222 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
223 : {
224 : struct nvme_passthru_cmd *passthru_cmd;
225 0 : struct iovec in_iov[3], out_iov[3];
226 0 : int in_iovcnt = 0, out_iovcnt = 0;
227 0 : const void *dptr = NULL, *mdptr = NULL;
228 : enum spdk_nvme_data_transfer data_transfer;
229 :
230 0 : in_iov[in_iovcnt].iov_base = (void *)arg;
231 0 : in_iov[in_iovcnt].iov_len = sizeof(*passthru_cmd);
232 0 : in_iovcnt += 1;
233 0 : if (in_bufsz == 0) {
234 0 : fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, NULL, out_iovcnt);
235 0 : return;
236 : }
237 :
238 0 : passthru_cmd = (struct nvme_passthru_cmd *)in_buf;
239 0 : data_transfer = spdk_nvme_opc_get_data_transfer(passthru_cmd->opcode);
240 :
241 0 : if (data_transfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
242 : /* Make data pointer accessible (RO) */
243 0 : if (passthru_cmd->addr != 0) {
244 0 : in_iov[in_iovcnt].iov_base = (void *)passthru_cmd->addr;
245 0 : in_iov[in_iovcnt].iov_len = passthru_cmd->data_len;
246 0 : in_iovcnt += 1;
247 : }
248 : /* Make metadata pointer accessible (RO) */
249 0 : if (passthru_cmd->metadata != 0) {
250 0 : in_iov[in_iovcnt].iov_base = (void *)passthru_cmd->metadata;
251 0 : in_iov[in_iovcnt].iov_len = passthru_cmd->metadata_len;
252 0 : in_iovcnt += 1;
253 : }
254 : }
255 :
256 0 : if (!fuse_check_req_size(req, in_iov, in_iovcnt)) {
257 0 : return;
258 : }
259 : /* Always make result field writeable regardless of data transfer bits */
260 0 : out_iov[out_iovcnt].iov_base = &((struct nvme_passthru_cmd *)arg)->result;
261 0 : out_iov[out_iovcnt].iov_len = sizeof(uint32_t);
262 0 : out_iovcnt += 1;
263 :
264 0 : if (data_transfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {
265 : /* Make data pointer accessible (WO) */
266 0 : if (passthru_cmd->data_len > 0) {
267 0 : out_iov[out_iovcnt].iov_base = (void *)passthru_cmd->addr;
268 0 : out_iov[out_iovcnt].iov_len = passthru_cmd->data_len;
269 0 : out_iovcnt += 1;
270 : }
271 : /* Make metadata pointer accessible (WO) */
272 0 : if (passthru_cmd->metadata_len > 0) {
273 0 : out_iov[out_iovcnt].iov_base = (void *)passthru_cmd->metadata;
274 0 : out_iov[out_iovcnt].iov_len = passthru_cmd->metadata_len;
275 0 : out_iovcnt += 1;
276 : }
277 : }
278 :
279 0 : if (!fuse_check_req_size(req, out_iov, out_iovcnt)) {
280 0 : return;
281 : }
282 :
283 0 : if (out_bufsz == 0) {
284 0 : fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, out_iov, out_iovcnt);
285 0 : return;
286 : }
287 :
288 0 : if (data_transfer == SPDK_NVME_DATA_BIDIRECTIONAL) {
289 0 : fuse_reply_err(req, EINVAL);
290 0 : return;
291 : }
292 :
293 0 : if (data_transfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
294 0 : dptr = (passthru_cmd->addr == 0) ? NULL : (uint8_t *)in_buf + sizeof(*passthru_cmd);
295 0 : mdptr = (passthru_cmd->metadata == 0) ? NULL : (uint8_t *)in_buf + sizeof(*passthru_cmd) +
296 0 : passthru_cmd->data_len;
297 : }
298 :
299 0 : cuse_nvme_passthru_cmd_send(req, passthru_cmd, dptr, mdptr, cmd);
300 : }
301 :
302 : static void
303 0 : cuse_nvme_reset_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
304 : {
305 : int rc;
306 0 : fuse_req_t req = arg;
307 :
308 0 : rc = spdk_nvme_ctrlr_reset(ctrlr);
309 0 : if (rc) {
310 0 : fuse_reply_err(req, rc);
311 0 : return;
312 : }
313 :
314 0 : fuse_reply_ioctl_iov(req, 0, NULL, 0);
315 : }
316 :
317 : static void
318 0 : cuse_nvme_subsys_reset_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
319 : {
320 : int rc;
321 0 : fuse_req_t req = arg;
322 :
323 0 : rc = spdk_nvme_ctrlr_reset_subsystem(ctrlr);
324 0 : if (rc) {
325 0 : fuse_reply_err(req, rc);
326 0 : return;
327 : }
328 :
329 0 : fuse_reply_ioctl_iov(req, 0, NULL, 0);
330 : }
331 :
332 : static void
333 2 : cuse_nvme_reset(fuse_req_t req, int cmd, void *arg,
334 : struct fuse_file_info *fi, unsigned flags,
335 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
336 : {
337 : int rv;
338 2 : struct cuse_device *cuse_device = fuse_req_userdata(req);
339 :
340 2 : if (cuse_device->nsid) {
341 1 : SPDK_ERRLOG("Namespace reset not supported\n");
342 1 : fuse_reply_err(req, EINVAL);
343 1 : return;
344 : }
345 :
346 1 : if (cmd == NVME_IOCTL_SUBSYS_RESET) {
347 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_SUBSYS_RESET\n");
348 0 : rv = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_subsys_reset_execute,
349 : (void *)req);
350 : } else {
351 1 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_RESET\n");
352 1 : rv = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_reset_execute, (void *)req);
353 : }
354 1 : if (rv) {
355 0 : SPDK_ERRLOG("Cannot send reset\n");
356 0 : fuse_reply_err(req, EINVAL);
357 : }
358 : }
359 :
360 : static void
361 0 : cuse_nvme_rescan_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
362 : {
363 0 : fuse_req_t req = arg;
364 :
365 0 : nvme_ctrlr_update_namespaces(ctrlr);
366 0 : fuse_reply_ioctl_iov(req, 0, NULL, 0);
367 0 : }
368 :
369 : static void
370 0 : cuse_nvme_rescan(fuse_req_t req, int cmd, void *arg,
371 : struct fuse_file_info *fi, unsigned flags,
372 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
373 : {
374 : int rv;
375 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
376 :
377 0 : if (cuse_device->nsid) {
378 0 : SPDK_ERRLOG("Namespace rescan not supported\n");
379 0 : fuse_reply_err(req, EINVAL);
380 0 : return;
381 : }
382 :
383 0 : rv = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_rescan_execute, (void *)req);
384 0 : if (rv) {
385 0 : SPDK_ERRLOG("Cannot send rescan\n");
386 0 : fuse_reply_err(req, EINVAL);
387 : }
388 : }
389 :
390 : /*****************************************************************************
391 : * Namespace IO requests
392 : */
393 :
394 : static void
395 0 : cuse_nvme_submit_io_write_done(void *ref, const struct spdk_nvme_cpl *cpl)
396 : {
397 0 : struct cuse_io_ctx *ctx = (struct cuse_io_ctx *)ref;
398 0 : uint16_t status_field = cpl->status_raw >> 1; /* Drop out phase bit */
399 :
400 0 : fuse_reply_ioctl_iov(ctx->req, status_field, NULL, 0);
401 :
402 0 : cuse_io_ctx_free(ctx);
403 0 : }
404 :
405 : static void
406 0 : cuse_nvme_submit_io_write_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
407 : {
408 : int rc;
409 0 : struct cuse_io_ctx *ctx = arg;
410 0 : struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
411 :
412 0 : rc = spdk_nvme_ns_cmd_write_with_md(ns, ctrlr->external_io_msgs_qpair, ctx->data, ctx->metadata,
413 : ctx->lba, /* LBA start */
414 : ctx->lba_count, /* number of LBAs */
415 : cuse_nvme_submit_io_write_done, ctx, 0,
416 0 : ctx->appmask, ctx->apptag);
417 :
418 0 : if (rc != 0) {
419 0 : SPDK_ERRLOG("write failed: rc = %d\n", rc);
420 0 : fuse_reply_err(ctx->req, rc);
421 0 : cuse_io_ctx_free(ctx);
422 0 : return;
423 : }
424 : }
425 :
426 : static void
427 3 : cuse_nvme_submit_io_write(struct cuse_device *cuse_device, fuse_req_t req, int cmd, void *arg,
428 : struct fuse_file_info *fi, unsigned flags, uint32_t block_size, uint32_t md_size,
429 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
430 : {
431 3 : const struct nvme_user_io *user_io = in_buf;
432 : struct cuse_io_ctx *ctx;
433 : int rc;
434 :
435 3 : ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
436 3 : if (!ctx) {
437 0 : SPDK_ERRLOG("Cannot allocate memory for context\n");
438 0 : fuse_reply_err(req, ENOMEM);
439 0 : return;
440 : }
441 :
442 3 : ctx->req = req;
443 3 : ctx->lba = user_io->slba;
444 3 : ctx->lba_count = user_io->nblocks + 1;
445 3 : ctx->data_len = ctx->lba_count * block_size;
446 :
447 3 : ctx->data = spdk_zmalloc(ctx->data_len, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY,
448 : SPDK_MALLOC_DMA);
449 3 : if (ctx->data == NULL) {
450 0 : SPDK_ERRLOG("Write buffer allocation failed\n");
451 0 : fuse_reply_err(ctx->req, ENOMEM);
452 0 : free(ctx);
453 0 : return;
454 : }
455 :
456 3 : memcpy(ctx->data, (uint8_t *)in_buf + sizeof(*user_io), ctx->data_len);
457 :
458 3 : if (user_io->metadata) {
459 1 : ctx->apptag = user_io->apptag;
460 1 : ctx->appmask = user_io->appmask;
461 1 : ctx->metadata_len = md_size * ctx->lba_count;
462 1 : ctx->metadata = spdk_zmalloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
463 :
464 1 : if (ctx->metadata == NULL) {
465 0 : SPDK_ERRLOG("Cannot allocate memory for metadata\n");
466 0 : if (ctx->metadata_len == 0) {
467 0 : SPDK_ERRLOG("Device format does not support metadata\n");
468 : }
469 0 : fuse_reply_err(req, ENOMEM);
470 0 : cuse_io_ctx_free(ctx);
471 0 : return;
472 : }
473 :
474 1 : memcpy(ctx->metadata, (uint8_t *)in_buf + sizeof(*user_io) + ctx->data_len,
475 1 : ctx->metadata_len);
476 : }
477 :
478 3 : rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_write_cb,
479 : ctx);
480 3 : if (rc < 0) {
481 0 : SPDK_ERRLOG("Cannot send write io\n");
482 0 : fuse_reply_err(ctx->req, rc);
483 0 : cuse_io_ctx_free(ctx);
484 : }
485 : }
486 :
487 : static void
488 0 : cuse_nvme_submit_io_read_done(void *ref, const struct spdk_nvme_cpl *cpl)
489 : {
490 0 : struct cuse_io_ctx *ctx = (struct cuse_io_ctx *)ref;
491 0 : struct iovec iov[2];
492 0 : int iovcnt = 0;
493 0 : uint16_t status_field = cpl->status_raw >> 1; /* Drop out phase bit */
494 :
495 0 : iov[iovcnt].iov_base = ctx->data;
496 0 : iov[iovcnt].iov_len = ctx->data_len;
497 0 : iovcnt += 1;
498 :
499 0 : if (ctx->metadata) {
500 0 : iov[iovcnt].iov_base = ctx->metadata;
501 0 : iov[iovcnt].iov_len = ctx->metadata_len;
502 0 : iovcnt += 1;
503 : }
504 :
505 0 : fuse_reply_ioctl_iov(ctx->req, status_field, iov, iovcnt);
506 :
507 0 : cuse_io_ctx_free(ctx);
508 0 : }
509 :
510 : static void
511 0 : cuse_nvme_submit_io_read_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void *arg)
512 : {
513 : int rc;
514 0 : struct cuse_io_ctx *ctx = arg;
515 0 : struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
516 :
517 0 : rc = spdk_nvme_ns_cmd_read_with_md(ns, ctrlr->external_io_msgs_qpair, ctx->data, ctx->metadata,
518 : ctx->lba, /* LBA start */
519 : ctx->lba_count, /* number of LBAs */
520 : cuse_nvme_submit_io_read_done, ctx, 0,
521 0 : ctx->appmask, ctx->apptag);
522 :
523 0 : if (rc != 0) {
524 0 : SPDK_ERRLOG("read failed: rc = %d\n", rc);
525 0 : fuse_reply_err(ctx->req, rc);
526 0 : cuse_io_ctx_free(ctx);
527 0 : return;
528 : }
529 : }
530 :
531 : static void
532 3 : cuse_nvme_submit_io_read(struct cuse_device *cuse_device, fuse_req_t req, int cmd, void *arg,
533 : struct fuse_file_info *fi, unsigned flags, uint32_t block_size, uint32_t md_size,
534 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
535 : {
536 : int rc;
537 : struct cuse_io_ctx *ctx;
538 3 : const struct nvme_user_io *user_io = in_buf;
539 :
540 3 : ctx = (struct cuse_io_ctx *)calloc(1, sizeof(struct cuse_io_ctx));
541 3 : if (!ctx) {
542 0 : SPDK_ERRLOG("Cannot allocate memory for context\n");
543 0 : fuse_reply_err(req, ENOMEM);
544 0 : return;
545 : }
546 :
547 3 : ctx->req = req;
548 3 : ctx->lba = user_io->slba;
549 3 : ctx->lba_count = user_io->nblocks + 1;
550 :
551 3 : ctx->data_len = ctx->lba_count * block_size;
552 3 : ctx->data = spdk_zmalloc(ctx->data_len, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY,
553 : SPDK_MALLOC_DMA);
554 3 : if (ctx->data == NULL) {
555 0 : SPDK_ERRLOG("Read buffer allocation failed\n");
556 0 : fuse_reply_err(ctx->req, ENOMEM);
557 0 : free(ctx);
558 0 : return;
559 : }
560 :
561 3 : if (user_io->metadata) {
562 1 : ctx->apptag = user_io->apptag;
563 1 : ctx->appmask = user_io->appmask;
564 1 : ctx->metadata_len = md_size * ctx->lba_count;
565 1 : ctx->metadata = spdk_zmalloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
566 :
567 1 : if (ctx->metadata == NULL) {
568 0 : SPDK_ERRLOG("Cannot allocate memory for metadata\n");
569 0 : if (ctx->metadata_len == 0) {
570 0 : SPDK_ERRLOG("Device format does not support metadata\n");
571 : }
572 0 : fuse_reply_err(req, ENOMEM);
573 0 : cuse_io_ctx_free(ctx);
574 0 : return;
575 : }
576 : }
577 :
578 3 : rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_read_cb, ctx);
579 3 : if (rc < 0) {
580 0 : SPDK_ERRLOG("Cannot send read io\n");
581 0 : fuse_reply_err(ctx->req, rc);
582 0 : cuse_io_ctx_free(ctx);
583 : }
584 : }
585 :
586 :
587 : static void
588 3 : cuse_nvme_submit_io(fuse_req_t req, int cmd, void *arg,
589 : struct fuse_file_info *fi, unsigned flags,
590 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
591 : {
592 : const struct nvme_user_io *user_io;
593 3 : struct iovec in_iov[3], out_iov[2];
594 3 : int in_iovcnt = 0, out_iovcnt = 0;
595 3 : struct cuse_device *cuse_device = fuse_req_userdata(req);
596 : struct spdk_nvme_ns *ns;
597 : uint32_t block_size;
598 : uint32_t md_size;
599 :
600 3 : in_iov[in_iovcnt].iov_base = (void *)arg;
601 3 : in_iov[in_iovcnt].iov_len = sizeof(*user_io);
602 3 : in_iovcnt += 1;
603 3 : if (in_bufsz == 0) {
604 0 : fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, NULL, 0);
605 0 : return;
606 : }
607 :
608 3 : user_io = in_buf;
609 :
610 3 : ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
611 3 : block_size = spdk_nvme_ns_get_sector_size(ns);
612 3 : md_size = spdk_nvme_ns_get_md_size(ns);
613 :
614 3 : switch (user_io->opcode) {
615 1 : case SPDK_NVME_OPC_READ:
616 1 : out_iov[out_iovcnt].iov_base = (void *)user_io->addr;
617 1 : out_iov[out_iovcnt].iov_len = (user_io->nblocks + 1) * block_size;
618 1 : out_iovcnt += 1;
619 1 : if (user_io->metadata != 0) {
620 0 : out_iov[out_iovcnt].iov_base = (void *)user_io->metadata;
621 0 : out_iov[out_iovcnt].iov_len = (user_io->nblocks + 1) * md_size;
622 0 : out_iovcnt += 1;
623 : }
624 1 : if (!fuse_check_req_size(req, out_iov, out_iovcnt)) {
625 0 : return;
626 : }
627 1 : if (out_bufsz == 0) {
628 0 : fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, out_iov, out_iovcnt);
629 0 : return;
630 : }
631 :
632 1 : cuse_nvme_submit_io_read(cuse_device, req, cmd, arg, fi, flags,
633 : block_size, md_size, in_buf, in_bufsz, out_bufsz);
634 1 : break;
635 1 : case SPDK_NVME_OPC_WRITE:
636 1 : in_iov[in_iovcnt].iov_base = (void *)user_io->addr;
637 1 : in_iov[in_iovcnt].iov_len = (user_io->nblocks + 1) * block_size;
638 1 : in_iovcnt += 1;
639 1 : if (user_io->metadata != 0) {
640 0 : in_iov[in_iovcnt].iov_base = (void *)user_io->metadata;
641 0 : in_iov[in_iovcnt].iov_len = (user_io->nblocks + 1) * md_size;
642 0 : in_iovcnt += 1;
643 : }
644 1 : if (!fuse_check_req_size(req, in_iov, in_iovcnt)) {
645 0 : return;
646 : }
647 1 : if (in_bufsz == sizeof(*user_io)) {
648 0 : fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, NULL, out_iovcnt);
649 0 : return;
650 : }
651 :
652 1 : cuse_nvme_submit_io_write(cuse_device, req, cmd, arg, fi, flags,
653 : block_size, md_size, in_buf, in_bufsz, out_bufsz);
654 1 : break;
655 1 : default:
656 1 : SPDK_ERRLOG("SUBMIT_IO: opc:%d not valid\n", user_io->opcode);
657 1 : fuse_reply_err(req, EINVAL);
658 1 : return;
659 : }
660 :
661 : }
662 :
663 : /*****************************************************************************
664 : * Other namespace IOCTLs
665 : */
666 : static void
667 0 : cuse_blkgetsize64(fuse_req_t req, int cmd, void *arg,
668 : struct fuse_file_info *fi, unsigned flags,
669 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
670 : {
671 0 : uint64_t size;
672 : struct spdk_nvme_ns *ns;
673 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
674 :
675 0 : FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, size);
676 :
677 0 : ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
678 0 : size = spdk_nvme_ns_get_num_sectors(ns);
679 0 : fuse_reply_ioctl(req, 0, &size, sizeof(size));
680 : }
681 :
682 : static void
683 0 : cuse_blkpbszget(fuse_req_t req, int cmd, void *arg,
684 : struct fuse_file_info *fi, unsigned flags,
685 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
686 : {
687 0 : int pbsz;
688 : struct spdk_nvme_ns *ns;
689 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
690 :
691 0 : FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, pbsz);
692 :
693 0 : ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
694 0 : pbsz = spdk_nvme_ns_get_sector_size(ns);
695 0 : fuse_reply_ioctl(req, 0, &pbsz, sizeof(pbsz));
696 : }
697 :
698 : static void
699 0 : cuse_blkgetsize(fuse_req_t req, int cmd, void *arg,
700 : struct fuse_file_info *fi, unsigned flags,
701 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
702 : {
703 0 : long size;
704 : struct spdk_nvme_ns *ns;
705 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
706 :
707 0 : FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, size);
708 :
709 0 : ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
710 :
711 : /* return size in 512 bytes blocks */
712 0 : size = spdk_nvme_ns_get_num_sectors(ns) * 512 / spdk_nvme_ns_get_sector_size(ns);
713 0 : fuse_reply_ioctl(req, 0, &size, sizeof(size));
714 : }
715 :
716 : static void
717 0 : cuse_blkgetsectorsize(fuse_req_t req, int cmd, void *arg,
718 : struct fuse_file_info *fi, unsigned flags,
719 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
720 : {
721 0 : int ssize;
722 : struct spdk_nvme_ns *ns;
723 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
724 :
725 0 : FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, ssize);
726 :
727 0 : ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);
728 0 : ssize = spdk_nvme_ns_get_sector_size(ns);
729 0 : fuse_reply_ioctl(req, 0, &ssize, sizeof(ssize));
730 : }
731 :
732 : static void
733 0 : cuse_getid(fuse_req_t req, int cmd, void *arg,
734 : struct fuse_file_info *fi, unsigned flags,
735 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
736 : {
737 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
738 :
739 0 : fuse_reply_ioctl(req, cuse_device->nsid, NULL, 0);
740 0 : }
741 :
742 : struct cuse_transport {
743 : char trstring[SPDK_NVMF_TRSTRING_MAX_LEN + 1];
744 : char traddr[SPDK_NVMF_TRADDR_MAX_LEN + 1];
745 : };
746 :
747 : #define SPDK_CUSE_GET_TRANSPORT _IOWR('n', 0x1, struct cuse_transport)
748 :
749 : static void
750 0 : cuse_get_transport(fuse_req_t req, int cmd, void *arg,
751 : struct fuse_file_info *fi, unsigned flags,
752 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
753 : {
754 0 : struct cuse_device *cuse_device = fuse_req_userdata(req);
755 0 : struct cuse_transport tr = {};
756 :
757 0 : FUSE_REPLY_CHECK_BUFFER(req, arg, out_bufsz, tr);
758 :
759 0 : memcpy(tr.trstring, cuse_device->ctrlr->trid.trstring, SPDK_NVMF_TRSTRING_MAX_LEN + 1);
760 0 : memcpy(tr.traddr, cuse_device->ctrlr->trid.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1);
761 :
762 0 : fuse_reply_ioctl(req, 0, &tr, sizeof(tr));
763 : }
764 :
765 : static void
766 0 : cuse_ctrlr_ioctl(fuse_req_t req, int cmd, void *arg,
767 : struct fuse_file_info *fi, unsigned flags,
768 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
769 : {
770 0 : if (flags & FUSE_IOCTL_COMPAT) {
771 0 : fuse_reply_err(req, ENOSYS);
772 0 : return;
773 : }
774 :
775 0 : switch ((unsigned int)cmd) {
776 0 : case NVME_IOCTL_ADMIN_CMD:
777 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_ADMIN_CMD\n");
778 0 : cuse_nvme_passthru_cmd(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
779 0 : break;
780 :
781 0 : case NVME_IOCTL_RESET:
782 : case NVME_IOCTL_SUBSYS_RESET:
783 0 : cuse_nvme_reset(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
784 0 : break;
785 :
786 0 : case NVME_IOCTL_RESCAN:
787 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_RESCAN\n");
788 0 : cuse_nvme_rescan(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
789 0 : break;
790 :
791 0 : case SPDK_CUSE_GET_TRANSPORT:
792 0 : SPDK_DEBUGLOG(nvme_cuse, "SPDK_CUSE_GET_TRANSPORT\n");
793 0 : cuse_get_transport(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
794 0 : break;
795 :
796 0 : default:
797 0 : SPDK_ERRLOG("Unsupported IOCTL 0x%X.\n", cmd);
798 0 : fuse_reply_err(req, ENOTTY);
799 : }
800 : }
801 :
802 : static void
803 0 : cuse_ns_ioctl(fuse_req_t req, int cmd, void *arg,
804 : struct fuse_file_info *fi, unsigned flags,
805 : const void *in_buf, size_t in_bufsz, size_t out_bufsz)
806 : {
807 0 : if (flags & FUSE_IOCTL_COMPAT) {
808 0 : fuse_reply_err(req, ENOSYS);
809 0 : return;
810 : }
811 :
812 0 : switch ((unsigned int)cmd) {
813 0 : case NVME_IOCTL_ADMIN_CMD:
814 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_ADMIN_CMD\n");
815 0 : cuse_nvme_passthru_cmd(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
816 0 : break;
817 :
818 0 : case NVME_IOCTL_SUBMIT_IO:
819 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_SUBMIT_IO\n");
820 0 : cuse_nvme_submit_io(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
821 0 : break;
822 :
823 0 : case NVME_IOCTL_IO_CMD:
824 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_IO_CMD\n");
825 0 : cuse_nvme_passthru_cmd(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
826 0 : break;
827 :
828 0 : case NVME_IOCTL_ID:
829 0 : SPDK_DEBUGLOG(nvme_cuse, "NVME_IOCTL_ID\n");
830 0 : cuse_getid(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
831 0 : break;
832 :
833 0 : case BLKPBSZGET:
834 0 : SPDK_DEBUGLOG(nvme_cuse, "BLKPBSZGET\n");
835 0 : cuse_blkpbszget(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
836 0 : break;
837 :
838 0 : case BLKSSZGET:
839 0 : SPDK_DEBUGLOG(nvme_cuse, "BLKSSZGET\n");
840 0 : cuse_blkgetsectorsize(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
841 0 : break;
842 :
843 0 : case BLKGETSIZE:
844 0 : SPDK_DEBUGLOG(nvme_cuse, "BLKGETSIZE\n");
845 : /* Returns the device size as a number of 512-byte blocks (returns pointer to long) */
846 0 : cuse_blkgetsize(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
847 0 : break;
848 :
849 0 : case BLKGETSIZE64:
850 0 : SPDK_DEBUGLOG(nvme_cuse, "BLKGETSIZE64\n");
851 : /* Returns the device size in sectors (returns pointer to uint64_t) */
852 0 : cuse_blkgetsize64(req, cmd, arg, fi, flags, in_buf, in_bufsz, out_bufsz);
853 0 : break;
854 :
855 0 : default:
856 0 : SPDK_ERRLOG("Unsupported IOCTL 0x%X.\n", cmd);
857 0 : fuse_reply_err(req, ENOTTY);
858 : }
859 : }
860 :
861 : /*****************************************************************************
862 : * CUSE threads initialization.
863 : */
864 :
865 : static void
866 0 : cuse_open(fuse_req_t req, struct fuse_file_info *fi)
867 : {
868 0 : fuse_reply_open(req, fi);
869 0 : }
870 :
871 : static const struct cuse_lowlevel_ops cuse_ctrlr_clop = {
872 : .open = cuse_open,
873 : .ioctl = cuse_ctrlr_ioctl,
874 : };
875 :
876 : static const struct cuse_lowlevel_ops cuse_ns_clop = {
877 : .open = cuse_open,
878 : .ioctl = cuse_ns_ioctl,
879 : };
880 :
881 : static int
882 0 : cuse_session_create(struct cuse_device *cuse_device)
883 : {
884 0 : char *cuse_argv[] = { "cuse", "-f" };
885 0 : int multithreaded;
886 0 : int cuse_argc = SPDK_COUNTOF(cuse_argv);
887 0 : struct cuse_info ci;
888 0 : char devname_arg[128 + 8];
889 0 : const char *dev_info_argv[] = { devname_arg };
890 :
891 0 : snprintf(devname_arg, sizeof(devname_arg), "DEVNAME=%s", cuse_device->dev_name);
892 :
893 0 : memset(&ci, 0, sizeof(ci));
894 0 : ci.dev_info_argc = 1;
895 0 : ci.dev_info_argv = dev_info_argv;
896 0 : ci.flags = CUSE_UNRESTRICTED_IOCTL;
897 :
898 0 : if (cuse_device->nsid) {
899 0 : cuse_device->session = cuse_lowlevel_setup(cuse_argc, cuse_argv, &ci, &cuse_ns_clop,
900 : &multithreaded, cuse_device);
901 : } else {
902 0 : cuse_device->session = cuse_lowlevel_setup(cuse_argc, cuse_argv, &ci, &cuse_ctrlr_clop,
903 : &multithreaded, cuse_device);
904 : }
905 :
906 0 : if (!cuse_device->session) {
907 0 : SPDK_ERRLOG("Cannot create cuse session\n");
908 0 : return -1;
909 : }
910 0 : SPDK_NOTICELOG("fuse session for device %s created\n", cuse_device->dev_name);
911 0 : return 0;
912 : }
913 :
914 : static void *
915 0 : cuse_thread(void *arg)
916 : {
917 0 : struct cuse_device *cuse_device = arg;
918 : int rc;
919 0 : struct fuse_buf buf = { .mem = NULL };
920 0 : struct pollfd fds;
921 0 : int timeout_msecs = 500;
922 :
923 0 : spdk_unaffinitize_thread();
924 :
925 : /* Receive and process fuse requests */
926 0 : fds.fd = fuse_session_fd(cuse_device->session);
927 0 : fds.events = POLLIN;
928 0 : while (!fuse_session_exited(cuse_device->session)) {
929 0 : rc = poll(&fds, 1, timeout_msecs);
930 0 : if (rc <= 0) {
931 0 : continue;
932 : }
933 0 : rc = fuse_session_receive_buf(cuse_device->session, &buf);
934 0 : if (rc > 0) {
935 0 : fuse_session_process_buf(cuse_device->session, &buf);
936 : }
937 : }
938 0 : free(buf.mem);
939 0 : fuse_session_reset(cuse_device->session);
940 0 : pthread_exit(NULL);
941 : }
942 :
943 : static struct cuse_device *nvme_cuse_get_cuse_ns_device(struct spdk_nvme_ctrlr *ctrlr,
944 : uint32_t nsid);
945 :
946 : /*****************************************************************************
947 : * CUSE devices management
948 : */
949 :
950 : static int
951 0 : cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid)
952 : {
953 : struct cuse_device *ns_device;
954 : int rv;
955 :
956 0 : ns_device = nvme_cuse_get_cuse_ns_device(ctrlr_device->ctrlr, nsid);
957 0 : if (ns_device != NULL) {
958 0 : return 0;
959 : }
960 :
961 0 : ns_device = calloc(1, sizeof(struct cuse_device));
962 0 : if (ns_device == NULL) {
963 0 : return -ENOMEM;
964 : }
965 :
966 0 : ns_device->ctrlr = ctrlr_device->ctrlr;
967 0 : ns_device->ctrlr_device = ctrlr_device;
968 0 : ns_device->nsid = nsid;
969 0 : rv = snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "%sn%d",
970 0 : ctrlr_device->dev_name, ns_device->nsid);
971 0 : if (rv < 0) {
972 0 : SPDK_ERRLOG("Device name too long.\n");
973 0 : free(ns_device);
974 0 : return -ENAMETOOLONG;
975 : }
976 0 : rv = cuse_session_create(ns_device);
977 0 : if (rv != 0) {
978 0 : free(ns_device);
979 0 : return rv;
980 : }
981 0 : rv = pthread_create(&ns_device->tid, NULL, cuse_thread, ns_device);
982 0 : if (rv != 0) {
983 0 : SPDK_ERRLOG("pthread_create failed\n");
984 0 : free(ns_device);
985 0 : return -rv;
986 : }
987 0 : TAILQ_INSERT_TAIL(&ctrlr_device->ns_devices, ns_device, tailq);
988 :
989 0 : return 0;
990 : }
991 :
992 : static void
993 2 : cuse_nvme_ns_stop(struct cuse_device *ctrlr_device, struct cuse_device *ns_device)
994 : {
995 2 : if (ns_device->session != NULL) {
996 0 : fuse_session_exit(ns_device->session);
997 : }
998 2 : pthread_join(ns_device->tid, NULL);
999 2 : TAILQ_REMOVE(&ctrlr_device->ns_devices, ns_device, tailq);
1000 2 : if (ns_device->session != NULL) {
1001 0 : cuse_lowlevel_teardown(ns_device->session);
1002 : }
1003 2 : free(ns_device);
1004 2 : }
1005 :
1006 : static int
1007 0 : nvme_cuse_claim(struct cuse_device *ctrlr_device, uint32_t index)
1008 : {
1009 : int dev_fd;
1010 : int pid;
1011 : void *dev_map;
1012 0 : struct flock cusedev_lock = {
1013 : .l_type = F_WRLCK,
1014 : .l_whence = SEEK_SET,
1015 : .l_start = 0,
1016 : .l_len = 0,
1017 : };
1018 :
1019 0 : snprintf(ctrlr_device->lock_name, sizeof(ctrlr_device->lock_name),
1020 : "/var/tmp/spdk_nvme_cuse_lock_%" PRIu32, index);
1021 :
1022 0 : dev_fd = open(ctrlr_device->lock_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
1023 0 : if (dev_fd == -1) {
1024 0 : SPDK_ERRLOG("could not open %s\n", ctrlr_device->lock_name);
1025 0 : return -errno;
1026 : }
1027 :
1028 0 : if (ftruncate(dev_fd, sizeof(int)) != 0) {
1029 0 : SPDK_ERRLOG("could not truncate %s\n", ctrlr_device->lock_name);
1030 0 : close(dev_fd);
1031 0 : return -errno;
1032 : }
1033 :
1034 0 : dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,
1035 : MAP_SHARED, dev_fd, 0);
1036 0 : if (dev_map == MAP_FAILED) {
1037 0 : SPDK_ERRLOG("could not mmap dev %s (%d)\n", ctrlr_device->lock_name, errno);
1038 0 : close(dev_fd);
1039 0 : return -errno;
1040 : }
1041 :
1042 0 : if (fcntl(dev_fd, F_SETLK, &cusedev_lock) != 0) {
1043 0 : pid = *(int *)dev_map;
1044 0 : SPDK_ERRLOG("Cannot create lock on device %s, probably"
1045 : " process %d has claimed it\n", ctrlr_device->lock_name, pid);
1046 0 : munmap(dev_map, sizeof(int));
1047 0 : close(dev_fd);
1048 : /* F_SETLK returns unspecified errnos, normalize them */
1049 0 : return -EACCES;
1050 : }
1051 :
1052 0 : *(int *)dev_map = (int)getpid();
1053 0 : munmap(dev_map, sizeof(int));
1054 0 : ctrlr_device->claim_fd = dev_fd;
1055 0 : ctrlr_device->index = index;
1056 : /* Keep dev_fd open to maintain the lock. */
1057 0 : return 0;
1058 : }
1059 :
1060 : static void
1061 1 : nvme_cuse_unclaim(struct cuse_device *ctrlr_device)
1062 : {
1063 1 : close(ctrlr_device->claim_fd);
1064 1 : ctrlr_device->claim_fd = -1;
1065 1 : unlink(ctrlr_device->lock_name);
1066 1 : }
1067 :
1068 : static void
1069 1 : cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device)
1070 : {
1071 : struct cuse_device *ns_device, *tmp;
1072 :
1073 3 : TAILQ_FOREACH_SAFE(ns_device, &ctrlr_device->ns_devices, tailq, tmp) {
1074 2 : cuse_nvme_ns_stop(ctrlr_device, ns_device);
1075 : }
1076 :
1077 1 : assert(TAILQ_EMPTY(&ctrlr_device->ns_devices));
1078 :
1079 1 : fuse_session_exit(ctrlr_device->session);
1080 1 : pthread_join(ctrlr_device->tid, NULL);
1081 1 : TAILQ_REMOVE(&g_ctrlr_ctx_head, ctrlr_device, tailq);
1082 1 : spdk_bit_array_clear(g_ctrlr_started, ctrlr_device->index);
1083 1 : if (spdk_bit_array_count_set(g_ctrlr_started) == 0) {
1084 1 : spdk_bit_array_free(&g_ctrlr_started);
1085 : }
1086 1 : nvme_cuse_unclaim(ctrlr_device);
1087 1 : if (ctrlr_device->session != NULL) {
1088 0 : cuse_lowlevel_teardown(ctrlr_device->session);
1089 : }
1090 1 : free(ctrlr_device);
1091 1 : }
1092 :
1093 : static int
1094 0 : cuse_nvme_ctrlr_update_namespaces(struct cuse_device *ctrlr_device)
1095 : {
1096 : struct cuse_device *ns_device, *tmp;
1097 : uint32_t nsid;
1098 :
1099 : /* Remove namespaces that have disappeared */
1100 0 : TAILQ_FOREACH_SAFE(ns_device, &ctrlr_device->ns_devices, tailq, tmp) {
1101 0 : if (!spdk_nvme_ctrlr_is_active_ns(ctrlr_device->ctrlr, ns_device->nsid)) {
1102 0 : cuse_nvme_ns_stop(ctrlr_device, ns_device);
1103 : }
1104 : }
1105 :
1106 : /* Add new namespaces */
1107 0 : nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr_device->ctrlr);
1108 0 : while (nsid != 0) {
1109 0 : if (cuse_nvme_ns_start(ctrlr_device, nsid) < 0) {
1110 0 : SPDK_ERRLOG("Cannot start CUSE namespace device.");
1111 0 : return -1;
1112 : }
1113 :
1114 0 : nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr_device->ctrlr, nsid);
1115 : }
1116 :
1117 0 : return 0;
1118 : }
1119 :
1120 : #ifdef FUSE_LOG_H_
1121 : static void
1122 : nvme_fuse_log_func(enum fuse_log_level level, const char *fmt, va_list ap)
1123 : {
1124 : /* fuse will unnecessarily print this log message when tearing down
1125 : * sessions, once for every session after the first. So use this custom
1126 : * log handler to silence that specific log message.
1127 : */
1128 : if (strstr(fmt, "fuse_remove_signal_handlers: unknown session") != NULL) {
1129 : return;
1130 : }
1131 :
1132 : vfprintf(stderr, fmt, ap);
1133 : }
1134 : #endif
1135 :
1136 : static int
1137 0 : nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr)
1138 : {
1139 0 : int rv = 0;
1140 : struct cuse_device *ctrlr_device;
1141 :
1142 0 : SPDK_NOTICELOG("Creating cuse device for controller\n");
1143 :
1144 0 : if (g_ctrlr_started == NULL) {
1145 0 : g_ctrlr_started = spdk_bit_array_create(128);
1146 0 : if (g_ctrlr_started == NULL) {
1147 0 : SPDK_ERRLOG("Cannot create bit array\n");
1148 0 : return -ENOMEM;
1149 : }
1150 : #ifdef FUSE_LOG_H_
1151 : /* Older versions of libfuse don't have fuse_set_log_func nor
1152 : * fuse_log.h, so this is the easiest way to check for it
1153 : * without adding a separate CONFIG flag.
1154 : */
1155 : fuse_set_log_func(nvme_fuse_log_func);
1156 : #endif
1157 : }
1158 :
1159 0 : ctrlr_device = (struct cuse_device *)calloc(1, sizeof(struct cuse_device));
1160 0 : if (!ctrlr_device) {
1161 0 : SPDK_ERRLOG("Cannot allocate memory for ctrlr_device.");
1162 0 : rv = -ENOMEM;
1163 0 : goto free_device;
1164 : }
1165 :
1166 0 : ctrlr_device->ctrlr = ctrlr;
1167 :
1168 : /* Check if device already exists, if not increment index until success */
1169 0 : ctrlr_device->index = 0;
1170 : while (1) {
1171 0 : ctrlr_device->index = spdk_bit_array_find_first_clear(g_ctrlr_started, ctrlr_device->index);
1172 0 : if (ctrlr_device->index == UINT32_MAX) {
1173 0 : SPDK_ERRLOG("Too many registered controllers\n");
1174 0 : goto free_device;
1175 : }
1176 :
1177 0 : if (nvme_cuse_claim(ctrlr_device, ctrlr_device->index) == 0) {
1178 0 : break;
1179 : }
1180 0 : ctrlr_device->index++;
1181 : }
1182 0 : spdk_bit_array_set(g_ctrlr_started, ctrlr_device->index);
1183 0 : snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), "spdk/nvme%d",
1184 : ctrlr_device->index);
1185 :
1186 0 : rv = cuse_session_create(ctrlr_device);
1187 0 : if (rv != 0) {
1188 0 : goto clear_and_free;
1189 : }
1190 :
1191 0 : rv = pthread_create(&ctrlr_device->tid, NULL, cuse_thread, ctrlr_device);
1192 0 : if (rv != 0) {
1193 0 : SPDK_ERRLOG("pthread_create failed\n");
1194 0 : rv = -rv;
1195 0 : goto clear_and_free;
1196 : }
1197 :
1198 0 : TAILQ_INSERT_TAIL(&g_ctrlr_ctx_head, ctrlr_device, tailq);
1199 :
1200 0 : TAILQ_INIT(&ctrlr_device->ns_devices);
1201 :
1202 : /* Start all active namespaces */
1203 0 : if (cuse_nvme_ctrlr_update_namespaces(ctrlr_device) < 0) {
1204 0 : SPDK_ERRLOG("Cannot start CUSE namespace devices.");
1205 0 : cuse_nvme_ctrlr_stop(ctrlr_device);
1206 0 : return -1;
1207 : }
1208 :
1209 0 : return 0;
1210 :
1211 0 : clear_and_free:
1212 0 : spdk_bit_array_clear(g_ctrlr_started, ctrlr_device->index);
1213 0 : free_device:
1214 0 : free(ctrlr_device);
1215 0 : if (spdk_bit_array_count_set(g_ctrlr_started) == 0) {
1216 0 : spdk_bit_array_free(&g_ctrlr_started);
1217 : }
1218 0 : return rv;
1219 : }
1220 :
1221 : static struct cuse_device *
1222 11 : nvme_cuse_get_cuse_ctrlr_device(struct spdk_nvme_ctrlr *ctrlr)
1223 : {
1224 11 : struct cuse_device *ctrlr_device = NULL;
1225 :
1226 13 : TAILQ_FOREACH(ctrlr_device, &g_ctrlr_ctx_head, tailq) {
1227 11 : if (ctrlr_device->ctrlr == ctrlr) {
1228 9 : break;
1229 : }
1230 : }
1231 :
1232 11 : return ctrlr_device;
1233 : }
1234 :
1235 : static struct cuse_device *
1236 7 : nvme_cuse_get_cuse_ns_device(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
1237 : {
1238 7 : struct cuse_device *ctrlr_device = NULL;
1239 : struct cuse_device *ns_device;
1240 :
1241 7 : ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
1242 7 : if (!ctrlr_device) {
1243 1 : return NULL;
1244 : }
1245 :
1246 10 : TAILQ_FOREACH(ns_device, &ctrlr_device->ns_devices, tailq) {
1247 7 : if (ns_device->nsid == nsid) {
1248 3 : return ns_device;
1249 : }
1250 : }
1251 :
1252 3 : return NULL;
1253 : }
1254 :
1255 : static void
1256 1 : nvme_cuse_stop(struct spdk_nvme_ctrlr *ctrlr)
1257 : {
1258 : struct cuse_device *ctrlr_device;
1259 :
1260 1 : assert(spdk_process_is_primary());
1261 :
1262 1 : pthread_mutex_lock(&g_cuse_mtx);
1263 :
1264 1 : ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
1265 1 : if (!ctrlr_device) {
1266 0 : SPDK_ERRLOG("Cannot find associated CUSE device\n");
1267 0 : pthread_mutex_unlock(&g_cuse_mtx);
1268 0 : return;
1269 : }
1270 :
1271 1 : cuse_nvme_ctrlr_stop(ctrlr_device);
1272 :
1273 1 : pthread_mutex_unlock(&g_cuse_mtx);
1274 : }
1275 :
1276 : static void
1277 0 : nvme_cuse_update(struct spdk_nvme_ctrlr *ctrlr)
1278 : {
1279 : struct cuse_device *ctrlr_device;
1280 :
1281 0 : assert(spdk_process_is_primary());
1282 :
1283 0 : pthread_mutex_lock(&g_cuse_mtx);
1284 :
1285 0 : ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
1286 0 : if (!ctrlr_device) {
1287 0 : pthread_mutex_unlock(&g_cuse_mtx);
1288 0 : return;
1289 : }
1290 :
1291 0 : cuse_nvme_ctrlr_update_namespaces(ctrlr_device);
1292 :
1293 0 : pthread_mutex_unlock(&g_cuse_mtx);
1294 : }
1295 :
1296 : static struct nvme_io_msg_producer cuse_nvme_io_msg_producer = {
1297 : .name = "cuse",
1298 : .stop = nvme_cuse_stop,
1299 : .update = nvme_cuse_update,
1300 : };
1301 :
1302 : int
1303 0 : spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr)
1304 : {
1305 : int rc;
1306 :
1307 0 : if (!spdk_process_is_primary()) {
1308 0 : SPDK_ERRLOG("only allowed from primary process\n");
1309 0 : return -EINVAL;
1310 : }
1311 :
1312 0 : rc = nvme_io_msg_ctrlr_register(ctrlr, &cuse_nvme_io_msg_producer);
1313 0 : if (rc) {
1314 0 : return rc;
1315 : }
1316 :
1317 0 : pthread_mutex_lock(&g_cuse_mtx);
1318 :
1319 0 : rc = nvme_cuse_start(ctrlr);
1320 0 : if (rc) {
1321 0 : nvme_io_msg_ctrlr_unregister(ctrlr, &cuse_nvme_io_msg_producer);
1322 : }
1323 :
1324 0 : pthread_mutex_unlock(&g_cuse_mtx);
1325 :
1326 0 : return rc;
1327 : }
1328 :
1329 : int
1330 0 : spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr)
1331 : {
1332 : struct cuse_device *ctrlr_device;
1333 :
1334 0 : if (!spdk_process_is_primary()) {
1335 0 : SPDK_ERRLOG("only allowed from primary process\n");
1336 0 : return -EINVAL;
1337 : }
1338 :
1339 0 : pthread_mutex_lock(&g_cuse_mtx);
1340 :
1341 0 : ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
1342 0 : if (!ctrlr_device) {
1343 0 : SPDK_ERRLOG("Cannot find associated CUSE device\n");
1344 0 : pthread_mutex_unlock(&g_cuse_mtx);
1345 0 : return -ENODEV;
1346 : }
1347 :
1348 0 : cuse_nvme_ctrlr_stop(ctrlr_device);
1349 :
1350 0 : pthread_mutex_unlock(&g_cuse_mtx);
1351 :
1352 0 : nvme_io_msg_ctrlr_unregister(ctrlr, &cuse_nvme_io_msg_producer);
1353 :
1354 0 : return 0;
1355 : }
1356 :
1357 : void
1358 0 : spdk_nvme_cuse_update_namespaces(struct spdk_nvme_ctrlr *ctrlr)
1359 : {
1360 0 : nvme_cuse_update(ctrlr);
1361 0 : }
1362 :
1363 : int
1364 3 : spdk_nvme_cuse_get_ctrlr_name(struct spdk_nvme_ctrlr *ctrlr, char *name, size_t *size)
1365 : {
1366 : struct cuse_device *ctrlr_device;
1367 : size_t req_len;
1368 :
1369 3 : pthread_mutex_lock(&g_cuse_mtx);
1370 :
1371 3 : ctrlr_device = nvme_cuse_get_cuse_ctrlr_device(ctrlr);
1372 3 : if (!ctrlr_device) {
1373 1 : pthread_mutex_unlock(&g_cuse_mtx);
1374 1 : return -ENODEV;
1375 : }
1376 :
1377 2 : req_len = strnlen(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name));
1378 2 : if (*size < req_len) {
1379 1 : *size = req_len;
1380 1 : pthread_mutex_unlock(&g_cuse_mtx);
1381 1 : return -ENOSPC;
1382 : }
1383 1 : snprintf(name, req_len + 1, "%s", ctrlr_device->dev_name);
1384 :
1385 1 : pthread_mutex_unlock(&g_cuse_mtx);
1386 :
1387 1 : return 0;
1388 : }
1389 :
1390 : int
1391 4 : spdk_nvme_cuse_get_ns_name(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, char *name, size_t *size)
1392 : {
1393 : struct cuse_device *ns_device;
1394 : size_t req_len;
1395 :
1396 4 : pthread_mutex_lock(&g_cuse_mtx);
1397 :
1398 4 : ns_device = nvme_cuse_get_cuse_ns_device(ctrlr, nsid);
1399 4 : if (!ns_device) {
1400 2 : pthread_mutex_unlock(&g_cuse_mtx);
1401 2 : return -ENODEV;
1402 : }
1403 :
1404 2 : req_len = strnlen(ns_device->dev_name, sizeof(ns_device->dev_name));
1405 2 : if (*size < req_len) {
1406 1 : *size = req_len;
1407 1 : pthread_mutex_unlock(&g_cuse_mtx);
1408 1 : return -ENOSPC;
1409 : }
1410 1 : snprintf(name, req_len + 1, "%s", ns_device->dev_name);
1411 :
1412 1 : pthread_mutex_unlock(&g_cuse_mtx);
1413 :
1414 1 : return 0;
1415 : }
1416 :
1417 1 : SPDK_LOG_REGISTER_COMPONENT(nvme_cuse)
1418 :
1419 : #else /* SPDK_CONFIG_NVME_CUSE */
1420 :
1421 : int
1422 : spdk_nvme_cuse_get_ctrlr_name(struct spdk_nvme_ctrlr *ctrlr, char *name, size_t *size)
1423 : {
1424 : SPDK_ERRLOG("spdk_nvme_cuse_get_ctrlr_name() is unsupported\n");
1425 : return -ENOTSUP;
1426 : }
1427 :
1428 : int
1429 : spdk_nvme_cuse_get_ns_name(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, char *name, size_t *size)
1430 : {
1431 : SPDK_ERRLOG("spdk_nvme_cuse_get_ns_name() is unsupported\n");
1432 : return -ENOTSUP;
1433 : }
1434 :
1435 : int
1436 : spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr)
1437 : {
1438 : SPDK_ERRLOG("spdk_nvme_cuse_register() is unsupported\n");
1439 : return -ENOTSUP;
1440 : }
1441 :
1442 : int
1443 : spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr)
1444 : {
1445 : SPDK_ERRLOG("spdk_nvme_cuse_unregister() is unsupported\n");
1446 : return -ENOTSUP;
1447 : }
1448 :
1449 : void
1450 : spdk_nvme_cuse_update_namespaces(struct spdk_nvme_ctrlr *ctrlr)
1451 : {
1452 : SPDK_ERRLOG("spdk_nvme_cuse_update_namespaces() is unsupported\n");
1453 : }
1454 :
1455 : #endif
|