Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2017 Intel Corporation.
3 : * All rights reserved.
4 : */
5 :
6 : #include "spdk/stdinc.h"
7 :
8 : #include "spdk/blobfs.h"
9 : #include "cache_tree.h"
10 :
11 : #include "spdk/queue.h"
12 : #include "spdk/thread.h"
13 : #include "spdk/assert.h"
14 : #include "spdk/env.h"
15 : #include "spdk/util.h"
16 : #include "spdk/log.h"
17 : #include "spdk/trace.h"
18 :
19 : #include "spdk_internal/trace_defs.h"
20 :
21 : #define BLOBFS_TRACE(file, str, args...) \
22 : SPDK_DEBUGLOG(blobfs, "file=%s " str, file->name, ##args)
23 :
24 : #define BLOBFS_TRACE_RW(file, str, args...) \
25 : SPDK_DEBUGLOG(blobfs_rw, "file=%s " str, file->name, ##args)
26 :
27 : #define BLOBFS_DEFAULT_CACHE_SIZE (4ULL * 1024 * 1024 * 1024)
28 : #define SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ (1024 * 1024)
29 :
30 : #define SPDK_BLOBFS_SIGNATURE "BLOBFS"
31 :
32 : static uint64_t g_fs_cache_size = BLOBFS_DEFAULT_CACHE_SIZE;
33 : static struct spdk_mempool *g_cache_pool;
34 : static TAILQ_HEAD(, spdk_file) g_caches = TAILQ_HEAD_INITIALIZER(g_caches);
35 : static struct spdk_poller *g_cache_pool_mgmt_poller;
36 : static struct spdk_thread *g_cache_pool_thread;
37 : #define BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US 1000ULL
38 : static int g_fs_count = 0;
39 : static pthread_mutex_t g_cache_init_lock = PTHREAD_MUTEX_INITIALIZER;
40 :
41 : static void
42 0 : blobfs_trace(void)
43 : {
44 0 : struct spdk_trace_tpoint_opts opts[] = {
45 : {
46 : "BLOBFS_XATTR_START", TRACE_BLOBFS_XATTR_START,
47 : OWNER_NONE, OBJECT_NONE, 0,
48 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
49 : },
50 : {
51 : "BLOBFS_XATTR_END", TRACE_BLOBFS_XATTR_END,
52 : OWNER_NONE, OBJECT_NONE, 0,
53 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
54 : },
55 : {
56 : "BLOBFS_OPEN", TRACE_BLOBFS_OPEN,
57 : OWNER_NONE, OBJECT_NONE, 0,
58 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
59 : },
60 : {
61 : "BLOBFS_CLOSE", TRACE_BLOBFS_CLOSE,
62 : OWNER_NONE, OBJECT_NONE, 0,
63 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
64 : },
65 : {
66 : "BLOBFS_DELETE_START", TRACE_BLOBFS_DELETE_START,
67 : OWNER_NONE, OBJECT_NONE, 0,
68 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
69 : },
70 : {
71 : "BLOBFS_DELETE_DONE", TRACE_BLOBFS_DELETE_DONE,
72 : OWNER_NONE, OBJECT_NONE, 0,
73 : {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
74 : }
75 : };
76 :
77 0 : spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
78 0 : }
79 2 : SPDK_TRACE_REGISTER_FN(blobfs_trace, "blobfs", TRACE_GROUP_BLOBFS)
80 :
81 : void
82 13 : cache_buffer_free(struct cache_buffer *cache_buffer)
83 : {
84 13 : spdk_mempool_put(g_cache_pool, cache_buffer->buf);
85 13 : free(cache_buffer);
86 13 : }
87 :
88 : #define CACHE_READAHEAD_THRESHOLD (128 * 1024)
89 :
90 : struct spdk_file {
91 : struct spdk_filesystem *fs;
92 : struct spdk_blob *blob;
93 : char *name;
94 : uint64_t length;
95 : bool is_deleted;
96 : bool open_for_writing;
97 : uint64_t length_flushed;
98 : uint64_t length_xattr;
99 : uint64_t append_pos;
100 : uint64_t seq_byte_count;
101 : uint64_t next_seq_offset;
102 : uint32_t priority;
103 : TAILQ_ENTRY(spdk_file) tailq;
104 : spdk_blob_id blobid;
105 : uint32_t ref_count;
106 : pthread_spinlock_t lock;
107 : struct cache_buffer *last;
108 : struct cache_tree *tree;
109 : TAILQ_HEAD(open_requests_head, spdk_fs_request) open_requests;
110 : TAILQ_HEAD(sync_requests_head, spdk_fs_request) sync_requests;
111 : TAILQ_ENTRY(spdk_file) cache_tailq;
112 : };
113 :
114 : struct spdk_deleted_file {
115 : spdk_blob_id id;
116 : TAILQ_ENTRY(spdk_deleted_file) tailq;
117 : };
118 :
119 : struct spdk_filesystem {
120 : struct spdk_blob_store *bs;
121 : TAILQ_HEAD(, spdk_file) files;
122 : struct spdk_bs_opts bs_opts;
123 : struct spdk_bs_dev *bdev;
124 : fs_send_request_fn send_request;
125 :
126 : struct {
127 : uint32_t max_ops;
128 : struct spdk_io_channel *sync_io_channel;
129 : struct spdk_fs_channel *sync_fs_channel;
130 : } sync_target;
131 :
132 : struct {
133 : uint32_t max_ops;
134 : struct spdk_io_channel *md_io_channel;
135 : struct spdk_fs_channel *md_fs_channel;
136 : } md_target;
137 :
138 : struct {
139 : uint32_t max_ops;
140 : } io_target;
141 : };
142 :
143 : struct spdk_fs_cb_args {
144 : union {
145 : spdk_fs_op_with_handle_complete fs_op_with_handle;
146 : spdk_fs_op_complete fs_op;
147 : spdk_file_op_with_handle_complete file_op_with_handle;
148 : spdk_file_op_complete file_op;
149 : spdk_file_stat_op_complete stat_op;
150 : } fn;
151 : void *arg;
152 : sem_t *sem;
153 : struct spdk_filesystem *fs;
154 : struct spdk_file *file;
155 : int rc;
156 : int *rwerrno;
157 : struct iovec *iovs;
158 : uint32_t iovcnt;
159 : struct iovec iov;
160 : union {
161 : struct {
162 : TAILQ_HEAD(, spdk_deleted_file) deleted_files;
163 : } fs_load;
164 : struct {
165 : uint64_t length;
166 : } truncate;
167 : struct {
168 : struct spdk_io_channel *channel;
169 : void *pin_buf;
170 : int is_read;
171 : off_t offset;
172 : size_t length;
173 : uint64_t start_lba;
174 : uint64_t num_lba;
175 : uint32_t blocklen;
176 : } rw;
177 : struct {
178 : const char *old_name;
179 : const char *new_name;
180 : } rename;
181 : struct {
182 : struct cache_buffer *cache_buffer;
183 : uint64_t length;
184 : } flush;
185 : struct {
186 : struct cache_buffer *cache_buffer;
187 : uint64_t length;
188 : uint64_t offset;
189 : } readahead;
190 : struct {
191 : /* offset of the file when the sync request was made */
192 : uint64_t offset;
193 : TAILQ_ENTRY(spdk_fs_request) tailq;
194 : bool xattr_in_progress;
195 : /* length written to the xattr for this file - this should
196 : * always be the same as the offset if only one thread is
197 : * writing to the file, but could differ if multiple threads
198 : * are appending
199 : */
200 : uint64_t length;
201 : } sync;
202 : struct {
203 : uint32_t num_clusters;
204 : } resize;
205 : struct {
206 : const char *name;
207 : uint32_t flags;
208 : TAILQ_ENTRY(spdk_fs_request) tailq;
209 : } open;
210 : struct {
211 : const char *name;
212 : struct spdk_blob *blob;
213 : } create;
214 : struct {
215 : const char *name;
216 : } delete;
217 : struct {
218 : const char *name;
219 : } stat;
220 : } op;
221 : };
222 :
223 : static void file_free(struct spdk_file *file);
224 : static void fs_io_device_unregister(struct spdk_filesystem *fs);
225 : static void fs_free_io_channels(struct spdk_filesystem *fs);
226 :
227 : void
228 0 : spdk_fs_opts_init(struct spdk_blobfs_opts *opts)
229 : {
230 0 : opts->cluster_sz = SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ;
231 0 : }
232 :
233 : static int _blobfs_cache_pool_reclaim(void *arg);
234 :
235 : static bool
236 0 : blobfs_cache_pool_need_reclaim(void)
237 : {
238 0 : size_t count;
239 :
240 0 : count = spdk_mempool_count(g_cache_pool);
241 : /* We define a aggressive policy here as the requirements from db_bench are batched, so start the poller
242 : * when the number of available cache buffer is less than 1/5 of total buffers.
243 : */
244 0 : if (count > (size_t)g_fs_cache_size / CACHE_BUFFER_SIZE / 5) {
245 0 : return false;
246 : }
247 :
248 0 : return true;
249 0 : }
250 :
251 : static void
252 20 : __start_cache_pool_mgmt(void *ctx)
253 : {
254 20 : assert(g_cache_pool == NULL);
255 :
256 20 : g_cache_pool = spdk_mempool_create("spdk_fs_cache",
257 20 : g_fs_cache_size / CACHE_BUFFER_SIZE,
258 20 : CACHE_BUFFER_SIZE,
259 : SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
260 : SPDK_ENV_SOCKET_ID_ANY);
261 20 : if (!g_cache_pool) {
262 0 : if (spdk_mempool_lookup("spdk_fs_cache") != NULL) {
263 0 : SPDK_ERRLOG("Unable to allocate mempool: already exists\n");
264 0 : SPDK_ERRLOG("Probably running in multiprocess environment, which is "
265 : "unsupported by the blobfs library\n");
266 0 : } else {
267 0 : SPDK_ERRLOG("Create mempool failed, you may "
268 : "increase the memory and try again\n");
269 : }
270 0 : assert(false);
271 : }
272 :
273 20 : assert(g_cache_pool_mgmt_poller == NULL);
274 20 : g_cache_pool_mgmt_poller = SPDK_POLLER_REGISTER(_blobfs_cache_pool_reclaim, NULL,
275 : BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
276 20 : }
277 :
278 : static void
279 20 : __stop_cache_pool_mgmt(void *ctx)
280 : {
281 20 : spdk_poller_unregister(&g_cache_pool_mgmt_poller);
282 :
283 20 : assert(g_cache_pool != NULL);
284 20 : assert(spdk_mempool_count(g_cache_pool) == g_fs_cache_size / CACHE_BUFFER_SIZE);
285 20 : spdk_mempool_free(g_cache_pool);
286 20 : g_cache_pool = NULL;
287 :
288 20 : spdk_thread_exit(g_cache_pool_thread);
289 20 : }
290 :
291 : static void
292 20 : initialize_global_cache(void)
293 : {
294 20 : pthread_mutex_lock(&g_cache_init_lock);
295 20 : if (g_fs_count == 0) {
296 20 : g_cache_pool_thread = spdk_thread_create("cache_pool_mgmt", NULL);
297 20 : assert(g_cache_pool_thread != NULL);
298 20 : spdk_thread_send_msg(g_cache_pool_thread, __start_cache_pool_mgmt, NULL);
299 20 : }
300 20 : g_fs_count++;
301 20 : pthread_mutex_unlock(&g_cache_init_lock);
302 20 : }
303 :
304 : static void
305 20 : free_global_cache(void)
306 : {
307 20 : pthread_mutex_lock(&g_cache_init_lock);
308 20 : g_fs_count--;
309 20 : if (g_fs_count == 0) {
310 20 : spdk_thread_send_msg(g_cache_pool_thread, __stop_cache_pool_mgmt, NULL);
311 20 : }
312 20 : pthread_mutex_unlock(&g_cache_init_lock);
313 20 : }
314 :
315 : static uint64_t
316 21 : __file_get_blob_size(struct spdk_file *file)
317 : {
318 21 : uint64_t cluster_sz;
319 :
320 21 : cluster_sz = file->fs->bs_opts.cluster_sz;
321 21 : return cluster_sz * spdk_blob_get_num_clusters(file->blob);
322 21 : }
323 :
324 : struct spdk_fs_request {
325 : struct spdk_fs_cb_args args;
326 : TAILQ_ENTRY(spdk_fs_request) link;
327 : struct spdk_fs_channel *channel;
328 : };
329 :
330 : struct spdk_fs_channel {
331 : struct spdk_fs_request *req_mem;
332 : TAILQ_HEAD(, spdk_fs_request) reqs;
333 : sem_t sem;
334 : struct spdk_filesystem *fs;
335 : struct spdk_io_channel *bs_channel;
336 : fs_send_request_fn send_request;
337 : bool sync;
338 : uint32_t outstanding_reqs;
339 : pthread_spinlock_t lock;
340 : };
341 :
342 : /* For now, this is effectively an alias. But eventually we'll shift
343 : * some data members over. */
344 : struct spdk_fs_thread_ctx {
345 : struct spdk_fs_channel ch;
346 : };
347 :
348 : static struct spdk_fs_request *
349 156 : alloc_fs_request_with_iov(struct spdk_fs_channel *channel, uint32_t iovcnt)
350 : {
351 156 : struct spdk_fs_request *req;
352 156 : struct iovec *iovs = NULL;
353 :
354 156 : if (iovcnt > 1) {
355 4 : iovs = calloc(iovcnt, sizeof(struct iovec));
356 4 : if (!iovs) {
357 0 : return NULL;
358 : }
359 4 : }
360 :
361 156 : if (channel->sync) {
362 66 : pthread_spin_lock(&channel->lock);
363 66 : }
364 :
365 156 : req = TAILQ_FIRST(&channel->reqs);
366 156 : if (req) {
367 156 : channel->outstanding_reqs++;
368 156 : TAILQ_REMOVE(&channel->reqs, req, link);
369 156 : }
370 :
371 156 : if (channel->sync) {
372 66 : pthread_spin_unlock(&channel->lock);
373 66 : }
374 :
375 156 : if (req == NULL) {
376 0 : SPDK_ERRLOG("Cannot allocate req on spdk_fs_channel =%p\n", channel);
377 0 : free(iovs);
378 0 : return NULL;
379 : }
380 156 : memset(req, 0, sizeof(*req));
381 156 : req->channel = channel;
382 156 : if (iovcnt > 1) {
383 4 : req->args.iovs = iovs;
384 4 : } else {
385 152 : req->args.iovs = &req->args.iov;
386 : }
387 156 : req->args.iovcnt = iovcnt;
388 :
389 156 : return req;
390 156 : }
391 :
392 : static struct spdk_fs_request *
393 148 : alloc_fs_request(struct spdk_fs_channel *channel)
394 : {
395 148 : return alloc_fs_request_with_iov(channel, 0);
396 : }
397 :
398 : static void
399 156 : free_fs_request(struct spdk_fs_request *req)
400 : {
401 156 : struct spdk_fs_channel *channel = req->channel;
402 :
403 156 : if (req->args.iovcnt > 1) {
404 4 : free(req->args.iovs);
405 4 : }
406 :
407 156 : if (channel->sync) {
408 66 : pthread_spin_lock(&channel->lock);
409 66 : }
410 :
411 156 : TAILQ_INSERT_HEAD(&req->channel->reqs, req, link);
412 156 : channel->outstanding_reqs--;
413 :
414 156 : if (channel->sync) {
415 66 : pthread_spin_unlock(&channel->lock);
416 66 : }
417 156 : }
418 :
419 : static int
420 53 : fs_channel_create(struct spdk_filesystem *fs, struct spdk_fs_channel *channel,
421 : uint32_t max_ops)
422 : {
423 53 : uint32_t i;
424 :
425 53 : channel->req_mem = calloc(max_ops, sizeof(struct spdk_fs_request));
426 53 : if (!channel->req_mem) {
427 0 : return -1;
428 : }
429 :
430 53 : channel->outstanding_reqs = 0;
431 53 : TAILQ_INIT(&channel->reqs);
432 53 : sem_init(&channel->sem, 0, 0);
433 :
434 27189 : for (i = 0; i < max_ops; i++) {
435 27136 : TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link);
436 27136 : }
437 :
438 53 : channel->fs = fs;
439 :
440 53 : return 0;
441 53 : }
442 :
443 : static int
444 20 : fs_md_channel_create(void *io_device, void *ctx_buf)
445 : {
446 20 : struct spdk_filesystem *fs;
447 20 : struct spdk_fs_channel *channel = ctx_buf;
448 :
449 20 : fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, md_target);
450 :
451 20 : return fs_channel_create(fs, channel, fs->md_target.max_ops);
452 20 : }
453 :
454 : static int
455 20 : fs_sync_channel_create(void *io_device, void *ctx_buf)
456 : {
457 20 : struct spdk_filesystem *fs;
458 20 : struct spdk_fs_channel *channel = ctx_buf;
459 :
460 20 : fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, sync_target);
461 :
462 20 : return fs_channel_create(fs, channel, fs->sync_target.max_ops);
463 20 : }
464 :
465 : static int
466 1 : fs_io_channel_create(void *io_device, void *ctx_buf)
467 : {
468 1 : struct spdk_filesystem *fs;
469 1 : struct spdk_fs_channel *channel = ctx_buf;
470 :
471 1 : fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, io_target);
472 :
473 1 : return fs_channel_create(fs, channel, fs->io_target.max_ops);
474 1 : }
475 :
476 : static void
477 53 : fs_channel_destroy(void *io_device, void *ctx_buf)
478 : {
479 53 : struct spdk_fs_channel *channel = ctx_buf;
480 :
481 53 : if (channel->outstanding_reqs > 0) {
482 0 : SPDK_ERRLOG("channel freed with %" PRIu32 " outstanding requests!\n",
483 : channel->outstanding_reqs);
484 0 : }
485 :
486 53 : free(channel->req_mem);
487 53 : if (channel->bs_channel != NULL) {
488 41 : spdk_bs_free_io_channel(channel->bs_channel);
489 41 : }
490 53 : }
491 :
492 : static void
493 0 : __send_request_direct(fs_request_fn fn, void *arg)
494 : {
495 0 : fn(arg);
496 0 : }
497 :
498 : static void
499 20 : common_fs_bs_init(struct spdk_filesystem *fs, struct spdk_blob_store *bs)
500 : {
501 20 : fs->bs = bs;
502 20 : fs->bs_opts.cluster_sz = spdk_bs_get_cluster_size(bs);
503 20 : fs->md_target.md_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
504 20 : fs->md_target.md_fs_channel->send_request = __send_request_direct;
505 20 : fs->sync_target.sync_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
506 20 : fs->sync_target.sync_fs_channel->send_request = __send_request_direct;
507 :
508 20 : initialize_global_cache();
509 20 : }
510 :
511 : static void
512 18 : init_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
513 : {
514 18 : struct spdk_fs_request *req = ctx;
515 18 : struct spdk_fs_cb_args *args = &req->args;
516 18 : struct spdk_filesystem *fs = args->fs;
517 :
518 18 : if (bserrno == 0) {
519 18 : common_fs_bs_init(fs, bs);
520 18 : } else {
521 0 : free(fs);
522 0 : fs = NULL;
523 : }
524 :
525 18 : args->fn.fs_op_with_handle(args->arg, fs, bserrno);
526 18 : free_fs_request(req);
527 18 : }
528 :
529 : static struct spdk_filesystem *
530 20 : fs_alloc(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn)
531 : {
532 20 : struct spdk_filesystem *fs;
533 :
534 20 : fs = calloc(1, sizeof(*fs));
535 20 : if (fs == NULL) {
536 0 : return NULL;
537 : }
538 :
539 20 : fs->bdev = dev;
540 20 : fs->send_request = send_request_fn;
541 20 : TAILQ_INIT(&fs->files);
542 :
543 20 : fs->md_target.max_ops = 512;
544 20 : spdk_io_device_register(&fs->md_target, fs_md_channel_create, fs_channel_destroy,
545 : sizeof(struct spdk_fs_channel), "blobfs_md");
546 20 : fs->md_target.md_io_channel = spdk_get_io_channel(&fs->md_target);
547 20 : fs->md_target.md_fs_channel = spdk_io_channel_get_ctx(fs->md_target.md_io_channel);
548 :
549 20 : fs->sync_target.max_ops = 512;
550 20 : spdk_io_device_register(&fs->sync_target, fs_sync_channel_create, fs_channel_destroy,
551 : sizeof(struct spdk_fs_channel), "blobfs_sync");
552 20 : fs->sync_target.sync_io_channel = spdk_get_io_channel(&fs->sync_target);
553 20 : fs->sync_target.sync_fs_channel = spdk_io_channel_get_ctx(fs->sync_target.sync_io_channel);
554 :
555 20 : fs->io_target.max_ops = 512;
556 20 : spdk_io_device_register(&fs->io_target, fs_io_channel_create, fs_channel_destroy,
557 : sizeof(struct spdk_fs_channel), "blobfs_io");
558 :
559 20 : return fs;
560 20 : }
561 :
562 : static void
563 51 : __wake_caller(void *arg, int fserrno)
564 : {
565 51 : struct spdk_fs_cb_args *args = arg;
566 :
567 51 : if ((args->rwerrno != NULL) && (*(args->rwerrno) == 0) && fserrno) {
568 0 : *(args->rwerrno) = fserrno;
569 0 : }
570 51 : args->rc = fserrno;
571 51 : sem_post(args->sem);
572 51 : }
573 :
574 : void
575 18 : spdk_fs_init(struct spdk_bs_dev *dev, struct spdk_blobfs_opts *opt,
576 : fs_send_request_fn send_request_fn,
577 : spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
578 : {
579 18 : struct spdk_filesystem *fs;
580 18 : struct spdk_fs_request *req;
581 18 : struct spdk_fs_cb_args *args;
582 18 : struct spdk_bs_opts opts = {};
583 :
584 18 : fs = fs_alloc(dev, send_request_fn);
585 18 : if (fs == NULL) {
586 0 : cb_fn(cb_arg, NULL, -ENOMEM);
587 0 : return;
588 : }
589 :
590 18 : req = alloc_fs_request(fs->md_target.md_fs_channel);
591 18 : if (req == NULL) {
592 0 : fs_free_io_channels(fs);
593 0 : fs_io_device_unregister(fs);
594 0 : cb_fn(cb_arg, NULL, -ENOMEM);
595 0 : return;
596 : }
597 :
598 18 : args = &req->args;
599 18 : args->fn.fs_op_with_handle = cb_fn;
600 18 : args->arg = cb_arg;
601 18 : args->fs = fs;
602 :
603 18 : spdk_bs_opts_init(&opts, sizeof(opts));
604 18 : snprintf(opts.bstype.bstype, sizeof(opts.bstype.bstype), SPDK_BLOBFS_SIGNATURE);
605 18 : if (opt) {
606 0 : opts.cluster_sz = opt->cluster_sz;
607 0 : }
608 18 : spdk_bs_init(dev, &opts, init_cb, req);
609 18 : }
610 :
611 : static struct spdk_file *
612 18 : file_alloc(struct spdk_filesystem *fs)
613 : {
614 18 : struct spdk_file *file;
615 :
616 18 : file = calloc(1, sizeof(*file));
617 18 : if (file == NULL) {
618 0 : return NULL;
619 : }
620 :
621 18 : file->tree = calloc(1, sizeof(*file->tree));
622 18 : if (file->tree == NULL) {
623 0 : free(file);
624 0 : return NULL;
625 : }
626 :
627 18 : if (pthread_spin_init(&file->lock, 0)) {
628 0 : free(file->tree);
629 0 : free(file);
630 0 : return NULL;
631 : }
632 :
633 18 : file->fs = fs;
634 18 : TAILQ_INIT(&file->open_requests);
635 18 : TAILQ_INIT(&file->sync_requests);
636 18 : TAILQ_INSERT_TAIL(&fs->files, file, tailq);
637 18 : file->priority = SPDK_FILE_PRIORITY_LOW;
638 18 : return file;
639 18 : }
640 :
641 : static void fs_load_done(void *ctx, int bserrno);
642 :
643 : static int
644 2 : _handle_deleted_files(struct spdk_fs_request *req)
645 : {
646 2 : struct spdk_fs_cb_args *args = &req->args;
647 2 : struct spdk_filesystem *fs = args->fs;
648 :
649 2 : if (!TAILQ_EMPTY(&args->op.fs_load.deleted_files)) {
650 0 : struct spdk_deleted_file *deleted_file;
651 :
652 0 : deleted_file = TAILQ_FIRST(&args->op.fs_load.deleted_files);
653 0 : TAILQ_REMOVE(&args->op.fs_load.deleted_files, deleted_file, tailq);
654 0 : spdk_bs_delete_blob(fs->bs, deleted_file->id, fs_load_done, req);
655 0 : free(deleted_file);
656 0 : return 0;
657 0 : }
658 :
659 2 : return 1;
660 2 : }
661 :
662 : static void
663 2 : fs_load_done(void *ctx, int bserrno)
664 : {
665 2 : struct spdk_fs_request *req = ctx;
666 2 : struct spdk_fs_cb_args *args = &req->args;
667 2 : struct spdk_filesystem *fs = args->fs;
668 :
669 : /* The filesystem has been loaded. Now check if there are any files that
670 : * were marked for deletion before last unload. Do not complete the
671 : * fs_load callback until all of them have been deleted on disk.
672 : */
673 2 : if (_handle_deleted_files(req) == 0) {
674 : /* We found a file that's been marked for deleting but not actually
675 : * deleted yet. This function will get called again once the delete
676 : * operation is completed.
677 : */
678 0 : return;
679 : }
680 :
681 2 : args->fn.fs_op_with_handle(args->arg, fs, 0);
682 2 : free_fs_request(req);
683 :
684 2 : }
685 :
686 : static void
687 2 : iter_cb(void *ctx, struct spdk_blob *blob, int rc)
688 : {
689 2 : struct spdk_fs_request *req = ctx;
690 2 : struct spdk_fs_cb_args *args = &req->args;
691 2 : struct spdk_filesystem *fs = args->fs;
692 2 : uint64_t *length;
693 2 : const char *name;
694 2 : uint32_t *is_deleted;
695 2 : size_t value_len;
696 :
697 2 : if (rc < 0) {
698 0 : args->fn.fs_op_with_handle(args->arg, fs, rc);
699 0 : free_fs_request(req);
700 0 : return;
701 : }
702 :
703 2 : rc = spdk_blob_get_xattr_value(blob, "name", (const void **)&name, &value_len);
704 2 : if (rc < 0) {
705 0 : args->fn.fs_op_with_handle(args->arg, fs, rc);
706 0 : free_fs_request(req);
707 0 : return;
708 : }
709 :
710 2 : rc = spdk_blob_get_xattr_value(blob, "length", (const void **)&length, &value_len);
711 2 : if (rc < 0) {
712 0 : args->fn.fs_op_with_handle(args->arg, fs, rc);
713 0 : free_fs_request(req);
714 0 : return;
715 : }
716 :
717 2 : assert(value_len == 8);
718 :
719 : /* This file could be deleted last time without close it, then app crashed, so we delete it now */
720 2 : rc = spdk_blob_get_xattr_value(blob, "is_deleted", (const void **)&is_deleted, &value_len);
721 2 : if (rc < 0) {
722 2 : struct spdk_file *f;
723 :
724 2 : f = file_alloc(fs);
725 2 : if (f == NULL) {
726 0 : SPDK_ERRLOG("Cannot allocate file to handle deleted file on disk\n");
727 0 : args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
728 0 : free_fs_request(req);
729 0 : return;
730 : }
731 :
732 2 : f->name = strdup(name);
733 2 : if (!f->name) {
734 0 : SPDK_ERRLOG("Cannot allocate memory for file name\n");
735 0 : args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
736 0 : free_fs_request(req);
737 0 : file_free(f);
738 0 : return;
739 : }
740 :
741 2 : f->blobid = spdk_blob_get_id(blob);
742 2 : f->length = *length;
743 2 : f->length_flushed = *length;
744 2 : f->length_xattr = *length;
745 2 : f->append_pos = *length;
746 2 : SPDK_DEBUGLOG(blobfs, "added file %s length=%ju\n", f->name, f->length);
747 2 : } else {
748 0 : struct spdk_deleted_file *deleted_file;
749 :
750 0 : deleted_file = calloc(1, sizeof(*deleted_file));
751 0 : if (deleted_file == NULL) {
752 0 : args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
753 0 : free_fs_request(req);
754 0 : return;
755 : }
756 0 : deleted_file->id = spdk_blob_get_id(blob);
757 0 : TAILQ_INSERT_TAIL(&args->op.fs_load.deleted_files, deleted_file, tailq);
758 0 : }
759 2 : }
760 :
761 : static void
762 2 : load_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
763 : {
764 2 : struct spdk_fs_request *req = ctx;
765 2 : struct spdk_fs_cb_args *args = &req->args;
766 2 : struct spdk_filesystem *fs = args->fs;
767 2 : struct spdk_bs_type bstype;
768 : static const struct spdk_bs_type blobfs_type = {SPDK_BLOBFS_SIGNATURE};
769 : static const struct spdk_bs_type zeros;
770 :
771 2 : if (bserrno != 0) {
772 0 : args->fn.fs_op_with_handle(args->arg, NULL, bserrno);
773 0 : free_fs_request(req);
774 0 : fs_free_io_channels(fs);
775 0 : fs_io_device_unregister(fs);
776 0 : return;
777 : }
778 :
779 2 : bstype = spdk_bs_get_bstype(bs);
780 :
781 2 : if (!memcmp(&bstype, &zeros, sizeof(bstype))) {
782 0 : SPDK_DEBUGLOG(blobfs, "assigning bstype\n");
783 0 : spdk_bs_set_bstype(bs, blobfs_type);
784 2 : } else if (memcmp(&bstype, &blobfs_type, sizeof(bstype))) {
785 0 : SPDK_ERRLOG("not blobfs\n");
786 0 : SPDK_LOGDUMP(blobfs, "bstype", &bstype, sizeof(bstype));
787 0 : args->fn.fs_op_with_handle(args->arg, NULL, -EINVAL);
788 0 : free_fs_request(req);
789 0 : fs_free_io_channels(fs);
790 0 : fs_io_device_unregister(fs);
791 0 : return;
792 : }
793 :
794 2 : common_fs_bs_init(fs, bs);
795 2 : fs_load_done(req, 0);
796 2 : }
797 :
798 : static void
799 20 : fs_io_device_unregister(struct spdk_filesystem *fs)
800 : {
801 20 : assert(fs != NULL);
802 20 : spdk_io_device_unregister(&fs->md_target, NULL);
803 20 : spdk_io_device_unregister(&fs->sync_target, NULL);
804 20 : spdk_io_device_unregister(&fs->io_target, NULL);
805 20 : free(fs);
806 20 : }
807 :
808 : static void
809 20 : fs_free_io_channels(struct spdk_filesystem *fs)
810 : {
811 20 : assert(fs != NULL);
812 20 : spdk_fs_free_io_channel(fs->md_target.md_io_channel);
813 20 : spdk_fs_free_io_channel(fs->sync_target.sync_io_channel);
814 20 : }
815 :
816 : void
817 2 : spdk_fs_load(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn,
818 : spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
819 : {
820 2 : struct spdk_filesystem *fs;
821 2 : struct spdk_fs_cb_args *args;
822 2 : struct spdk_fs_request *req;
823 2 : struct spdk_bs_opts bs_opts;
824 :
825 2 : fs = fs_alloc(dev, send_request_fn);
826 2 : if (fs == NULL) {
827 0 : cb_fn(cb_arg, NULL, -ENOMEM);
828 0 : return;
829 : }
830 :
831 2 : req = alloc_fs_request(fs->md_target.md_fs_channel);
832 2 : if (req == NULL) {
833 0 : fs_free_io_channels(fs);
834 0 : fs_io_device_unregister(fs);
835 0 : cb_fn(cb_arg, NULL, -ENOMEM);
836 0 : return;
837 : }
838 :
839 2 : args = &req->args;
840 2 : args->fn.fs_op_with_handle = cb_fn;
841 2 : args->arg = cb_arg;
842 2 : args->fs = fs;
843 2 : TAILQ_INIT(&args->op.fs_load.deleted_files);
844 2 : spdk_bs_opts_init(&bs_opts, sizeof(bs_opts));
845 2 : bs_opts.iter_cb_fn = iter_cb;
846 2 : bs_opts.iter_cb_arg = req;
847 2 : spdk_bs_load(dev, &bs_opts, load_cb, req);
848 2 : }
849 :
850 : static void
851 20 : unload_cb(void *ctx, int bserrno)
852 : {
853 20 : struct spdk_fs_request *req = ctx;
854 20 : struct spdk_fs_cb_args *args = &req->args;
855 20 : struct spdk_filesystem *fs = args->fs;
856 20 : struct spdk_file *file, *tmp;
857 :
858 26 : TAILQ_FOREACH_SAFE(file, &fs->files, tailq, tmp) {
859 6 : TAILQ_REMOVE(&fs->files, file, tailq);
860 6 : file_free(file);
861 6 : }
862 :
863 20 : free_global_cache();
864 :
865 20 : args->fn.fs_op(args->arg, bserrno);
866 20 : free(req);
867 :
868 20 : fs_io_device_unregister(fs);
869 20 : }
870 :
871 : void
872 20 : spdk_fs_unload(struct spdk_filesystem *fs, spdk_fs_op_complete cb_fn, void *cb_arg)
873 : {
874 20 : struct spdk_fs_request *req;
875 20 : struct spdk_fs_cb_args *args;
876 :
877 : /*
878 : * We must free the md_channel before unloading the blobstore, so just
879 : * allocate this request from the general heap.
880 : */
881 20 : req = calloc(1, sizeof(*req));
882 20 : if (req == NULL) {
883 0 : cb_fn(cb_arg, -ENOMEM);
884 0 : return;
885 : }
886 :
887 20 : args = &req->args;
888 20 : args->fn.fs_op = cb_fn;
889 20 : args->arg = cb_arg;
890 20 : args->fs = fs;
891 :
892 20 : fs_free_io_channels(fs);
893 20 : spdk_bs_unload(fs->bs, unload_cb, req);
894 20 : }
895 :
896 : static struct spdk_file *
897 75 : fs_find_file(struct spdk_filesystem *fs, const char *name)
898 : {
899 75 : struct spdk_file *file;
900 :
901 82 : TAILQ_FOREACH(file, &fs->files, tailq) {
902 48 : if (!strncmp(name, file->name, SPDK_FILE_NAME_MAX)) {
903 41 : return file;
904 : }
905 7 : }
906 :
907 34 : return NULL;
908 75 : }
909 :
910 : void
911 4 : spdk_fs_file_stat_async(struct spdk_filesystem *fs, const char *name,
912 : spdk_file_stat_op_complete cb_fn, void *cb_arg)
913 : {
914 4 : struct spdk_file_stat stat;
915 4 : struct spdk_file *f = NULL;
916 :
917 4 : if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
918 0 : cb_fn(cb_arg, NULL, -ENAMETOOLONG);
919 0 : return;
920 : }
921 :
922 4 : f = fs_find_file(fs, name);
923 4 : if (f != NULL) {
924 4 : stat.blobid = f->blobid;
925 4 : stat.size = f->append_pos >= f->length ? f->append_pos : f->length;
926 4 : cb_fn(cb_arg, &stat, 0);
927 4 : return;
928 : }
929 :
930 0 : cb_fn(cb_arg, NULL, -ENOENT);
931 4 : }
932 :
933 : static void
934 4 : __copy_stat(void *arg, struct spdk_file_stat *stat, int fserrno)
935 : {
936 4 : struct spdk_fs_request *req = arg;
937 4 : struct spdk_fs_cb_args *args = &req->args;
938 :
939 4 : args->rc = fserrno;
940 4 : if (fserrno == 0) {
941 4 : memcpy(args->arg, stat, sizeof(*stat));
942 4 : }
943 4 : sem_post(args->sem);
944 4 : }
945 :
946 : static void
947 4 : __file_stat(void *arg)
948 : {
949 4 : struct spdk_fs_request *req = arg;
950 4 : struct spdk_fs_cb_args *args = &req->args;
951 :
952 8 : spdk_fs_file_stat_async(args->fs, args->op.stat.name,
953 4 : args->fn.stat_op, req);
954 4 : }
955 :
956 : int
957 4 : spdk_fs_file_stat(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
958 : const char *name, struct spdk_file_stat *stat)
959 : {
960 4 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
961 4 : struct spdk_fs_request *req;
962 4 : int rc;
963 :
964 4 : req = alloc_fs_request(channel);
965 4 : if (req == NULL) {
966 0 : SPDK_ERRLOG("Cannot allocate stat req on file=%s\n", name);
967 0 : return -ENOMEM;
968 : }
969 :
970 4 : req->args.fs = fs;
971 4 : req->args.op.stat.name = name;
972 4 : req->args.fn.stat_op = __copy_stat;
973 4 : req->args.arg = stat;
974 4 : req->args.sem = &channel->sem;
975 4 : channel->send_request(__file_stat, req);
976 4 : sem_wait(&channel->sem);
977 :
978 4 : rc = req->args.rc;
979 4 : free_fs_request(req);
980 :
981 4 : return rc;
982 4 : }
983 :
984 : static void
985 16 : fs_create_blob_close_cb(void *ctx, int bserrno)
986 : {
987 16 : int rc;
988 16 : struct spdk_fs_request *req = ctx;
989 16 : struct spdk_fs_cb_args *args = &req->args;
990 :
991 16 : rc = args->rc ? args->rc : bserrno;
992 16 : args->fn.file_op(args->arg, rc);
993 16 : free_fs_request(req);
994 16 : }
995 :
996 : static void
997 16 : fs_create_blob_resize_cb(void *ctx, int bserrno)
998 : {
999 16 : struct spdk_fs_request *req = ctx;
1000 16 : struct spdk_fs_cb_args *args = &req->args;
1001 16 : struct spdk_file *f = args->file;
1002 16 : struct spdk_blob *blob = args->op.create.blob;
1003 16 : uint64_t length = 0;
1004 :
1005 16 : args->rc = bserrno;
1006 16 : if (bserrno) {
1007 0 : spdk_blob_close(blob, fs_create_blob_close_cb, args);
1008 0 : return;
1009 : }
1010 :
1011 16 : spdk_blob_set_xattr(blob, "name", f->name, strlen(f->name) + 1);
1012 16 : spdk_blob_set_xattr(blob, "length", &length, sizeof(length));
1013 :
1014 16 : spdk_blob_close(blob, fs_create_blob_close_cb, args);
1015 16 : }
1016 :
1017 : static void
1018 16 : fs_create_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
1019 : {
1020 16 : struct spdk_fs_request *req = ctx;
1021 16 : struct spdk_fs_cb_args *args = &req->args;
1022 :
1023 16 : if (bserrno) {
1024 0 : args->fn.file_op(args->arg, bserrno);
1025 0 : free_fs_request(req);
1026 0 : return;
1027 : }
1028 :
1029 16 : args->op.create.blob = blob;
1030 16 : spdk_blob_resize(blob, 1, fs_create_blob_resize_cb, req);
1031 16 : }
1032 :
1033 : static void
1034 16 : fs_create_blob_create_cb(void *ctx, spdk_blob_id blobid, int bserrno)
1035 : {
1036 16 : struct spdk_fs_request *req = ctx;
1037 16 : struct spdk_fs_cb_args *args = &req->args;
1038 16 : struct spdk_file *f = args->file;
1039 :
1040 16 : if (bserrno) {
1041 0 : args->fn.file_op(args->arg, bserrno);
1042 0 : free_fs_request(req);
1043 0 : return;
1044 : }
1045 :
1046 16 : f->blobid = blobid;
1047 16 : spdk_bs_open_blob(f->fs->bs, blobid, fs_create_blob_open_cb, req);
1048 16 : }
1049 :
1050 : void
1051 19 : spdk_fs_create_file_async(struct spdk_filesystem *fs, const char *name,
1052 : spdk_file_op_complete cb_fn, void *cb_arg)
1053 : {
1054 19 : struct spdk_file *file;
1055 19 : struct spdk_fs_request *req;
1056 19 : struct spdk_fs_cb_args *args;
1057 :
1058 19 : if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
1059 1 : cb_fn(cb_arg, -ENAMETOOLONG);
1060 1 : return;
1061 : }
1062 :
1063 18 : file = fs_find_file(fs, name);
1064 18 : if (file != NULL) {
1065 2 : cb_fn(cb_arg, -EEXIST);
1066 2 : return;
1067 : }
1068 :
1069 16 : file = file_alloc(fs);
1070 16 : if (file == NULL) {
1071 0 : SPDK_ERRLOG("Cannot allocate new file for creation\n");
1072 0 : cb_fn(cb_arg, -ENOMEM);
1073 0 : return;
1074 : }
1075 :
1076 16 : req = alloc_fs_request(fs->md_target.md_fs_channel);
1077 16 : if (req == NULL) {
1078 0 : SPDK_ERRLOG("Cannot allocate create async req for file=%s\n", name);
1079 0 : TAILQ_REMOVE(&fs->files, file, tailq);
1080 0 : file_free(file);
1081 0 : cb_fn(cb_arg, -ENOMEM);
1082 0 : return;
1083 : }
1084 :
1085 16 : args = &req->args;
1086 16 : args->file = file;
1087 16 : args->fn.file_op = cb_fn;
1088 16 : args->arg = cb_arg;
1089 :
1090 16 : file->name = strdup(name);
1091 16 : if (!file->name) {
1092 0 : SPDK_ERRLOG("Cannot allocate file->name for file=%s\n", name);
1093 0 : free_fs_request(req);
1094 0 : TAILQ_REMOVE(&fs->files, file, tailq);
1095 0 : file_free(file);
1096 0 : cb_fn(cb_arg, -ENOMEM);
1097 0 : return;
1098 : }
1099 16 : spdk_bs_create_blob(fs->bs, fs_create_blob_create_cb, args);
1100 19 : }
1101 :
1102 : static void
1103 2 : __fs_create_file_done(void *arg, int fserrno)
1104 : {
1105 2 : struct spdk_fs_request *req = arg;
1106 2 : struct spdk_fs_cb_args *args = &req->args;
1107 :
1108 2 : SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
1109 2 : __wake_caller(args, fserrno);
1110 2 : }
1111 :
1112 : static void
1113 2 : __fs_create_file(void *arg)
1114 : {
1115 2 : struct spdk_fs_request *req = arg;
1116 2 : struct spdk_fs_cb_args *args = &req->args;
1117 :
1118 2 : SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
1119 2 : spdk_fs_create_file_async(args->fs, args->op.create.name, __fs_create_file_done, req);
1120 2 : }
1121 :
1122 : int
1123 2 : spdk_fs_create_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx, const char *name)
1124 : {
1125 2 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
1126 2 : struct spdk_fs_request *req;
1127 2 : struct spdk_fs_cb_args *args;
1128 2 : int rc;
1129 :
1130 2 : SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
1131 :
1132 2 : req = alloc_fs_request(channel);
1133 2 : if (req == NULL) {
1134 0 : SPDK_ERRLOG("Cannot allocate req to create file=%s\n", name);
1135 0 : return -ENOMEM;
1136 : }
1137 :
1138 2 : args = &req->args;
1139 2 : args->fs = fs;
1140 2 : args->op.create.name = name;
1141 2 : args->sem = &channel->sem;
1142 2 : fs->send_request(__fs_create_file, req);
1143 2 : sem_wait(&channel->sem);
1144 2 : rc = args->rc;
1145 2 : free_fs_request(req);
1146 :
1147 2 : return rc;
1148 2 : }
1149 :
1150 : static void
1151 17 : fs_open_blob_done(void *ctx, struct spdk_blob *blob, int bserrno)
1152 : {
1153 17 : struct spdk_fs_request *req = ctx;
1154 17 : struct spdk_fs_cb_args *args = &req->args;
1155 17 : struct spdk_file *f = args->file;
1156 :
1157 17 : f->blob = blob;
1158 34 : while (!TAILQ_EMPTY(&f->open_requests)) {
1159 17 : req = TAILQ_FIRST(&f->open_requests);
1160 17 : args = &req->args;
1161 17 : TAILQ_REMOVE(&f->open_requests, req, args.op.open.tailq);
1162 17 : spdk_trace_record(TRACE_BLOBFS_OPEN, 0, 0, 0, f->name);
1163 17 : args->fn.file_op_with_handle(args->arg, f, bserrno);
1164 17 : free_fs_request(req);
1165 : }
1166 17 : }
1167 :
1168 : static void
1169 17 : fs_open_blob_create_cb(void *ctx, int bserrno)
1170 : {
1171 17 : struct spdk_fs_request *req = ctx;
1172 17 : struct spdk_fs_cb_args *args = &req->args;
1173 17 : struct spdk_file *file = args->file;
1174 17 : struct spdk_filesystem *fs = args->fs;
1175 :
1176 17 : if (file == NULL) {
1177 : /*
1178 : * This is from an open with CREATE flag - the file
1179 : * is now created so look it up in the file list for this
1180 : * filesystem.
1181 : */
1182 13 : file = fs_find_file(fs, args->op.open.name);
1183 13 : assert(file != NULL);
1184 13 : args->file = file;
1185 13 : }
1186 :
1187 17 : file->ref_count++;
1188 17 : TAILQ_INSERT_TAIL(&file->open_requests, req, args.op.open.tailq);
1189 17 : if (file->ref_count == 1) {
1190 17 : assert(file->blob == NULL);
1191 17 : spdk_bs_open_blob(fs->bs, file->blobid, fs_open_blob_done, req);
1192 17 : } else if (file->blob != NULL) {
1193 0 : fs_open_blob_done(req, file->blob, 0);
1194 0 : } else {
1195 : /*
1196 : * The blob open for this file is in progress due to a previous
1197 : * open request. When that open completes, it will invoke the
1198 : * open callback for this request.
1199 : */
1200 : }
1201 17 : }
1202 :
1203 : void
1204 21 : spdk_fs_open_file_async(struct spdk_filesystem *fs, const char *name, uint32_t flags,
1205 : spdk_file_op_with_handle_complete cb_fn, void *cb_arg)
1206 : {
1207 21 : struct spdk_file *f = NULL;
1208 21 : struct spdk_fs_request *req;
1209 21 : struct spdk_fs_cb_args *args;
1210 :
1211 21 : if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
1212 1 : cb_fn(cb_arg, NULL, -ENAMETOOLONG);
1213 1 : return;
1214 : }
1215 :
1216 20 : f = fs_find_file(fs, name);
1217 20 : if (f == NULL && !(flags & SPDK_BLOBFS_OPEN_CREATE)) {
1218 2 : cb_fn(cb_arg, NULL, -ENOENT);
1219 2 : return;
1220 : }
1221 :
1222 18 : if (f != NULL && f->is_deleted == true) {
1223 1 : cb_fn(cb_arg, NULL, -ENOENT);
1224 1 : return;
1225 : }
1226 :
1227 17 : req = alloc_fs_request(fs->md_target.md_fs_channel);
1228 17 : if (req == NULL) {
1229 0 : SPDK_ERRLOG("Cannot allocate async open req for file=%s\n", name);
1230 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1231 0 : return;
1232 : }
1233 :
1234 17 : args = &req->args;
1235 17 : args->fn.file_op_with_handle = cb_fn;
1236 17 : args->arg = cb_arg;
1237 17 : args->file = f;
1238 17 : args->fs = fs;
1239 17 : args->op.open.name = name;
1240 :
1241 17 : if (f == NULL) {
1242 13 : spdk_fs_create_file_async(fs, name, fs_open_blob_create_cb, req);
1243 13 : } else {
1244 4 : fs_open_blob_create_cb(req, 0);
1245 : }
1246 21 : }
1247 :
1248 : static void
1249 13 : __fs_open_file_done(void *arg, struct spdk_file *file, int bserrno)
1250 : {
1251 13 : struct spdk_fs_request *req = arg;
1252 13 : struct spdk_fs_cb_args *args = &req->args;
1253 :
1254 13 : args->file = file;
1255 13 : SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
1256 13 : __wake_caller(args, bserrno);
1257 13 : }
1258 :
1259 : static void
1260 13 : __fs_open_file(void *arg)
1261 : {
1262 13 : struct spdk_fs_request *req = arg;
1263 13 : struct spdk_fs_cb_args *args = &req->args;
1264 :
1265 13 : SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
1266 13 : spdk_fs_open_file_async(args->fs, args->op.open.name, args->op.open.flags,
1267 13 : __fs_open_file_done, req);
1268 13 : }
1269 :
1270 : int
1271 13 : spdk_fs_open_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
1272 : const char *name, uint32_t flags, struct spdk_file **file)
1273 : {
1274 13 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
1275 13 : struct spdk_fs_request *req;
1276 13 : struct spdk_fs_cb_args *args;
1277 13 : int rc;
1278 :
1279 13 : SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
1280 :
1281 13 : req = alloc_fs_request(channel);
1282 13 : if (req == NULL) {
1283 0 : SPDK_ERRLOG("Cannot allocate req for opening file=%s\n", name);
1284 0 : return -ENOMEM;
1285 : }
1286 :
1287 13 : args = &req->args;
1288 13 : args->fs = fs;
1289 13 : args->op.open.name = name;
1290 13 : args->op.open.flags = flags;
1291 13 : args->sem = &channel->sem;
1292 13 : fs->send_request(__fs_open_file, req);
1293 13 : sem_wait(&channel->sem);
1294 13 : rc = args->rc;
1295 13 : if (rc == 0) {
1296 11 : *file = args->file;
1297 11 : } else {
1298 2 : *file = NULL;
1299 : }
1300 13 : free_fs_request(req);
1301 :
1302 13 : return rc;
1303 13 : }
1304 :
1305 : static void
1306 2 : fs_rename_blob_close_cb(void *ctx, int bserrno)
1307 : {
1308 2 : struct spdk_fs_request *req = ctx;
1309 2 : struct spdk_fs_cb_args *args = &req->args;
1310 :
1311 2 : args->fn.fs_op(args->arg, bserrno);
1312 2 : free_fs_request(req);
1313 2 : }
1314 :
1315 : static void
1316 2 : fs_rename_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
1317 : {
1318 2 : struct spdk_fs_request *req = ctx;
1319 2 : struct spdk_fs_cb_args *args = &req->args;
1320 2 : const char *new_name = args->op.rename.new_name;
1321 :
1322 2 : spdk_blob_set_xattr(blob, "name", new_name, strlen(new_name) + 1);
1323 2 : spdk_blob_close(blob, fs_rename_blob_close_cb, req);
1324 2 : }
1325 :
1326 : static void
1327 2 : _fs_md_rename_file(struct spdk_fs_request *req)
1328 : {
1329 2 : struct spdk_fs_cb_args *args = &req->args;
1330 2 : struct spdk_file *f;
1331 :
1332 2 : f = fs_find_file(args->fs, args->op.rename.old_name);
1333 2 : if (f == NULL) {
1334 0 : args->fn.fs_op(args->arg, -ENOENT);
1335 0 : free_fs_request(req);
1336 0 : return;
1337 : }
1338 :
1339 2 : free(f->name);
1340 2 : f->name = strdup(args->op.rename.new_name);
1341 2 : if (!f->name) {
1342 0 : SPDK_ERRLOG("Cannot allocate memory for file name\n");
1343 0 : args->fn.fs_op(args->arg, -ENOMEM);
1344 0 : free_fs_request(req);
1345 0 : return;
1346 : }
1347 :
1348 2 : args->file = f;
1349 2 : spdk_bs_open_blob(args->fs->bs, f->blobid, fs_rename_blob_open_cb, req);
1350 2 : }
1351 :
1352 : static void
1353 1 : fs_rename_delete_done(void *arg, int fserrno)
1354 : {
1355 1 : _fs_md_rename_file(arg);
1356 1 : }
1357 :
1358 : void
1359 2 : spdk_fs_rename_file_async(struct spdk_filesystem *fs,
1360 : const char *old_name, const char *new_name,
1361 : spdk_file_op_complete cb_fn, void *cb_arg)
1362 : {
1363 2 : struct spdk_file *f;
1364 2 : struct spdk_fs_request *req;
1365 2 : struct spdk_fs_cb_args *args;
1366 :
1367 2 : SPDK_DEBUGLOG(blobfs, "old=%s new=%s\n", old_name, new_name);
1368 2 : if (strnlen(new_name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
1369 0 : cb_fn(cb_arg, -ENAMETOOLONG);
1370 0 : return;
1371 : }
1372 :
1373 2 : req = alloc_fs_request(fs->md_target.md_fs_channel);
1374 2 : if (req == NULL) {
1375 0 : SPDK_ERRLOG("Cannot allocate rename async req for renaming file from %s to %s\n", old_name,
1376 : new_name);
1377 0 : cb_fn(cb_arg, -ENOMEM);
1378 0 : return;
1379 : }
1380 :
1381 2 : args = &req->args;
1382 2 : args->fn.fs_op = cb_fn;
1383 2 : args->fs = fs;
1384 2 : args->arg = cb_arg;
1385 2 : args->op.rename.old_name = old_name;
1386 2 : args->op.rename.new_name = new_name;
1387 :
1388 2 : f = fs_find_file(fs, new_name);
1389 2 : if (f == NULL) {
1390 1 : _fs_md_rename_file(req);
1391 1 : return;
1392 : }
1393 :
1394 : /*
1395 : * The rename overwrites an existing file. So delete the existing file, then
1396 : * do the actual rename.
1397 : */
1398 1 : spdk_fs_delete_file_async(fs, new_name, fs_rename_delete_done, req);
1399 2 : }
1400 :
1401 : static void
1402 1 : __fs_rename_file_done(void *arg, int fserrno)
1403 : {
1404 1 : struct spdk_fs_request *req = arg;
1405 1 : struct spdk_fs_cb_args *args = &req->args;
1406 :
1407 1 : __wake_caller(args, fserrno);
1408 1 : }
1409 :
1410 : static void
1411 1 : __fs_rename_file(void *arg)
1412 : {
1413 1 : struct spdk_fs_request *req = arg;
1414 1 : struct spdk_fs_cb_args *args = &req->args;
1415 :
1416 1 : spdk_fs_rename_file_async(args->fs, args->op.rename.old_name, args->op.rename.new_name,
1417 1 : __fs_rename_file_done, req);
1418 1 : }
1419 :
1420 : int
1421 1 : spdk_fs_rename_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
1422 : const char *old_name, const char *new_name)
1423 : {
1424 1 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
1425 1 : struct spdk_fs_request *req;
1426 1 : struct spdk_fs_cb_args *args;
1427 1 : int rc;
1428 :
1429 1 : req = alloc_fs_request(channel);
1430 1 : if (req == NULL) {
1431 0 : SPDK_ERRLOG("Cannot allocate rename req for file=%s\n", old_name);
1432 0 : return -ENOMEM;
1433 : }
1434 :
1435 1 : args = &req->args;
1436 :
1437 1 : args->fs = fs;
1438 1 : args->op.rename.old_name = old_name;
1439 1 : args->op.rename.new_name = new_name;
1440 1 : args->sem = &channel->sem;
1441 1 : fs->send_request(__fs_rename_file, req);
1442 1 : sem_wait(&channel->sem);
1443 1 : rc = args->rc;
1444 1 : free_fs_request(req);
1445 1 : return rc;
1446 1 : }
1447 :
1448 : static void
1449 16 : blob_delete_cb(void *ctx, int bserrno)
1450 : {
1451 16 : struct spdk_fs_request *req = ctx;
1452 16 : struct spdk_fs_cb_args *args = &req->args;
1453 :
1454 16 : args->fn.file_op(args->arg, bserrno);
1455 16 : free_fs_request(req);
1456 16 : }
1457 :
1458 : void
1459 16 : spdk_fs_delete_file_async(struct spdk_filesystem *fs, const char *name,
1460 : spdk_file_op_complete cb_fn, void *cb_arg)
1461 : {
1462 16 : struct spdk_file *f;
1463 16 : spdk_blob_id blobid;
1464 16 : struct spdk_fs_request *req;
1465 16 : struct spdk_fs_cb_args *args;
1466 :
1467 16 : SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
1468 :
1469 16 : if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
1470 0 : cb_fn(cb_arg, -ENAMETOOLONG);
1471 0 : return;
1472 : }
1473 :
1474 16 : f = fs_find_file(fs, name);
1475 16 : if (f == NULL) {
1476 2 : SPDK_ERRLOG("Cannot find the file=%s to deleted\n", name);
1477 2 : cb_fn(cb_arg, -ENOENT);
1478 2 : return;
1479 : }
1480 :
1481 14 : req = alloc_fs_request(fs->md_target.md_fs_channel);
1482 14 : if (req == NULL) {
1483 0 : SPDK_ERRLOG("Cannot allocate the req for the file=%s to deleted\n", name);
1484 0 : cb_fn(cb_arg, -ENOMEM);
1485 0 : return;
1486 : }
1487 :
1488 14 : args = &req->args;
1489 14 : args->fn.file_op = cb_fn;
1490 14 : args->arg = cb_arg;
1491 :
1492 14 : if (f->ref_count > 0) {
1493 : /* If the ref > 0, we mark the file as deleted and delete it when we close it. */
1494 2 : f->is_deleted = true;
1495 2 : spdk_blob_set_xattr(f->blob, "is_deleted", &f->is_deleted, sizeof(bool));
1496 2 : spdk_blob_sync_md(f->blob, blob_delete_cb, req);
1497 2 : return;
1498 : }
1499 :
1500 12 : blobid = f->blobid;
1501 12 : TAILQ_REMOVE(&fs->files, f, tailq);
1502 :
1503 12 : file_free(f);
1504 :
1505 12 : spdk_bs_delete_blob(fs->bs, blobid, blob_delete_cb, req);
1506 16 : }
1507 :
1508 : static void
1509 8 : __fs_delete_file_done(void *arg, int fserrno)
1510 : {
1511 8 : struct spdk_fs_request *req = arg;
1512 8 : struct spdk_fs_cb_args *args = &req->args;
1513 :
1514 8 : spdk_trace_record(TRACE_BLOBFS_DELETE_DONE, 0, 0, 0, args->op.delete.name);
1515 8 : __wake_caller(args, fserrno);
1516 8 : }
1517 :
1518 : static void
1519 8 : __fs_delete_file(void *arg)
1520 : {
1521 8 : struct spdk_fs_request *req = arg;
1522 8 : struct spdk_fs_cb_args *args = &req->args;
1523 :
1524 8 : spdk_trace_record(TRACE_BLOBFS_DELETE_START, 0, 0, 0, args->op.delete.name);
1525 8 : spdk_fs_delete_file_async(args->fs, args->op.delete.name, __fs_delete_file_done, req);
1526 8 : }
1527 :
1528 : int
1529 8 : spdk_fs_delete_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
1530 : const char *name)
1531 : {
1532 8 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
1533 8 : struct spdk_fs_request *req;
1534 8 : struct spdk_fs_cb_args *args;
1535 8 : int rc;
1536 :
1537 8 : req = alloc_fs_request(channel);
1538 8 : if (req == NULL) {
1539 0 : SPDK_DEBUGLOG(blobfs, "Cannot allocate req to delete file=%s\n", name);
1540 0 : return -ENOMEM;
1541 : }
1542 :
1543 8 : args = &req->args;
1544 8 : args->fs = fs;
1545 8 : args->op.delete.name = name;
1546 8 : args->sem = &channel->sem;
1547 8 : fs->send_request(__fs_delete_file, req);
1548 8 : sem_wait(&channel->sem);
1549 8 : rc = args->rc;
1550 8 : free_fs_request(req);
1551 :
1552 8 : return rc;
1553 8 : }
1554 :
1555 : spdk_fs_iter
1556 1 : spdk_fs_iter_first(struct spdk_filesystem *fs)
1557 : {
1558 1 : struct spdk_file *f;
1559 :
1560 1 : f = TAILQ_FIRST(&fs->files);
1561 2 : return f;
1562 1 : }
1563 :
1564 : spdk_fs_iter
1565 1 : spdk_fs_iter_next(spdk_fs_iter iter)
1566 : {
1567 1 : struct spdk_file *f = iter;
1568 :
1569 1 : if (f == NULL) {
1570 0 : return NULL;
1571 : }
1572 :
1573 1 : f = TAILQ_NEXT(f, tailq);
1574 1 : return f;
1575 1 : }
1576 :
1577 : const char *
1578 2 : spdk_file_get_name(struct spdk_file *file)
1579 : {
1580 2 : return file->name;
1581 : }
1582 :
1583 : uint64_t
1584 6 : spdk_file_get_length(struct spdk_file *file)
1585 : {
1586 6 : uint64_t length;
1587 :
1588 6 : assert(file != NULL);
1589 :
1590 6 : length = file->append_pos >= file->length ? file->append_pos : file->length;
1591 6 : SPDK_DEBUGLOG(blobfs, "file=%s length=0x%jx\n", file->name, length);
1592 12 : return length;
1593 6 : }
1594 :
1595 : static void
1596 8 : fs_truncate_complete_cb(void *ctx, int bserrno)
1597 : {
1598 8 : struct spdk_fs_request *req = ctx;
1599 8 : struct spdk_fs_cb_args *args = &req->args;
1600 :
1601 8 : args->fn.file_op(args->arg, bserrno);
1602 8 : free_fs_request(req);
1603 8 : }
1604 :
1605 : static void
1606 8 : fs_truncate_resize_cb(void *ctx, int bserrno)
1607 : {
1608 8 : struct spdk_fs_request *req = ctx;
1609 8 : struct spdk_fs_cb_args *args = &req->args;
1610 8 : struct spdk_file *file = args->file;
1611 8 : uint64_t *length = &args->op.truncate.length;
1612 :
1613 8 : if (bserrno) {
1614 0 : args->fn.file_op(args->arg, bserrno);
1615 0 : free_fs_request(req);
1616 0 : return;
1617 : }
1618 :
1619 8 : spdk_blob_set_xattr(file->blob, "length", length, sizeof(*length));
1620 :
1621 8 : file->length = *length;
1622 8 : if (file->append_pos > file->length) {
1623 0 : file->append_pos = file->length;
1624 0 : }
1625 :
1626 8 : spdk_blob_sync_md(file->blob, fs_truncate_complete_cb, req);
1627 8 : }
1628 :
1629 : static uint64_t
1630 8 : __bytes_to_clusters(uint64_t length, uint64_t cluster_sz)
1631 : {
1632 8 : return (length + cluster_sz - 1) / cluster_sz;
1633 : }
1634 :
1635 : void
1636 9 : spdk_file_truncate_async(struct spdk_file *file, uint64_t length,
1637 : spdk_file_op_complete cb_fn, void *cb_arg)
1638 : {
1639 9 : struct spdk_filesystem *fs;
1640 9 : size_t num_clusters;
1641 9 : struct spdk_fs_request *req;
1642 9 : struct spdk_fs_cb_args *args;
1643 :
1644 9 : SPDK_DEBUGLOG(blobfs, "file=%s old=0x%jx new=0x%jx\n", file->name, file->length, length);
1645 9 : if (length == file->length) {
1646 1 : cb_fn(cb_arg, 0);
1647 1 : return;
1648 : }
1649 :
1650 8 : req = alloc_fs_request(file->fs->md_target.md_fs_channel);
1651 8 : if (req == NULL) {
1652 0 : cb_fn(cb_arg, -ENOMEM);
1653 0 : return;
1654 : }
1655 :
1656 8 : args = &req->args;
1657 8 : args->fn.file_op = cb_fn;
1658 8 : args->arg = cb_arg;
1659 8 : args->file = file;
1660 8 : args->op.truncate.length = length;
1661 8 : fs = file->fs;
1662 :
1663 8 : num_clusters = __bytes_to_clusters(length, fs->bs_opts.cluster_sz);
1664 :
1665 8 : spdk_blob_resize(file->blob, num_clusters, fs_truncate_resize_cb, req);
1666 9 : }
1667 :
1668 : static void
1669 3 : __truncate(void *arg)
1670 : {
1671 3 : struct spdk_fs_request *req = arg;
1672 3 : struct spdk_fs_cb_args *args = &req->args;
1673 :
1674 6 : spdk_file_truncate_async(args->file, args->op.truncate.length,
1675 3 : args->fn.file_op, args);
1676 3 : }
1677 :
1678 : int
1679 3 : spdk_file_truncate(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
1680 : uint64_t length)
1681 : {
1682 3 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
1683 3 : struct spdk_fs_request *req;
1684 3 : struct spdk_fs_cb_args *args;
1685 3 : int rc;
1686 :
1687 3 : req = alloc_fs_request(channel);
1688 3 : if (req == NULL) {
1689 0 : return -ENOMEM;
1690 : }
1691 :
1692 3 : args = &req->args;
1693 :
1694 3 : args->file = file;
1695 3 : args->op.truncate.length = length;
1696 3 : args->fn.file_op = __wake_caller;
1697 3 : args->sem = &channel->sem;
1698 :
1699 3 : channel->send_request(__truncate, req);
1700 3 : sem_wait(&channel->sem);
1701 3 : rc = args->rc;
1702 3 : free_fs_request(req);
1703 :
1704 3 : return rc;
1705 3 : }
1706 :
1707 : static void
1708 7 : __rw_done(void *ctx, int bserrno)
1709 : {
1710 7 : struct spdk_fs_request *req = ctx;
1711 7 : struct spdk_fs_cb_args *args = &req->args;
1712 :
1713 7 : spdk_free(args->op.rw.pin_buf);
1714 7 : args->fn.file_op(args->arg, bserrno);
1715 7 : free_fs_request(req);
1716 7 : }
1717 :
1718 : static void
1719 6 : __read_done(void *ctx, int bserrno)
1720 : {
1721 6 : struct spdk_fs_request *req = ctx;
1722 6 : struct spdk_fs_cb_args *args = &req->args;
1723 6 : void *buf;
1724 :
1725 6 : if (bserrno) {
1726 0 : __rw_done(req, bserrno);
1727 0 : return;
1728 : }
1729 :
1730 6 : assert(req != NULL);
1731 6 : buf = (void *)((uintptr_t)args->op.rw.pin_buf + (args->op.rw.offset & (args->op.rw.blocklen - 1)));
1732 6 : if (args->op.rw.is_read) {
1733 3 : spdk_copy_buf_to_iovs(args->iovs, args->iovcnt, buf, args->op.rw.length);
1734 3 : __rw_done(req, 0);
1735 3 : } else {
1736 3 : spdk_copy_iovs_to_buf(buf, args->op.rw.length, args->iovs, args->iovcnt);
1737 6 : spdk_blob_io_write(args->file->blob, args->op.rw.channel,
1738 3 : args->op.rw.pin_buf,
1739 3 : args->op.rw.start_lba, args->op.rw.num_lba,
1740 3 : __rw_done, req);
1741 : }
1742 6 : }
1743 :
1744 : static void
1745 6 : __do_blob_read(void *ctx, int fserrno)
1746 : {
1747 6 : struct spdk_fs_request *req = ctx;
1748 6 : struct spdk_fs_cb_args *args = &req->args;
1749 :
1750 6 : if (fserrno) {
1751 0 : __rw_done(req, fserrno);
1752 0 : return;
1753 : }
1754 12 : spdk_blob_io_read(args->file->blob, args->op.rw.channel,
1755 6 : args->op.rw.pin_buf,
1756 6 : args->op.rw.start_lba, args->op.rw.num_lba,
1757 6 : __read_done, req);
1758 6 : }
1759 :
1760 : static void
1761 17 : __get_page_parameters(struct spdk_file *file, uint64_t offset, uint64_t length,
1762 : uint64_t *start_lba, uint32_t *lba_size, uint64_t *num_lba)
1763 : {
1764 17 : uint64_t end_lba;
1765 :
1766 17 : *lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
1767 17 : *start_lba = offset / *lba_size;
1768 17 : end_lba = (offset + length - 1) / *lba_size;
1769 17 : *num_lba = (end_lba - *start_lba + 1);
1770 17 : }
1771 :
1772 : static bool
1773 1 : __is_lba_aligned(struct spdk_file *file, uint64_t offset, uint64_t length)
1774 : {
1775 1 : uint32_t lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
1776 :
1777 1 : if ((offset % lba_size == 0) && (length % lba_size == 0)) {
1778 1 : return true;
1779 : }
1780 :
1781 0 : return false;
1782 1 : }
1783 :
1784 : static void
1785 7 : _fs_request_setup_iovs(struct spdk_fs_request *req, struct iovec *iovs, uint32_t iovcnt)
1786 : {
1787 7 : uint32_t i;
1788 :
1789 18 : for (i = 0; i < iovcnt; i++) {
1790 11 : req->args.iovs[i].iov_base = iovs[i].iov_base;
1791 11 : req->args.iovs[i].iov_len = iovs[i].iov_len;
1792 11 : }
1793 7 : }
1794 :
1795 : static void
1796 7 : __readvwritev(struct spdk_file *file, struct spdk_io_channel *_channel,
1797 : struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
1798 : spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
1799 : {
1800 7 : struct spdk_fs_request *req;
1801 7 : struct spdk_fs_cb_args *args;
1802 7 : struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
1803 7 : uint64_t start_lba, num_lba, pin_buf_length;
1804 7 : uint32_t lba_size;
1805 :
1806 7 : if (is_read && offset + length > file->length) {
1807 0 : cb_fn(cb_arg, -EINVAL);
1808 0 : return;
1809 : }
1810 :
1811 7 : req = alloc_fs_request_with_iov(channel, iovcnt);
1812 7 : if (req == NULL) {
1813 0 : cb_fn(cb_arg, -ENOMEM);
1814 0 : return;
1815 : }
1816 :
1817 7 : __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
1818 :
1819 7 : args = &req->args;
1820 7 : args->fn.file_op = cb_fn;
1821 7 : args->arg = cb_arg;
1822 7 : args->file = file;
1823 7 : args->op.rw.channel = channel->bs_channel;
1824 7 : _fs_request_setup_iovs(req, iovs, iovcnt);
1825 7 : args->op.rw.is_read = is_read;
1826 7 : args->op.rw.offset = offset;
1827 7 : args->op.rw.blocklen = lba_size;
1828 :
1829 7 : pin_buf_length = num_lba * lba_size;
1830 7 : args->op.rw.length = pin_buf_length;
1831 7 : args->op.rw.pin_buf = spdk_malloc(pin_buf_length, lba_size, NULL,
1832 : SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
1833 7 : if (args->op.rw.pin_buf == NULL) {
1834 0 : SPDK_DEBUGLOG(blobfs, "Failed to allocate buf for: file=%s offset=%jx length=%jx\n",
1835 : file->name, offset, length);
1836 0 : free_fs_request(req);
1837 0 : cb_fn(cb_arg, -ENOMEM);
1838 0 : return;
1839 : }
1840 :
1841 7 : args->op.rw.start_lba = start_lba;
1842 7 : args->op.rw.num_lba = num_lba;
1843 :
1844 7 : if (!is_read && file->length < offset + length) {
1845 3 : spdk_file_truncate_async(file, offset + length, __do_blob_read, req);
1846 7 : } else if (!is_read && __is_lba_aligned(file, offset, length)) {
1847 1 : spdk_copy_iovs_to_buf(args->op.rw.pin_buf, args->op.rw.length, args->iovs, args->iovcnt);
1848 2 : spdk_blob_io_write(args->file->blob, args->op.rw.channel,
1849 1 : args->op.rw.pin_buf,
1850 1 : args->op.rw.start_lba, args->op.rw.num_lba,
1851 1 : __rw_done, req);
1852 1 : } else {
1853 3 : __do_blob_read(req, 0);
1854 : }
1855 7 : }
1856 :
1857 : static void
1858 3 : __readwrite(struct spdk_file *file, struct spdk_io_channel *channel,
1859 : void *payload, uint64_t offset, uint64_t length,
1860 : spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
1861 : {
1862 3 : struct iovec iov;
1863 :
1864 3 : iov.iov_base = payload;
1865 3 : iov.iov_len = (size_t)length;
1866 :
1867 3 : __readvwritev(file, channel, &iov, 1, offset, length, cb_fn, cb_arg, is_read);
1868 3 : }
1869 :
1870 : void
1871 2 : spdk_file_write_async(struct spdk_file *file, struct spdk_io_channel *channel,
1872 : void *payload, uint64_t offset, uint64_t length,
1873 : spdk_file_op_complete cb_fn, void *cb_arg)
1874 : {
1875 2 : __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 0);
1876 2 : }
1877 :
1878 : void
1879 2 : spdk_file_writev_async(struct spdk_file *file, struct spdk_io_channel *channel,
1880 : struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
1881 : spdk_file_op_complete cb_fn, void *cb_arg)
1882 : {
1883 2 : SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
1884 : file->name, offset, length);
1885 :
1886 2 : __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 0);
1887 2 : }
1888 :
1889 : void
1890 1 : spdk_file_read_async(struct spdk_file *file, struct spdk_io_channel *channel,
1891 : void *payload, uint64_t offset, uint64_t length,
1892 : spdk_file_op_complete cb_fn, void *cb_arg)
1893 : {
1894 1 : SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
1895 : file->name, offset, length);
1896 :
1897 1 : __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 1);
1898 1 : }
1899 :
1900 : void
1901 2 : spdk_file_readv_async(struct spdk_file *file, struct spdk_io_channel *channel,
1902 : struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
1903 : spdk_file_op_complete cb_fn, void *cb_arg)
1904 : {
1905 2 : SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
1906 : file->name, offset, length);
1907 :
1908 2 : __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 1);
1909 2 : }
1910 :
1911 : struct spdk_io_channel *
1912 1 : spdk_fs_alloc_io_channel(struct spdk_filesystem *fs)
1913 : {
1914 1 : struct spdk_io_channel *io_channel;
1915 1 : struct spdk_fs_channel *fs_channel;
1916 :
1917 1 : io_channel = spdk_get_io_channel(&fs->io_target);
1918 1 : fs_channel = spdk_io_channel_get_ctx(io_channel);
1919 1 : fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
1920 1 : fs_channel->send_request = __send_request_direct;
1921 :
1922 2 : return io_channel;
1923 1 : }
1924 :
1925 : void
1926 41 : spdk_fs_free_io_channel(struct spdk_io_channel *channel)
1927 : {
1928 41 : spdk_put_io_channel(channel);
1929 41 : }
1930 :
1931 : struct spdk_fs_thread_ctx *
1932 12 : spdk_fs_alloc_thread_ctx(struct spdk_filesystem *fs)
1933 : {
1934 12 : struct spdk_fs_thread_ctx *ctx;
1935 :
1936 12 : ctx = calloc(1, sizeof(*ctx));
1937 12 : if (!ctx) {
1938 0 : return NULL;
1939 : }
1940 :
1941 12 : if (pthread_spin_init(&ctx->ch.lock, 0)) {
1942 0 : free(ctx);
1943 0 : return NULL;
1944 : }
1945 :
1946 12 : fs_channel_create(fs, &ctx->ch, 512);
1947 :
1948 12 : ctx->ch.send_request = fs->send_request;
1949 12 : ctx->ch.sync = 1;
1950 :
1951 12 : return ctx;
1952 12 : }
1953 :
1954 :
1955 : void
1956 12 : spdk_fs_free_thread_ctx(struct spdk_fs_thread_ctx *ctx)
1957 : {
1958 12 : assert(ctx->ch.sync == 1);
1959 :
1960 12 : while (true) {
1961 12 : pthread_spin_lock(&ctx->ch.lock);
1962 12 : if (ctx->ch.outstanding_reqs == 0) {
1963 12 : pthread_spin_unlock(&ctx->ch.lock);
1964 12 : break;
1965 : }
1966 0 : pthread_spin_unlock(&ctx->ch.lock);
1967 0 : usleep(1000);
1968 : }
1969 :
1970 12 : fs_channel_destroy(NULL, &ctx->ch);
1971 12 : free(ctx);
1972 12 : }
1973 :
1974 : int
1975 0 : spdk_fs_set_cache_size(uint64_t size_in_mb)
1976 : {
1977 : /* setting g_fs_cache_size is only permitted if cache pool
1978 : * is already freed or hasn't been initialized
1979 : */
1980 0 : if (g_cache_pool != NULL) {
1981 0 : return -EPERM;
1982 : }
1983 :
1984 0 : g_fs_cache_size = size_in_mb * 1024 * 1024;
1985 :
1986 0 : return 0;
1987 0 : }
1988 :
1989 : uint64_t
1990 0 : spdk_fs_get_cache_size(void)
1991 : {
1992 0 : return g_fs_cache_size / (1024 * 1024);
1993 : }
1994 :
1995 : static void __file_flush(void *ctx);
1996 :
1997 : /* Try to free some cache buffers from this file.
1998 : */
1999 : static int
2000 0 : reclaim_cache_buffers(struct spdk_file *file)
2001 : {
2002 0 : int rc;
2003 :
2004 0 : BLOBFS_TRACE(file, "free=%s\n", file->name);
2005 :
2006 : /* The function is safe to be called with any threads, while the file
2007 : * lock maybe locked by other thread for now, so try to get the file
2008 : * lock here.
2009 : */
2010 0 : rc = pthread_spin_trylock(&file->lock);
2011 0 : if (rc != 0) {
2012 0 : return -1;
2013 : }
2014 :
2015 0 : if (file->tree->present_mask == 0) {
2016 0 : pthread_spin_unlock(&file->lock);
2017 0 : return -1;
2018 : }
2019 0 : tree_free_buffers(file->tree);
2020 :
2021 0 : TAILQ_REMOVE(&g_caches, file, cache_tailq);
2022 : /* If not freed, put it in the end of the queue */
2023 0 : if (file->tree->present_mask != 0) {
2024 0 : TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
2025 0 : } else {
2026 0 : file->last = NULL;
2027 : }
2028 0 : pthread_spin_unlock(&file->lock);
2029 :
2030 0 : return 0;
2031 0 : }
2032 :
2033 : static int
2034 0 : _blobfs_cache_pool_reclaim(void *arg)
2035 : {
2036 0 : struct spdk_file *file, *tmp;
2037 0 : int rc;
2038 :
2039 0 : if (!blobfs_cache_pool_need_reclaim()) {
2040 0 : return SPDK_POLLER_IDLE;
2041 : }
2042 :
2043 0 : TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
2044 0 : if (!file->open_for_writing &&
2045 0 : file->priority == SPDK_FILE_PRIORITY_LOW) {
2046 0 : rc = reclaim_cache_buffers(file);
2047 0 : if (rc < 0) {
2048 0 : continue;
2049 : }
2050 0 : if (!blobfs_cache_pool_need_reclaim()) {
2051 0 : return SPDK_POLLER_BUSY;
2052 : }
2053 0 : break;
2054 : }
2055 0 : }
2056 :
2057 0 : TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
2058 0 : if (!file->open_for_writing) {
2059 0 : rc = reclaim_cache_buffers(file);
2060 0 : if (rc < 0) {
2061 0 : continue;
2062 : }
2063 0 : if (!blobfs_cache_pool_need_reclaim()) {
2064 0 : return SPDK_POLLER_BUSY;
2065 : }
2066 0 : break;
2067 : }
2068 0 : }
2069 :
2070 0 : TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
2071 0 : rc = reclaim_cache_buffers(file);
2072 0 : if (rc < 0) {
2073 0 : continue;
2074 : }
2075 0 : break;
2076 : }
2077 :
2078 0 : return SPDK_POLLER_BUSY;
2079 0 : }
2080 :
2081 : static void
2082 5 : _add_file_to_cache_pool(void *ctx)
2083 : {
2084 5 : struct spdk_file *file = ctx;
2085 :
2086 5 : TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
2087 5 : }
2088 :
2089 : static void
2090 0 : _remove_file_from_cache_pool(void *ctx)
2091 : {
2092 0 : struct spdk_file *file = ctx;
2093 :
2094 0 : TAILQ_REMOVE(&g_caches, file, cache_tailq);
2095 0 : }
2096 :
2097 : static struct cache_buffer *
2098 10 : cache_insert_buffer(struct spdk_file *file, uint64_t offset)
2099 : {
2100 10 : struct cache_buffer *buf;
2101 10 : int count = 0;
2102 10 : bool need_update = false;
2103 :
2104 10 : buf = calloc(1, sizeof(*buf));
2105 10 : if (buf == NULL) {
2106 0 : SPDK_DEBUGLOG(blobfs, "calloc failed\n");
2107 0 : return NULL;
2108 : }
2109 :
2110 10 : do {
2111 10 : buf->buf = spdk_mempool_get(g_cache_pool);
2112 10 : if (buf->buf) {
2113 10 : break;
2114 : }
2115 0 : if (count++ == 100) {
2116 0 : SPDK_ERRLOG("Could not allocate cache buffer for file=%p on offset=%jx\n",
2117 : file, offset);
2118 0 : free(buf);
2119 0 : return NULL;
2120 : }
2121 0 : usleep(BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
2122 0 : } while (true);
2123 :
2124 10 : buf->buf_size = CACHE_BUFFER_SIZE;
2125 10 : buf->offset = offset;
2126 :
2127 10 : if (file->tree->present_mask == 0) {
2128 5 : need_update = true;
2129 5 : }
2130 10 : file->tree = tree_insert_buffer(file->tree, buf);
2131 :
2132 10 : if (need_update) {
2133 5 : spdk_thread_send_msg(g_cache_pool_thread, _add_file_to_cache_pool, file);
2134 5 : }
2135 :
2136 10 : return buf;
2137 10 : }
2138 :
2139 : static struct cache_buffer *
2140 10 : cache_append_buffer(struct spdk_file *file)
2141 : {
2142 10 : struct cache_buffer *last;
2143 :
2144 10 : assert(file->last == NULL || file->last->bytes_filled == file->last->buf_size);
2145 10 : assert((file->append_pos % CACHE_BUFFER_SIZE) == 0);
2146 :
2147 10 : last = cache_insert_buffer(file, file->append_pos);
2148 10 : if (last == NULL) {
2149 0 : SPDK_DEBUGLOG(blobfs, "cache_insert_buffer failed\n");
2150 0 : return NULL;
2151 : }
2152 :
2153 10 : file->last = last;
2154 :
2155 10 : return last;
2156 10 : }
2157 :
2158 : static void __check_sync_reqs(struct spdk_file *file);
2159 :
2160 : static void
2161 7 : __file_cache_finish_sync(void *ctx, int bserrno)
2162 : {
2163 7 : struct spdk_file *file;
2164 7 : struct spdk_fs_request *sync_req = ctx;
2165 7 : struct spdk_fs_cb_args *sync_args;
2166 :
2167 7 : sync_args = &sync_req->args;
2168 7 : file = sync_args->file;
2169 7 : pthread_spin_lock(&file->lock);
2170 7 : file->length_xattr = sync_args->op.sync.length;
2171 7 : assert(sync_args->op.sync.offset <= file->length_flushed);
2172 7 : spdk_trace_record(TRACE_BLOBFS_XATTR_END, 0, sync_args->op.sync.offset,
2173 : 0, file->name);
2174 7 : BLOBFS_TRACE(file, "sync done offset=%jx\n", sync_args->op.sync.offset);
2175 7 : TAILQ_REMOVE(&file->sync_requests, sync_req, args.op.sync.tailq);
2176 7 : pthread_spin_unlock(&file->lock);
2177 :
2178 7 : sync_args->fn.file_op(sync_args->arg, bserrno);
2179 :
2180 7 : free_fs_request(sync_req);
2181 7 : __check_sync_reqs(file);
2182 7 : }
2183 :
2184 : static void
2185 25 : __check_sync_reqs(struct spdk_file *file)
2186 : {
2187 25 : struct spdk_fs_request *sync_req;
2188 :
2189 25 : pthread_spin_lock(&file->lock);
2190 :
2191 25 : TAILQ_FOREACH(sync_req, &file->sync_requests, args.op.sync.tailq) {
2192 14 : if (sync_req->args.op.sync.offset <= file->length_flushed) {
2193 14 : break;
2194 : }
2195 0 : }
2196 :
2197 25 : if (sync_req != NULL && !sync_req->args.op.sync.xattr_in_progress) {
2198 7 : BLOBFS_TRACE(file, "set xattr length 0x%jx\n", file->length_flushed);
2199 7 : sync_req->args.op.sync.xattr_in_progress = true;
2200 7 : sync_req->args.op.sync.length = file->length_flushed;
2201 7 : spdk_blob_set_xattr(file->blob, "length", &file->length_flushed,
2202 : sizeof(file->length_flushed));
2203 :
2204 7 : pthread_spin_unlock(&file->lock);
2205 7 : spdk_trace_record(TRACE_BLOBFS_XATTR_START, 0, file->length_flushed,
2206 : 0, file->name);
2207 7 : spdk_blob_sync_md(file->blob, __file_cache_finish_sync, sync_req);
2208 7 : } else {
2209 18 : pthread_spin_unlock(&file->lock);
2210 : }
2211 25 : }
2212 :
2213 : static void
2214 10 : __file_flush_done(void *ctx, int bserrno)
2215 : {
2216 10 : struct spdk_fs_request *req = ctx;
2217 10 : struct spdk_fs_cb_args *args = &req->args;
2218 10 : struct spdk_file *file = args->file;
2219 10 : struct cache_buffer *next = args->op.flush.cache_buffer;
2220 :
2221 10 : BLOBFS_TRACE(file, "length=%jx\n", args->op.flush.length);
2222 :
2223 10 : pthread_spin_lock(&file->lock);
2224 10 : next->in_progress = false;
2225 10 : next->bytes_flushed += args->op.flush.length;
2226 10 : file->length_flushed += args->op.flush.length;
2227 10 : if (file->length_flushed > file->length) {
2228 0 : file->length = file->length_flushed;
2229 0 : }
2230 10 : if (next->bytes_flushed == next->buf_size) {
2231 5 : BLOBFS_TRACE(file, "write buffer fully flushed 0x%jx\n", file->length_flushed);
2232 5 : next = tree_find_buffer(file->tree, file->length_flushed);
2233 5 : }
2234 :
2235 : /*
2236 : * Assert that there is no cached data that extends past the end of the underlying
2237 : * blob.
2238 : */
2239 10 : assert(next == NULL || next->offset < __file_get_blob_size(file) ||
2240 : next->bytes_filled == 0);
2241 :
2242 10 : pthread_spin_unlock(&file->lock);
2243 :
2244 10 : __check_sync_reqs(file);
2245 :
2246 10 : __file_flush(req);
2247 10 : }
2248 :
2249 : static void
2250 20 : __file_flush(void *ctx)
2251 : {
2252 20 : struct spdk_fs_request *req = ctx;
2253 20 : struct spdk_fs_cb_args *args = &req->args;
2254 20 : struct spdk_file *file = args->file;
2255 20 : struct cache_buffer *next;
2256 20 : uint64_t offset, length, start_lba, num_lba;
2257 20 : uint32_t lba_size;
2258 :
2259 20 : pthread_spin_lock(&file->lock);
2260 20 : next = tree_find_buffer(file->tree, file->length_flushed);
2261 34 : if (next == NULL || next->in_progress ||
2262 19 : ((next->bytes_filled < next->buf_size) && TAILQ_EMPTY(&file->sync_requests))) {
2263 : /*
2264 : * There is either no data to flush, a flush I/O is already in
2265 : * progress, or the next buffer is partially filled but there's no
2266 : * outstanding request to sync it.
2267 : * So return immediately - if a flush I/O is in progress we will flush
2268 : * more data after that is completed, or a partial buffer will get flushed
2269 : * when it is either filled or the file is synced.
2270 : */
2271 3 : free_fs_request(req);
2272 3 : if (next == NULL) {
2273 : /*
2274 : * For cases where a file's cache was evicted, and then the
2275 : * file was later appended, we will write the data directly
2276 : * to disk and bypass cache. So just update length_flushed
2277 : * here to reflect that all data was already written to disk.
2278 : */
2279 1 : file->length_flushed = file->append_pos;
2280 1 : }
2281 3 : pthread_spin_unlock(&file->lock);
2282 3 : if (next == NULL) {
2283 : /*
2284 : * There is no data to flush, but we still need to check for any
2285 : * outstanding sync requests to make sure metadata gets updated.
2286 : */
2287 1 : __check_sync_reqs(file);
2288 1 : }
2289 3 : return;
2290 : }
2291 :
2292 17 : offset = next->offset + next->bytes_flushed;
2293 17 : length = next->bytes_filled - next->bytes_flushed;
2294 17 : if (length == 0) {
2295 7 : free_fs_request(req);
2296 7 : pthread_spin_unlock(&file->lock);
2297 : /*
2298 : * There is no data to flush, but we still need to check for any
2299 : * outstanding sync requests to make sure metadata gets updated.
2300 : */
2301 7 : __check_sync_reqs(file);
2302 7 : return;
2303 : }
2304 10 : args->op.flush.length = length;
2305 10 : args->op.flush.cache_buffer = next;
2306 :
2307 10 : __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
2308 :
2309 10 : next->in_progress = true;
2310 10 : BLOBFS_TRACE(file, "offset=0x%jx length=0x%jx page start=0x%jx num=0x%jx\n",
2311 : offset, length, start_lba, num_lba);
2312 10 : pthread_spin_unlock(&file->lock);
2313 20 : spdk_blob_io_write(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
2314 10 : next->buf + (start_lba * lba_size) - next->offset,
2315 10 : start_lba, num_lba, __file_flush_done, req);
2316 20 : }
2317 :
2318 : static void
2319 0 : __file_extend_done(void *arg, int bserrno)
2320 : {
2321 0 : struct spdk_fs_cb_args *args = arg;
2322 :
2323 0 : __wake_caller(args, bserrno);
2324 0 : }
2325 :
2326 : static void
2327 0 : __file_extend_resize_cb(void *_args, int bserrno)
2328 : {
2329 0 : struct spdk_fs_cb_args *args = _args;
2330 0 : struct spdk_file *file = args->file;
2331 :
2332 0 : if (bserrno) {
2333 0 : __wake_caller(args, bserrno);
2334 0 : return;
2335 : }
2336 :
2337 0 : spdk_blob_sync_md(file->blob, __file_extend_done, args);
2338 0 : }
2339 :
2340 : static void
2341 0 : __file_extend_blob(void *_args)
2342 : {
2343 0 : struct spdk_fs_cb_args *args = _args;
2344 0 : struct spdk_file *file = args->file;
2345 :
2346 0 : spdk_blob_resize(file->blob, args->op.resize.num_clusters, __file_extend_resize_cb, args);
2347 0 : }
2348 :
2349 : static void
2350 1 : __rw_from_file_done(void *ctx, int bserrno)
2351 : {
2352 1 : struct spdk_fs_request *req = ctx;
2353 :
2354 1 : __wake_caller(&req->args, bserrno);
2355 1 : free_fs_request(req);
2356 1 : }
2357 :
2358 : static void
2359 1 : __rw_from_file(void *ctx)
2360 : {
2361 1 : struct spdk_fs_request *req = ctx;
2362 1 : struct spdk_fs_cb_args *args = &req->args;
2363 1 : struct spdk_file *file = args->file;
2364 :
2365 1 : if (args->op.rw.is_read) {
2366 0 : spdk_file_read_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
2367 0 : args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
2368 0 : __rw_from_file_done, req);
2369 0 : } else {
2370 2 : spdk_file_write_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
2371 1 : args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
2372 1 : __rw_from_file_done, req);
2373 : }
2374 1 : }
2375 :
2376 : struct rw_from_file_arg {
2377 : struct spdk_fs_channel *channel;
2378 : int rwerrno;
2379 : };
2380 :
2381 : static int
2382 1 : __send_rw_from_file(struct spdk_file *file, void *payload,
2383 : uint64_t offset, uint64_t length, bool is_read,
2384 : struct rw_from_file_arg *arg)
2385 : {
2386 1 : struct spdk_fs_request *req;
2387 1 : struct spdk_fs_cb_args *args;
2388 :
2389 1 : req = alloc_fs_request_with_iov(arg->channel, 1);
2390 1 : if (req == NULL) {
2391 0 : sem_post(&arg->channel->sem);
2392 0 : return -ENOMEM;
2393 : }
2394 :
2395 1 : args = &req->args;
2396 1 : args->file = file;
2397 1 : args->sem = &arg->channel->sem;
2398 1 : args->iovs[0].iov_base = payload;
2399 1 : args->iovs[0].iov_len = (size_t)length;
2400 1 : args->op.rw.offset = offset;
2401 1 : args->op.rw.is_read = is_read;
2402 1 : args->rwerrno = &arg->rwerrno;
2403 1 : file->fs->send_request(__rw_from_file, req);
2404 1 : return 0;
2405 1 : }
2406 :
2407 : int
2408 11 : spdk_file_write(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
2409 : void *payload, uint64_t offset, uint64_t length)
2410 : {
2411 11 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
2412 11 : struct spdk_fs_request *flush_req;
2413 11 : uint64_t rem_length, copy, blob_size, cluster_sz;
2414 11 : uint32_t cache_buffers_filled = 0;
2415 11 : uint8_t *cur_payload;
2416 11 : struct cache_buffer *last;
2417 :
2418 11 : BLOBFS_TRACE_RW(file, "offset=%jx length=%jx\n", offset, length);
2419 :
2420 11 : if (length == 0) {
2421 1 : return 0;
2422 : }
2423 :
2424 10 : if (offset != file->append_pos) {
2425 0 : BLOBFS_TRACE(file, " error offset=%jx append_pos=%jx\n", offset, file->append_pos);
2426 0 : return -EINVAL;
2427 : }
2428 :
2429 10 : pthread_spin_lock(&file->lock);
2430 10 : file->open_for_writing = true;
2431 :
2432 10 : if ((file->last == NULL) && (file->append_pos % CACHE_BUFFER_SIZE == 0)) {
2433 5 : cache_append_buffer(file);
2434 5 : }
2435 :
2436 10 : if (file->last == NULL) {
2437 1 : struct rw_from_file_arg arg = {};
2438 1 : int rc;
2439 :
2440 1 : arg.channel = channel;
2441 1 : arg.rwerrno = 0;
2442 1 : file->append_pos += length;
2443 1 : pthread_spin_unlock(&file->lock);
2444 1 : rc = __send_rw_from_file(file, payload, offset, length, false, &arg);
2445 1 : if (rc != 0) {
2446 0 : return rc;
2447 : }
2448 1 : sem_wait(&channel->sem);
2449 1 : return arg.rwerrno;
2450 1 : }
2451 :
2452 9 : blob_size = __file_get_blob_size(file);
2453 :
2454 9 : if ((offset + length) > blob_size) {
2455 0 : struct spdk_fs_cb_args extend_args = {};
2456 :
2457 0 : cluster_sz = file->fs->bs_opts.cluster_sz;
2458 0 : extend_args.sem = &channel->sem;
2459 0 : extend_args.op.resize.num_clusters = __bytes_to_clusters((offset + length), cluster_sz);
2460 0 : extend_args.file = file;
2461 0 : BLOBFS_TRACE(file, "start resize to %u clusters\n", extend_args.op.resize.num_clusters);
2462 0 : pthread_spin_unlock(&file->lock);
2463 0 : file->fs->send_request(__file_extend_blob, &extend_args);
2464 0 : sem_wait(&channel->sem);
2465 0 : if (extend_args.rc) {
2466 0 : return extend_args.rc;
2467 : }
2468 0 : pthread_spin_lock(&file->lock);
2469 0 : }
2470 :
2471 9 : flush_req = alloc_fs_request(channel);
2472 9 : if (flush_req == NULL) {
2473 0 : pthread_spin_unlock(&file->lock);
2474 0 : return -ENOMEM;
2475 : }
2476 :
2477 9 : last = file->last;
2478 9 : rem_length = length;
2479 9 : cur_payload = payload;
2480 22 : while (rem_length > 0) {
2481 13 : copy = last->buf_size - last->bytes_filled;
2482 13 : if (copy > rem_length) {
2483 8 : copy = rem_length;
2484 8 : }
2485 13 : BLOBFS_TRACE_RW(file, " fill offset=%jx length=%jx\n", file->append_pos, copy);
2486 13 : memcpy(&last->buf[last->bytes_filled], cur_payload, copy);
2487 13 : file->append_pos += copy;
2488 13 : if (file->length < file->append_pos) {
2489 12 : file->length = file->append_pos;
2490 12 : }
2491 13 : cur_payload += copy;
2492 13 : last->bytes_filled += copy;
2493 13 : rem_length -= copy;
2494 13 : if (last->bytes_filled == last->buf_size) {
2495 5 : cache_buffers_filled++;
2496 5 : last = cache_append_buffer(file);
2497 5 : if (last == NULL) {
2498 0 : BLOBFS_TRACE(file, "nomem\n");
2499 0 : free_fs_request(flush_req);
2500 0 : pthread_spin_unlock(&file->lock);
2501 0 : return -ENOMEM;
2502 : }
2503 5 : }
2504 : }
2505 :
2506 9 : pthread_spin_unlock(&file->lock);
2507 :
2508 9 : if (cache_buffers_filled == 0) {
2509 6 : free_fs_request(flush_req);
2510 6 : return 0;
2511 : }
2512 :
2513 3 : flush_req->args.file = file;
2514 3 : file->fs->send_request(__file_flush, flush_req);
2515 3 : return 0;
2516 11 : }
2517 :
2518 : static void
2519 0 : __readahead_done(void *ctx, int bserrno)
2520 : {
2521 0 : struct spdk_fs_request *req = ctx;
2522 0 : struct spdk_fs_cb_args *args = &req->args;
2523 0 : struct cache_buffer *cache_buffer = args->op.readahead.cache_buffer;
2524 0 : struct spdk_file *file = args->file;
2525 :
2526 0 : BLOBFS_TRACE(file, "offset=%jx\n", cache_buffer->offset);
2527 :
2528 0 : pthread_spin_lock(&file->lock);
2529 0 : cache_buffer->bytes_filled = args->op.readahead.length;
2530 0 : cache_buffer->bytes_flushed = args->op.readahead.length;
2531 0 : cache_buffer->in_progress = false;
2532 0 : pthread_spin_unlock(&file->lock);
2533 :
2534 0 : free_fs_request(req);
2535 0 : }
2536 :
2537 : static void
2538 0 : __readahead(void *ctx)
2539 : {
2540 0 : struct spdk_fs_request *req = ctx;
2541 0 : struct spdk_fs_cb_args *args = &req->args;
2542 0 : struct spdk_file *file = args->file;
2543 0 : uint64_t offset, length, start_lba, num_lba;
2544 0 : uint32_t lba_size;
2545 :
2546 0 : offset = args->op.readahead.offset;
2547 0 : length = args->op.readahead.length;
2548 0 : assert(length > 0);
2549 :
2550 0 : __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
2551 :
2552 0 : BLOBFS_TRACE(file, "offset=%jx length=%jx page start=%jx num=%jx\n",
2553 : offset, length, start_lba, num_lba);
2554 0 : spdk_blob_io_read(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
2555 0 : args->op.readahead.cache_buffer->buf,
2556 0 : start_lba, num_lba, __readahead_done, req);
2557 0 : }
2558 :
2559 : static uint64_t
2560 0 : __next_cache_buffer_offset(uint64_t offset)
2561 : {
2562 0 : return (offset + CACHE_BUFFER_SIZE) & ~(CACHE_TREE_LEVEL_MASK(0));
2563 : }
2564 :
2565 : static void
2566 0 : check_readahead(struct spdk_file *file, uint64_t offset,
2567 : struct spdk_fs_channel *channel)
2568 : {
2569 0 : struct spdk_fs_request *req;
2570 0 : struct spdk_fs_cb_args *args;
2571 :
2572 0 : offset = __next_cache_buffer_offset(offset);
2573 0 : if (tree_find_buffer(file->tree, offset) != NULL || file->length <= offset) {
2574 0 : return;
2575 : }
2576 :
2577 0 : req = alloc_fs_request(channel);
2578 0 : if (req == NULL) {
2579 0 : return;
2580 : }
2581 0 : args = &req->args;
2582 :
2583 0 : BLOBFS_TRACE(file, "offset=%jx\n", offset);
2584 :
2585 0 : args->file = file;
2586 0 : args->op.readahead.offset = offset;
2587 0 : args->op.readahead.cache_buffer = cache_insert_buffer(file, offset);
2588 0 : if (!args->op.readahead.cache_buffer) {
2589 0 : BLOBFS_TRACE(file, "Cannot allocate buf for offset=%jx\n", offset);
2590 0 : free_fs_request(req);
2591 0 : return;
2592 : }
2593 :
2594 0 : args->op.readahead.cache_buffer->in_progress = true;
2595 0 : if (file->length < (offset + CACHE_BUFFER_SIZE)) {
2596 0 : args->op.readahead.length = file->length & (CACHE_BUFFER_SIZE - 1);
2597 0 : } else {
2598 0 : args->op.readahead.length = CACHE_BUFFER_SIZE;
2599 : }
2600 0 : file->fs->send_request(__readahead, req);
2601 0 : }
2602 :
2603 : int64_t
2604 1 : spdk_file_read(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
2605 : void *payload, uint64_t offset, uint64_t length)
2606 : {
2607 1 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
2608 1 : uint64_t final_offset, final_length;
2609 1 : uint32_t sub_reads = 0;
2610 1 : struct cache_buffer *buf;
2611 1 : uint64_t read_len;
2612 1 : struct rw_from_file_arg arg = {};
2613 :
2614 1 : pthread_spin_lock(&file->lock);
2615 :
2616 1 : BLOBFS_TRACE_RW(file, "offset=%ju length=%ju\n", offset, length);
2617 :
2618 1 : file->open_for_writing = false;
2619 :
2620 1 : if (length == 0 || offset >= file->append_pos) {
2621 0 : pthread_spin_unlock(&file->lock);
2622 0 : return 0;
2623 : }
2624 :
2625 1 : if (offset + length > file->append_pos) {
2626 0 : length = file->append_pos - offset;
2627 0 : }
2628 :
2629 1 : if (offset != file->next_seq_offset) {
2630 0 : file->seq_byte_count = 0;
2631 0 : }
2632 1 : file->seq_byte_count += length;
2633 1 : file->next_seq_offset = offset + length;
2634 1 : if (file->seq_byte_count >= CACHE_READAHEAD_THRESHOLD) {
2635 0 : check_readahead(file, offset, channel);
2636 0 : check_readahead(file, offset + CACHE_BUFFER_SIZE, channel);
2637 0 : }
2638 :
2639 1 : arg.channel = channel;
2640 1 : arg.rwerrno = 0;
2641 1 : final_length = 0;
2642 1 : final_offset = offset + length;
2643 2 : while (offset < final_offset) {
2644 1 : int ret = 0;
2645 1 : length = NEXT_CACHE_BUFFER_OFFSET(offset) - offset;
2646 1 : if (length > (final_offset - offset)) {
2647 1 : length = final_offset - offset;
2648 1 : }
2649 :
2650 1 : buf = tree_find_filled_buffer(file->tree, offset);
2651 1 : if (buf == NULL) {
2652 0 : pthread_spin_unlock(&file->lock);
2653 0 : ret = __send_rw_from_file(file, payload, offset, length, true, &arg);
2654 0 : pthread_spin_lock(&file->lock);
2655 0 : if (ret == 0) {
2656 0 : sub_reads++;
2657 0 : }
2658 0 : } else {
2659 1 : read_len = length;
2660 1 : if ((offset + length) > (buf->offset + buf->bytes_filled)) {
2661 0 : read_len = buf->offset + buf->bytes_filled - offset;
2662 0 : }
2663 1 : BLOBFS_TRACE(file, "read %p offset=%ju length=%ju\n", payload, offset, read_len);
2664 1 : memcpy(payload, &buf->buf[offset - buf->offset], read_len);
2665 1 : if ((offset + read_len) % CACHE_BUFFER_SIZE == 0) {
2666 0 : tree_remove_buffer(file->tree, buf);
2667 0 : if (file->tree->present_mask == 0) {
2668 0 : spdk_thread_send_msg(g_cache_pool_thread, _remove_file_from_cache_pool, file);
2669 0 : }
2670 0 : }
2671 : }
2672 :
2673 1 : if (ret == 0) {
2674 1 : final_length += length;
2675 1 : } else {
2676 0 : arg.rwerrno = ret;
2677 0 : break;
2678 : }
2679 1 : payload += length;
2680 1 : offset += length;
2681 1 : }
2682 1 : pthread_spin_unlock(&file->lock);
2683 1 : while (sub_reads > 0) {
2684 0 : sem_wait(&channel->sem);
2685 0 : sub_reads--;
2686 : }
2687 1 : if (arg.rwerrno == 0) {
2688 1 : return final_length;
2689 : } else {
2690 0 : return arg.rwerrno;
2691 : }
2692 1 : }
2693 :
2694 : static void
2695 18 : _file_sync(struct spdk_file *file, struct spdk_fs_channel *channel,
2696 : spdk_file_op_complete cb_fn, void *cb_arg)
2697 : {
2698 18 : struct spdk_fs_request *sync_req;
2699 18 : struct spdk_fs_request *flush_req;
2700 18 : struct spdk_fs_cb_args *sync_args;
2701 18 : struct spdk_fs_cb_args *flush_args;
2702 :
2703 18 : BLOBFS_TRACE(file, "offset=%jx\n", file->append_pos);
2704 :
2705 18 : pthread_spin_lock(&file->lock);
2706 18 : if (file->append_pos <= file->length_xattr) {
2707 11 : BLOBFS_TRACE(file, "done - file already synced\n");
2708 11 : pthread_spin_unlock(&file->lock);
2709 11 : cb_fn(cb_arg, 0);
2710 11 : return;
2711 : }
2712 :
2713 7 : sync_req = alloc_fs_request(channel);
2714 7 : if (!sync_req) {
2715 0 : SPDK_ERRLOG("Cannot allocate sync req for file=%s\n", file->name);
2716 0 : pthread_spin_unlock(&file->lock);
2717 0 : cb_fn(cb_arg, -ENOMEM);
2718 0 : return;
2719 : }
2720 7 : sync_args = &sync_req->args;
2721 :
2722 7 : flush_req = alloc_fs_request(channel);
2723 7 : if (!flush_req) {
2724 0 : SPDK_ERRLOG("Cannot allocate flush req for file=%s\n", file->name);
2725 0 : free_fs_request(sync_req);
2726 0 : pthread_spin_unlock(&file->lock);
2727 0 : cb_fn(cb_arg, -ENOMEM);
2728 0 : return;
2729 : }
2730 7 : flush_args = &flush_req->args;
2731 :
2732 7 : sync_args->file = file;
2733 7 : sync_args->fn.file_op = cb_fn;
2734 7 : sync_args->arg = cb_arg;
2735 7 : sync_args->op.sync.offset = file->append_pos;
2736 7 : sync_args->op.sync.xattr_in_progress = false;
2737 7 : TAILQ_INSERT_TAIL(&file->sync_requests, sync_req, args.op.sync.tailq);
2738 7 : pthread_spin_unlock(&file->lock);
2739 :
2740 7 : flush_args->file = file;
2741 7 : channel->send_request(__file_flush, flush_req);
2742 18 : }
2743 :
2744 : int
2745 12 : spdk_file_sync(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
2746 : {
2747 12 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
2748 12 : struct spdk_fs_cb_args args = {};
2749 :
2750 12 : args.sem = &channel->sem;
2751 12 : _file_sync(file, channel, __wake_caller, &args);
2752 12 : sem_wait(&channel->sem);
2753 :
2754 12 : return args.rc;
2755 12 : }
2756 :
2757 : void
2758 6 : spdk_file_sync_async(struct spdk_file *file, struct spdk_io_channel *_channel,
2759 : spdk_file_op_complete cb_fn, void *cb_arg)
2760 : {
2761 6 : struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
2762 :
2763 6 : _file_sync(file, channel, cb_fn, cb_arg);
2764 6 : }
2765 :
2766 : void
2767 0 : spdk_file_set_priority(struct spdk_file *file, uint32_t priority)
2768 : {
2769 0 : BLOBFS_TRACE(file, "priority=%u\n", priority);
2770 0 : file->priority = priority;
2771 :
2772 0 : }
2773 :
2774 : /*
2775 : * Close routines
2776 : */
2777 :
2778 : static void
2779 17 : __file_close_async_done(void *ctx, int bserrno)
2780 : {
2781 17 : struct spdk_fs_request *req = ctx;
2782 17 : struct spdk_fs_cb_args *args = &req->args;
2783 17 : struct spdk_file *file = args->file;
2784 :
2785 17 : spdk_trace_record(TRACE_BLOBFS_CLOSE, 0, 0, 0, file->name);
2786 :
2787 17 : if (file->is_deleted) {
2788 2 : spdk_fs_delete_file_async(file->fs, file->name, blob_delete_cb, ctx);
2789 2 : return;
2790 : }
2791 :
2792 15 : args->fn.file_op(args->arg, bserrno);
2793 15 : free_fs_request(req);
2794 17 : }
2795 :
2796 : static void
2797 17 : __file_close_async(struct spdk_file *file, struct spdk_fs_request *req)
2798 : {
2799 17 : struct spdk_blob *blob;
2800 :
2801 17 : pthread_spin_lock(&file->lock);
2802 17 : if (file->ref_count == 0) {
2803 0 : pthread_spin_unlock(&file->lock);
2804 0 : __file_close_async_done(req, -EBADF);
2805 0 : return;
2806 : }
2807 :
2808 17 : file->ref_count--;
2809 17 : if (file->ref_count > 0) {
2810 0 : pthread_spin_unlock(&file->lock);
2811 0 : req->args.fn.file_op(req->args.arg, 0);
2812 0 : free_fs_request(req);
2813 0 : return;
2814 : }
2815 :
2816 17 : pthread_spin_unlock(&file->lock);
2817 :
2818 17 : blob = file->blob;
2819 17 : file->blob = NULL;
2820 17 : spdk_blob_close(blob, __file_close_async_done, req);
2821 17 : }
2822 :
2823 : static void
2824 6 : __file_close_async__sync_done(void *arg, int fserrno)
2825 : {
2826 6 : struct spdk_fs_request *req = arg;
2827 6 : struct spdk_fs_cb_args *args = &req->args;
2828 :
2829 6 : __file_close_async(args->file, req);
2830 6 : }
2831 :
2832 : void
2833 6 : spdk_file_close_async(struct spdk_file *file, spdk_file_op_complete cb_fn, void *cb_arg)
2834 : {
2835 6 : struct spdk_fs_request *req;
2836 6 : struct spdk_fs_cb_args *args;
2837 :
2838 6 : req = alloc_fs_request(file->fs->md_target.md_fs_channel);
2839 6 : if (req == NULL) {
2840 0 : SPDK_ERRLOG("Cannot allocate close async req for file=%s\n", file->name);
2841 0 : cb_fn(cb_arg, -ENOMEM);
2842 0 : return;
2843 : }
2844 :
2845 6 : args = &req->args;
2846 6 : args->file = file;
2847 6 : args->fn.file_op = cb_fn;
2848 6 : args->arg = cb_arg;
2849 :
2850 6 : spdk_file_sync_async(file, file->fs->md_target.md_io_channel, __file_close_async__sync_done, req);
2851 6 : }
2852 :
2853 : static void
2854 11 : __file_close(void *arg)
2855 : {
2856 11 : struct spdk_fs_request *req = arg;
2857 11 : struct spdk_fs_cb_args *args = &req->args;
2858 11 : struct spdk_file *file = args->file;
2859 :
2860 11 : __file_close_async(file, req);
2861 11 : }
2862 :
2863 : int
2864 11 : spdk_file_close(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
2865 : {
2866 11 : struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
2867 11 : struct spdk_fs_request *req;
2868 11 : struct spdk_fs_cb_args *args;
2869 :
2870 11 : req = alloc_fs_request(channel);
2871 11 : if (req == NULL) {
2872 0 : SPDK_ERRLOG("Cannot allocate close req for file=%s\n", file->name);
2873 0 : return -ENOMEM;
2874 : }
2875 :
2876 11 : args = &req->args;
2877 :
2878 11 : spdk_file_sync(file, ctx);
2879 11 : BLOBFS_TRACE(file, "name=%s\n", file->name);
2880 11 : args->file = file;
2881 11 : args->sem = &channel->sem;
2882 11 : args->fn.file_op = __wake_caller;
2883 11 : args->arg = args;
2884 11 : channel->send_request(__file_close, req);
2885 11 : sem_wait(&channel->sem);
2886 :
2887 11 : return args->rc;
2888 11 : }
2889 :
2890 : int
2891 0 : spdk_file_get_id(struct spdk_file *file, void *id, size_t size)
2892 : {
2893 0 : if (size < sizeof(spdk_blob_id)) {
2894 0 : return -EINVAL;
2895 : }
2896 :
2897 0 : memcpy(id, &file->blobid, sizeof(spdk_blob_id));
2898 :
2899 0 : return sizeof(spdk_blob_id);
2900 0 : }
2901 :
2902 : static void
2903 5 : _file_free(void *ctx)
2904 : {
2905 5 : struct spdk_file *file = ctx;
2906 :
2907 5 : TAILQ_REMOVE(&g_caches, file, cache_tailq);
2908 :
2909 5 : free(file->name);
2910 5 : free(file->tree);
2911 5 : free(file);
2912 5 : }
2913 :
2914 : static void
2915 18 : file_free(struct spdk_file *file)
2916 : {
2917 18 : BLOBFS_TRACE(file, "free=%s\n", file->name);
2918 18 : pthread_spin_lock(&file->lock);
2919 18 : if (file->tree->present_mask == 0) {
2920 13 : pthread_spin_unlock(&file->lock);
2921 13 : free(file->name);
2922 13 : free(file->tree);
2923 13 : free(file);
2924 13 : return;
2925 : }
2926 :
2927 5 : tree_free_buffers(file->tree);
2928 5 : assert(file->tree->present_mask == 0);
2929 5 : spdk_thread_send_msg(g_cache_pool_thread, _file_free, file);
2930 5 : pthread_spin_unlock(&file->lock);
2931 18 : }
2932 :
2933 2 : SPDK_LOG_REGISTER_COMPONENT(blobfs)
2934 2 : SPDK_LOG_REGISTER_COMPONENT(blobfs_rw)
|